Merge pull request ruby#468 from rhenium/ky/openssl-3.0.0-part1

Miscellaneous changes for OpenSSL 3.0 support

Author: rhenium

ext/openssl/extconf.rb

@@ -160,7 +160,7 @@ def find_openssl_library
 have_func("TS_STATUS_INFO_get0_status")
 have_func("TS_STATUS_INFO_get0_text")
 have_func("TS_STATUS_INFO_get0_failure_info")
-have_func("TS_VERIFY_CTS_set_certs")
+have_func("TS_VERIFY_CTS_set_certs(NULL, NULL)", "openssl/ts.h")
 have_func("TS_VERIFY_CTX_set_store")
 have_func("TS_VERIFY_CTX_add_flags")
 have_func("TS_RESP_CTX_set_time_cb")
@@ -172,6 +172,13 @@ def find_openssl_library
 
 # added in 3.0.0
 have_func("SSL_set0_tmp_dh_pkey")
+have_func("ERR_get_error_all")
+have_func("TS_VERIFY_CTX_set_certs(NULL, NULL)", "openssl/ts.h")
+have_func("SSL_CTX_load_verify_file")
+have_func("BN_check_prime")
+have_func("EVP_MD_CTX_get0_md")
+have_func("EVP_MD_CTX_get_pkey_ctx")
+have_func("EVP_PKEY_eq")
 
 Logging::message "=== Checking done. ===\n"
 

ext/openssl/openssl_missing.h

@@ -21,10 +21,6 @@
 #  define EVP_MD_CTX_free EVP_MD_CTX_destroy
 #endif
 
-#if !defined(HAVE_EVP_MD_CTX_PKEY_CTX)
-#  define EVP_MD_CTX_pkey_ctx(x) (x)->pctx
-#endif
-
 #if !defined(HAVE_X509_STORE_GET_EX_DATA)
 #  define X509_STORE_get_ex_data(x, idx) \
 	CRYPTO_get_ex_data(&(x)->ex_data, (idx))
@@ -214,4 +210,29 @@ IMPL_PKEY_GETTER(EC_KEY, ec)
     } while (0)
 #endif
 
+/* added in 3.0.0 */
+#if !defined(HAVE_TS_VERIFY_CTX_SET_CERTS)
+#  define TS_VERIFY_CTX_set_certs(ctx, crts) TS_VERIFY_CTS_set_certs(ctx, crts)
+#endif
+
+#ifndef HAVE_EVP_MD_CTX_GET0_MD
+#  define EVP_MD_CTX_get0_md(ctx) EVP_MD_CTX_md(ctx)
+#endif
+
+/*
+ * OpenSSL 1.1.0 added EVP_MD_CTX_pkey_ctx(), and then it was renamed to
+ * EVP_MD_CTX_get_pkey_ctx(x) in OpenSSL 3.0.
+ */
+#ifndef HAVE_EVP_MD_CTX_GET_PKEY_CTX
+# ifdef HAVE_EVP_MD_CTX_PKEY_CTX
+#  define EVP_MD_CTX_get_pkey_ctx(x) EVP_MD_CTX_pkey_ctx(x)
+# else
+#  define EVP_MD_CTX_get_pkey_ctx(x) (x)->pctx
+# endif
+#endif
+
+#ifndef HAVE_EVP_PKEY_EQ
+#  define EVP_PKEY_eq(a, b) EVP_PKEY_cmp(a, b)
+#endif
+
 #endif /* _OSSL_OPENSSL_MISSING_H_ */

ext/openssl/ossl.c

@@ -313,27 +313,31 @@ void
 ossl_clear_error(void)
 {
     if (dOSSL == Qtrue) {
-	unsigned long e;
-	const char *file, *data, *errstr;
-	int line, flags;
-
-	while ((e = ERR_get_error_line_data(&file, &line, &data, &flags))) {
-	    errstr = ERR_error_string(e, NULL);
-	    if (!errstr)
-		errstr = "(null)";
-
-	    if (flags & ERR_TXT_STRING) {
-		if (!data)
-		    data = "(null)";
-		rb_warn("error on stack: %s (%s)", errstr, data);
-	    }
-	    else {
-		rb_warn("error on stack: %s", errstr);
-	    }
-	}
+        unsigned long e;
+        const char *file, *data, *func, *lib, *reason;
+        char append[256] = "";
+        int line, flags;
+
+#ifdef HAVE_ERR_GET_ERROR_ALL
+        while ((e = ERR_get_error_all(&file, &line, &func, &data, &flags))) {
+#else
+        while ((e = ERR_get_error_line_data(&file, &line, &data, &flags))) {
+            func = ERR_func_error_string(e);
+#endif
+            lib = ERR_lib_error_string(e);
+            reason = ERR_reason_error_string(e);
+
+            if (flags & ERR_TXT_STRING) {
+                if (!data)
+                    data = "(null)";
+                snprintf(append, sizeof(append), " (%s)", data);
+            }
+            rb_warn("error on stack: error:%08lX:%s:%s:%s%s", e, lib ? lib : "",
+                    func ? func : "", reason ? reason : "", append);
+        }
     }
     else {
-	ERR_clear_error();
+        ERR_clear_error();
     }
 }
 

ext/openssl/ossl.h

@@ -42,6 +42,18 @@
 #include <openssl/evp.h>
 #include <openssl/dh.h>
 
+#ifndef LIBRESSL_VERSION_NUMBER
+# define OSSL_IS_LIBRESSL 0
+# define OSSL_OPENSSL_PREREQ(maj, min, pat) \
+      (OPENSSL_VERSION_NUMBER >= (maj << 28) | (min << 20) | (pat << 12))
+# define OSSL_LIBRESSL_PREREQ(maj, min, pat) 0
+#else
+# define OSSL_IS_LIBRESSL 1
+# define OSSL_OPENSSL_PREREQ(maj, min, pat) 0
+# define OSSL_LIBRESSL_PREREQ(maj, min, pat) \
+      (LIBRESSL_VERSION_NUMBER >= (maj << 28) | (min << 20) | (pat << 12))
+#endif
+
 /*
  * Common Module
  */

ext/openssl/ossl_bn.c

@@ -792,78 +792,64 @@ BIGNUM_SELF_SHIFT(lshift)
  */
 BIGNUM_SELF_SHIFT(rshift)
 
-#define BIGNUM_RAND(func)					\
-    static VALUE						\
-    ossl_bn_s_##func(int argc, VALUE *argv, VALUE klass)	\
-    {								\
-	BIGNUM *result;						\
-	int bottom = 0, top = 0, b;				\
-	VALUE bits, fill, odd, obj;				\
-								\
-	switch (rb_scan_args(argc, argv, "12", &bits, &fill, &odd)) {	\
-	case 3:							\
-	    bottom = (odd == Qtrue) ? 1 : 0;			\
-	    /* FALLTHROUGH */					\
-	case 2:							\
-	    top = NUM2INT(fill);				\
-	}							\
-	b = NUM2INT(bits);					\
-	obj = NewBN(klass);					\
-	if (!(result = BN_new())) {				\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	if (BN_##func(result, b, top, bottom) <= 0) {		\
-	    BN_free(result);					\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	SetBN(obj, result);					\
-	return obj;						\
-    }
-
-/*
- * Document-method: OpenSSL::BN.rand
- *   BN.rand(bits [, fill [, odd]]) -> aBN
- */
-BIGNUM_RAND(rand)
-
 /*
- * Document-method: OpenSSL::BN.pseudo_rand
- *   BN.pseudo_rand(bits [, fill [, odd]]) -> aBN
- */
-BIGNUM_RAND(pseudo_rand)
-
-#define BIGNUM_RAND_RANGE(func)					\
-    static VALUE						\
-    ossl_bn_s_##func##_range(VALUE klass, VALUE range)		\
-    {								\
-	BIGNUM *bn = GetBNPtr(range), *result;			\
-	VALUE obj = NewBN(klass);				\
-	if (!(result = BN_new())) {				\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	if (BN_##func##_range(result, bn) <= 0) {		\
-	    BN_free(result);					\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	SetBN(obj, result);					\
-	return obj;						\
-    }
-
-/*
- * Document-method: OpenSSL::BN.rand_range
  * call-seq:
- *   BN.rand_range(range) -> aBN
+ *    BN.rand(bits [, fill [, odd]]) -> aBN
+ *
+ * Generates a cryptographically strong pseudo-random number of +bits+.
  *
+ * See also the man page BN_rand(3).
  */
-BIGNUM_RAND_RANGE(rand)
+static VALUE
+ossl_bn_s_rand(int argc, VALUE *argv, VALUE klass)
+{
+    BIGNUM *result;
+    int bottom = 0, top = 0, b;
+    VALUE bits, fill, odd, obj;
+
+    switch (rb_scan_args(argc, argv, "12", &bits, &fill, &odd)) {
+      case 3:
+        bottom = (odd == Qtrue) ? 1 : 0;
+        /* FALLTHROUGH */
+      case 2:
+        top = NUM2INT(fill);
+    }
+    b = NUM2INT(bits);
+    obj = NewBN(klass);
+    if (!(result = BN_new())) {
+        ossl_raise(eBNError, "BN_new");
+    }
+    if (BN_rand(result, b, top, bottom) <= 0) {
+        BN_free(result);
+        ossl_raise(eBNError, "BN_rand");
+    }
+    SetBN(obj, result);
+    return obj;
+}
 
 /*
- * Document-method: OpenSSL::BN.pseudo_rand_range
  * call-seq:
- *   BN.pseudo_rand_range(range) -> aBN
+ *    BN.rand_range(range) -> aBN
  *
+ * Generates a cryptographically strong pseudo-random number in the range
+ * 0...+range+.
+ *
+ * See also the man page BN_rand_range(3).
  */
-BIGNUM_RAND_RANGE(pseudo_rand)
+static VALUE
+ossl_bn_s_rand_range(VALUE klass, VALUE range)
+{
+    BIGNUM *bn = GetBNPtr(range), *result;
+    VALUE obj = NewBN(klass);
+    if (!(result = BN_new()))
+        ossl_raise(eBNError, "BN_new");
+    if (BN_rand_range(result, bn) <= 0) {
+        BN_free(result);
+        ossl_raise(eBNError, "BN_rand_range");
+    }
+    SetBN(obj, result);
+    return obj;
+}
 
 /*
  * call-seq:
@@ -1118,34 +1104,29 @@ ossl_bn_hash(VALUE self)
  *    bn.prime? => true | false
  *    bn.prime?(checks) => true | false
  *
- * Performs a Miller-Rabin probabilistic primality test with _checks_
- * iterations. If _checks_ is not specified, a number of iterations is used
- * that yields a false positive rate of at most 2^-80 for random input.
+ * Performs a Miller-Rabin probabilistic primality test for +bn+.
  *
- * === Parameters
- * * _checks_ - integer
+ * <b>+checks+ parameter is deprecated in version 3.0.</b> It has no effect.
  */
 static VALUE
 ossl_bn_is_prime(int argc, VALUE *argv, VALUE self)
 {
     BIGNUM *bn;
-    VALUE vchecks;
-    int checks = BN_prime_checks;
+    int ret;
 
-    if (rb_scan_args(argc, argv, "01", &vchecks) == 1) {
-	checks = NUM2INT(vchecks);
-    }
+    rb_check_arity(argc, 0, 1);
     GetBN(self, bn);
-    switch (BN_is_prime_ex(bn, checks, ossl_bn_ctx, NULL)) {
-    case 1:
-	return Qtrue;
-    case 0:
-	return Qfalse;
-    default:
-	ossl_raise(eBNError, NULL);
-    }
-    /* not reachable */
-    return Qnil;
+
+#ifdef HAVE_BN_CHECK_PRIME
+    ret = BN_check_prime(bn, ossl_bn_ctx, NULL);
+    if (ret < 0)
+        ossl_raise(eBNError, "BN_check_prime");
+#else
+    ret = BN_is_prime_fasttest_ex(bn, BN_prime_checks, ossl_bn_ctx, 1, NULL);
+    if (ret < 0)
+        ossl_raise(eBNError, "BN_is_prime_fasttest_ex");
+#endif
+    return ret ? Qtrue : Qfalse;
 }
 
 /*
@@ -1154,40 +1135,17 @@ ossl_bn_is_prime(int argc, VALUE *argv, VALUE self)
  *    bn.prime_fasttest?(checks) => true | false
  *    bn.prime_fasttest?(checks, trial_div) => true | false
  *
- * Performs a Miller-Rabin primality test. This is same as #prime? except this
- * first attempts trial divisions with some small primes.
+ * Performs a Miller-Rabin probabilistic primality test for +bn+.
  *
- * === Parameters
- * * _checks_ - integer
- * * _trial_div_ - boolean
+ * <b>Deprecated in version 3.0.</b> Use #prime? instead.
+ *
+ * +checks+ and +trial_div+ parameters no longer have any effect.
  */
 static VALUE
 ossl_bn_is_prime_fasttest(int argc, VALUE *argv, VALUE self)
 {
-    BIGNUM *bn;
-    VALUE vchecks, vtrivdiv;
-    int checks = BN_prime_checks, do_trial_division = 1;
-
-    rb_scan_args(argc, argv, "02", &vchecks, &vtrivdiv);
-
-    if (!NIL_P(vchecks)) {
-	checks = NUM2INT(vchecks);
-    }
-    GetBN(self, bn);
-    /* handle true/false */
-    if (vtrivdiv == Qfalse) {
-	do_trial_division = 0;
-    }
-    switch (BN_is_prime_fasttest_ex(bn, checks, ossl_bn_ctx, do_trial_division, NULL)) {
-    case 1:
-	return Qtrue;
-    case 0:
-	return Qfalse;
-    default:
-	ossl_raise(eBNError, NULL);
-    }
-    /* not reachable */
-    return Qnil;
+    rb_check_arity(argc, 0, 2);
+    return ossl_bn_is_prime(0, argv, self);
 }
 
 /*
@@ -1306,9 +1264,9 @@ Init_ossl_bn(void)
      * get_word */
 
     rb_define_singleton_method(cBN, "rand", ossl_bn_s_rand, -1);
-    rb_define_singleton_method(cBN, "pseudo_rand", ossl_bn_s_pseudo_rand, -1);
     rb_define_singleton_method(cBN, "rand_range", ossl_bn_s_rand_range, 1);
-    rb_define_singleton_method(cBN, "pseudo_rand_range", ossl_bn_s_pseudo_rand_range, 1);
+    rb_define_alias(rb_singleton_class(cBN), "pseudo_rand", "rand");
+    rb_define_alias(rb_singleton_class(cBN), "pseudo_rand_range", "rand_range");
 
     rb_define_singleton_method(cBN, "generate_prime", ossl_bn_s_generate_prime, -1);
     rb_define_method(cBN, "prime?", ossl_bn_is_prime, -1);

ext/openssl/ossl_digest.c

@@ -63,7 +63,7 @@ ossl_evp_get_digestbyname(VALUE obj)
 
         GetDigest(obj, ctx);
 
-        md = EVP_MD_CTX_md(ctx);
+        md = EVP_MD_CTX_get0_md(ctx);
     }
 
     return md;
@@ -176,7 +176,7 @@ ossl_digest_reset(VALUE self)
     EVP_MD_CTX *ctx;
 
     GetDigest(self, ctx);
-    if (EVP_DigestInit_ex(ctx, EVP_MD_CTX_md(ctx), NULL) != 1) {
+    if (EVP_DigestInit_ex(ctx, EVP_MD_CTX_get0_md(ctx), NULL) != 1) {
 	ossl_raise(eDigestError, "Digest initialization failed.");
     }
 
@@ -259,7 +259,7 @@ ossl_digest_name(VALUE self)
 
     GetDigest(self, ctx);
 
-    return rb_str_new2(EVP_MD_name(EVP_MD_CTX_md(ctx)));
+    return rb_str_new_cstr(EVP_MD_name(EVP_MD_CTX_get0_md(ctx)));
 }
 
 /*