Miscellaneous changes for OpenSSL 3.0 support

ext/openssl/extconf.rb

@@ -160,7 +160,7 @@ def find_openssl_library
 have_func("TS_STATUS_INFO_get0_status")
 have_func("TS_STATUS_INFO_get0_text")
 have_func("TS_STATUS_INFO_get0_failure_info")
-have_func("TS_VERIFY_CTS_set_certs")
+have_func("TS_VERIFY_CTS_set_certs(NULL, NULL)", "openssl/ts.h")
 have_func("TS_VERIFY_CTX_set_store")
 have_func("TS_VERIFY_CTX_add_flags")
 have_func("TS_RESP_CTX_set_time_cb")
@@ -172,6 +172,13 @@ def find_openssl_library
 
 # added in 3.0.0
 have_func("SSL_set0_tmp_dh_pkey")
+have_func("ERR_get_error_all")
+have_func("TS_VERIFY_CTX_set_certs(NULL, NULL)", "openssl/ts.h")
+have_func("SSL_CTX_load_verify_file")
+have_func("BN_check_prime")
+have_func("EVP_MD_CTX_get0_md")
+have_func("EVP_MD_CTX_get_pkey_ctx")
+have_func("EVP_PKEY_eq")
 
 Logging::message "=== Checking done. ===\n"
 

ext/openssl/openssl_missing.h

@@ -21,10 +21,6 @@
 # define EVP_MD_CTX_free EVP_MD_CTX_destroy
 #endif
 
-#if !defined(HAVE_EVP_MD_CTX_PKEY_CTX)
-# define EVP_MD_CTX_pkey_ctx(x) (x)->pctx
-#endif
-
 #if !defined(HAVE_X509_STORE_GET_EX_DATA)
 # define X509_STORE_get_ex_data(x, idx) \
  CRYPTO_get_ex_data(&(x)->ex_data, (idx))
@@ -214,4 +210,29 @@ IMPL_PKEY_GETTER(EC_KEY, ec)
  } while (0)
 #endif
 
+/* added in 3.0.0 */
+#if !defined(HAVE_TS_VERIFY_CTX_SET_CERTS)
+# define TS_VERIFY_CTX_set_certs(ctx, crts) TS_VERIFY_CTS_set_certs(ctx, crts)
+#endif
+
+#ifndef HAVE_EVP_MD_CTX_GET0_MD
+# define EVP_MD_CTX_get0_md(ctx) EVP_MD_CTX_md(ctx)
+#endif
+
+/*
+ * OpenSSL 1.1.0 added EVP_MD_CTX_pkey_ctx(), and then it was renamed to
+ * EVP_MD_CTX_get_pkey_ctx(x) in OpenSSL 3.0.
+ */
+#ifndef HAVE_EVP_MD_CTX_GET_PKEY_CTX
+# ifdef HAVE_EVP_MD_CTX_PKEY_CTX
+# define EVP_MD_CTX_get_pkey_ctx(x) EVP_MD_CTX_pkey_ctx(x)
+# else
+# define EVP_MD_CTX_get_pkey_ctx(x) (x)->pctx
+# endif
+#endif
+
+#ifndef HAVE_EVP_PKEY_EQ
+# define EVP_PKEY_eq(a, b) EVP_PKEY_cmp(a, b)
+#endif
+
 #endif /* _OSSL_OPENSSL_MISSING_H_ */

ext/openssl/ossl.c

@@ -313,27 +313,31 @@ void
 ossl_clear_error(void)
 {
  if (dOSSL == Qtrue) {
- unsigned long e;
- const char *file, *data, *errstr;
- int line, flags;
-
- while ((e = ERR_get_error_line_data(&file, &line, &data, &flags))) {
- errstr = ERR_error_string(e, NULL);
- if (!errstr)
- errstr = "(null)";
-
- if (flags & ERR_TXT_STRING) {
- if (!data)
- data = "(null)";
- rb_warn("error on stack: %s (%s)", errstr, data);
- }
- else {
- rb_warn("error on stack: %s", errstr);
- }
- }
+ unsigned long e;
+ const char *file, *data, *func, *lib, *reason;
+ char append[256] = "";
+ int line, flags;
+
+#ifdef HAVE_ERR_GET_ERROR_ALL
+ while ((e = ERR_get_error_all(&file, &line, &func, &data, &flags))) {
+#else
+ while ((e = ERR_get_error_line_data(&file, &line, &data, &flags))) {
+ func = ERR_func_error_string(e);
+#endif
+ lib = ERR_lib_error_string(e);
+ reason = ERR_reason_error_string(e);
+
+ if (flags & ERR_TXT_STRING) {
+ if (!data)
+ data = "(null)";
+ snprintf(append, sizeof(append), " (%s)", data);
+ }
+ rb_warn("error on stack: error:%08lX:%s:%s:%s%s", e, lib ? lib : "",
+ func ? func : "", reason ? reason : "", append);
+ }
  }
  else {
- ERR_clear_error();
+ ERR_clear_error();
  }
 }
 

ext/openssl/ossl.h

@@ -42,6 +42,18 @@
 #include <openssl/evp.h>
 #include <openssl/dh.h>
 
+#ifndef LIBRESSL_VERSION_NUMBER
+# define OSSL_IS_LIBRESSL 0
+# define OSSL_OPENSSL_PREREQ(maj, min, pat) \
+ (OPENSSL_VERSION_NUMBER >= (maj << 28) | (min << 20) | (pat << 12))
+# define OSSL_LIBRESSL_PREREQ(maj, min, pat) 0
+#else
+# define OSSL_IS_LIBRESSL 1
+# define OSSL_OPENSSL_PREREQ(maj, min, pat) 0
+# define OSSL_LIBRESSL_PREREQ(maj, min, pat) \
+ (LIBRESSL_VERSION_NUMBER >= (maj << 28) | (min << 20) | (pat << 12))
+#endif
+
 /*
  * Common Module
  */

ext/openssl/ossl_bn.c

@@ -792,78 +792,64 @@ BIGNUM_SELF_SHIFT(lshift)
  */
 BIGNUM_SELF_SHIFT(rshift)
 
-#define BIGNUM_RAND(func) \
- static VALUE \
- ossl_bn_s_##func(int argc, VALUE *argv, VALUE klass) \
- { \
- BIGNUM *result; \
- int bottom = 0, top = 0, b; \
- VALUE bits, fill, odd, obj; \
- \
- switch (rb_scan_args(argc, argv, "12", &bits, &fill, &odd)) { \
- case 3: \
- bottom = (odd == Qtrue) ? 1 : 0; \
- /* FALLTHROUGH */ \
- case 2: \
- top = NUM2INT(fill); \
- } \
- b = NUM2INT(bits); \
- obj = NewBN(klass); \
- if (!(result = BN_new())) { \
- ossl_raise(eBNError, NULL); \
- } \
- if (BN_##func(result, b, top, bottom) <= 0) { \
- BN_free(result); \
- ossl_raise(eBNError, NULL); \
- } \
- SetBN(obj, result); \
- return obj; \
- }
-
-/*
- * Document-method: OpenSSL::BN.rand
- * BN.rand(bits [, fill [, odd]]) -> aBN
- */
-BIGNUM_RAND(rand)
-
 /*
- * Document-method: OpenSSL::BN.pseudo_rand
- * BN.pseudo_rand(bits [, fill [, odd]]) -> aBN
- */
-BIGNUM_RAND(pseudo_rand)
-
-#define BIGNUM_RAND_RANGE(func) \
- static VALUE \
- ossl_bn_s_##func##_range(VALUE klass, VALUE range) \
- { \
- BIGNUM *bn = GetBNPtr(range), *result; \
- VALUE obj = NewBN(klass); \
- if (!(result = BN_new())) { \
- ossl_raise(eBNError, NULL); \
- } \
- if (BN_##func##_range(result, bn) <= 0) { \
- BN_free(result); \
- ossl_raise(eBNError, NULL); \
- } \
- SetBN(obj, result); \
- return obj; \
- }
-
-/*
- * Document-method: OpenSSL::BN.rand_range
  * call-seq:
- * BN.rand_range(range) -> aBN
+ * BN.rand(bits [, fill [, odd]]) -> aBN
+ *
+ * Generates a cryptographically strong pseudo-random number of +bits+.
  *
+ * See also the man page BN_rand(3).
  */
-BIGNUM_RAND_RANGE(rand)
+static VALUE
+ossl_bn_s_rand(int argc, VALUE *argv, VALUE klass)
+{
+ BIGNUM *result;
+ int bottom = 0, top = 0, b;
+ VALUE bits, fill, odd, obj;
+
+ switch (rb_scan_args(argc, argv, "12", &bits, &fill, &odd)) {
+ case 3:
+ bottom = (odd == Qtrue) ? 1 : 0;
+ /* FALLTHROUGH */
+ case 2:
+ top = NUM2INT(fill);
+ }
+ b = NUM2INT(bits);
+ obj = NewBN(klass);
+ if (!(result = BN_new())) {
+ ossl_raise(eBNError, "BN_new");
+ }
+ if (BN_rand(result, b, top, bottom) <= 0) {
+ BN_free(result);
+ ossl_raise(eBNError, "BN_rand");
+ }
+ SetBN(obj, result);
+ return obj;
+}
 
 /*
- * Document-method: OpenSSL::BN.pseudo_rand_range
  * call-seq:
- * BN.pseudo_rand_range(range) -> aBN
+ *  BN.rand_range(range) -> aBN
  *
+ * Generates a cryptographically strong pseudo-random number in the range
+ * 0...+range+.
+ *
+ * See also the man page BN_rand_range(3).
  */
-BIGNUM_RAND_RANGE(pseudo_rand)
+static VALUE
+ossl_bn_s_rand_range(VALUE klass, VALUE range)
+{
+ BIGNUM *bn = GetBNPtr(range), *result;
+ VALUE obj = NewBN(klass);
+ if (!(result = BN_new()))
+ ossl_raise(eBNError, "BN_new");
+ if (BN_rand_range(result, bn) <= 0) {
+ BN_free(result);
+ ossl_raise(eBNError, "BN_rand_range");
+ }
+ SetBN(obj, result);
+ return obj;
+}
 
 /*
  * call-seq:
@@ -1118,34 +1104,29 @@ ossl_bn_hash(VALUE self)
  * bn.prime? => true | false
  * bn.prime?(checks) => true | false
  *
- * Performs a Miller-Rabin probabilistic primality test with _checks_
- * iterations. If _checks_ is not specified, a number of iterations is used
- * that yields a false positive rate of at most 2^-80 for random input.
+ * Performs a Miller-Rabin probabilistic primality test for +bn+.
  *
- * === Parameters
- * * _checks_ - integer
+ * <b>+checks+ parameter is deprecated in version 3.0.</b> It has no effect.
  */
 static VALUE
 ossl_bn_is_prime(int argc, VALUE *argv, VALUE self)
 {
  BIGNUM *bn;
- VALUE vchecks;
- int checks = BN_prime_checks;
+ int ret;
 
- if (rb_scan_args(argc, argv, "01", &vchecks) == 1) {
- checks = NUM2INT(vchecks);
- }
+ rb_check_arity(argc, 0, 1);
  GetBN(self, bn);
- switch (BN_is_prime_ex(bn, checks, ossl_bn_ctx, NULL)) {
- case 1:
- return Qtrue;
- case 0:
- return Qfalse;
- default:
- ossl_raise(eBNError, NULL);
- }
- /* not reachable */
- return Qnil;
+
+#ifdef HAVE_BN_CHECK_PRIME
+ ret = BN_check_prime(bn, ossl_bn_ctx, NULL);
+ if (ret < 0)
+ ossl_raise(eBNError, "BN_check_prime");
+#else
+ ret = BN_is_prime_fasttest_ex(bn, BN_prime_checks, ossl_bn_ctx, 1, NULL);
+ if (ret < 0)
+ ossl_raise(eBNError, "BN_is_prime_fasttest_ex");
+#endif
+ return ret ? Qtrue : Qfalse;
 }
 
 /*
@@ -1154,40 +1135,17 @@ ossl_bn_is_prime(int argc, VALUE *argv, VALUE self)
  * bn.prime_fasttest?(checks) => true | false
  * bn.prime_fasttest?(checks, trial_div) => true | false
  *
- * Performs a Miller-Rabin primality test. This is same as #prime? except this
- * first attempts trial divisions with some small primes.
+ * Performs a Miller-Rabin probabilistic primality test for +bn+.
  *
- * === Parameters
- * * _checks_ - integer
- * * _trial_div_ - boolean
+ * <b>Deprecated in version 3.0.</b> Use #prime? instead.
+ *
+ * +checks+ and +trial_div+ parameters no longer have any effect.
  */
 static VALUE
 ossl_bn_is_prime_fasttest(int argc, VALUE *argv, VALUE self)
 {
- BIGNUM *bn;
- VALUE vchecks, vtrivdiv;
- int checks = BN_prime_checks, do_trial_division = 1;
-
- rb_scan_args(argc, argv, "02", &vchecks, &vtrivdiv);
-
- if (!NIL_P(vchecks)) {
- checks = NUM2INT(vchecks);
- }
- GetBN(self, bn);
- /* handle true/false */
- if (vtrivdiv == Qfalse) {
- do_trial_division = 0;
- }
- switch (BN_is_prime_fasttest_ex(bn, checks, ossl_bn_ctx, do_trial_division, NULL)) {
- case 1:
- return Qtrue;
- case 0:
- return Qfalse;
- default:
- ossl_raise(eBNError, NULL);
- }
- /* not reachable */
- return Qnil;
+ rb_check_arity(argc, 0, 2);
+ return ossl_bn_is_prime(0, argv, self);
 }
 
 /*
@@ -1306,9 +1264,9 @@ Init_ossl_bn(void)
  * get_word */
 
  rb_define_singleton_method(cBN, "rand", ossl_bn_s_rand, -1);
- rb_define_singleton_method(cBN, "pseudo_rand", ossl_bn_s_pseudo_rand, -1);
  rb_define_singleton_method(cBN, "rand_range", ossl_bn_s_rand_range, 1);
- rb_define_singleton_method(cBN, "pseudo_rand_range", ossl_bn_s_pseudo_rand_range, 1);
+ rb_define_alias(rb_singleton_class(cBN), "pseudo_rand", "rand");
+ rb_define_alias(rb_singleton_class(cBN), "pseudo_rand_range", "rand_range");
 
  rb_define_singleton_method(cBN, "generate_prime", ossl_bn_s_generate_prime, -1);
  rb_define_method(cBN, "prime?", ossl_bn_is_prime, -1);

ext/openssl/ossl_digest.c

@@ -63,7 +63,7 @@ ossl_evp_get_digestbyname(VALUE obj)
 
  GetDigest(obj, ctx);
 
- md = EVP_MD_CTX_md(ctx);
+ md = EVP_MD_CTX_get0_md(ctx);
  }
 
  return md;
@@ -176,7 +176,7 @@ ossl_digest_reset(VALUE self)
  EVP_MD_CTX *ctx;
 
  GetDigest(self, ctx);
- if (EVP_DigestInit_ex(ctx, EVP_MD_CTX_md(ctx), NULL) != 1) {
+ if (EVP_DigestInit_ex(ctx, EVP_MD_CTX_get0_md(ctx), NULL) != 1) {
  ossl_raise(eDigestError, "Digest initialization failed.");
  }
 
@@ -259,7 +259,7 @@ ossl_digest_name(VALUE self)
 
  GetDigest(self, ctx);
 
- return rb_str_new2(EVP_MD_name(EVP_MD_CTX_md(ctx)));
+ return rb_str_new_cstr(EVP_MD_name(EVP_MD_CTX_get0_md(ctx)));
 }
 
 /*