Fix OpenSSL::PKey.read that cannot parse PKey in the FIPS mode.

.github/workflows/test.yml

@@ -149,5 +149,7 @@ jobs:
  # Run only the passing tests on the FIPS mode as a temporary workaround.
  # TODO Fix other tests, and run all the tests on FIPS mode.
  - name: test on fips mode
- run: ruby -Ilib test/openssl/test_fips.rb
+ run: |
+ ruby -I./lib -ropenssl \
+ -e 'Dir.glob "./test/openssl/{test_fips.rb,test_pkey.rb}", &method(:require)'
  if: matrix.fips_enabled

ext/openssl/ossl_pkey.c

@@ -101,10 +101,9 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass)
  goto out;
  OSSL_BIO_reset(bio);
 
- /* Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed */
- if (OSSL_DECODER_CTX_set_input_type(dctx, "PEM") != 1)
- goto out;
  /*
+ * Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed.
+ *
  * First check for private key formats. This is to keep compatibility with
  * ruby/openssl < 3.0 which decoded the following as a private key.
  *
@@ -124,8 +123,19 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass)
  *
  * Note that normally, the input is supposed to contain a single decodable
  * PEM block only, so this special handling should not create a new problem.
+ *
+ * Note that we need to create the OSSL_DECODER_CTX variable each time when
+ * we use the different selection as a workaround.
+ * https://github.com/openssl/openssl/issues/20657
  */
- OSSL_DECODER_CTX_set_selection(dctx, EVP_PKEY_KEYPAIR);
+ OSSL_DECODER_CTX_free(dctx);
+ dctx = NULL;
+ dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL,
+ EVP_PKEY_KEYPAIR, NULL, NULL);
+ if (!dctx)
+ goto out;
+ if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
+ goto out;
  while (1) {
  if (OSSL_DECODER_from_bio(dctx, bio) == 1)
  goto out;
@@ -139,7 +149,13 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass)
  }
 
  OSSL_BIO_reset(bio);
- OSSL_DECODER_CTX_set_selection(dctx, 0);
+ OSSL_DECODER_CTX_free(dctx);
+ dctx = NULL;
+ dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL, 0, NULL, NULL);
+ if (!dctx)
+ goto out;
+ if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
+ goto out;
  while (1) {
  if (OSSL_DECODER_from_bio(dctx, bio) == 1)
  goto out;

test/openssl/test_pkey.rb

@@ -82,6 +82,9 @@ def test_hmac_sign_verify
  end
 
  def test_ed25519
+ # https://github.com/openssl/openssl/issues/20758
+ pend('Not supported on FIPS mode enabled') if OpenSSL.fips_mode
+
  # Test vector from RFC 8032 Section 7.1 TEST 2
  priv_pem = <<~EOF
  -----BEGIN PRIVATE KEY-----
@@ -127,6 +130,8 @@ def test_ed25519
  end
 
  def test_x25519
+ pend('Not supported on FIPS mode enabled') if OpenSSL.fips_mode
+
  # Test vector from RFC 7748 Section 6.1
  alice_pem = <<~EOF
  -----BEGIN PRIVATE KEY-----
@@ -153,6 +158,8 @@ def test_x25519
  end
 
  def test_compare?
+ pend('Not supported on FIPS mode enabled') if OpenSSL.fips_mode
+
  key1 = Fixtures.pkey("rsa1024")
  key2 = Fixtures.pkey("rsa1024")
  key3 = Fixtures.pkey("rsa2048")

test/openssl/utils.rb

@@ -1,11 +1,6 @@
 # frozen_string_literal: true
 begin
  require "openssl"
-
- # Disable FIPS mode for tests for installations
- # where FIPS mode would be enabled by default.
- # Has no effect on all other installations.
- OpenSSL.fips_mode=false
 rescue LoadError
 end