pkey: deprecate PKey::*#set_* and PKey::{DH,EC}#generate_key!

ext/openssl/ossl_pkey.h

@@ -124,6 +124,7 @@ static VALUE ossl_##_keytype##_get_##_name(VALUE self)			\
 	OSSL_PKEY_BN_DEF_GETTER0(_keytype, _type, a2,			\
 		_type##_get0_##_group(obj, NULL, &bn))
 
+#if !OSSL_OPENSSL_PREREQ(3, 0, 0)
 #define OSSL_PKEY_BN_DEF_SETTER3(_keytype, _type, _group, a1, a2, a3)	\
 /*									\
  *  call-seq:								\
@@ -181,6 +182,21 @@ static VALUE ossl_##_keytype##_set_##_group(VALUE self, VALUE v1, VALUE v2) \
 	}								\
 	return self;							\
 }
+#else
+#define OSSL_PKEY_BN_DEF_SETTER3(_keytype, _type, _group, a1, a2, a3)	\
+static VALUE ossl_##_keytype##_set_##_group(VALUE self, VALUE v1, VALUE v2, VALUE v3) \
+{									\
+        rb_raise(ePKeyError,						\
+                 #_keytype"#set_"#_group"= is incompatible with OpenSSL 3.0"); \
+}
+
+#define OSSL_PKEY_BN_DEF_SETTER2(_keytype, _type, _group, a1, a2)	\
+static VALUE ossl_##_keytype##_set_##_group(VALUE self, VALUE v1, VALUE v2) \
+{									\
+        rb_raise(ePKeyError,						\
+                 #_keytype"#set_"#_group"= is incompatible with OpenSSL 3.0"); \
+}
+#endif
 
 #define OSSL_PKEY_BN_DEF3(_keytype, _type, _group, a1, a2, a3)		\
 	OSSL_PKEY_BN_DEF_GETTER3(_keytype, _type, _group, a1, a2, a3)	\

ext/openssl/ossl_pkey_dh.c

@@ -58,15 +58,16 @@ VALUE eDHError;
  *
  * Examples:
  *   # Creating an instance from scratch
- *   dh = DH.new
+ *   # Note that this is deprecated and will not work on OpenSSL 3.0 or later.
+ *   dh = OpenSSL::PKey::DH.new
  *   dh.set_pqg(bn_p, nil, bn_g)
  *
  *   # Generating a parameters and a key pair
- *   dh = DH.new(2048) # An alias of DH.generate(2048)
+ *   dh = OpenSSL::PKey::DH.new(2048) # An alias of OpenSSL::PKey::DH.generate(2048)
  *
  *   # Reading DH parameters
- *   dh = DH.new(File.read('parameters.pem')) # -> dh, but no public/private key yet
- *   dh.generate_key! # -> dh with public and private key
+ *   dh_params = OpenSSL::PKey::DH.new(File.read('parameters.pem')) # loads parameters only
+ *   dh = OpenSSL::PKey.generate_key(dh_params) # generates a key pair
  */
 static VALUE
 ossl_dh_initialize(int argc, VALUE *argv, VALUE self)

ext/openssl/ossl_pkey_ec.c

@@ -237,6 +237,9 @@ ossl_ec_key_get_group(VALUE self)
 static VALUE
 ossl_ec_key_set_group(VALUE self, VALUE group_v)
 {
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+    rb_raise(ePKeyError, "pkeys are immutable on OpenSSL 3.0");
+#else
     EC_KEY *ec;
     EC_GROUP *group;
 
@@ -247,6 +250,7 @@ ossl_ec_key_set_group(VALUE self, VALUE group_v)
         ossl_raise(eECError, "EC_KEY_set_group");
 
     return group_v;
+#endif
 }
 
 /*
@@ -275,6 +279,9 @@ static VALUE ossl_ec_key_get_private_key(VALUE self)
  */
 static VALUE ossl_ec_key_set_private_key(VALUE self, VALUE private_key)
 {
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+    rb_raise(ePKeyError, "pkeys are immutable on OpenSSL 3.0");
+#else
     EC_KEY *ec;
     BIGNUM *bn = NULL;
 
@@ -294,6 +301,7 @@ static VALUE ossl_ec_key_set_private_key(VALUE self, VALUE private_key)
     }
 
     return private_key;
+#endif
 }
 
 /*
@@ -322,6 +330,9 @@ static VALUE ossl_ec_key_get_public_key(VALUE self)
  */
 static VALUE ossl_ec_key_set_public_key(VALUE self, VALUE public_key)
 {
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+    rb_raise(ePKeyError, "pkeys are immutable on OpenSSL 3.0");
+#else
     EC_KEY *ec;
     EC_POINT *point = NULL;
 
@@ -341,6 +352,7 @@ static VALUE ossl_ec_key_set_public_key(VALUE self, VALUE public_key)
     }
 
     return public_key;
+#endif
 }
 
 /*
@@ -430,13 +442,17 @@ ossl_ec_key_to_der(VALUE self)
  */
 static VALUE ossl_ec_key_generate_key(VALUE self)
 {
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+    rb_raise(ePKeyError, "pkeys are immutable on OpenSSL 3.0");
+#else
     EC_KEY *ec;
 
     GetEC(self, ec);
     if (EC_KEY_generate_key(ec) != 1)
 	ossl_raise(eECError, "EC_KEY_generate_key");
 
     return self;
+#endif
 }
 
 /*

lib/openssl/pkey.rb

@@ -47,9 +47,19 @@ def public_key
     # * _pub_bn_ is a OpenSSL::BN, *not* the DH instance returned by
     #   DH#public_key as that contains the DH parameters only.
     def compute_key(pub_bn)
-      peer = dup
-      peer.set_key(pub_bn, nil)
-      derive(peer)
+      # FIXME: This is constructing an X.509 SubjectPublicKeyInfo and is very
+      # inefficient
+      obj = OpenSSL::ASN1.Sequence([
+        OpenSSL::ASN1.Sequence([
+          OpenSSL::ASN1.ObjectId("dhKeyAgreement"),
+          OpenSSL::ASN1.Sequence([
+            OpenSSL::ASN1.Integer(p),
+            OpenSSL::ASN1.Integer(g),
+          ]),
+        ]),
+        OpenSSL::ASN1.BitString(OpenSSL::ASN1.Integer(pub_bn).to_der),
+      ])
+      derive(OpenSSL::PKey.read(obj.to_der))
     end
 
     # :call-seq:
@@ -61,14 +71,29 @@ def compute_key(pub_bn)
     # called first in order to generate the per-session keys before performing
     # the actual key exchange.
     #
+    # <b>Deprecated in version 3.0</b>. This method is incompatible with
+    # OpenSSL 3.0.0 or later.
+    #
     # See also OpenSSL::PKey.generate_key.
     #
     # Example:
-    #   dh = OpenSSL::PKey::DH.new(2048)
-    #   public_key = dh.public_key #contains no private/public key yet
-    #   public_key.generate_key!
-    #   puts public_key.private? # => true
+    #   # DEPRECATED USAGE: This will not work on OpenSSL 3.0 or later
+    #   dh0 = OpenSSL::PKey::DH.new(2048)
+    #   dh = dh0.public_key # #public_key only copies the DH parameters (contrary to the name)
+    #   dh.generate_key!
+    #   puts dh.private? # => true
+    #   puts dh0.pub_key == dh.pub_key #=> false
+    #
+    #   # With OpenSSL::PKey.generate_key
+    #   dh0 = OpenSSL::PKey::DH.new(2048)
+    #   dh = OpenSSL::PKey.generate_key(dh0)
+    #   puts dh0.pub_key == dh.pub_key #=> false
     def generate_key!
+      if OpenSSL::OPENSSL_VERSION_NUMBER >= 0x30000000
+        raise DHError, "OpenSSL::PKey::DH is immutable on OpenSSL 3.0; " \
+        "use OpenSSL::PKey.generate_key instead"
+      end
+
       unless priv_key
         tmp = OpenSSL::PKey.generate_key(self)
         set_key(tmp.pub_key, tmp.priv_key)
@@ -249,9 +274,14 @@ def dsa_verify_asn1(data, sig)
     # This method is provided for backwards compatibility, and calls #derive
     # internally.
     def dh_compute_key(pubkey)
-      peer = OpenSSL::PKey::EC.new(group)
-      peer.public_key = pubkey
-      derive(peer)
+      obj = OpenSSL::ASN1.Sequence([
+        OpenSSL::ASN1.Sequence([
+          OpenSSL::ASN1.ObjectId("id-ecPublicKey"),
+          group.to_der,
+        ]),
+        OpenSSL::ASN1.BitString(pubkey.to_octet_string(:uncompressed)),
+      ])
+      derive(OpenSSL::PKey.read(obj.to_der))
     end
   end
 

test/openssl/test_pkey_dh.rb

@@ -26,14 +26,19 @@ def test_new_break
   end
 
   def test_derive_key
-    dh1 = Fixtures.pkey("dh1024").generate_key!
-    dh2 = Fixtures.pkey("dh1024").generate_key!
+    params = Fixtures.pkey("dh1024")
+    dh1 = OpenSSL::PKey.generate_key(params)
+    dh2 = OpenSSL::PKey.generate_key(params)
     dh1_pub = OpenSSL::PKey.read(dh1.public_to_der)
     dh2_pub = OpenSSL::PKey.read(dh2.public_to_der)
+
     z = dh1.g.mod_exp(dh1.priv_key, dh1.p).mod_exp(dh2.priv_key, dh1.p).to_s(2)
     assert_equal z, dh1.derive(dh2_pub)
     assert_equal z, dh2.derive(dh1_pub)
 
+    assert_raise(OpenSSL::PKey::PKeyError) { params.derive(dh1_pub) }
+    assert_raise(OpenSSL::PKey::PKeyError) { dh1_pub.derive(params) }
+
     assert_equal z, dh1.compute_key(dh2.pub_key)
     assert_equal z, dh2.compute_key(dh1.pub_key)
   end
@@ -74,19 +79,16 @@ def test_public_key
   end
 
   def test_generate_key
-    dh = Fixtures.pkey("dh1024").public_key # creates a copy
+    # Deprecated in v3.0.0; incompatible with OpenSSL 3.0
+    dh = Fixtures.pkey("dh1024").public_key # creates a copy with params only
     assert_no_key(dh)
     dh.generate_key!
     assert_key(dh)
-  end
 
-  def test_key_exchange
-    dh = Fixtures.pkey("dh1024")
     dh2 = dh.public_key
-    dh.generate_key!
     dh2.generate_key!
     assert_equal(dh.compute_key(dh2.pub_key), dh2.compute_key(dh.pub_key))
-  end
+  end if !openssl?(3, 0, 0)
 
   def test_params_ok?
     dh0 = Fixtures.pkey("dh1024")
@@ -105,13 +107,32 @@ def test_params_ok?
   end
 
   def test_dup
-    dh = Fixtures.pkey("dh1024")
-    dh2 = dh.dup
-    assert_equal dh.to_der, dh2.to_der # params
-    assert_equal_params dh, dh2 # keys
-    dh2.set_pqg(dh2.p + 1, nil, dh2.g)
-    assert_not_equal dh2.p, dh.p
-    assert_equal dh2.g, dh.g
+    # Parameters only
+    dh1 = Fixtures.pkey("dh1024")
+    dh2 = dh1.dup
+    assert_equal dh1.to_der, dh2.to_der
+    assert_not_equal nil, dh1.p
+    assert_not_equal nil, dh1.g
+    assert_equal [dh1.p, dh1.g], [dh2.p, dh2.g]
+    assert_equal nil, dh1.pub_key
+    assert_equal nil, dh1.priv_key
+    assert_equal [dh1.pub_key, dh1.priv_key], [dh2.pub_key, dh2.priv_key]
+
+    # PKey is immutable in OpenSSL >= 3.0
+    if !openssl?(3, 0, 0)
+      dh2.set_pqg(dh2.p + 1, nil, dh2.g)
+      assert_not_equal dh2.p, dh1.p
+    end
+
+    # With a key pair
+    dh3 = OpenSSL::PKey.generate_key(Fixtures.pkey("dh1024"))
+    dh4 = dh3.dup
+    assert_equal dh3.to_der, dh4.to_der
+    assert_equal dh1.to_der, dh4.to_der # encodes parameters only
+    assert_equal [dh1.p, dh1.g], [dh4.p, dh4.g]
+    assert_not_equal nil, dh3.pub_key
+    assert_not_equal nil, dh3.priv_key
+    assert_equal [dh3.pub_key, dh3.priv_key], [dh4.pub_key, dh4.priv_key]
   end
 
   def test_marshal
@@ -123,11 +144,6 @@ def test_marshal
 
   private
 
-  def assert_equal_params(dh1, dh2)
-    assert_equal(dh1.g, dh2.g)
-    assert_equal(dh1.p, dh2.p)
-  end
-
   def assert_no_key(dh)
     assert_equal(false, dh.public?)
     assert_equal(false, dh.private?)

test/openssl/test_pkey_dsa.rb

@@ -208,8 +208,12 @@ def test_dup
     key = Fixtures.pkey("dsa1024")
     key2 = key.dup
     assert_equal key.params, key2.params
-    key2.set_pqg(key2.p + 1, key2.q, key2.g)
-    assert_not_equal key.params, key2.params
+
+    # PKey is immutable in OpenSSL >= 3.0
+    if !openssl?(3, 0, 0)
+      key2.set_pqg(key2.p + 1, key2.q, key2.g)
+      assert_not_equal key.params, key2.params
+    end
   end
 
   def test_marshal