Use EVP_PKEY for RSA key generation (native side)

Author: bartonjs

src/libraries/Native/Unix/System.Security.Cryptography.Native/apibridge.c

@@ -788,4 +788,20 @@ int local_BIO_up_ref(BIO *bio)
 
     return CRYPTO_add_lock(&bio->references, 1, CRYPTO_LOCK_BIO, __FILE__, __LINE__) > 1;
 }
+
+int32_t local_RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2)
+{
+    if (ctx != NULL)
+    {
+        EVP_PKEY* pkey = EVP_PKEY_CTX_get0_pkey(ctx);
+
+        if (EVP_PKEY_base_id(pkey) != NID_rsaEncryption)
+        {
+            return -1;
+        }
+    }
+
+    return EVP_PKEY_CTX_ctrl(ctx, -1, optype, cmd, p1, p2);
+}
+
 #endif

src/libraries/Native/Unix/System.Security.Cryptography.Native/apibridge.h

@@ -26,6 +26,7 @@ int32_t local_RSA_meth_get_flags(const RSA_METHOD* meth);
 int32_t local_RSA_set0_crt_params(RSA* rsa, BIGNUM* dmp1, BIGNUM* dmq1, BIGNUM* iqmp);
 int32_t local_RSA_set0_factors(RSA* rsa, BIGNUM* p, BIGNUM* q);
 int32_t local_RSA_set0_key(RSA* rsa, BIGNUM* n, BIGNUM* e, BIGNUM* d);
+int32_t local_RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2);
 int32_t local_SSL_is_init_finished(const SSL* ssl);
 int32_t local_SSL_CTX_config(SSL_CTX* ctx, const char* name);
 unsigned long local_SSL_CTX_set_options(SSL_CTX* ctx, unsigned long options);

src/libraries/Native/Unix/System.Security.Cryptography.Native/entrypoints.c

@@ -222,7 +222,7 @@ static const Entry s_cryptoNative[] =
     DllImportEntry(CryptoNative_RecursiveFreeX509Stack)
     DllImportEntry(CryptoNative_RsaCreate)
     DllImportEntry(CryptoNative_RsaDestroy)
-    DllImportEntry(CryptoNative_RsaGenerateKeyEx)
+    DllImportEntry(CryptoNative_RsaGenerateKey)
     DllImportEntry(CryptoNative_RsaPrivateDecrypt)
     DllImportEntry(CryptoNative_RsaPublicEncrypt)
     DllImportEntry(CryptoNative_RsaSign)

src/libraries/Native/Unix/System.Security.Cryptography.Native/opensslshim.h

@@ -176,6 +176,13 @@ const X509_ALGOR* X509_get0_tbs_sigalg(const X509* x509);
 X509_PUBKEY* X509_get_X509_PUBKEY(const X509* x509);
 int32_t X509_get_version(const X509* x509);
 int32_t X509_up_ref(X509* x509);
+
+// Redefine EVP_PKEY_CTX_set_rsa operations to use (local_)RSA_pkey_ctx_ctrl so the path is the same
+// for 1.0-built on 1.1 as on 1.1-built on 1.1.
+#undef EVP_PKEY_CTX_set_rsa_keygen_bits
+#define EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits) \
+    RSA_pkey_ctx_ctrl(ctx, EVP_PKEY_OP_KEYGEN, EVP_PKEY_CTRL_RSA_KEYGEN_BITS, bits, NULL)
+
 #endif
 
 #if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_1_0_2_RTM
@@ -362,15 +369,21 @@ void SSL_get0_alpn_selected(const SSL* ssl, const unsigned char** protocol, unsi
     RENAMED_FUNCTION(EVP_MD_CTX_free, EVP_MD_CTX_destroy) \
     RENAMED_FUNCTION(EVP_MD_CTX_new, EVP_MD_CTX_create) \
     REQUIRED_FUNCTION(EVP_MD_size) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_ctrl) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_free) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_get0_pkey) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_new) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_new_id) \
+    REQUIRED_FUNCTION(EVP_PKEY_base_id) \
     REQUIRED_FUNCTION(EVP_PKEY_derive_set_peer) \
     REQUIRED_FUNCTION(EVP_PKEY_derive_init) \
     REQUIRED_FUNCTION(EVP_PKEY_derive) \
     REQUIRED_FUNCTION(EVP_PKEY_free) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_DSA) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_EC_KEY) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_RSA) \
+    REQUIRED_FUNCTION(EVP_PKEY_keygen) \
+    REQUIRED_FUNCTION(EVP_PKEY_keygen_init) \
     REQUIRED_FUNCTION(EVP_PKEY_new) \
     REQUIRED_FUNCTION(EVP_PKEY_set1_DSA) \
     REQUIRED_FUNCTION(EVP_PKEY_set1_EC_KEY) \
@@ -453,6 +466,7 @@ void SSL_get0_alpn_selected(const SSL* ssl, const unsigned char** protocol, unsi
     FALLBACK_FUNCTION(RSA_get0_key) \
     FALLBACK_FUNCTION(RSA_meth_get_flags) \
     REQUIRED_FUNCTION(RSA_new) \
+    FALLBACK_FUNCTION(RSA_pkey_ctx_ctrl) \
     RENAMED_FUNCTION(RSA_PKCS1_OpenSSL, RSA_PKCS1_SSLeay) \
     REQUIRED_FUNCTION(RSA_private_decrypt) \
     REQUIRED_FUNCTION(RSA_private_encrypt) \
@@ -773,15 +787,21 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define EVP_MD_CTX_free EVP_MD_CTX_free_ptr
 #define EVP_MD_CTX_new EVP_MD_CTX_new_ptr
 #define EVP_MD_size EVP_MD_size_ptr
+#define EVP_PKEY_CTX_ctrl EVP_PKEY_CTX_ctrl_ptr
 #define EVP_PKEY_CTX_free EVP_PKEY_CTX_free_ptr
+#define EVP_PKEY_CTX_get0_pkey EVP_PKEY_CTX_get0_pkey_ptr
 #define EVP_PKEY_CTX_new EVP_PKEY_CTX_new_ptr
+#define EVP_PKEY_CTX_new_id EVP_PKEY_CTX_new_id_ptr
+#define EVP_PKEY_base_id EVP_PKEY_base_id_ptr
 #define EVP_PKEY_derive_set_peer EVP_PKEY_derive_set_peer_ptr
 #define EVP_PKEY_derive_init EVP_PKEY_derive_init_ptr
 #define EVP_PKEY_derive EVP_PKEY_derive_ptr
 #define EVP_PKEY_free EVP_PKEY_free_ptr
 #define EVP_PKEY_get1_DSA EVP_PKEY_get1_DSA_ptr
 #define EVP_PKEY_get1_EC_KEY EVP_PKEY_get1_EC_KEY_ptr
 #define EVP_PKEY_get1_RSA EVP_PKEY_get1_RSA_ptr
+#define EVP_PKEY_keygen EVP_PKEY_keygen_ptr
+#define EVP_PKEY_keygen_init EVP_PKEY_keygen_init_ptr
 #define EVP_PKEY_new EVP_PKEY_new_ptr
 #define EVP_PKEY_set1_DSA EVP_PKEY_set1_DSA_ptr
 #define EVP_PKEY_set1_EC_KEY EVP_PKEY_set1_EC_KEY_ptr
@@ -864,6 +884,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define RSA_get_method RSA_get_method_ptr
 #define RSA_meth_get_flags RSA_meth_get_flags_ptr
 #define RSA_new RSA_new_ptr
+#define RSA_pkey_ctx_ctrl RSA_pkey_ctx_ctrl_ptr
 #define RSA_PKCS1_OpenSSL RSA_PKCS1_OpenSSL_ptr
 #define RSA_private_decrypt RSA_private_decrypt_ptr
 #define RSA_private_encrypt RSA_private_encrypt_ptr
@@ -1084,6 +1105,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define RSA_set0_crt_params local_RSA_set0_crt_params
 #define RSA_set0_factors local_RSA_set0_factors
 #define RSA_set0_key local_RSA_set0_key
+#define RSA_pkey_ctx_ctrl local_RSA_pkey_ctx_ctrl
 #define SSL_CTX_set_security_level local_SSL_CTX_set_security_level
 #define SSL_is_init_finished local_SSL_is_init_finished
 #define X509_CRL_get0_nextUpdate local_X509_CRL_get0_nextUpdate

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c

@@ -3,6 +3,31 @@
 
 #include "pal_evp_pkey_rsa.h"
 
+EVP_PKEY* CryptoNative_RsaGenerateKey(int keySize)
+{
+    EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
+
+    if (ctx == NULL)
+    {
+        return NULL;
+    }
+
+    EVP_PKEY* pkey = NULL;
+    int success = 1;
+    success = success && (1 == EVP_PKEY_keygen_init(ctx));
+    success = success && (1 == EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, keySize));
+    success = success && (1 == EVP_PKEY_keygen(ctx, &pkey));
+
+    if (pkey != NULL && !success)
+    {
+        EVP_PKEY_free(pkey);
+        pkey = NULL;
+    }
+
+    EVP_PKEY_CTX_free(ctx);
+    return pkey;
+}
+
 RSA* CryptoNative_EvpPkeyGetRsa(EVP_PKEY* pkey)
 {
     return EVP_PKEY_get1_RSA(pkey);

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.h

@@ -5,6 +5,11 @@
 #include "pal_compiler.h"
 #include "opensslshim.h"
 
+/*
+Creates an RSA key of the requested size.
+*/
+PALEXPORT EVP_PKEY* CryptoNative_RsaGenerateKey(int32_t keySize);
+
 /*
 Shims the EVP_PKEY_get1_RSA method.
 

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_rsa.c

@@ -144,11 +144,6 @@ int32_t CryptoNative_RsaSize(RSA* rsa)
     return RSA_size(rsa);
 }
 
-int32_t CryptoNative_RsaGenerateKeyEx(RSA* rsa, int32_t bits, BIGNUM* e)
-{
-    return RSA_generate_key_ex(rsa, bits, e, NULL);
-}
-
 int32_t
 CryptoNative_RsaSign(int32_t type, const uint8_t* m, int32_t mlen, uint8_t* sigret, int32_t* siglen, RSA* rsa)
 {

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_rsa.h

@@ -85,13 +85,6 @@ Returns the RSA modulus size in bytes.
 */
 PALEXPORT int32_t CryptoNative_RsaSize(RSA* rsa);
 
-/*
-Shims the RSA_generate_key_ex method.
-
-Returns 1 upon success, otherwise 0.
-*/
-PALEXPORT int32_t CryptoNative_RsaGenerateKeyEx(RSA* rsa, int32_t bits, BIGNUM* e);
-
 /*
 Shims the RSA_sign method.
 

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_ssl.c

@@ -644,16 +644,22 @@ int32_t CryptoNative_SslGetCurrentCipherId(SSL* ssl, int32_t* cipherId)
 // This function generates key pair and creates simple certificate.
 static int MakeSelfSignedCertificate(X509 * cert, EVP_PKEY* evp)
 {
-    RSA* rsa = CryptoNative_RsaCreate();
+    RSA* rsa = NULL;
     ASN1_TIME* time = ASN1_TIME_new();
-    BIGNUM* bn = BN_new();
-    BN_set_word(bn, RSA_F4);
     X509_NAME * asnName;
     unsigned char * name = (unsigned char*)"localhost";
 
     int ret = 0;
 
-    if (rsa != NULL && CryptoNative_RsaGenerateKeyEx(rsa, 2048, bn) == 1)
+    EVP_PKEY* pkey = CryptoNative_RsaGenerateKey(2048);
+
+    if (pkey != NULL)
+    {
+        rsa = EVP_PKEY_get1_RSA(pkey);
+        EVP_PKEY_free(pkey);
+    }
+
+    if (rsa != NULL)
     {
         if (CryptoNative_EvpPkeySetRsa(evp, rsa) == 1)
         {
@@ -675,11 +681,6 @@ static int MakeSelfSignedCertificate(X509 * cert, EVP_PKEY* evp)
         ret = X509_sign(cert, evp, EVP_sha256());
     }
 
-    if (bn != NULL)
-    {
-        BN_free(bn);
-    }
-
     if (rsa != NULL)
     {
         RSA_free(rsa);