[release/3.1] WIP: Openssl 3.0 support

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EVP.DigestAlgs.cs

@@ -0,0 +1,58 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.InteropServices;
+using System.Security.Cryptography;
+
+internal static partial class Interop
+{
+    internal static partial class Crypto
+    {
+        private static volatile IntPtr s_evpMd5;
+        private static volatile IntPtr s_evpSha1;
+        private static volatile IntPtr s_evpSha256;
+        private static volatile IntPtr s_evpSha384;
+        private static volatile IntPtr s_evpSha512;
+
+        [DllImport(Libraries.CryptoNative)]
+        private static extern IntPtr CryptoNative_EvpMd5();
+
+        internal static IntPtr EvpMd5() =>
+            s_evpMd5 != IntPtr.Zero ? s_evpMd5 : (s_evpMd5 = CryptoNative_EvpMd5());
+
+        [DllImport(Libraries.CryptoNative)]
+        internal static extern IntPtr CryptoNative_EvpSha1();
+
+        internal static IntPtr EvpSha1() =>
+            s_evpSha1 != IntPtr.Zero ? s_evpSha1 : (s_evpSha1 = CryptoNative_EvpSha1());
+
+        [DllImport(Libraries.CryptoNative)]
+        internal static extern IntPtr CryptoNative_EvpSha256();
+
+        internal static IntPtr EvpSha256() =>
+            s_evpSha256 != IntPtr.Zero ? s_evpSha256 : (s_evpSha256 = CryptoNative_EvpSha256());
+
+        [DllImport(Libraries.CryptoNative)]
+        internal static extern IntPtr CryptoNative_EvpSha384();
+
+        internal static IntPtr EvpSha384() =>
+            s_evpSha384 != IntPtr.Zero ? s_evpSha384 : (s_evpSha384 = CryptoNative_EvpSha384());
+
+        [DllImport(Libraries.CryptoNative)]
+        internal static extern IntPtr CryptoNative_EvpSha512();
+
+        internal static IntPtr EvpSha512() =>
+            s_evpSha512 != IntPtr.Zero ? s_evpSha512 : (s_evpSha512 = CryptoNative_EvpSha512());
+
+        internal static IntPtr HashAlgorithmToEvp(string hashAlgorithmId) => hashAlgorithmId switch
+        {
+            nameof(HashAlgorithmName.SHA1) => EvpSha1(),
+            nameof(HashAlgorithmName.SHA256) => EvpSha256(),
+            nameof(HashAlgorithmName.SHA384) => EvpSha384(),
+            nameof(HashAlgorithmName.SHA512) => EvpSha512(),
+            nameof(HashAlgorithmName.MD5) => EvpMd5(),
+            _ => throw new CryptographicException(SR.Format(SR.Cryptography_UnknownHashAlgorithm, hashAlgorithmId))
+        };
+    }
+}

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EVP.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.Runtime.InteropServices;
+using System.Security.Cryptography;
 using Microsoft.Win32.SafeHandles;
 
 internal static partial class Interop
@@ -31,23 +32,6 @@ internal static int EvpDigestUpdate(SafeEvpMdCtxHandle ctx, ReadOnlySpan<byte> d
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpMdSize")]
         internal extern static int EvpMdSize(IntPtr md);
 
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpMd5")]
-        internal extern static IntPtr EvpMd5();
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpSha1")]
-        internal extern static IntPtr EvpSha1();
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpSha256")]
-        internal extern static IntPtr EvpSha256();
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpSha384")]
-        internal extern static IntPtr EvpSha384();
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpSha512")]
-        internal extern static IntPtr EvpSha512();
-
-
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_GetMaxMdSize")]
         private extern static int GetMaxMdSize();
 

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs

@@ -2,6 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
+using System.Diagnostics;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using Microsoft.Win32.SafeHandles;
@@ -10,6 +12,92 @@ internal static partial class Interop
 {
     internal static partial class Crypto
     {
+        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaGenerateKey")]
+        private static extern SafeEvpPKeyHandle CryptoNative_RsaGenerateKey(int keySize);
+
+        internal static SafeEvpPKeyHandle RsaGenerateKey(int keySize)
+        {
+            SafeEvpPKeyHandle pkey = CryptoNative_RsaGenerateKey(keySize);
+
+            if (pkey.IsInvalid)
+            {
+                pkey.Dispose();
+                throw CreateOpenSslCryptographicException();
+            }
+
+            return pkey;
+        }
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaDecrypt")]
+        private static extern int CryptoNative_RsaDecrypt(
+            SafeEvpPKeyHandle pkey,
+            ref byte source,
+            int sourceLength,
+            RSAEncryptionPaddingMode paddingMode,
+            IntPtr digestAlgorithm,
+            ref byte destination,
+            int destinationLength);
+
+        internal static int RsaDecrypt(
+            SafeEvpPKeyHandle pkey,
+            ReadOnlySpan<byte> source,
+            RSAEncryptionPaddingMode paddingMode,
+            IntPtr digestAlgorithm,
+            Span<byte> destination)
+        {
+            int written = CryptoNative_RsaDecrypt(
+                pkey,
+                ref MemoryMarshal.GetReference(source),
+                source.Length,
+                paddingMode,
+                digestAlgorithm,
+                ref MemoryMarshal.GetReference(destination),
+                destination.Length);
+
+            if (written < 0)
+            {
+                Debug.Assert(written == -1);
+                throw CreateOpenSslCryptographicException();
+            }
+
+            return written;
+        }
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaSignHash")]
+        private static extern int CryptoNative_RsaSignHash(
+            SafeEvpPKeyHandle pkey,
+            RSASignaturePaddingMode paddingMode,
+            IntPtr digestAlgorithm,
+            ref byte hash,
+            int hashLength,
+            ref byte destination,
+            int destinationLength);
+
+        internal static int RsaSignHash(
+            SafeEvpPKeyHandle pkey,
+            RSASignaturePaddingMode paddingMode,
+            IntPtr digestAlgorithm,
+            ReadOnlySpan<byte> hash,
+            Span<byte> destination)
+        {
+            int written = CryptoNative_RsaSignHash(
+                pkey,
+                paddingMode,
+                digestAlgorithm,
+                ref MemoryMarshal.GetReference(hash),
+                hash.Length,
+                ref MemoryMarshal.GetReference(destination),
+                destination.Length);
+
+            if (written < 0)
+            {
+                Debug.Assert(written == -1);
+                throw CreateOpenSslCryptographicException();
+            }
+
+            return written;
+        }
+
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpPkeyGetRsa")]
         internal static extern SafeRsaHandle EvpPkeyGetRsa(SafeEvpPKeyHandle pkey);
 

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.cs

@@ -16,6 +16,9 @@ internal static partial class Crypto
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpPkeyDestroy")]
         internal static extern void EvpPkeyDestroy(IntPtr pkey);
 
+        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpPKeySize")]
+        internal static extern int EvpPKeySize(SafeEvpPKeyHandle pkey);
+
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_UpRefEvpPkey")]
         internal static extern int UpRefEvpPkey(SafeEvpPKeyHandle handle);
     }

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.LegacyAlgorithms.cs

@@ -0,0 +1,31 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+    internal static partial class Crypto
+    {
+        private static volatile bool s_loadedLegacy;
+        private static readonly object s_legacyLoadLock = new object();
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RegisterLegacyAlgorithms")]
+        private static extern void CryptoNative_RegisterLegacyAlgorithms();
+
+        internal static void EnsureLegacyAlgorithmsRegistered()
+        {
+            if (!s_loadedLegacy)
+            {
+                lock (s_legacyLoadLock)
+                {
+                    if (!s_loadedLegacy)
+                    {
+                        CryptoNative_RegisterLegacyAlgorithms();
+                        s_loadedLegacy = true;
+                    }
+                }
+            }
+        }
+    }
+}

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OCSP.cs

@@ -43,7 +43,7 @@ internal static X509VerifyStatusCode X509ChainGetCachedOcspStatus(SafeX509StoreC
         {
             X509VerifyStatusCode response = CryptoNative_X509ChainGetCachedOcspStatus(ctx, cachePath);
 
-            if (response < 0)
+            if (response.Code < 0)
             {
                 Debug.Fail($"Unexpected response from X509ChainGetCachedOcspSuccess: {response}");
                 throw new CryptographicException();
@@ -67,7 +67,7 @@ internal static X509VerifyStatusCode X509ChainVerifyOcsp(
         {
             X509VerifyStatusCode response = CryptoNative_X509ChainVerifyOcsp(ctx, req, resp, cachePath);
 
-            if (response < 0)
+            if (response.Code < 0)
             {
                 Debug.Fail($"Unexpected response from X509ChainGetCachedOcspSuccess: {response}");
                 throw new CryptographicException();

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Rsa.cs

@@ -44,35 +44,6 @@ private extern static int RsaPublicEncrypt(
             SafeRsaHandle rsa,
             RsaPadding padding);
 
-        internal static int RsaPrivateDecrypt(
-            int flen,
-            ReadOnlySpan<byte> from,
-            Span<byte> to,
-            SafeRsaHandle rsa,
-            RsaPadding padding) =>
-            RsaPrivateDecrypt(flen, ref MemoryMarshal.GetReference(from), ref MemoryMarshal.GetReference(to), rsa, padding);
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaPrivateDecrypt")]
-        private extern static int RsaPrivateDecrypt(
-            int flen,
-            ref byte from,
-            ref byte to,
-            SafeRsaHandle rsa,
-            RsaPadding padding);
-
-        internal static int RsaSignPrimitive(
-            ReadOnlySpan<byte> from,
-            Span<byte> to,
-            SafeRsaHandle rsa) =>
-            RsaSignPrimitive(from.Length, ref MemoryMarshal.GetReference(from), ref MemoryMarshal.GetReference(to), rsa);
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaSignPrimitive")]
-        private static extern int RsaSignPrimitive(
-            int flen,
-            ref byte from,
-            ref byte to,
-            SafeRsaHandle rsa);
-
         internal static int RsaVerificationPrimitive(
             ReadOnlySpan<byte> from,
             Span<byte> to,
@@ -89,16 +60,6 @@ private static extern int RsaVerificationPrimitive(
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaSize")]
         internal static extern int RsaSize(SafeRsaHandle rsa);
 
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaGenerateKeyEx")]
-        internal static extern int RsaGenerateKeyEx(SafeRsaHandle rsa, int bits, SafeBignumHandle e);
-
-        internal static bool RsaSign(int type, ReadOnlySpan<byte> m, int m_len, Span<byte> sigret, out int siglen, SafeRsaHandle rsa) =>
-            RsaSign(type, ref MemoryMarshal.GetReference(m), m_len, ref MemoryMarshal.GetReference(sigret), out siglen, rsa);
-
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaSign")]
-        [return: MarshalAs(UnmanagedType.Bool)]
-        private static extern bool RsaSign(int type, ref byte m, int m_len, ref byte sigret, out int siglen, SafeRsaHandle rsa);
-
         internal static bool RsaVerify(int type, ReadOnlySpan<byte> m, ReadOnlySpan<byte> sigbuf, SafeRsaHandle rsa)
         {
             bool ret = RsaVerify(