[AndroidCrypto] Fix up RSA and some EC tests (#48930)

Co-authored-by: Jeremy Barton <jbarton@microsoft.com>

src/libraries/Common/src/Interop/Android/System.Security.Cryptography.Native.Android/Interop.Rsa.cs

@@ -0,0 +1,189 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Security.Cryptography;
+using Microsoft.Win32.SafeHandles;
+
+internal static partial class Interop
+{
+    internal static partial class AndroidCrypto
+    {
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaCreate")]
+        internal static extern SafeRsaHandle RsaCreate();
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaUpRef")]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        internal static extern bool RsaUpRef(IntPtr rsa);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaDestroy")]
+        internal static extern void RsaDestroy(IntPtr rsa);
+
+        internal static SafeRsaHandle DecodeRsaPublicKey(ReadOnlySpan<byte> buf) =>
+            DecodeRsaPublicKey(ref MemoryMarshal.GetReference(buf), buf.Length);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_DecodeRsaPublicKey")]
+        private static extern SafeRsaHandle DecodeRsaPublicKey(ref byte buf, int len);
+
+        internal static int RsaPublicEncrypt(
+            int flen,
+            ReadOnlySpan<byte> from,
+            Span<byte> to,
+            SafeRsaHandle rsa,
+            RsaPadding padding) =>
+            RsaPublicEncrypt(flen, ref MemoryMarshal.GetReference(from), ref MemoryMarshal.GetReference(to), rsa, padding);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaPublicEncrypt")]
+        private static extern int RsaPublicEncrypt(
+            int flen,
+            ref byte from,
+            ref byte to,
+            SafeRsaHandle rsa,
+            RsaPadding padding);
+
+        internal static int RsaPrivateDecrypt(
+            int flen,
+            ReadOnlySpan<byte> from,
+            Span<byte> to,
+            SafeRsaHandle rsa,
+            RsaPadding padding) =>
+            RsaPrivateDecrypt(flen, ref MemoryMarshal.GetReference(from), ref MemoryMarshal.GetReference(to), rsa, padding);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaPrivateDecrypt")]
+        private static extern int RsaPrivateDecrypt(
+            int flen,
+            ref byte from,
+            ref byte to,
+            SafeRsaHandle rsa,
+            RsaPadding padding);
+
+        internal static int RsaSignPrimitive(
+            ReadOnlySpan<byte> from,
+            Span<byte> to,
+            SafeRsaHandle rsa) =>
+            RsaSignPrimitive(from.Length, ref MemoryMarshal.GetReference(from), ref MemoryMarshal.GetReference(to), rsa);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaSignPrimitive")]
+        private static extern int RsaSignPrimitive(
+            int flen,
+            ref byte from,
+            ref byte to,
+            SafeRsaHandle rsa);
+
+        internal static int RsaVerificationPrimitive(
+            ReadOnlySpan<byte> from,
+            Span<byte> to,
+            SafeRsaHandle rsa) =>
+            RsaVerificationPrimitive(from.Length, ref MemoryMarshal.GetReference(from), ref MemoryMarshal.GetReference(to), rsa);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaVerificationPrimitive")]
+        private static extern int RsaVerificationPrimitive(
+            int flen,
+            ref byte from,
+            ref byte to,
+            SafeRsaHandle rsa);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaSize")]
+        internal static extern int RsaSize(SafeRsaHandle rsa);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_RsaGenerateKeyEx")]
+        internal static extern int RsaGenerateKeyEx(SafeRsaHandle rsa, int bits);
+
+        internal static RSAParameters ExportRsaParameters(SafeRsaHandle key, bool includePrivateParameters)
+        {
+            Debug.Assert(
+                key != null && !key.IsInvalid,
+                "Callers should check the key is invalid and throw an exception with a message");
+
+            if (key == null || key.IsInvalid)
+            {
+                throw new CryptographicException();
+            }
+
+            bool addedRef = false;
+
+            try
+            {
+                key.DangerousAddRef(ref addedRef);
+
+                IntPtr n, e, d, p, dmp1, q, dmq1, iqmp;
+                if (!GetRsaParameters(key, out n, out e, out d, out p, out dmp1, out q, out dmq1, out iqmp))
+                {
+                    throw new CryptographicException();
+                }
+
+                int modulusSize = RsaSize(key);
+
+                // RSACryptoServiceProvider expects P, DP, Q, DQ, and InverseQ to all
+                // be padded up to half the modulus size.
+                int halfModulus = modulusSize / 2;
+
+                RSAParameters rsaParameters = new RSAParameters
+                {
+                    Modulus = Crypto.ExtractBignum(n, modulusSize)!,
+                    Exponent = Crypto.ExtractBignum(e, 0)!,
+                };
+
+                if (includePrivateParameters)
+                {
+                    rsaParameters.D = Crypto.ExtractBignum(d, modulusSize);
+                    rsaParameters.P = Crypto.ExtractBignum(p, halfModulus);
+                    rsaParameters.DP = Crypto.ExtractBignum(dmp1, halfModulus);
+                    rsaParameters.Q = Crypto.ExtractBignum(q, halfModulus);
+                    rsaParameters.DQ = Crypto.ExtractBignum(dmq1, halfModulus);
+                    rsaParameters.InverseQ = Crypto.ExtractBignum(iqmp, halfModulus);
+                }
+
+                return rsaParameters;
+            }
+            finally
+            {
+                if (addedRef)
+                    key.DangerousRelease();
+            }
+        }
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_GetRsaParameters")]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        private static extern bool GetRsaParameters(
+            SafeRsaHandle key,
+            out IntPtr n,
+            out IntPtr e,
+            out IntPtr d,
+            out IntPtr p,
+            out IntPtr dmp1,
+            out IntPtr q,
+            out IntPtr dmq1,
+            out IntPtr iqmp);
+
+        [DllImport(Libraries.CryptoNative, EntryPoint = "AndroidCryptoNative_SetRsaParameters")]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        internal static extern bool SetRsaParameters(
+            SafeRsaHandle key,
+            byte[]? n,
+            int nLength,
+            byte[]? e,
+            int eLength,
+            byte[]? d,
+            int dLength,
+            byte[]? p,
+            int pLength,
+            byte[]? dmp1,
+            int dmp1Length,
+            byte[]? q,
+            int qLength,
+            byte[]? dmq1,
+            int dmq1Length,
+            byte[]? iqmp,
+            int iqmpLength);
+
+        internal enum RsaPadding : int
+        {
+            Pkcs1 = 0,
+            OaepSHA1 = 1,
+            NoPadding = 2,
+        }
+    }
+}

src/libraries/Common/src/Microsoft/Win32/SafeHandles/SafeRsaHandle.Android.cs

@@ -0,0 +1,47 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.Security;
+using System.Runtime.InteropServices;
+
+namespace System.Security.Cryptography
+{
+    internal sealed class SafeRsaHandle : SafeHandle
+    {
+        public SafeRsaHandle() :
+            base(IntPtr.Zero, ownsHandle: true)
+        {
+        }
+
+        protected override bool ReleaseHandle()
+        {
+            Interop.AndroidCrypto.RsaDestroy(handle);
+            SetHandle(IntPtr.Zero);
+            return true;
+        }
+
+        public override bool IsInvalid
+        {
+            get { return handle == IntPtr.Zero; }
+        }
+
+        internal static SafeRsaHandle DuplicateHandle(IntPtr handle)
+        {
+            Debug.Assert(handle != IntPtr.Zero);
+
+            // Reliability: Allocate the SafeHandle before calling RSA_up_ref so
+            // that we don't lose a tracked reference in low-memory situations.
+            SafeRsaHandle safeHandle = new SafeRsaHandle();
+
+            if (!Interop.AndroidCrypto.RsaUpRef(handle))
+            {
+                throw new CryptographicException();
+            }
+
+            safeHandle.SetHandle(handle);
+            return safeHandle;
+        }
+    }
+}

src/libraries/Common/src/System/Security/Cryptography/RsaPaddingProcessor.cs

@@ -9,16 +9,56 @@ namespace System.Security.Cryptography
 {
     internal sealed class RsaPaddingProcessor
     {
+        // DigestInfo header values taken from https://tools.ietf.org/html/rfc3447#section-9.2, Note 1.
+        private static readonly byte[] s_digestInfoMD5 =
+            {
+                0x30, 0x20, 0x30, 0x0C, 0x06, 0x08, 0x2A, 0x86,
+                0x48, 0x86, 0xF7, 0x0D, 0x02, 0x05, 0x05, 0x00,
+                0x04, 0x10,
+            };
+
+        private static readonly byte[] s_digestInfoSha1 =
+            {
+                0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2B, 0x0E, 0x03,
+                0x02, 0x1A, 0x05, 0x00, 0x04, 0x14,
+            };
+
+        private static readonly byte[] s_digestInfoSha256 =
+            {
+                0x30, 0x31, 0x30, 0x0D, 0x06, 0x09, 0x60, 0x86, 0x48,
+                0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04,
+                0x20,
+            };
+
+        private static readonly byte[] s_digestInfoSha384 =
+            {
+                0x30, 0x41, 0x30, 0x0D, 0x06, 0x09, 0x60, 0x86, 0x48,
+                0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04,
+                0x30,
+            };
+
+        private static readonly byte[] s_digestInfoSha512 =
+            {
+                0x30, 0x41, 0x30, 0x0D, 0x06, 0x09, 0x60, 0x86, 0x48,
+                0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04,
+                0x40,
+            };
+
         private static readonly ConcurrentDictionary<HashAlgorithmName, RsaPaddingProcessor> s_lookup =
             new ConcurrentDictionary<HashAlgorithmName, RsaPaddingProcessor>();
 
         private readonly HashAlgorithmName _hashAlgorithmName;
         private readonly int _hLen;
+        private readonly ReadOnlyMemory<byte> _digestInfoPrefix;
 
-        private RsaPaddingProcessor(HashAlgorithmName hashAlgorithmName, int hLen)
+        private RsaPaddingProcessor(
+            HashAlgorithmName hashAlgorithmName,
+            int hLen,
+            ReadOnlyMemory<byte> digestInfoPrefix)
         {
             _hashAlgorithmName = hashAlgorithmName;
             _hLen = hLen;
+            _digestInfoPrefix = digestInfoPrefix;
         }
 
         internal static int BytesRequiredForBitCount(int keySizeInBits)
@@ -40,14 +80,41 @@ internal static RsaPaddingProcessor OpenProcessor(HashAlgorithmName hashAlgorith
                     {
                         // SHA-2-512 is the biggest we expect
                         Span<byte> stackDest = stackalloc byte[512 / 8];
+                        ReadOnlyMemory<byte> digestInfoPrefix;
+
+                        if (hashAlgorithmName == HashAlgorithmName.MD5)
+                        {
+                            digestInfoPrefix = s_digestInfoMD5;
+                        }
+                        else if (hashAlgorithmName == HashAlgorithmName.SHA1)
+                        {
+                            digestInfoPrefix = s_digestInfoSha1;
+                        }
+                        else if (hashAlgorithmName == HashAlgorithmName.SHA256)
+                        {
+                            digestInfoPrefix = s_digestInfoSha256;
+                        }
+                        else if (hashAlgorithmName == HashAlgorithmName.SHA384)
+                        {
+                            digestInfoPrefix = s_digestInfoSha384;
+                        }
+                        else if (hashAlgorithmName == HashAlgorithmName.SHA512)
+                        {
+                            digestInfoPrefix = s_digestInfoSha512;
+                        }
+                        else
+                        {
+                            Debug.Fail("Unknown digest algorithm");
+                            throw new CryptographicException();
+                        }
 
                         if (hasher.TryGetHashAndReset(stackDest, out int bytesWritten))
                         {
-                            return new RsaPaddingProcessor(hashAlgorithmName, bytesWritten);
+                            return new RsaPaddingProcessor(hashAlgorithmName, bytesWritten, digestInfoPrefix);
                         }
 
                         byte[] big = hasher.GetHashAndReset();
-                        return new RsaPaddingProcessor(hashAlgorithmName, big.Length);
+                        return new RsaPaddingProcessor(hashAlgorithmName, big.Length, digestInfoPrefix);
                     }
                 });
         }
@@ -80,6 +147,44 @@ internal static void PadPkcs1Encryption(
             source.CopyTo(mInEM);
         }
 
+        internal void PadPkcs1Signature(
+            ReadOnlySpan<byte> source,
+            Span<byte> destination)
+        {
+            // https://tools.ietf.org/html/rfc3447#section-9.2
+
+            // 1. H = Hash(M)
+            // Done by the caller.
+
+            // 2. Encode the DigestInfo value
+            ReadOnlySpan<byte> digestInfoPrefix = _digestInfoPrefix.Span;
+            int expectedLength = digestInfoPrefix[^1];
+
+            if (source.Length != expectedLength)
+            {
+                throw new CryptographicException(SR.Cryptography_SignHash_WrongSize);
+            }
+
+            int tLen = digestInfoPrefix.Length + expectedLength;
+
+            // 3. If emLen < tLen + 11, fail
+            if (destination.Length - 11 < tLen)
+            {
+                throw new CryptographicException(SR.Cryptography_KeyTooSmall);
+            }
+
+            // 4. Generate emLen - tLen - 3 bytes of 0xFF as "PS"
+            int paddingLength = destination.Length - tLen - 3;
+
+            // 5. EM = 0x00 || 0x01 || PS || 0x00 || T
+            destination[0] = 0;
+            destination[1] = 1;
+            destination.Slice(2, paddingLength).Fill(0xFF);
+            destination[paddingLength + 2] = 0;
+            digestInfoPrefix.CopyTo(destination.Slice(paddingLength + 3));
+            source.CopyTo(destination.Slice(paddingLength + 3 + digestInfoPrefix.Length));
+        }
+
         internal void PadOaep(
             ReadOnlySpan<byte> source,
             Span<byte> destination)

src/libraries/Native/Unix/Common/pal_config.h.in

@@ -31,6 +31,8 @@
 #cmakedefine01 HAVE_NON_LEGACY_STATFS
 #cmakedefine01 HAVE_STRCPY_S
 #cmakedefine01 HAVE_STRLCPY
+#cmakedefine01 HAVE_STRCAT_S
+#cmakedefine01 HAVE_STRLCAT
 #cmakedefine01 HAVE_SHM_OPEN_THAT_WORKS_WELL_ENOUGH_WITH_MMAP
 #cmakedefine01 HAVE_POSIX_ADVISE
 #cmakedefine01 PRIORITY_REQUIRES_INT_WHO