diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 76075a5e66dc48..c23c9781dcb853 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -489,8 +489,8 @@ public static unsafe nuint GetIndexOfFirstNonAsciiChar(char* pBuffer, nuint buff // pmovmskb which we know are optimized, and (b) we can avoid downclocking the processor while // this method is running. - return (Sse2.IsSupported) - ? GetIndexOfFirstNonAsciiChar_Sse2(pBuffer, bufferLength) + return (BitConverter.IsLittleEndian && (Sse2.IsSupported || AdvSimd.Arm64.IsSupported)) + ? GetIndexOfFirstNonAsciiChar_Sse2OrArm64(pBuffer, bufferLength) : GetIndexOfFirstNonAsciiChar_Default(pBuffer, bufferLength); } @@ -630,9 +630,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Default(char* pBuffer, n goto Finish; } - private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuint bufferLength /* in chars */) + private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2OrArm64(char* pBuffer, nuint bufferLength /* in chars */) { - // This method contains logic optimized for both SSE2 and SSE41. Much of the logic in this method + // This method contains logic optimized for SSE2, SSE41 and ARM64. Much of the logic in this method // will be elided by JIT once we determine which specific ISAs we support. // Quick check for empty inputs. @@ -647,9 +647,10 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin uint SizeOfVector128InBytes = (uint)Unsafe.SizeOf>(); uint SizeOfVector128InChars = SizeOfVector128InBytes / sizeof(char); - Debug.Assert(Sse2.IsSupported, "Should've been checked by caller."); - Debug.Assert(BitConverter.IsLittleEndian, "SSE2 assumes little-endian."); + Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported, "Sse2 or AdvSimd64 required."); + Debug.Assert(BitConverter.IsLittleEndian, "This SSE2/Arm64 implementation assumes little-endian."); + Vector128 bitmask = Vector128.Create((ushort)0x1001).AsByte(); Vector128 firstVector, secondVector; uint currentMask; char* pOriginalBuffer = pBuffer; @@ -673,13 +674,35 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin // Read the first vector unaligned. - firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load + if (Sse2.IsSupported) + { + firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load + } + else if (AdvSimd.Arm64.IsSupported) + { + firstVector = AdvSimd.LoadVector128((ushort*)pBuffer); // unaligned load + } + else + { + throw new PlatformNotSupportedException(); + } // The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element - // has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order + // has value >= 0x0080 (non-ASCII). Then we'll treat the vector as a BYTE vector in order // to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored. - currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte()); + if (Sse2.IsSupported) + { + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte()); + } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = Unicode.Utf16Utility.GetNonAsciiBytes(AdvSimd.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte(), bitmask); + } + else + { + throw new PlatformNotSupportedException(); + } if ((currentMask & NonAsciiDataSeenMask) != 0) { @@ -725,9 +748,23 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin do { - firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer); - secondVector = Sse2.LoadAlignedVector128((ushort*)pBuffer + SizeOfVector128InChars); - Vector128 combinedVector = Sse2.Or(firstVector, secondVector); + Vector128 combinedVector; + if (Sse2.IsSupported) + { + firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer); + secondVector = Sse2.LoadAlignedVector128((ushort*)pBuffer + SizeOfVector128InChars); + combinedVector = Sse2.Or(firstVector, secondVector); + } + else if (AdvSimd.Arm64.IsSupported) + { + firstVector = AdvSimd.LoadVector128((ushort*)pBuffer); + secondVector = AdvSimd.LoadVector128((ushort*)pBuffer + SizeOfVector128InChars); + combinedVector = AdvSimd.Or(firstVector, secondVector); + } + else + { + throw new PlatformNotSupportedException(); + } if (Sse41.IsSupported) { @@ -738,7 +775,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInFirstOrSecondVector; } } - else + else if (Sse2.IsSupported) { // See comment earlier in the method for an explanation of how the below logic works. currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(combinedVector, asciiMaskForAddSaturate).AsByte()); @@ -747,6 +784,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInFirstOrSecondVector; } } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = Unicode.Utf16Utility.GetNonAsciiBytes(AdvSimd.AddSaturate(combinedVector, asciiMaskForAddSaturate).AsByte(), bitmask); + if ((currentMask & NonAsciiDataSeenMask) != 0) + { + goto FoundNonAsciiDataInFirstOrSecondVector; + } + } + else + { + throw new PlatformNotSupportedException(); + } pBuffer += 2 * SizeOfVector128InChars; } while (pBuffer <= pFinalVectorReadPos); @@ -770,7 +819,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin // At least one full vector's worth of data remains, so we can safely read it. // Remember, at this point pBuffer is still aligned. - firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer); + if (Sse2.IsSupported) + { + firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer); + } + else if (AdvSimd.Arm64.IsSupported) + { + firstVector = AdvSimd.LoadVector128((ushort*)pBuffer); + } + else + { + throw new PlatformNotSupportedException(); + } if (Sse41.IsSupported) { @@ -781,7 +841,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInFirstVector; } } - else + else if (Sse2.IsSupported) { // See comment earlier in the method for an explanation of how the below logic works. currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte()); @@ -790,6 +850,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInCurrentMask; } } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = Unicode.Utf16Utility.GetNonAsciiBytes(AdvSimd.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte(), bitmask); + if ((currentMask & NonAsciiDataSeenMask) != 0) + { + goto FoundNonAsciiDataInCurrentMask; + } + } + else + { + throw new PlatformNotSupportedException(); + } IncrementCurrentOffsetBeforeFinalUnalignedVectorRead: @@ -803,7 +875,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin // We need to adjust the pointer because we're re-reading data. pBuffer = (char*)((byte*)pBuffer + (bufferLength & (SizeOfVector128InBytes - 1)) - SizeOfVector128InBytes); - firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load + if (Sse2.IsSupported) + { + firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load + } + else if (AdvSimd.Arm64.IsSupported) + { + firstVector = AdvSimd.LoadVector128((ushort*)pBuffer); // unaligned load + } + else + { + throw new PlatformNotSupportedException(); + } if (Sse41.IsSupported) { @@ -814,7 +897,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInFirstVector; } } - else + else if (Sse2.IsSupported) { // See comment earlier in the method for an explanation of how the below logic works. currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte()); @@ -823,6 +906,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInCurrentMask; } } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = Unicode.Utf16Utility.GetNonAsciiBytes(AdvSimd.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte(), bitmask); + if ((currentMask & NonAsciiDataSeenMask) != 0) + { + goto FoundNonAsciiDataInCurrentMask; + } + } + else + { + throw new PlatformNotSupportedException(); + } pBuffer += SizeOfVector128InChars; } @@ -846,7 +941,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInFirstVector; } } - else + else if (Sse2.IsSupported) { currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte()); if ((currentMask & NonAsciiDataSeenMask) != 0) @@ -854,6 +949,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin goto FoundNonAsciiDataInCurrentMask; } } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = Unicode.Utf16Utility.GetNonAsciiBytes(AdvSimd.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte(), bitmask); + if ((currentMask & NonAsciiDataSeenMask) != 0) + { + goto FoundNonAsciiDataInCurrentMask; + } + } + else + { + throw new PlatformNotSupportedException(); + } // Wasn't the first vector; must be the second. @@ -863,7 +970,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin FoundNonAsciiDataInFirstVector: // See comment earlier in the method for an explanation of how the below logic works. - currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte()); + if (Sse2.IsSupported) + { + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte()); + } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = Unicode.Utf16Utility.GetNonAsciiBytes(AdvSimd.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte(), bitmask); + } + else + { + throw new PlatformNotSupportedException(); + } FoundNonAsciiDataInCurrentMask: diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs index f2df0ccdf53c42..90c91ed2acd8ad 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs @@ -486,7 +486,7 @@ static Utf16Utility() } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint GetNonAsciiBytes(Vector128 value, Vector128 bitMask128) + internal static uint GetNonAsciiBytes(Vector128 value, Vector128 bitMask128) { Debug.Assert(AdvSimd.Arm64.IsSupported);