Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement Vector128 version of System.Buffers.Text.Base64 DecodeFromUtf8 and EncodeToUtf8 #70654

Merged
merged 11 commits into from
Jun 20, 2022
Merged

Implement Vector128 version of System.Buffers.Text.Base64 DecodeFromUtf8 and EncodeToUtf8 #70654

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
bd353ba
Implement Vector128 version of System.Buffers.Text.Base64.DecodeFromUtf8
a74nh May 20, 2022
ccec758
SSE3 improvements
a74nh Jun 13, 2022
b6ef98a
Remove superfluous bitwise And
a74nh Jun 13, 2022
a33084f
Add comment to SimdShuffle
a74nh Jun 13, 2022
e679183
Inline SimdShuffle
a74nh Jun 13, 2022
15327f6
Implement Vector128 version of System.Buffers.Text.Base64.EncodeToUtf8
a74nh Jun 14, 2022
da20c97
Ensure masking on SSE3
a74nh Jun 15, 2022
8af1976
Restore asserts and move zero inside the loop
a74nh Jun 17, 2022
6e4c2e5
Neater C# code
a74nh Jun 20, 2022
4a3bf89
Make SimdShuffle consistent across X64 and Arm64
a74nh Jun 20, 2022
e2c1f34
Better looking multiply
a74nh Jun 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 8 additions & 27 deletions src/libraries/System.Memory/src/System/Buffers/Text/Base64Decoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,29 +574,10 @@ private static unsafe void Vector128Decode(ref byte* srcBytes, ref byte* destByt
// 1111 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10

// The JIT won't hoist these "constants", so help it
Vector128<byte> lutHi = Vector128.Create(
0x10, 0x10, 0x01, 0x02,
0x04, 0x08, 0x04, 0x08,
0x10, 0x10, 0x10, 0x10,
0x10, 0x10, 0x10, 0x10).AsByte();

Vector128<byte> lutLo = Vector128.Create(
0x15, 0x11, 0x11, 0x11,
0x11, 0x11, 0x11, 0x11,
0x11, 0x11, 0x13, 0x1A,
0x1B, 0x1B, 0x1B, 0x1A).AsByte();

Vector128<sbyte> lutShift = Vector128.Create(
0, 16, 19, 4,
-65, -65, -71, -71,
0, 0, 0, 0,
0, 0, 0, 0);

Vector128<sbyte> packBytesMask = Vector128.Create(
2, 1, 0, 6,
5, 4, 10, 9,
8, 14, 13, 12,
-1, -1, -1, -1);
Vector128<byte> lutHi = Vector128.Create(0x02011010, 0x08040804, 0x10101010, 0x10101010).AsByte();
Vector128<byte> lutLo = Vector128.Create(0x11111115, 0x11111111, 0x1A131111, 0x1A1B1B1B).AsByte();
Vector128<sbyte> lutShift = Vector128.Create(0x04131000, 0xb9b9bfbf, 0x00000000, 0x00000000).AsSByte();
Vector128<sbyte> packBytesMask = Vector128.Create(0x06000102, 0x090A0405, 0x0C0D0E08, 0xffffffff).AsSByte();

Vector128<byte> mask2F = Vector128.Create((byte)'/');
Vector128<byte> mergeConstant0 = Vector128.Create(0x01400140).AsByte();
Expand All @@ -618,21 +599,21 @@ private static unsafe void Vector128Decode(ref byte* srcBytes, ref byte* destByt
if (Ssse3.IsSupported)
{
// Ensure the high bits are masked off for the shuffle.
hiNibbles = Vector128.BitwiseAnd(hiNibbles, mask2F);
hiNibbles &= mask2F;
}
Vector128<byte> hi = SimdShuffle(lutHi, hiNibbles, f);
Vector128<byte> lo = SimdShuffle(lutLo, str, f);

// Check for invalid input: if any "and" values from lo and hi are not zero,
// fall back on bytewise code to do error checking and reporting:
if (Vector128.BitwiseAnd(lo, hi).AsUInt64().ToScalar() != 0)
if ((lo & hi) != Vector128<byte>.Zero)
break;

Vector128<byte> eq2F = Vector128.Equals(str, mask2F);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and here too: str == mask2F.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one can't be unless its immediately followed by the relevant ExtractMostSignificantBits and corresponding compare sequence.

== is equivalent to EqualsAll and returns a bool not a vector

Vector128<byte> shift = SimdShuffle(lutShift.AsByte(), Vector128.Add(eq2F, hiNibbles), f);
Vector128<byte> shift = SimdShuffle(lutShift.AsByte(), (eq2F + hiNibbles), f);

// Now simply add the delta values to the input:
str = Vector128.Add(str, shift);
str += shift;

// in, bits, upper case are most significant bits, lower case are least significant bits
// 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ
Expand Down
29 changes: 10 additions & 19 deletions src/libraries/System.Memory/src/System/Buffers/Text/Base64Encoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
namespace System.Buffers.Text
{
// AVX2 version based on https://github.com/aklomp/base64/tree/e516d769a2a432c08404f1981e73b431566057be/lib/arch/avx2
// SSSE3 version based on https://github.com/aklomp/base64/tree/e516d769a2a432c08404f1981e73b431566057be/lib/arch/ssse3
// Vector128 version based on https://github.com/aklomp/base64/tree/e516d769a2a432c08404f1981e73b431566057be/lib/arch/ssse3

/// <summary>
/// Convert between binary data and UTF-8 encoded text that is represented in base 64.
Expand Down Expand Up @@ -406,18 +406,8 @@ private static unsafe void Vector128Encode(ref byte* srcBytes, ref byte* destByt
// 0 0 0 0 l k j i h g f e d c b a

// The JIT won't hoist these "constants", so help it
Vector128<byte> shuffleVec = Vector128.Create(
1, 0, 2, 1,
4, 3, 5, 4,
7, 6, 8, 7,
10, 9, 11, 10).AsByte();

Vector128<byte> lut = Vector128.Create(
65, 71, -4, -4,
-4, -4, -4, -4,
-4, -4, -4, -4,
-19, -16, 0, 0).AsByte();

Vector128<byte> shuffleVec = Vector128.Create(0x01020001, 0x04050304, 0x07080607, 0x0A0B090A).AsByte();
Vector128<byte> lut = Vector128.Create(0xFCFC4741, 0xFCFCFCFC, 0xFCFCFCFC, 0x0000F0ED).AsByte();
Vector128<byte> maskAC = Vector128.Create(0x0fc0fc00).AsByte();
Vector128<byte> maskBB = Vector128.Create(0x003f03f0).AsByte();
Vector128<ushort> shiftAC = Vector128.Create(0x04000040).AsUInt16();
Expand All @@ -432,6 +422,7 @@ private static unsafe void Vector128Encode(ref byte* srcBytes, ref byte* destByt
//while (remaining >= 16)
do
{
AssertRead<Vector128<sbyte>>(src, srcStart, sourceLength);
Vector128<byte> str = Vector128.LoadUnsafe(ref *src);

// Reshuffle
Expand All @@ -442,14 +433,14 @@ private static unsafe void Vector128Encode(ref byte* srcBytes, ref byte* destByt
// e f d e
// b c a b

Vector128<byte> t0 = Vector128.BitwiseAnd(str, maskAC);
Vector128<byte> t0 = str & maskAC;
// bits, upper case are most significant bits, lower case are least significant bits
// 0000kkkk LL000000 JJJJJJ00 00000000
// 0000hhhh II000000 GGGGGG00 00000000
// 0000eeee FF000000 DDDDDD00 00000000
// 0000bbbb CC000000 AAAAAA00 00000000

Vector128<byte> t2 = Vector128.BitwiseAnd(str, maskBB);
Vector128<byte> t2 = str & maskBB;
// 00000000 00llllll 000000jj KKKK0000
// 00000000 00iiiiii 000000gg HHHH0000
// 00000000 00ffffff 000000dd EEEE0000
Expand All @@ -472,13 +463,12 @@ private static unsafe void Vector128Encode(ref byte* srcBytes, ref byte* destByt
// 00000000 00bbbbCC 00000000 00AAAAAA

Vector128<short> t3 = Vector128.Multiply(t2.AsInt16(), shiftBB);

// 00llllll 00000000 00jjKKKK 00000000
// 00iiiiii 00000000 00ggHHHH 00000000
// 00ffffff 00000000 00ddEEEE 00000000
// 00cccccc 00000000 00aaBBBB 00000000

str = Vector128.BitwiseOr(t1.AsByte(), t3.AsByte());
str = t1.AsByte() | t3.AsByte();
// 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ
// 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG
// 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD
Expand Down Expand Up @@ -510,11 +500,12 @@ private static unsafe void Vector128Encode(ref byte* srcBytes, ref byte* destByt
Vector128<sbyte> mask = Vector128.GreaterThan(str.AsSByte(), const25);

// substract -1, so add 1 to indices for range #[1..4], All indices are now correct:
Vector128<sbyte> tmp = Vector128.Subtract(indices.AsSByte(), mask);
Vector128<sbyte> tmp = indices.AsSByte() - mask;

// Add offsets to input values:
str = Vector128.Add(str, SimdShuffle(lut, tmp.AsByte(), f));
str += SimdShuffle(lut, tmp.AsByte(), f);

AssertWrite<Vector128<sbyte>>(dest, destStart, destLength);
str.Store(dest);

src += 12;
Expand Down