From f0ad6afb5eb247250d1159867f61211efb208ac9 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 5 Aug 2022 14:29:35 -0400 Subject: [PATCH 1/2] Vectorize IndexOfAnyException(T value) --- .../src/System/MemoryExtensions.cs | 39 ++++++++-- .../src/System/SpanHelpers.T.cs | 71 +++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index f7787949c61901..2b8f1988b50917 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -500,17 +500,48 @@ public static int IndexOfAnyExcept(this Span span, ReadOnlySpan values) /// The index in the span of the first occurrence of any value other than . /// If all of the values are , returns -1. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int IndexOfAnyExcept(this ReadOnlySpan span, T value) where T : IEquatable { - for (int i = 0; i < span.Length; i++) + if (RuntimeHelpers.IsBitwiseEquatable()) { - if (!EqualityComparer.Default.Equals(span[i], value)) + if (Unsafe.SizeOf() == sizeof(byte)) { - return i; + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + + if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + + if (Unsafe.SizeOf() == sizeof(int)) + { + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + + if (Unsafe.SizeOf() == sizeof(long)) + { + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); } } - return -1; + return SpanHelpers.IndexOfAnyExcept( + ref MemoryMarshal.GetReference(span), + value, + span.Length); } /// Searches for the first index of any value other than the specified or . diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 8b372fe952b320..708fa7ede340d3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1147,6 +1147,77 @@ public static int LastIndexOfAny(ref T searchSpace, int searchSpaceLength, re return -1; // not found } + public static int IndexOfAnyExcept(ref T searchSpace, T value0, int length) where T : IEquatable + { + Debug.Assert(length >= 0, "Expected non-negative length"); + + for (int i = 0; i < length; i++) + { + if (!Unsafe.Add(ref searchSpace, i).Equals(value0)) + { + return i; + } + } + + return -1; + } + + public static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, int length) where T : struct, IEquatable + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + for (int i = 0; i < length; i++) + { + if (!Unsafe.Add(ref searchSpace, i).Equals(value0)) + { + return i; + } + } + } + else + { + Vector128 notEquals, value0Vector = Vector128.Create(value0); + ref T current = ref searchSpace; + ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + notEquals = ~Vector128.Equals(value0Vector, Vector128.LoadUnsafe(ref current)); + if (notEquals != Vector128.Zero) + { + return ComputeIndex(ref searchSpace, ref current, notEquals); + } + + current = ref Unsafe.Add(ref current, Vector128.Count); + } + while (!Unsafe.IsAddressGreaterThan(ref current, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the last vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + notEquals = ~Vector128.Equals(value0Vector, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)); + if (notEquals != Vector128.Zero) + { + return ComputeIndex(ref searchSpace, ref oneVectorAwayFromEnd, notEquals); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int ComputeIndex(ref T searchSpace, ref T current, Vector128 notEquals) + { + uint notEqualsElements = notEquals.ExtractMostSignificantBits(); + int index = BitOperations.TrailingZeroCount(notEqualsElements); + return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()); + } + } + + return -1; + } + public static bool SequenceEqual(ref T first, ref T second, int length) where T : IEquatable { Debug.Assert(length >= 0); From 4bf352f9009ff8ee7ab755fcb7d0ac94d00683a6 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 5 Aug 2022 15:21:49 -0400 Subject: [PATCH 2/2] Fix comparison handling null --- .../tests/Span/IndexOfAnyExcept.T.cs | 43 ++++++++++++++++++- .../src/System/SpanHelpers.T.cs | 5 ++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Memory/tests/Span/IndexOfAnyExcept.T.cs b/src/libraries/System.Memory/tests/Span/IndexOfAnyExcept.T.cs index 0c4d7d6cf0801f..6eee1dd362bb54 100644 --- a/src/libraries/System.Memory/tests/Span/IndexOfAnyExcept.T.cs +++ b/src/libraries/System.Memory/tests/Span/IndexOfAnyExcept.T.cs @@ -11,8 +11,49 @@ public class IndexOfAnyExceptTests_Byte : IndexOfAnyExceptTests { protecte public class IndexOfAnyExceptTests_Char : IndexOfAnyExceptTests { protected override char Create(int value) => (char)value; } public class IndexOfAnyExceptTests_Int32 : IndexOfAnyExceptTests { protected override int Create(int value) => value; } public class IndexOfAnyExceptTests_Int64 : IndexOfAnyExceptTests { protected override long Create(int value) => value; } - public class IndexOfAnyExceptTests_String : IndexOfAnyExceptTests { protected override string Create(int value) => ((char)value).ToString(); } public class IndexOfAnyExceptTests_Record : IndexOfAnyExceptTests { protected override SimpleRecord Create(int value) => new SimpleRecord(value); } + public class IndexOfAnyExceptTests_String : IndexOfAnyExceptTests + { + protected override string Create(int value) => ((char)value).ToString(); + + [Theory] + + [InlineData(new string[] { null, "a", "a", "a", "a" }, new string[] { "a" }, 0)] + [InlineData(new string[] { "a", "a", null, "a", "a" }, new string[] { "a" }, 2)] + [InlineData(new string[] { "a", "a", "a", "a", "a" }, new string[] { null }, 0)] + [InlineData(new string[] { null, null, null, null, null }, new string[] { null }, -1)] + [InlineData(new string[] { null, null, null, null, "a" }, new string[] { null }, 4)] + + [InlineData(new string[] { null, "a", "a", "a", "a" }, new string[] { "a", null }, -1)] + [InlineData(new string[] { null, null, null, null, "a" }, new string[] { null, "a" }, -1)] + [InlineData(new string[] { "a", "a", null, "a", "a" }, new string[] { "a", "b" }, 2)] + [InlineData(new string[] { "a", "a", "a", "a", "a" }, new string[] { null, null, }, 0)] + [InlineData(new string[] { null, null, null, null, null }, new string[] { null, null }, -1)] + + [InlineData(new string[] { null, "a", "a", "a", "a" }, new string[] { "a", null, null }, -1)] + [InlineData(new string[] { null, null, null, null, "a" }, new string[] { null, null, "a" }, -1)] + [InlineData(new string[] { "a", "a", null, "a", "a" }, new string[] { "a", "b", null }, -1)] + [InlineData(new string[] { "a", "a", null, "a", "a" }, new string[] { "a", "a", "a" }, 2)] + [InlineData(new string[] { "a", "a", "a", "a", "a" }, new string[] { null, null, null }, 0)] + [InlineData(new string[] { null, null, null, null, null }, new string[] { null, null, null }, -1)] + + public void SearchingNulls(string[] input, string[] targets, int expected) + { + Assert.Equal(expected, input.AsSpan().IndexOfAnyExcept(targets)); + switch (targets.Length) + { + case 1: + Assert.Equal(expected, input.AsSpan().IndexOfAnyExcept(targets[0])); + break; + case 2: + Assert.Equal(expected, input.AsSpan().IndexOfAnyExcept(targets[0], targets[1])); + break; + case 3: + Assert.Equal(expected, input.AsSpan().IndexOfAnyExcept(targets[0], targets[1], targets[2])); + break; + } + } + } public record SimpleRecord(int Value); diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 708fa7ede340d3..386d44c0111169 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; @@ -1147,13 +1148,13 @@ public static int LastIndexOfAny(ref T searchSpace, int searchSpaceLength, re return -1; // not found } - public static int IndexOfAnyExcept(ref T searchSpace, T value0, int length) where T : IEquatable + public static int IndexOfAnyExcept(ref T searchSpace, T value0, int length) { Debug.Assert(length >= 0, "Expected non-negative length"); for (int i = 0; i < length; i++) { - if (!Unsafe.Add(ref searchSpace, i).Equals(value0)) + if (!EqualityComparer.Default.Equals(Unsafe.Add(ref searchSpace, i), value0)) { return i; }