From 92a86b05548424b41e4e9238079348a3290be47b Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 26 Jul 2024 10:02:59 -0400 Subject: [PATCH] =?UTF-8?q?Revert=20"Add=20SearchValues=20implementa?= =?UTF-8?q?tion=20for=20two=20sets=20of=20128=20chars=20(#103=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 13abb446cfda5a37bf193b4c0bc3cd372986fb88. --- .../System.Memory/tests/Span/SearchValues.cs | 1 - .../System.Private.CoreLib.Shared.projitems | 1 - .../AsciiWithSecondSetCharSearchValues.cs | 75 --- .../SearchValues/IndexOfAnyAsciiSearcher.cs | 428 +----------------- .../src/System/SearchValues/SearchValues.cs | 15 +- 5 files changed, 22 insertions(+), 498 deletions(-) delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiWithSecondSetCharSearchValues.cs diff --git a/src/libraries/System.Memory/tests/Span/SearchValues.cs b/src/libraries/System.Memory/tests/Span/SearchValues.cs index e4afec5381c3e7..8f39a95bd3c0f4 100644 --- a/src/libraries/System.Memory/tests/Span/SearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/SearchValues.cs @@ -82,7 +82,6 @@ public static IEnumerable Values_MemberData() { yield return Pair(value); yield return Pair('a' + value); - yield return Pair('\uFF00' + value); // Test some more duplicates if (value.Length > 0) diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index ffc8b5925122a9..533ff2cb96b213 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -441,7 +441,6 @@ - diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiWithSecondSetCharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiWithSecondSetCharSearchValues.cs deleted file mode 100644 index f83f2c5f241932..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiWithSecondSetCharSearchValues.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; - -namespace System.Buffers -{ - internal sealed class AsciiWithSecondSetCharSearchValues : SearchValues - where TOptimizations : struct, IndexOfAnyAsciiSearcher.IOptimizations - { - private IndexOfAnyAsciiSearcher.AsciiWithSecondSetState _state; - - public AsciiWithSecondSetCharSearchValues(IndexOfAnyAsciiSearcher.AsciiWithSecondSetState state) => - _state = state; - - internal override char[] GetValues() => - _state.Lookup.GetValues(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) => - _state.Lookup.FastContains(value); - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.IndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.IndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.LastIndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.LastIndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsAny(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.ContainsAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsAnyExcept(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.ContainsAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs index af67df918ec737..36965c24da7e54 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs @@ -16,39 +16,21 @@ internal static class IndexOfAnyAsciiSearcher public struct AsciiState(Vector128 bitmap, BitVector256 lookup) { public Vector256 Bitmap = Vector256.Create(bitmap); - public readonly BitVector256 Lookup = lookup; + public BitVector256 Lookup = lookup; public readonly AsciiState CreateInverse() => new AsciiState(~Bitmap._lower, Lookup.CreateInverse()); } - public readonly struct AsciiWithSecondSetState(Vector128 asciiBitmap, ushort offset, Vector128 secondBitmap, ProbabilisticMapState lookup) + public struct AnyByteState(Vector128 bitmap0, Vector128 bitmap1, BitVector256 lookup) { - public readonly ushort Offset = offset; - public readonly Vector256 AsciiBitmap = Vector256.Create(asciiBitmap, asciiBitmap); - public readonly Vector256 SecondBitmap = Vector256.Create(secondBitmap, secondBitmap); - public readonly ProbabilisticMapState Lookup = lookup; // Only used for single-character checks. - } - - public readonly struct AnyByteState(Vector128 bitmap0, Vector128 bitmap1, BitVector256 lookup) - { - public readonly Vector256 Bitmap0 = Vector256.Create(bitmap0); - public readonly Vector256 Bitmap1 = Vector256.Create(bitmap1); - public readonly BitVector256 Lookup = lookup; + public Vector256 Bitmap0 = Vector256.Create(bitmap0); + public Vector256 Bitmap1 = Vector256.Create(bitmap1); + public BitVector256 Lookup = lookup; } internal static bool IsVectorizationSupported => Ssse3.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe void SetBitmapBit(byte* bitmap, int value) - { - Debug.Assert((uint)value <= 127); - - int highNibble = value >> 4; - int lowNibble = value & 0xF; - bitmap[(uint)lowNibble] |= (byte)(1 << highNibble); - } - internal static unsafe void ComputeAnyByteState(ReadOnlySpan values, out AnyByteState state) { // The exact format of these bitmaps differs from the other ComputeBitmap overloads as it's meant for the full [0, 255] range algorithm. @@ -64,13 +46,16 @@ internal static unsafe void ComputeAnyByteState(ReadOnlySpan values, out A { lookupLocal.Set(b); - if (b < 128) + int highNibble = b >> 4; + int lowNibble = b & 0xF; + + if (highNibble < 8) { - SetBitmapBit(bitmapLocal0, b); + bitmapLocal0[(uint)lowNibble] |= (byte)(1 << highNibble); } else { - SetBitmapBit(bitmapLocal1, b - 128); + bitmapLocal1[(uint)lowNibble] |= (byte)(1 << (highNibble - 8)); } } @@ -96,64 +81,14 @@ internal static unsafe void ComputeAsciiState(ReadOnlySpan values, out Asc } lookupLocal.Set(value); - SetBitmapBit(bitmapLocal, value); - } - state = new AsciiState(bitmapSpace, lookupLocal); - } - - /// - /// Tests if the values fit within two ranges of 128 characters each, with the first one being fixed to [0, 127]. - /// - internal static unsafe bool TryComputeAsciiWithSecondSetState(ReadOnlySpan values, int maxInclusive, out AsciiWithSecondSetState state) - { - Debug.Assert(maxInclusive > 127); - Debug.Assert(values.ContainsAnyInRange((char)0, (char)127)); + int highNibble = value >> 4; + int lowNibble = value & 0xF; - int offset = maxInclusive - 127; - - // The check below is inclusive of offset because we're not allowing the 0th character to be set in the second set. - // Allowing it would mean having to flow an extra TOptimizations generic parameter to the IndexOfAny implementation. - // The sets may overlap, so if all the values are < 255, we know we can cover all values with two sets of 128 chars. - if (maxInclusive >= 255 && - values.ContainsAnyInRange((char)128, (char)offset)) - { - state = default; - return false; + bitmapLocal[(uint)lowNibble] |= (byte)(1 << highNibble); } - Vector128 asciiBitmapSpace = default; - Vector128 secondBitmapSpace = default; - byte* asciiBitmapLocal = (byte*)&asciiBitmapSpace; - byte* secondBitmapLocal = (byte*)&secondBitmapSpace; - - foreach (char c in values) - { - if (c < 128) - { - SetBitmapBit(asciiBitmapLocal, c); - } - else - { - char second = (char)(c - offset); - if (second >= 128) - { - // The values were modified concurrent with the call to SearchValues.Create. - ThrowHelper.ThrowInvalidOperationException_InvalidOperation_EnumFailedVersion(); - } - - SetBitmapBit(secondBitmapLocal, second); - } - } - - // The 0th bit in second bitmap shouldn't be set while searching because we're using "Default" packing, which can't handle 0s. - // It's possible the bit is set here if the two sets overlap. If that's the case, we must clear it. - // This is still okay as that character will also be matched by the ASCII set, and we're ORing the results together. - Debug.Assert((secondBitmapSpace[0] & 1) == 0 || maxInclusive < 255); - secondBitmapLocal[0] &= 0xFF - 1; - - state = new AsciiWithSecondSetState(asciiBitmapSpace, (ushort)offset, secondBitmapSpace, new ProbabilisticMapState(values, maxInclusive)); - return true; + state = new AsciiState(bitmapSpace, lookupLocal); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -169,7 +104,10 @@ private static unsafe bool TryComputeBitmap(ReadOnlySpan values, byte* bit return false; } - SetBitmapBit(bitmapLocal, c); + int highNibble = c >> 4; + int lowNibble = c & 0xF; + + bitmapLocal[(uint)lowNibble] |= (byte)(1 << highNibble); } needleContainsZero = (bitmap[0] & 1) != 0; @@ -530,298 +468,6 @@ public static int LastIndexOfAny(ref short searchSpace return -1; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - public static bool ContainsAny(ref short searchSpace, int searchSpaceLength, ref AsciiWithSecondSetState state) - where TNegator : struct, INegator - where TOptimizations : struct, IOptimizations => - IndexOfAnyCore>(ref searchSpace, searchSpaceLength, ref state); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - public static int IndexOfAny(ref short searchSpace, int searchSpaceLength, ref AsciiWithSecondSetState state) - where TNegator : struct, INegator - where TOptimizations : struct, IOptimizations => - IndexOfAnyCore>(ref searchSpace, searchSpaceLength, ref state); - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - private static TResult IndexOfAnyCore(ref short searchSpace, int searchSpaceLength, ref AsciiWithSecondSetState state) - where TResult : struct - where TNegator : struct, INegator - where TOptimizations : struct, IOptimizations - where TResultMapper : struct, IResultMapper - { - ref short currentSearchSpace = ref searchSpace; - - if (searchSpaceLength < Vector128.Count) - { - ref short searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength); - - while (!Unsafe.AreSame(ref currentSearchSpace, ref searchSpaceEnd)) - { - char c = (char)currentSearchSpace; - if (TNegator.NegateIfNeeded(state.Lookup.FastContains(c))) - { - return TResultMapper.ScalarResult(ref searchSpace, ref currentSearchSpace); - } - - currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 1); - } - - return TResultMapper.NotFound; - } - -#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough // The behavior of the rest of the function remains the same if Avx2.IsSupported is false - if (Avx2.IsSupported && searchSpaceLength > 2 * Vector128.Count) -#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough - { - Vector256 asciiBitmap256 = state.AsciiBitmap; - Vector256 secondBitmap256 = state.SecondBitmap; - Vector256 offset256 = Vector256.Create(state.Offset); - - if (searchSpaceLength > 2 * Vector256.Count) - { - // Process the input in chunks of 32 characters (2 * Vector256). - // We're mainly interested in a single byte of each character, and the core lookup operates on a Vector256. - // As packing two Vector256s into a Vector256 is cheap compared to the lookup, we can effectively double the throughput. - // If the input length is a multiple of 32, don't consume the last 32 characters in this loop. - // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". - ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - (2 * Vector256.Count)); - - do - { - Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); - Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - - Vector256 result = IndexOfAnyLookup(source0, source1, asciiBitmap256, secondBitmap256, offset256); - if (result != Vector256.Zero) - { - return TResultMapper.FirstIndex(ref searchSpace, ref currentSearchSpace, result); - } - - currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); - } - while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); - } - - // We have 1-32 characters remaining. Process the first and last vector in the search space. - // They may overlap, but we'll handle that in the index calculation if we do get a match. - Debug.Assert(searchSpaceLength >= Vector256.Count, "We expect that the input is long enough for us to load a whole vector."); - { - ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - Vector256.Count); - - ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) - ? ref oneVectorAwayFromEnd - : ref currentSearchSpace; - - Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); - Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); - - Vector256 result = IndexOfAnyLookup(source0, source1, asciiBitmap256, secondBitmap256, offset256); - if (result != Vector256.Zero) - { - return TResultMapper.FirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); - } - } - - return TResultMapper.NotFound; - } - - Vector128 asciiBitmap = state.AsciiBitmap._lower; - Vector128 secondBitmap = state.SecondBitmap._lower; - Vector128 offset = Vector128.Create(state.Offset); - -#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough // The behavior of the rest of the function remains the same if Avx2.IsSupported is false - if (!Avx2.IsSupported && searchSpaceLength > 2 * Vector128.Count) -#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough - { - // Process the input in chunks of 16 characters (2 * Vector128). - // We're mainly interested in a single byte of each character, and the core lookup operates on a Vector128. - // As packing two Vector128s into a Vector128 is cheap compared to the lookup, we can effectively double the throughput. - // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. - // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". - ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - (2 * Vector128.Count)); - - do - { - Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); - Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - - Vector128 result = IndexOfAnyLookup(source0, source1, asciiBitmap, secondBitmap, offset); - if (result != Vector128.Zero) - { - return TResultMapper.FirstIndex(ref searchSpace, ref currentSearchSpace, result); - } - - currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); - } - while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); - } - - // We have 1-16 characters remaining. Process the first and last vector in the search space. - // They may overlap, but we'll handle that in the index calculation if we do get a match. - Debug.Assert(searchSpaceLength >= Vector128.Count, "We expect that the input is long enough for us to load a whole vector."); - { - ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - Vector128.Count); - - ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) - ? ref oneVectorAwayFromEnd - : ref currentSearchSpace; - - Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); - Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - - Vector128 result = IndexOfAnyLookup(source0, source1, asciiBitmap, secondBitmap, offset); - if (result != Vector128.Zero) - { - return TResultMapper.FirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); - } - } - - return TResultMapper.NotFound; - } - - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - public static int LastIndexOfAny(ref short searchSpace, int searchSpaceLength, ref AsciiWithSecondSetState state) - where TNegator : struct, INegator - where TOptimizations : struct, IOptimizations - { - if (searchSpaceLength < Vector128.Count) - { - for (int i = searchSpaceLength - 1; i >= 0; i--) - { - char c = (char)Unsafe.Add(ref searchSpace, i); - if (TNegator.NegateIfNeeded(state.Lookup.FastContains(c))) - { - return i; - } - } - - return -1; - } - - ref short currentSearchSpace = ref Unsafe.Add(ref searchSpace, searchSpaceLength); - -#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough // The else clause is semantically equivalent - if (Avx2.IsSupported && searchSpaceLength > 2 * Vector128.Count) -#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough - { - Vector256 asciiBitmap256 = state.AsciiBitmap; - Vector256 secondBitmap256 = state.SecondBitmap; - Vector256 offset256 = Vector256.Create(state.Offset); - - if (searchSpaceLength > 2 * Vector256.Count) - { - // Process the input in chunks of 32 characters (2 * Vector256). - // We're mainly interested in a single byte of each character, and the core lookup operates on a Vector256. - // As packing two Vector256s into a Vector256 is cheap compared to the lookup, we can effectively double the throughput. - // If the input length is a multiple of 32, don't consume the last 32 characters in this loop. - // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". - ref short twoVectorsAfterStart = ref Unsafe.Add(ref searchSpace, 2 * Vector256.Count); - - do - { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, 2 * Vector256.Count); - - Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); - Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - - Vector256 result = IndexOfAnyLookup(source0, source1, asciiBitmap256, secondBitmap256, offset256); - if (result != Vector256.Zero) - { - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, result); - } - } - while (Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref twoVectorsAfterStart)); - } - - // We have 1-32 characters remaining. Process the first and last vector in the search space. - // They may overlap, but we'll handle that in the index calculation if we do get a match. - Debug.Assert(searchSpaceLength >= Vector256.Count, "We expect that the input is long enough for us to load a whole vector."); - { - ref short oneVectorAfterStart = ref Unsafe.Add(ref searchSpace, Vector256.Count); - - ref short secondVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAfterStart) - ? ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count) - : ref searchSpace; - - Vector256 source0 = Vector256.LoadUnsafe(ref searchSpace); - Vector256 source1 = Vector256.LoadUnsafe(ref secondVector); - - Vector256 result = IndexOfAnyLookup(source0, source1, asciiBitmap256, secondBitmap256, offset256); - if (result != Vector256.Zero) - { - return ComputeLastIndexOverlapped(ref searchSpace, ref secondVector, result); - } - } - - return -1; - } - - Vector128 asciiBitmap = state.AsciiBitmap._lower; - Vector128 secondBitmap = state.SecondBitmap._lower; - Vector128 offset = Vector128.Create(state.Offset); - - if (!Avx2.IsSupported && searchSpaceLength > 2 * Vector128.Count) - { - // Process the input in chunks of 16 characters (2 * Vector128). - // We're mainly interested in a single byte of each character, and the core lookup operates on a Vector128. - // As packing two Vector128s into a Vector128 is cheap compared to the lookup, we can effectively double the throughput. - // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. - // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". - ref short twoVectorsAfterStart = ref Unsafe.Add(ref searchSpace, 2 * Vector128.Count); - - do - { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, 2 * Vector128.Count); - - Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); - Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - - Vector128 result = IndexOfAnyLookup(source0, source1, asciiBitmap, secondBitmap, offset); - if (result != Vector128.Zero) - { - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, result); - } - } - while (Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref twoVectorsAfterStart)); - } - - // We have 1-16 characters remaining. Process the first and last vector in the search space. - // They may overlap, but we'll handle that in the index calculation if we do get a match. - Debug.Assert(searchSpaceLength >= Vector128.Count, "We expect that the input is long enough for us to load a whole vector."); - { - ref short oneVectorAfterStart = ref Unsafe.Add(ref searchSpace, Vector128.Count); - - ref short secondVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAfterStart) - ? ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count) - : ref searchSpace; - - Vector128 source0 = Vector128.LoadUnsafe(ref searchSpace); - Vector128 source1 = Vector128.LoadUnsafe(ref secondVector); - - Vector128 result = IndexOfAnyLookup(source0, source1, asciiBitmap, secondBitmap, offset); - if (result != Vector128.Zero) - { - return ComputeLastIndexOverlapped(ref searchSpace, ref secondVector, result); - } - } - - return -1; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd))] @@ -1385,25 +1031,6 @@ private static Vector128 IndexOfAnyLookup(Vector return TNegator.NegateIfNeeded(result); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Sse2))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - private static Vector128 IndexOfAnyLookup(Vector128 source0, Vector128 source1, Vector128 bitmapLookup0, Vector128 bitmapLookup1, Vector128 offset) - where TNegator : struct, INegator - where TOptimizations : struct, IOptimizations - { - Debug.Assert((bitmapLookup1[0] & 1) == 0, "The 0th bit in second bitmap shouldn't be set."); - - Vector128 packed0 = TOptimizations.PackSources(source0.AsUInt16(), source1.AsUInt16()); - Vector128 packed1 = Default.PackSources(source0.AsUInt16() - offset, source1.AsUInt16() - offset); - - Vector128 result0 = IndexOfAnyLookupCore(packed0, bitmapLookup0); - Vector128 result1 = IndexOfAnyLookupCore(packed1, bitmapLookup1); - - return TNegator.NegateIfNeeded(result0 | result1); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd))] @@ -1447,23 +1074,6 @@ private static Vector256 IndexOfAnyLookup(Vector return TNegator.NegateIfNeeded(result); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - private static Vector256 IndexOfAnyLookup(Vector256 source0, Vector256 source1, Vector256 bitmapLookup0, Vector256 bitmapLookup1, Vector256 offset) - where TNegator : struct, INegator - where TOptimizations : struct, IOptimizations - { - Debug.Assert((bitmapLookup1[0] & 1) == 0, "The 0th bit in second bitmap shouldn't be set."); - - Vector256 packed0 = TOptimizations.PackSources(source0.AsUInt16(), source1.AsUInt16()); - Vector256 packed1 = Default.PackSources(source0.AsUInt16() - offset, source1.AsUInt16() - offset); - - Vector256 result0 = IndexOfAnyLookupCore(packed0, bitmapLookup0); - Vector256 result1 = IndexOfAnyLookupCore(packed1, bitmapLookup1); - - return TNegator.NegateIfNeeded(result0 | result1); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx2))] private static Vector256 IndexOfAnyLookupCore(Vector256 source, Vector256 bitmapLookup) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs index c566244a2dd3e5..e02192a70ebbe4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs @@ -159,19 +159,10 @@ public static SearchValues Create(params ReadOnlySpan values) if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && minInclusive < 128) { - // We have a mix of ASCII and non-ASCII characters. + // If we have both ASCII and non-ASCII characters, use an implementation that + // does an optimistic ASCII fast-path and then falls back to the ProbabilisticMap. - if (IndexOfAnyAsciiSearcher.TryComputeAsciiWithSecondSetState(values, maxInclusive, out IndexOfAnyAsciiSearcher.AsciiWithSecondSetState state)) - { - // All of the non-ASCII values fit within a range of 128 characters. - // Use an implementation that checks against two 128-bit bitmaps, with the second one at a variable offset to the start of the non-ASCII set. - return (Ssse3.IsSupported || PackedSimd.IsSupported) && minInclusive == 0 - ? new AsciiWithSecondSetCharSearchValues(state) - : new AsciiWithSecondSetCharSearchValues(state); - } - - // Use an implementation that does an optimistic ASCII fast-path and then falls back to the ProbabilisticMap. - return (Ssse3.IsSupported || PackedSimd.IsSupported) && minInclusive == 0 + return (Ssse3.IsSupported || PackedSimd.IsSupported) && values.Contains('\0') ? new ProbabilisticWithAsciiCharSearchValues(values, maxInclusive) : new ProbabilisticWithAsciiCharSearchValues(values, maxInclusive); }