Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Vectorize {Last}IndexOf{Any} and {Last}IndexOfAnyExcept without code duplication #73768

Merged
merged 37 commits into from
Aug 17, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
9d01665
Vectorize LastIndexOf
adamsitnik Aug 11, 2022
14224a3
use structs to get performant code without code duplication
adamsitnik Aug 11, 2022
d05cf56
use it in all possible places
adamsitnik Aug 11, 2022
22f71ff
vectorize LastIndexOfAny(value0, value1)
adamsitnik Aug 11, 2022
bbc64e0
vectorize LastIndexOfAnyExcept(value0, value1)
adamsitnik Aug 11, 2022
f265d28
simplify it to make it easier to add 3 and 4 values overloads
adamsitnik Aug 11, 2022
b0f259c
vectorize LastIndexOfAny and LastIndexOfAnyExcept for 3 values
adamsitnik Aug 11, 2022
7dc03c3
rename (I am not convinced it's the best name yet)
adamsitnik Aug 11, 2022
628d429
vectorize Contains
adamsitnik Aug 12, 2022
114ca88
hide the implementation details
adamsitnik Aug 12, 2022
95a1b62
address review from Jan, don't cast Span to ROS. Introduce new helper…
adamsitnik Aug 12, 2022
ab8df3d
vectorize IndexOf(value)
adamsitnik Aug 12, 2022
bb13957
rename IndexOf used only by strlen to IndexOfNullByte and optimize it…
adamsitnik Aug 12, 2022
bbb7e4c
vectorize IndexOfAny<T>(value0, value1) and IndexOfAnyExcept<T>(value…
adamsitnik Aug 12, 2022
9186b8f
vectorize IndexOfAny<T>(value0, value1, value2) and IndexOfAnyExcept<…
adamsitnik Aug 12, 2022
5a39d21
vectorize IndexOfAny<T>(value0, value1, value2, value3) and IndexOfAn…
adamsitnik Aug 12, 2022
9099ef9
vectorize IndexOfAny<T>(value0, value1, value2, value3, value4)
adamsitnik Aug 12, 2022
5661793
remove dead code
adamsitnik Aug 12, 2022
8cb8208
use built-in helpers
adamsitnik Aug 12, 2022
ec447c7
Revert "address review from Jan, don't cast Span to ROS. Introduce ne…
adamsitnik Aug 12, 2022
62f16c1
add/remove special handling of some types
adamsitnik Aug 12, 2022
26c188d
fix NativeAOT build
adamsitnik Aug 12, 2022
2af01e3
address code review feedback
adamsitnik Aug 13, 2022
b305c54
perform manual loop unrolling in order to avoid perf regression for s…
adamsitnik Aug 13, 2022
f954ac5
Merge remote-tracking branch 'upstream/main' into vectorizeLastIndexOf
adamsitnik Aug 14, 2022
846e4fc
restore Vector<T> code path as Vector128 is not accelerated by Mono x…
adamsitnik Aug 14, 2022
8a74f28
remove redundant for loop that was left here by mistake
adamsitnik Aug 14, 2022
fab3143
apply manual loop unrolling to LastIndexOfAny methods
adamsitnik Aug 15, 2022
4274f54
vectorize LastIndexOf(4) and LastIndexOfAnyExcept(4)
adamsitnik Aug 15, 2022
59c867f
Apply suggestions from code review
adamsitnik Aug 15, 2022
3049470
Merge remote-tracking branch 'upstream/main' into vectorizeLastIndexOf
adamsitnik Aug 15, 2022
afc09bd
fix the build
adamsitnik Aug 15, 2022
efdff0e
fix the NativeAOT build
adamsitnik Aug 15, 2022
5831e09
Merge remote-tracking branch 'upstream/main' into vectorizeLastIndexOf
adamsitnik Aug 16, 2022
7f003da
optimize Except methods for small inputs
adamsitnik Aug 16, 2022
0dc257f
address code review feedback:
adamsitnik Aug 17, 2022
0c6b01d
reduce the regression for IndexOfAnyExcept(3) and LastIndexOfAnyExcep…
adamsitnik Aug 17, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ internal static unsafe void ConvertFixedToNative(int flags, string strManaged, I

internal static unsafe string ConvertFixedToManaged(IntPtr cstr, int length)
{
int end = SpanHelpers.IndexOf(ref *(byte*)cstr, 0, length);
int end = new ReadOnlySpan<byte>((byte*)cstr, length).IndexOf((byte)0);
if (end >= 0)
{
length = end;
Expand Down Expand Up @@ -450,7 +450,7 @@ internal static unsafe void ConvertToNative(string? strManaged, IntPtr nativeHom

internal static unsafe string ConvertToManaged(IntPtr nativeHome, int length)
{
int end = SpanHelpers.IndexOf(ref *(char*)nativeHome, '\0', length);
int end = new ReadOnlySpan<char>((char*)nativeHome, length).IndexOf('\0');
if (end >= 0)
{
length = end;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ internal static unsafe void StringToByValAnsiString(string str, byte* pNative, i

public static unsafe string ByValAnsiStringToString(byte* buffer, int length)
{
int end = SpanHelpers.IndexOf(ref *(byte*)buffer, 0, length);
if (end != -1)
int end = new ReadOnlySpan<byte>(buffer, length).IndexOf((byte)0);
if (end >= 0)
{
length = end;
}
Expand All @@ -77,8 +77,8 @@ internal static unsafe void StringToUnicodeFixedArray(string str, ushort* buffer

internal static unsafe string UnicodeToStringFixedArray(ushort* buffer, int length)
{
int end = SpanHelpers.IndexOf(ref *(char*)buffer, '\0', length);
if (end != -1)
int end = new ReadOnlySpan<char>(buffer, length).IndexOf('\0');
if (end >= 0)
{
length = end;
}
Expand Down
55 changes: 55 additions & 0 deletions src/libraries/System.Memory/tests/Span/IndexOf.T.cs
Original file line number Diff line number Diff line change
Expand Up @@ -192,5 +192,60 @@ public static void IndexOfNull_String(string[] spanInput, int expected)
Span<string> theStrings = spanInput;
Assert.Equal(expected, theStrings.IndexOf((string)null));
}

[Fact]
public static void NotBitwiseEquatableUsesCustomIEquatableImplementationForActualComparison()
{
const byte Ten = 10, NotTen = 11;
for (int length = 1; length < 100; length++)
{
TwoBytes[] array = new TwoBytes[length];
for (int i = 0; i < length; i++)
{
array[i] = new TwoBytes(Ten, (byte)i);
}

Span<TwoBytes> span = new Span<TwoBytes>(array);
ReadOnlySpan<TwoBytes> ros = new ReadOnlySpan<TwoBytes>(array);

ReadOnlySpan<TwoBytes> noMatch2 = new TwoBytes[2] { new TwoBytes(10, NotTen), new TwoBytes(10, NotTen) };
Assert.Equal(-1, span.IndexOfAny(noMatch2));
Assert.Equal(-1, ros.IndexOfAny(noMatch2));
Assert.Equal(-1, span.LastIndexOfAny(noMatch2));
Assert.Equal(-1, ros.LastIndexOfAny(noMatch2));

ReadOnlySpan<TwoBytes> noMatch3 = new TwoBytes[3] { new TwoBytes(10, NotTen), new TwoBytes(10, NotTen), new TwoBytes(10, NotTen) };
Assert.Equal(-1, span.IndexOfAny(noMatch3));
Assert.Equal(-1, ros.IndexOfAny(noMatch3));
Assert.Equal(-1, span.LastIndexOfAny(noMatch3));
Assert.Equal(-1, ros.LastIndexOfAny(noMatch3));

ReadOnlySpan<TwoBytes> match2 = new TwoBytes[2] { new TwoBytes(0, Ten), new TwoBytes(0, Ten) };
Assert.Equal(0, span.IndexOfAny(match2));
Assert.Equal(0, ros.IndexOfAny(match2));
Assert.Equal(0, span.LastIndexOfAny(match2));
Assert.Equal(0, ros.LastIndexOfAny(match2));

ReadOnlySpan<TwoBytes> match3 = new TwoBytes[3] { new TwoBytes(0, Ten), new TwoBytes(0, Ten), new TwoBytes(0, Ten) };
Assert.Equal(0, span.IndexOfAny(match3));
Assert.Equal(0, ros.IndexOfAny(match3));
Assert.Equal(0, span.LastIndexOfAny(match3));
Assert.Equal(0, ros.LastIndexOfAny(match3));
}
}

private readonly struct TwoBytes : IEquatable<TwoBytes>
{
private readonly byte _first, _second;

public TwoBytes(byte first, byte second)
{
_first = first;
_second = second;
}

// it compares different fields on purpose
public bool Equals(TwoBytes other) => _first == other._second && _second == other._first;
}
}
}
24 changes: 12 additions & 12 deletions src/libraries/System.Private.CoreLib/src/System/Array.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1328,17 +1328,17 @@ public static int IndexOf<T>(T[] array, T value, int startIndex, int count)
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
int result = SpanHelpers.IndexOf(
int result = SpanHelpers.IndexOfValueType(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array)), startIndex),
Unsafe.As<T, byte>(ref value),
count);
return (result >= 0 ? startIndex : 0) + result;
}
else if (Unsafe.SizeOf<T>() == sizeof(char))
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
int result = SpanHelpers.IndexOf(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<char[]>(array)), startIndex),
Unsafe.As<T, char>(ref value),
int result = SpanHelpers.IndexOfValueType(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)), startIndex),
Unsafe.As<T, short>(ref value),
count);
return (result >= 0 ? startIndex : 0) + result;
}
Expand Down Expand Up @@ -1586,27 +1586,27 @@ public static int LastIndexOf<T>(T[] array, T value, int startIndex, int count)
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
int result = SpanHelpers.LastIndexOfValueType(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array)), endIndex),
Unsafe.As<T, byte>(ref value),
count);

return (result >= 0 ? endIndex : 0) + result;
}
else if (Unsafe.SizeOf<T>() == sizeof(char))
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<char[]>(array)), endIndex),
Unsafe.As<T, char>(ref value),
int result = SpanHelpers.LastIndexOfValueType(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)), endIndex),
Unsafe.As<T, short>(ref value),
count);

return (result >= 0 ? endIndex : 0) + result;
}
else if (Unsafe.SizeOf<T>() == sizeof(int))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
int result = SpanHelpers.LastIndexOfValueType(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)), endIndex),
Unsafe.As<T, int>(ref value),
count);
Expand All @@ -1616,7 +1616,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)),
else if (Unsafe.SizeOf<T>() == sizeof(long))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
int result = SpanHelpers.LastIndexOfValueType(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<long[]>(array)), endIndex),
Unsafe.As<T, long>(ref value),
count);
Expand Down
2 changes: 1 addition & 1 deletion src/libraries/System.Private.CoreLib/src/System/Enum.cs
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ private static int FindDefinedIndex(ulong[] ulValues, ulong ulValue)
int ulValuesLength = ulValues.Length;
ref ulong start = ref MemoryMarshal.GetArrayDataReference(ulValues);
return ulValuesLength <= NumberOfValuesThreshold ?
SpanHelpers.IndexOf(ref start, ulValue, ulValuesLength) :
SpanHelpers.IndexOfValueType(ref Unsafe.As<ulong, long>(ref start), (long)ulValue, ulValuesLength) :
SpanHelpers.BinarySearch(ref start, ulValuesLength, ulValue);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,8 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan<char> source, ReadOnly
{
// Do a quick search for the first element of "value".
int relativeIndex = isLetter ?
SpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) :
SpanHelpers.IndexOf(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength);
SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) :
SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength);
if (relativeIndex < 0)
{
break;
Expand Down
Loading