Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit 005c990

Browse files
benaadamsbenaadams
committed
Intrinsicify SequenceCompareTo(char)
1 parent ad68763 commit 005c990

File tree

1 file changed

+153
-28
lines changed

1 file changed

+153
-28
lines changed

‎src/System.Private.CoreLib/shared/System/SpanHelpers.Char.cs

+153-28
Original file line numberDiff line numberDiff line change
@@ -66,66 +66,191 @@ ref Unsafe.As<char, byte>(ref valueTail),
6666
}
6767

6868
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
69-
public static unsafe int SequenceCompareTo(ref char first, int firstLength, ref char second, int secondLength)
69+
public static unsafe int SequenceCompareTo(ref char firstStart, int firstLength, ref char secondStart, int secondLength)
7070
{
7171
Debug.Assert(firstLength >= 0);
7272
Debug.Assert(secondLength >= 0);
7373

74-
int lengthDelta = firstLength - secondLength;
75-
76-
if (Unsafe.AreSame(ref first, ref second))
74+
if (Unsafe.AreSame(ref firstStart, ref secondStart))
7775
goto Equal;
7876

79-
IntPtr minLength = (IntPtr)((firstLength < secondLength) ? firstLength : secondLength);
80-
IntPtr i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations
77+
int minLength = (firstLength < secondLength) ? firstLength : secondLength;
78+
79+
int offset = 0;
80+
int lengthToExamine = minLength;
8181

82-
if ((byte*)minLength >= (byte*)(sizeof(UIntPtr) / sizeof(char)))
82+
if (Avx2.IsSupported)
8383
{
84-
if (Vector.IsHardwareAccelerated && (byte*)minLength >= (byte*)Vector<ushort>.Count)
84+
if (lengthToExamine >= Vector256<ushort>.Count)
85+
{
86+
lengthToExamine -= Vector256<ushort>.Count;
87+
uint matches;
88+
while (lengthToExamine > offset)
89+
{
90+
matches = (uint)Avx2.MoveMask(Avx2.CompareEqual(LoadVector256(ref firstStart, offset), LoadVector256(ref secondStart, offset)).AsByte());
91+
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
92+
// So the bit position in 'matches' corresponds to the element offset.
93+
94+
// 32 elements in Vector256<byte> so we compare to uint.MaxValue to check if everything matched
95+
if (matches == uint.MaxValue)
96+
{
97+
// All matched
98+
offset += Vector256<ushort>.Count;
99+
continue;
100+
}
101+
102+
goto Difference;
103+
}
104+
// Move to Vector length from end for final compare
105+
offset = lengthToExamine;
106+
// Same as method as above
107+
matches = (uint)Avx2.MoveMask(Avx2.CompareEqual(LoadVector256(ref firstStart, offset), LoadVector256(ref secondStart, offset)).AsByte());
108+
if (matches == uint.MaxValue)
109+
{
110+
// All matched
111+
goto Equal;
112+
}
113+
Difference:
114+
// Invert matches to find differences
115+
uint differences = ~matches;
116+
// Find bitflag offset of first difference and add to current offset,
117+
// flags are in bytes so divide for chars
118+
offset = offset + BitOperations.TrailingZeroCount((int)differences) / sizeof(char);
119+
120+
int result = Unsafe.Add(ref firstStart, offset).CompareTo(Unsafe.Add(ref secondStart, offset));
121+
Debug.Assert(result != 0);
122+
123+
return result;
124+
}
125+
126+
if (lengthToExamine >= Vector128<ushort>.Count)
85127
{
86-
IntPtr nLength = minLength - Vector<ushort>.Count;
87-
do
128+
lengthToExamine -= Vector128<ushort>.Count;
129+
uint matches;
130+
if (lengthToExamine > offset)
88131
{
89-
if (Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref first, i))) !=
90-
Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref second, i))))
132+
matches = (uint)Sse2.MoveMask(Sse2.CompareEqual(LoadVector128(ref firstStart, offset), LoadVector128(ref secondStart, offset)).AsByte());
133+
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
134+
// So the bit position in 'matches' corresponds to the element offset.
135+
136+
// 16 elements in Vector128<byte> so we compare to ushort.MaxValue to check if everything matched
137+
if (matches == ushort.MaxValue)
91138
{
92-
break;
139+
// All matched
140+
offset += Vector128<ushort>.Count;
141+
}
142+
else
143+
{
144+
goto Difference;
93145
}
94-
i += Vector<ushort>.Count;
95146
}
96-
while ((byte*)nLength >= (byte*)i);
147+
// Move to Vector length from end for final compare
148+
offset = lengthToExamine;
149+
// Same as method as above
150+
matches = (uint)Sse2.MoveMask(Sse2.CompareEqual(LoadVector128(ref firstStart, offset), LoadVector128(ref secondStart, offset)).AsByte());
151+
if (matches == ushort.MaxValue)
152+
{
153+
// All matched
154+
goto Equal;
155+
}
156+
Difference:
157+
// Invert matches to find differences
158+
uint differences = ~matches;
159+
// Find bitflag offset of first difference and add to current offset,
160+
// flags are in bytes so divide for chars
161+
offset = offset + BitOperations.TrailingZeroCount((int)differences) / sizeof(char);
162+
163+
int result = Unsafe.Add(ref firstStart, offset).CompareTo(Unsafe.Add(ref secondStart, offset));
164+
Debug.Assert(result != 0);
165+
166+
return result;
97167
}
168+
}
169+
else if (Sse2.IsSupported)
170+
{
171+
if (lengthToExamine >= Vector128<ushort>.Count)
172+
{
173+
lengthToExamine -= Vector128<ushort>.Count;
174+
uint matches;
175+
while (lengthToExamine > offset)
176+
{
177+
matches = (uint)Sse2.MoveMask(Sse2.CompareEqual(LoadVector128(ref firstStart, offset), LoadVector128(ref secondStart, offset)).AsByte());
178+
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
179+
// So the bit position in 'matches' corresponds to the element offset.
180+
181+
// 16 elements in Vector128<byte> so we compare to ushort.MaxValue to check if everything matched
182+
if (matches == ushort.MaxValue)
183+
{
184+
// All matched
185+
offset += Vector128<ushort>.Count;
186+
continue;
187+
}
188+
189+
goto Difference;
190+
}
191+
// Move to Vector length from end for final compare
192+
offset = lengthToExamine;
193+
// Same as method as above
194+
matches = (uint)Sse2.MoveMask(Sse2.CompareEqual(LoadVector128(ref firstStart, offset), LoadVector128(ref secondStart, offset)).AsByte());
195+
if (matches == ushort.MaxValue)
196+
{
197+
// All matched
198+
goto Equal;
199+
}
200+
Difference:
201+
// Invert matches to find differences
202+
uint differences = ~matches;
203+
// Find bitflag offset of first difference and add to current offset,
204+
// flags are in bytes so divide for chars
205+
offset = offset + BitOperations.TrailingZeroCount((int)differences) / sizeof(char);
206+
207+
int result = Unsafe.Add(ref firstStart, offset).CompareTo(Unsafe.Add(ref secondStart, offset));
208+
Debug.Assert(result != 0);
98209

99-
while ((byte*)minLength >= (byte*)(i + sizeof(UIntPtr) / sizeof(char)))
210+
return result;
211+
}
212+
}
213+
else if (Vector.IsHardwareAccelerated)
214+
{
215+
if (lengthToExamine > Vector<ushort>.Count)
100216
{
101-
if (Unsafe.ReadUnaligned<UIntPtr>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref first, i))) !=
102-
Unsafe.ReadUnaligned<UIntPtr>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref second, i))))
217+
lengthToExamine -= Vector<ushort>.Count;
218+
while (lengthToExamine > offset)
103219
{
104-
break;
220+
if (LoadVector(ref firstStart, offset) != LoadVector(ref secondStart, offset))
221+
{
222+
goto CharwiseCheck;
223+
}
224+
offset += Vector<ushort>.Count;
105225
}
106-
i += sizeof(UIntPtr) / sizeof(char);
226+
goto CharwiseCheck;
107227
}
108228
}
109229

110-
if (sizeof(UIntPtr) > sizeof(int) && (byte*)minLength >= (byte*)(i + sizeof(int) / sizeof(char)))
230+
if (lengthToExamine > sizeof(UIntPtr) / sizeof(char))
111231
{
112-
if (Unsafe.ReadUnaligned<int>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref first, i))) ==
113-
Unsafe.ReadUnaligned<int>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref second, i))))
232+
lengthToExamine -= sizeof(UIntPtr) / sizeof(char);
233+
while (lengthToExamine > offset)
114234
{
115-
i += sizeof(int) / sizeof(char);
235+
if (LoadUIntPtr(ref firstStart, offset) != LoadUIntPtr(ref secondStart, offset))
236+
{
237+
goto CharwiseCheck;
238+
}
239+
offset += sizeof(UIntPtr) / sizeof(char);
116240
}
117241
}
118242

119-
while ((byte*)i < (byte*)minLength)
243+
CharwiseCheck:
244+
while (minLength > offset)
120245
{
121-
int result = Unsafe.Add(ref first, i).CompareTo(Unsafe.Add(ref second, i));
246+
int result = Unsafe.Add(ref firstStart, offset).CompareTo(Unsafe.Add(ref secondStart, offset));
122247
if (result != 0)
123248
return result;
124-
i += 1;
249+
offset += 1;
125250
}
126251

127252
Equal:
128-
return lengthDelta;
253+
return firstLength - secondLength;
129254
}
130255

131256
// Adapted from IndexOf(...)

0 commit comments

Comments
 (0)