From e29deddd67ff344399947afbf532490c1cf14f3e Mon Sep 17 00:00:00 2001 From: kzrnm Date: Thu, 3 Jul 2025 02:01:35 +0900 Subject: [PATCH 1/4] Add test caces for BitArray.*Shift --- .../tests/BitArray/BitArray_OperatorsTests.cs | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs b/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs index cf4ae6c477a68c..9d8e91f46a8c1f 100644 --- a/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs +++ b/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs @@ -245,13 +245,33 @@ public static void Xor_With_Resize(BitArray left, BitArray right, int newLeftLen public static IEnumerable Shift_Data() { - foreach (int size in new[] { 0, 1, BitsPerInt32 / 2, BitsPerInt32, BitsPerInt32 + 1, 2 * BitsPerInt32, 2 * BitsPerInt32 + 1 }) + Random random = new Random(0); + foreach (int size in new[] { + 0, + 1, + BitsPerInt32 / 2, + BitsPerInt32, + BitsPerInt32 + 1, + 2 * BitsPerInt32 - 1, + 2 * BitsPerInt32 + 1, + 1023, + 1024, + 1025, + }) { - foreach (int shift in new[] { 0, 1, size / 2, size - 1, size }.Where(s => s >= 0).Distinct()) + foreach (int shift in new[] { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + size / 3, size / 2, size / 2 + 1, size - 1, size, + }.Where(s => s >= 0).Distinct()) { yield return new object[] { size, new int[] { /* deliberately empty */ }, shift }; yield return new object[] { size, Enumerable.Range(0, size), shift }; + int[] nums = Enumerable.Range(0, size).ToArray(); + random.Shuffle(nums); + yield return new object[] { size, nums.Take(size / 2), shift }; + if (size > 1) { foreach (int position in new[] { 0, size / 2, size - 1 }) From a01656a44f8179be46264acd9b57ebc2403d6fd9 Mon Sep 17 00:00:00 2001 From: kzrnm Date: Thu, 3 Jul 2025 02:20:05 +0900 Subject: [PATCH 2/4] hardware intrinsic in BitArray.*Shift --- .../src/System/Collections/BitArray.cs | 171 +++++++++++++----- 1 file changed, 129 insertions(+), 42 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs index 66fe6f6c37eff6..e79a3ef2727d95 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs @@ -513,51 +513,90 @@ public BitArray RightShift(int count) return this; } - Span intSpan = MemoryMarshal.Cast((Span)_array); - + Span thisSpan = new Span(_array, 0, GetByteArrayLengthFromBitLength(_bitLength)); int toIndex = 0; - int ints = GetInt32ArrayLengthFromBitLength(_bitLength); + if (count < _bitLength) { - // We can not use Math.DivRem without taking a dependency on System.Runtime.Extensions - (int fromIndex, int shiftCount) = Math.DivRem(count, 32); - int extraBits = (int)((uint)_bitLength % 32); + (int fromIndex, int shiftCount) = Math.DivRem(count, BitsPerByte); if (shiftCount == 0) { - // Cannot use `(1u << extraBits) - 1u` as the mask - // because for extraBits == 0, we need the mask to be 111...111, not 0. - // In that case, we are shifting a uint by 32, which could be considered undefined. - // The result of a shift operation is undefined ... if the right operand - // is greater than or equal to the width in bits of the promoted left operand, - // https://learn.microsoft.com/cpp/c-language/bitwise-shift-operators?view=vs-2017 - // However, the compiler protects us from undefined behaviour by constraining the - // right operand to between 0 and width - 1 (inclusive), i.e. right_operand = (right_operand % width). - uint mask = uint.MaxValue >> (BitsPerInt32 - extraBits); - intSpan[ints - 1] &= ReverseIfBE((int)mask); - - intSpan.Slice((int)fromIndex, ints - fromIndex).CopyTo(intSpan); - toIndex = ints - fromIndex; + thisSpan.Slice(fromIndex).CopyTo(thisSpan); + toIndex = thisSpan.Length - fromIndex; } else { - int lastIndex = ints - 1; + if (Vector512.IsHardwareAccelerated) + { + toIndex = Apply>(shiftCount, fromIndex, thisSpan); + } + else if (Vector256.IsHardwareAccelerated) + { + toIndex = Apply>(shiftCount, fromIndex, thisSpan); + } + else if (Vector128.IsHardwareAccelerated) + { + toIndex = Apply>(shiftCount, fromIndex, thisSpan); + } + fromIndex += toIndex; + + ref byte p = ref MemoryMarshal.GetReference(thisSpan); - while (fromIndex < lastIndex) + int carry32Count = BitsPerInt32 - shiftCount; + while (fromIndex < thisSpan.Length - 4) { - uint right = (uint)ReverseIfBE(intSpan[fromIndex]) >> shiftCount; - int left = ReverseIfBE(intSpan[++fromIndex]) << (BitsPerInt32 - shiftCount); - intSpan[toIndex++] = ReverseIfBE(left | (int)right); + int lo = ReverseIfBE(Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)fromIndex))) >>> shiftCount; + int hi = Unsafe.AddByteOffset(ref p, (uint)(fromIndex + 4)) << carry32Count; + int result = ReverseIfBE(hi | lo); + Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref p, toIndex), result); + + fromIndex += 4; + toIndex += 4; } - uint mask = uint.MaxValue >> (BitsPerInt32 - extraBits); - mask &= (uint)ReverseIfBE(intSpan[fromIndex]); - intSpan[toIndex++] = ReverseIfBE((int)(mask >> shiftCount)); + int carryCount = BitsPerByte - shiftCount; + while (fromIndex < thisSpan.Length) + { + int lo = thisSpan[fromIndex] >>> shiftCount; + int hi = + fromIndex + 1 < thisSpan.Length + ? thisSpan[fromIndex + 1] << carryCount + : 0; + + thisSpan[toIndex] = (byte)(hi | lo); + + fromIndex++; + toIndex++; + } } } - intSpan.Slice(toIndex, ints - toIndex).Clear(); + thisSpan.Slice(toIndex).Clear(); _version++; return this; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int Apply(int shiftCount, int fromIndex, Span thisSpan) + where TVector : ISimdVector + { + ref byte p = ref MemoryMarshal.GetReference(thisSpan); + int carryCount = BitsPerByte - shiftCount; + + int toIndex = 0; + + while (fromIndex <= thisSpan.Length - (TVector.ElementCount + 1)) + { + TVector lo = TVector.LoadUnsafe(ref p, (uint)fromIndex) >>> shiftCount; + TVector hi = TVector.LoadUnsafe(ref p, (uint)(fromIndex + 1)) << carryCount; + TVector result = lo | hi; + result.StoreUnsafe(ref p, (uint)toIndex); + + fromIndex += TVector.ElementCount; + toIndex += TVector.ElementCount; + } + + return toIndex; + } } /// @@ -576,41 +615,89 @@ public BitArray LeftShift(int count) return this; } - Span intSpan = MemoryMarshal.Cast((Span)_array); + Span thisSpan = new Span(_array, 0, GetByteArrayLengthFromBitLength(_bitLength)); int lengthToClear; if (count < _bitLength) { - int lastIndex = (int)((uint)(_bitLength - 1) / BitsPerInt32); - - (lengthToClear, int shiftCount) = Math.DivRem(count, BitsPerInt32); + (lengthToClear, int shiftCount) = Math.DivRem(count, BitsPerByte); if (shiftCount == 0) { - intSpan.Slice(0, lastIndex + 1 - lengthToClear).CopyTo(intSpan.Slice(lengthToClear)); + thisSpan.Slice(0, thisSpan.Length - lengthToClear).CopyTo(thisSpan.Slice(lengthToClear)); } else { - int fromindex = lastIndex - lengthToClear; + int toIndex = thisSpan.Length; + int fromIndex = toIndex - lengthToClear; + + if (Vector512.IsHardwareAccelerated) + { + toIndex = Apply>(shiftCount, fromIndex, thisSpan); + } + else if (Vector256.IsHardwareAccelerated) + { + toIndex = Apply>(shiftCount, fromIndex, thisSpan); + } + else if (Vector128.IsHardwareAccelerated) + { + toIndex = Apply>(shiftCount, fromIndex, thisSpan); + } + fromIndex = toIndex - lengthToClear; + + ref byte p = ref MemoryMarshal.GetReference(thisSpan); + + int carryCount = BitsPerByte - shiftCount; + while (fromIndex >= 5) + { + int hi = ReverseIfBE(Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)(fromIndex -= 4)))) << shiftCount; + int lo = Unsafe.AddByteOffset(ref p, (uint)(fromIndex - 1)) >>> carryCount; + int result = ReverseIfBE(hi | lo); + Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref p, toIndex -= 4), result); + } - while (fromindex > 0) + while (--fromIndex >= 0) { - int left = ReverseIfBE(intSpan[fromindex]) << shiftCount; - uint right = (uint)ReverseIfBE(intSpan[--fromindex]) >> (BitsPerInt32 - shiftCount); - intSpan[lastIndex] = ReverseIfBE(left | (int)right); - lastIndex--; + int hi = thisSpan[fromIndex] << shiftCount; + int lo = + fromIndex > 0 + ? thisSpan[fromIndex - 1] >>> carryCount + : 0; + + thisSpan[--toIndex] = (byte)(hi | lo); } - intSpan[lastIndex] = ReverseIfBE(ReverseIfBE(intSpan[fromindex]) << shiftCount); + + Debug.Assert(toIndex == lengthToClear); } } else { - lengthToClear = GetInt32ArrayLengthFromBitLength(_bitLength); // Clear all + lengthToClear = thisSpan.Length; // Clear all } - intSpan.Slice(0, lengthToClear).Clear(); + thisSpan.Slice(0, lengthToClear).Clear(); _version++; return this; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int Apply(int shiftCount, int fromIndex, Span thisSpan) + where TVector : ISimdVector + { + ref byte p = ref MemoryMarshal.GetReference(thisSpan); + int carryCount = BitsPerByte - shiftCount; + + int toIndex = thisSpan.Length; + + while (fromIndex >= TVector.ElementCount + 1) + { + TVector hi = TVector.LoadUnsafe(ref p, (nuint)(fromIndex -= TVector.ElementCount)) << shiftCount; + TVector lo = TVector.LoadUnsafe(ref p, (nuint)(fromIndex - 1)) >>> carryCount; + TVector result = hi | lo; + result.StoreUnsafe(ref p, (nuint)(toIndex -= TVector.ElementCount)); + } + + return toIndex; + } } /// From a48270170184f1b9b43903051c9ac86b56e73f95 Mon Sep 17 00:00:00 2001 From: kzrnm Date: Thu, 3 Jul 2025 20:47:26 +0900 Subject: [PATCH 3/4] ClearHighExtraBits --- .../tests/BitArray/BitArray_OperatorsTests.cs | 9 +++++++++ .../src/System/Collections/BitArray.cs | 1 + 2 files changed, 10 insertions(+) diff --git a/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs b/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs index 9d8e91f46a8c1f..c6957d75a0d480 100644 --- a/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs +++ b/src/libraries/System.Collections/tests/BitArray/BitArray_OperatorsTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; +using System.Runtime.InteropServices; using Xunit; namespace System.Collections.Tests @@ -310,6 +311,14 @@ public static void LeftShift(int length, IEnumerable set, int shift) int index = 0; Assert.All(ba.Cast(), bit => Assert.Equal(expected[index++], bit)); + + (int byteIndex, int bitOffeset) = Math.DivRem(length, BitsPerByte); + if (bitOffeset != 0) + { + Span bs = CollectionsMarshal.AsBytes(ba); + Assert.Equal(byteIndex + 1, bs.Length); + Assert.Equal(0, bs[byteIndex] >> bitOffeset); + } } private static bool[] GetBoolArray(int length, IEnumerable set) diff --git a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs index e79a3ef2727d95..141ed209a0fd5d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs @@ -676,6 +676,7 @@ public BitArray LeftShift(int count) } thisSpan.Slice(0, lengthToClear).Clear(); + ClearHighExtraBits(); _version++; return this; From e96a2b8758ba2046c458c23dd6da5f12a025d40d Mon Sep 17 00:00:00 2001 From: kzrnm Date: Fri, 4 Jul 2025 01:59:53 +0900 Subject: [PATCH 4/4] Update logic of Int32 --- .../src/System/Collections/BitArray.cs | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs index 141ed209a0fd5d..7c10f711dea3ba 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs @@ -540,21 +540,25 @@ public BitArray RightShift(int count) } fromIndex += toIndex; + int carryCount = BitsPerByte - shiftCount; + ref byte p = ref MemoryMarshal.GetReference(thisSpan); - int carry32Count = BitsPerInt32 - shiftCount; + const uint shiftUnit = 0x01010101u; + uint shiftMask = (shiftUnit << carryCount) - shiftUnit; + uint carryMask = ~shiftMask; + while (fromIndex < thisSpan.Length - 4) { - int lo = ReverseIfBE(Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)fromIndex))) >>> shiftCount; - int hi = Unsafe.AddByteOffset(ref p, (uint)(fromIndex + 4)) << carry32Count; - int result = ReverseIfBE(hi | lo); + uint lo = (Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)fromIndex)) >>> shiftCount) & shiftMask; + uint hi = (Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)(fromIndex + 1))) << carryCount) & carryMask; + uint result = hi | lo; Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref p, toIndex), result); fromIndex += 4; toIndex += 4; } - int carryCount = BitsPerByte - shiftCount; while (fromIndex < thisSpan.Length) { int lo = thisSpan[fromIndex] >>> shiftCount; @@ -645,14 +649,19 @@ public BitArray LeftShift(int count) } fromIndex = toIndex - lengthToClear; + int carryCount = BitsPerByte - shiftCount; + ref byte p = ref MemoryMarshal.GetReference(thisSpan); - int carryCount = BitsPerByte - shiftCount; + const uint shiftUnit = 0x01010101u; + uint carryMask = (shiftUnit << shiftCount) - shiftUnit; + uint shiftMask = ~carryMask; + while (fromIndex >= 5) { - int hi = ReverseIfBE(Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)(fromIndex -= 4)))) << shiftCount; - int lo = Unsafe.AddByteOffset(ref p, (uint)(fromIndex - 1)) >>> carryCount; - int result = ReverseIfBE(hi | lo); + uint lo = (Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)(fromIndex -= 4))) << shiftCount) & shiftMask; + uint hi = (Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref p, (uint)(fromIndex - 1))) >>> carryCount) & carryMask; + uint result = hi | lo; Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref p, toIndex -= 4), result); }