From 0af60f0076256810864f3955b050b2800a205c1e Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Tue, 16 Apr 2024 01:19:59 +1200 Subject: [PATCH] GH-40517: [C#] Fix writing sliced arrays to IPC format (#41197) ### Rationale for this change Fixes writing sliced arrays to IPC files or streams, so that they can be successfully read back in. Previously, writing such data would succeed but then couldn't be read. ### What changes are included in this PR? * Fixes `BinaryViewArray.GetBytes` to account for the array offset * Fixes `FixedSizeBinaryArray.GetBytes` to account for the array offset * Updates `ArrowStreamWriter` so that it writes slices of buffers when required, and handles slicing bitmap arrays by creating a copy if the offset isn't a multiple of 8 * Refactors `ArrowStreamWriter`, making the `ArrowRecordBatchFlatBufferBuilder` class responsible for building a list of field nodes as well as buffers. This was required to avoid having to duplicate logic for handling array types with child data between the `ArrowRecordBatchFlatBufferBuilder` class and the `CreateSelfAndChildrenFieldNodes` method, which I've removed. Note that after this change, we still write more data than required when writing a slice of a `ListArray`, `BinaryArray`, `ListViewArray`, `BinaryViewArray` or `DenseUnionArray`. When writing a `ListArray` for example, we write slices of the null bitmap and value offsets and write the full values array. Ideally we should write a slice of the values and adjust the value offsets so they start at zero. The C++ implementation for example handles this [here](https://github.com/apache/arrow/blob/18c74b0733c9ff473a211259cf10705b2c9be891/cpp/src/arrow/ipc/writer.cc#L316). I will make a follow-up issue for this once this PR is merged. ### Are these changes tested? Yes, I've added new unit tests for this. ### Are there any user-facing changes? Yes, this is a user-facing bug fix. * GitHub Issue: #40517 Authored-by: Adam Reeve Signed-off-by: Curt Hagenlocher --- .../src/Apache.Arrow/Ipc/ArrowStreamWriter.cs | 151 +----------------- .../ArrowArrayConcatenatorTests.cs | 6 + .../ArrowFileWriterTests.cs | 66 -------- .../Apache.Arrow.Tests/ArrowReaderVerifier.cs | 94 ++++++----- 4 files changed, 58 insertions(+), 259 deletions(-) diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs index 1cdd97e4ac6c9..6127c5a662dfe 100644 --- a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs +++ b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs @@ -163,40 +163,18 @@ public void Visit(BooleanArray array) public void Visit(ListArray array) { _buffers.Add(CreateBitmapBuffer(array.NullBitmapBuffer, array.Offset, array.Length)); - _buffers.Add(CreateBuffer(GetZeroBasedValueOffsets(array.ValueOffsetsBuffer, array.Offset, array.Length))); + _buffers.Add(CreateSlicedBuffer(array.ValueOffsetsBuffer, array.Offset, array.Length + 1)); - int valuesOffset = 0; - int valuesLength = 0; - if (array.Length > 0) - { - valuesOffset = array.ValueOffsets[0]; - valuesLength = array.ValueOffsets[array.Length] - valuesOffset; - } - - var values = array.Values; - if (valuesOffset > 0 || valuesLength < values.Length) - { - values = ArrowArrayFactory.Slice(values, valuesOffset, valuesLength); - } - - VisitArray(values); + VisitArray(array.Values); } public void Visit(ListViewArray array) { - var (valueOffsetsBuffer, minOffset, maxEnd) = GetZeroBasedListViewOffsets(array); - _buffers.Add(CreateBitmapBuffer(array.NullBitmapBuffer, array.Offset, array.Length)); - _buffers.Add(CreateBuffer(valueOffsetsBuffer)); + _buffers.Add(CreateSlicedBuffer(array.ValueOffsetsBuffer, array.Offset, array.Length)); _buffers.Add(CreateSlicedBuffer(array.SizesBuffer, array.Offset, array.Length)); - IArrowArray values = array.Values; - if (minOffset != 0 || values.Length != maxEnd) - { - values = ArrowArrayFactory.Slice(values, minOffset, maxEnd - minOffset); - } - - VisitArray(values); + VisitArray(array.Values); } public void Visit(FixedSizeListArray array) @@ -217,17 +195,8 @@ public void Visit(FixedSizeListArray array) public void Visit(BinaryArray array) { _buffers.Add(CreateBitmapBuffer(array.NullBitmapBuffer, array.Offset, array.Length)); - _buffers.Add(CreateBuffer(GetZeroBasedValueOffsets(array.ValueOffsetsBuffer, array.Offset, array.Length))); - - int valuesOffset = 0; - int valuesLength = 0; - if (array.Length > 0) - { - valuesOffset = array.ValueOffsets[0]; - valuesLength = array.ValueOffsets[array.Length] - valuesOffset; - } - - _buffers.Add(CreateSlicedBuffer(array.ValueBuffer, valuesOffset, valuesLength)); + _buffers.Add(CreateSlicedBuffer(array.ValueOffsetsBuffer, array.Offset, array.Length + 1)); + _buffers.Add(CreateBuffer(array.ValueBuffer)); } public void Visit(BinaryViewArray array) @@ -294,91 +263,6 @@ public void Visit(NullArray array) // There are no buffers for a NullArray } - private ArrowBuffer GetZeroBasedValueOffsets(ArrowBuffer valueOffsetsBuffer, int arrayOffset, int arrayLength) - { - var requiredBytes = CalculatePaddedBufferLength(sizeof(int) * (arrayLength + 1)); - - if (arrayOffset != 0) - { - // Array has been sliced, so we need to shift and adjust the offsets - var originalOffsets = valueOffsetsBuffer.Span.CastTo().Slice(arrayOffset, arrayLength + 1); - var firstOffset = arrayLength > 0 ? originalOffsets[0] : 0; - - var newValueOffsetsBuffer = _allocator.Allocate(requiredBytes); - var newValueOffsets = newValueOffsetsBuffer.Memory.Span.CastTo(); - - for (int i = 0; i < arrayLength + 1; ++i) - { - newValueOffsets[i] = originalOffsets[i] - firstOffset; - } - - return new ArrowBuffer(newValueOffsetsBuffer); - } - else if (valueOffsetsBuffer.Length > requiredBytes) - { - // Array may have been sliced but the offset is zero, - // so we can truncate the existing offsets - return new ArrowBuffer(valueOffsetsBuffer.Memory.Slice(0, requiredBytes)); - } - else - { - // Use the full buffer - return valueOffsetsBuffer; - } - } - - private (ArrowBuffer Buffer, int minOffset, int maxEnd) GetZeroBasedListViewOffsets(ListViewArray array) - { - if (array.Length == 0) - { - return (ArrowBuffer.Empty, 0, 0); - } - - var offsets = array.ValueOffsets; - var sizes = array.Sizes; - - int minOffset = offsets[0]; - int maxEnd = offsets[array.Length - 1] + sizes[array.Length - 1]; - - // Min possible offset is zero, and max possible end is the values length. - // If these match the first offset and last end we don't need to do anything further, - // but otherwise we need to iterate over each index in case the offsets aren't ordered. - if (minOffset != 0 || maxEnd != array.Values.Length) - { - for (int i = 0; i < array.Length; ++i) - { - minOffset = Math.Min(minOffset, offsets[i]); - maxEnd = Math.Max(maxEnd, offsets[i] + sizes[i]); - } - } - - var requiredBytes = CalculatePaddedBufferLength(sizeof(int) * array.Length); - - if (minOffset == 0) - { - // No need to adjust the offsets, but we may need to slice the offsets buffer. - ArrowBuffer buffer = array.ValueOffsetsBuffer; - if (array.Offset != 0 || buffer.Length > requiredBytes) - { - var byteOffset = sizeof(int) * array.Offset; - var sliceLength = Math.Min(requiredBytes, buffer.Length - byteOffset); - buffer = new ArrowBuffer(buffer.Memory.Slice(byteOffset, sliceLength)); - } - - return (buffer, minOffset, maxEnd); - } - - // Compute shifted offsets - var newOffsetsBuffer = _allocator.Allocate(requiredBytes); - var newOffsets = newOffsetsBuffer.Memory.Span.CastTo(); - for (int i = 0; i < array.Length; ++i) - { - newOffsets[i] = offsets[i] - minOffset; - } - - return (new ArrowBuffer(newOffsetsBuffer), minOffset, maxEnd); - } - private Buffer CreateBitmapBuffer(ArrowBuffer buffer, int offset, int length) { if (buffer.IsEmpty) @@ -557,29 +441,6 @@ public ArrowStreamWriter(Stream baseStream, Schema schema, bool leaveOpen, IpcOp } } - private void CreateSelfAndChildrenFieldNodes(ArrayData data) - { - if (data.DataType is NestedType) - { - // flatbuffer struct vectors have to be created in reverse order - for (int i = data.Children.Length - 1; i >= 0; i--) - { - CreateSelfAndChildrenFieldNodes(data.Children[i]); - } - } - Flatbuf.FieldNode.CreateFieldNode(Builder, data.Length, data.GetNullCount()); - } - - private static int CountAllNodes(IReadOnlyList fields) - { - int count = 0; - foreach (Field arrowArray in fields) - { - CountSelfAndChildrenNodes(arrowArray.DataType, ref count); - } - return count; - } - private Offset GetBodyCompression() { if (_options.CompressionCodec == null) diff --git a/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs b/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs index 2437d3d94c446..80c5e586963d2 100644 --- a/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs +++ b/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs @@ -29,6 +29,12 @@ public void TestStandardCases() { foreach ((List testTargetArrayList, IArrowArray expectedArray) in GenerateTestData()) { + if (expectedArray is UnionArray) + { + // Union array concatenation is incorrect. See https://github.com/apache/arrow/issues/41198 + continue; + } + IArrowArray actualArray = ArrowArrayConcatenator.Concatenate(testTargetArrayList); ArrowReaderVerifier.CompareArrays(expectedArray, actualArray); } diff --git a/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs b/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs index 7b25910114882..e1efc6475226a 100644 --- a/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs +++ b/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs @@ -113,7 +113,6 @@ public async Task WritesFooterAlignedMultipleOf8Async() [InlineData(0, 45)] [InlineData(3, 45)] [InlineData(16, 45)] - [InlineData(10, 0)] public async Task WriteSlicedArrays(int sliceOffset, int sliceLength) { var originalBatch = TestData.CreateSampleRecordBatch(length: 100); @@ -135,71 +134,6 @@ public async Task WriteSlicedArrays(int sliceOffset, int sliceLength) await ValidateRecordBatchFile(stream, slicedBatch, strictCompare: false); } - [Theory] - [InlineData(0, 100)] - [InlineData(0, 50)] - [InlineData(50, 50)] - [InlineData(25, 50)] - public async Task WriteListViewDataWithUnorderedOffsets(int sliceOffset, int sliceLength) - { - // A list-view array doesn't require that offsets are ordered, - // so verify that we can round trip a list-view array with out-of-order offsets. - const int length = 100; - var random = new Random(); - - var randomizedIndices = Enumerable.Range(0, length).ToArray(); - Shuffle(randomizedIndices, random); - - var offsetsBuilder = new ArrowBuffer.Builder().Resize(length); - var sizesBuilder = new ArrowBuffer.Builder().Resize(length); - var validityBuilder = new ArrowBuffer.BitmapBuilder().Reserve(length); - - var valuesLength = 0; - for (int i = 0; i < length; ++i) - { - var index = randomizedIndices[i]; - var listLength = random.Next(0, 10); - offsetsBuilder.Span[index] = valuesLength; - sizesBuilder.Span[index] = listLength; - valuesLength += listLength; - - validityBuilder.Append(random.NextDouble() < 0.9); - } - - var valuesBuilder = new Int64Array.Builder().Reserve(valuesLength); - for (int i = 0; i < valuesLength; ++i) - { - valuesBuilder.Append(random.Next(0, 1_000)); - } - - var type = new ListViewType(new Int64Type()); - var offsets = offsetsBuilder.Build(); - var sizes = sizesBuilder.Build(); - var values = valuesBuilder.Build(); - var nullCount = validityBuilder.UnsetBitCount; - var validityBuffer = validityBuilder.Build(); - - IArrowArray listViewArray = new ListViewArray( - type, length, offsets, sizes, values, validityBuffer, nullCount); - - if (sliceOffset != 0 || sliceLength != length) - { - listViewArray = ArrowArrayFactory.Slice(listViewArray, sliceOffset, sliceLength); - } - - var recordBatch = new RecordBatch.Builder().Append("x", true, listViewArray).Build(); - - var stream = new MemoryStream(); - var writer = new ArrowFileWriter(stream, recordBatch.Schema, leaveOpen: true); - - await writer.WriteRecordBatchAsync(recordBatch); - await writer.WriteEndAsync(); - - stream.Position = 0; - - await ValidateRecordBatchFile(stream, recordBatch, strictCompare: false); - } - private async Task ValidateRecordBatchFile(Stream stream, RecordBatch recordBatch, bool strictCompare = true) { var reader = new ArrowFileReader(stream); diff --git a/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs b/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs index c084b5f9185e3..502e28a505890 100644 --- a/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs +++ b/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs @@ -160,6 +160,7 @@ public void Visit(StructArray array) Assert.Equal(expectedArray.Length, array.Length); Assert.Equal(expectedArray.NullCount, array.NullCount); + Assert.Equal(0, array.Offset); Assert.Equal(expectedArray.Data.Children.Length, array.Data.Children.Length); Assert.Equal(expectedArray.Fields.Count, array.Fields.Count); @@ -182,12 +183,12 @@ public void Visit(UnionArray array) Assert.Equal(expectedArray.Mode, array.Mode); Assert.Equal(expectedArray.Length, array.Length); Assert.Equal(expectedArray.NullCount, array.NullCount); + Assert.Equal(0, array.Offset); Assert.Equal(expectedArray.Data.Children.Length, array.Data.Children.Length); Assert.Equal(expectedArray.Fields.Count, array.Fields.Count); if (_strictCompare) { - Assert.Equal(expectedArray.Offset, array.Offset); Assert.True(expectedArray.TypeBuffer.Span.SequenceEqual(array.TypeBuffer.Span)); } else @@ -256,8 +257,9 @@ private void CompareBinaryArrays(BinaryArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(0, actualArray.Offset); - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); if (_strictCompare) { @@ -288,13 +290,9 @@ private void CompareVariadicArrays(BinaryViewArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(0, actualArray.Offset); - if (_strictCompare) - { - Assert.Equal(expectedArray.Offset, actualArray.Offset); - } - - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); Assert.True(expectedArray.Views.SequenceEqual(actualArray.Views)); @@ -317,8 +315,9 @@ private void CompareArrays(FixedSizeBinaryArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(0, actualArray.Offset); - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); if (_strictCompare) { @@ -346,8 +345,9 @@ private void CompareArrays(PrimitiveArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(0, actualArray.Offset); - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); if (_strictCompare) { @@ -378,8 +378,9 @@ private void CompareArrays(BooleanArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(0, actualArray.Offset); - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); if (_strictCompare) { @@ -405,8 +406,9 @@ private void CompareArrays(ListArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(0, actualArray.Offset); - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); if (_strictCompare) { @@ -416,19 +418,9 @@ private void CompareArrays(ListArray actualArray) } else { - for (int i = 0; i < actualArray.Length; ++i) - { - if (expectedArray.IsNull(i)) - { - Assert.True(actualArray.IsNull(i)); - } - else - { - var expectedList = expectedArray.GetSlicedValues(i); - var actualList = actualArray.GetSlicedValues(i); - actualList.Accept(new ArrayComparer(expectedList, _strictCompare)); - } - } + int offsetsStart = (expectedArray.Offset) * sizeof(int); + int offsetsLength = (expectedArray.Length + 1) * sizeof(int); + Assert.True(expectedArray.ValueOffsetsBuffer.Span.Slice(offsetsStart, offsetsLength).SequenceEqual(actualArray.ValueOffsetsBuffer.Span.Slice(0, offsetsLength))); } } @@ -441,8 +433,9 @@ private void CompareArrays(ListViewArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(0, actualArray.Offset); - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); if (_strictCompare) { @@ -453,19 +446,10 @@ private void CompareArrays(ListViewArray actualArray) } else { - for (int i = 0; i < actualArray.Length; ++i) - { - if (expectedArray.IsNull(i)) - { - Assert.True(actualArray.IsNull(i)); - } - else - { - var expectedList = expectedArray.GetSlicedValues(i); - var actualList = actualArray.GetSlicedValues(i); - actualList.Accept(new ArrayComparer(expectedList, _strictCompare)); - } - } + int start = expectedArray.Offset * sizeof(int); + int length = expectedArray.Length * sizeof(int); + Assert.True(expectedArray.ValueOffsetsBuffer.Span.Slice(start, length).SequenceEqual(actualArray.ValueOffsetsBuffer.Span.Slice(0, length))); + Assert.True(expectedArray.SizesBuffer.Span.Slice(start, length).SequenceEqual(actualArray.SizesBuffer.Span.Slice(0, length))); } } @@ -478,12 +462,9 @@ private void CompareArrays(FixedSizeListArray actualArray) Assert.Equal(expectedArray.Length, actualArray.Length); Assert.Equal(expectedArray.NullCount, actualArray.NullCount); - if (_strictCompare) - { - Assert.Equal(expectedArray.Offset, actualArray.Offset); - } + Assert.Equal(0, actualArray.Offset); - CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset); + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer); var listSize = ((FixedSizeListType)expectedArray.Data.DataType).ListSize; var expectedValuesSlice = ArrowArrayFactory.Slice( @@ -491,16 +472,21 @@ private void CompareArrays(FixedSizeListArray actualArray) actualArray.Values.Accept(new ArrayComparer(expectedValuesSlice, _strictCompare)); } - private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer expectedValidityBuffer, int expectedBufferOffset, ArrowBuffer actualValidityBuffer, int actualBufferOffset) + private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer expectedValidityBuffer, int expectedBufferOffset, ArrowBuffer actualValidityBuffer) { if (_strictCompare) { Assert.True(expectedValidityBuffer.Span.SequenceEqual(actualValidityBuffer.Span)); } - else if (nullCount != 0 && arrayLength > 0) + else if (actualValidityBuffer.IsEmpty) + { + Assert.True(nullCount == 0 || arrayLength == 0); + } + else if (expectedBufferOffset % 8 == 0) { int validityBitmapByteCount = BitUtility.ByteCount(arrayLength); - ReadOnlySpan expectedSpanPartial = expectedValidityBuffer.Span.Slice(0, validityBitmapByteCount - 1); + int byteOffset = BitUtility.ByteCount(expectedBufferOffset); + ReadOnlySpan expectedSpanPartial = expectedValidityBuffer.Span.Slice(byteOffset, validityBitmapByteCount - 1); ReadOnlySpan actualSpanPartial = actualValidityBuffer.Span.Slice(0, validityBitmapByteCount - 1); // Compare the first validityBitmapByteCount - 1 bytes @@ -510,7 +496,7 @@ private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer e // Compare the last byte bitwise (because there is no guarantee about the value of // bits outside the range [0, arrayLength]) - ReadOnlySpan expectedSpanFull = expectedValidityBuffer.Span.Slice(0, validityBitmapByteCount); + ReadOnlySpan expectedSpanFull = expectedValidityBuffer.Span.Slice(byteOffset, validityBitmapByteCount); ReadOnlySpan actualSpanFull = actualValidityBuffer.Span.Slice(0, validityBitmapByteCount); for (int i = 8 * (validityBitmapByteCount - 1); i < arrayLength; i++) { @@ -519,6 +505,18 @@ private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer e string.Format("Bit at index {0}/{1} is not equal", i, arrayLength)); } } + else + { + // Have to compare all values bitwise + var expectedSpan = expectedValidityBuffer.Span; + var actualSpan = actualValidityBuffer.Span; + for (int i = 0; i < arrayLength; i++) + { + Assert.True( + BitUtility.GetBit(expectedSpan, expectedBufferOffset + i) == BitUtility.GetBit(actualSpan, i), + string.Format("Bit at index {0}/{1} is not equal", i, arrayLength)); + } + } } } }