Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support .NET 4.7.2 #124

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/DeltaLake/Bridge/ByteArrayRef.cs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,11 @@ public static RentedByteArrayRef RentUtf8(string s)
}

var bytes = ArrayPool<byte>.Shared.Rent(StrictUTF8.GetByteCount(s));
#if NETCOREAPP
var length = StrictUTF8.GetBytes(s, bytes);
#else
var length = StrictUTF8.GetBytes(s, 0, s.Length, bytes, 0);
#endif
return new RentedByteArrayRef(new ByteArrayRef(bytes.AsMemory(0, length)), bytes, ArrayPool<byte>.Shared);
}

Expand Down
8 changes: 8 additions & 0 deletions src/DeltaLake/Bridge/RecordBatchReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,19 @@ public ValueTask<RecordBatch> ReadNextRecordBatchAsync(System.Threading.Cancella
{
if (_enumerator.MoveNext())
{
#if NETCOREAPP
return ValueTask.FromResult(_enumerator.Current);
#else
return new ValueTask<RecordBatch>(_enumerator.Current);
#endif
}

#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type.
#if NETCOREAPP
return ValueTask.FromResult<RecordBatch>(default);
#else
return new ValueTask<RecordBatch>(default(RecordBatch));
#endif
#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type.
}
}
Expand Down
4 changes: 4 additions & 0 deletions src/DeltaLake/Bridge/Runtime.cs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ internal virtual async Task<IntPtr> LoadTablePtrAsync(
System.Threading.CancellationToken cancellationToken)
{
var buffer = ArrayPool<byte>.Shared.Rent(System.Text.Encoding.UTF8.GetByteCount(options.TableLocation));
#if NETCOREAPP
var encodedLength = System.Text.Encoding.UTF8.GetBytes(options.TableLocation, buffer);
#else
var encodedLength = System.Text.Encoding.UTF8.GetBytes(options.TableLocation, 0, options.TableLocation.Length, buffer, 0);
#endif
try
{
return await LoadTablePtrAsync(buffer.AsMemory(0, encodedLength), options, cancellationToken).ConfigureAwait(false);
Expand Down
8 changes: 8 additions & 0 deletions src/DeltaLake/Bridge/Table.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ namespace DeltaLake.Bridge
/// </summary>
internal class Table : SafeHandle
{
#if !NETCOREAPP
private unsafe delegate void MetadataRelease(Interop.TableMetadata* metadata);
#endif

internal static readonly ByteArrayRef SaveModeAppend = ByteArrayRef.FromUTF8("append");

internal static readonly ByteArrayRef SaveModeOverwrite = ByteArrayRef.FromUTF8("overwrite");
Expand Down Expand Up @@ -563,7 +567,11 @@ internal virtual DeltaLake.Table.TableMetadata Metadata()
}
finally
{
#if NETCOREAPP
var release = (delegate* unmanaged<Interop.TableMetadata*, void>)result.metadata->release;
#else
var release = Marshal.GetDelegateForFunctionPointer<MetadataRelease>(result.metadata->release);
#endif
release(result.metadata);
}
}
Expand Down
5 changes: 3 additions & 2 deletions src/DeltaLake/DeltaLake.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
<Description>DeltaLake for .NET</Description>
<IncludeSymbols>true</IncludeSymbols>
<LangVersion>9.0</LangVersion>
<EnablePackageValidation Condition="'$(TargetFramework)' == 'net6.0'">true</EnablePackageValidation>
<EnablePackageValidation Condition="'$(TargetFramework)' != 'net8.0'">true</EnablePackageValidation>
<PackageReadmeFile>README.md</PackageReadmeFile>
<PublishRepositoryUrl>true</PublishRepositoryUrl>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
<Nullable>enable</Nullable>
<TargetFrameworks>net6.0;net8.0;</TargetFrameworks>
<TargetFrameworks>net472;net8.0;</TargetFrameworks>
<VersionSuffix>rc-1</VersionSuffix>
<PackageTags>deltalake csharp</PackageTags>
<PackageId>DeltaLake.Net</PackageId>
Expand All @@ -20,6 +20,7 @@
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Apache.Arrow" Version="18.1.0" />
<PackageReference Include="System.Text.Json" Version="9.0.0" />
</ItemGroup>

<ItemGroup>
Expand Down
95 changes: 95 additions & 0 deletions src/DeltaLake/Extensions/MarshalExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// -----------------------------------------------------------------------------
// <summary>
// Runtime-specific helpers for marshaling to and from UTF8 strings
// </summary>
//
// <copyright company="The Delta Lake Project Authors">
// Copyright (2025) The Delta Lake Project Authors. All rights reserved.
// Licensed under the Apache license. See LICENSE file in the project root for full license information.
// </copyright>
// -----------------------------------------------------------------------------

using System;
#pragma warning disable IDE0005 // Using directive is unnecessary.
using System.Diagnostics.CodeAnalysis;
using System.Runtime.InteropServices;
using System.Text;
#pragma warning restore IDE0005 // Using directive is unnecessary.

namespace DeltaLake.Extensions
{
internal static class MarshalExtensions
{
#if !NETCOREAPP
public static unsafe string? PtrToStringUTF8(IntPtr intPtr)
{
if (intPtr == IntPtr.Zero)
{
return null;
}

byte* source = (byte*)intPtr;
int length = 0;

while (source[length] != 0)
{
length++;
}

return PtrToStringUTF8(intPtr, length);
}

public static unsafe string? PtrToStringUTF8(IntPtr intPtr, int length)
{
if (intPtr == IntPtr.Zero)
{
return null;
}

byte[] bytes = new byte[length];
Marshal.Copy(intPtr, bytes, 0, length);

return Encoding.UTF8.GetString(bytes);
}

public static unsafe IntPtr StringToCoTaskMemUTF8(string? s)
{
if (s is null)
{
return IntPtr.Zero;
}

int nb = Encoding.UTF8.GetMaxByteCount(s.Length);

IntPtr pMem = Marshal.AllocHGlobal(nb + 1);

int nbWritten;
byte* pbMem = (byte*)pMem;

fixed (char* firstChar = s)
{
nbWritten = Encoding.UTF8.GetBytes(firstChar, s.Length, pbMem, nb);
}

pbMem[nbWritten] = 0;

return pMem;
}
#else
public static unsafe string? PtrToStringUTF8(IntPtr intPtr)
{
return Marshal.PtrToStringUTF8(intPtr);
}

public static IntPtr StringToCoTaskMemUTF8(string? s)
{
return Marshal.StringToCoTaskMemUTF8(s);
}

public static unsafe string? PtrToStringUTF8(IntPtr intPtr, int length)
{
return Marshal.PtrToStringUTF8(intPtr, length);
}
#endif
}
}
126 changes: 126 additions & 0 deletions src/DeltaLake/Extensions/NetFxShims.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
// -----------------------------------------------------------------------------
// <summary>
// Shims for .NET 6+ functionality that's not available to Desktop Framework
// </summary>
//
// <copyright company="The Delta Lake Project Authors">
// Copyright (2025) The Delta Lake Project Authors. All rights reserved.
// Licensed under the Apache license. See LICENSE file in the project root for full license information.
// </copyright>
// -----------------------------------------------------------------------------

#if !NETCOREAPP

#pragma warning disable IDE0130 // Namespace does not match folder structure

using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace System.Runtime.CompilerServices
{
internal static class IsExternalInit { }
}

namespace System.Collections.Generic
{
internal static class KeyValuePairExtensions
{
public static void Deconstruct<TKey, TValue>(this KeyValuePair<TKey, TValue> pair, out TKey key, out TValue value)
{
key = pair.Key;
value = pair.Value;
}
}

internal static class DictionaryExtensions
{
public static bool TryAdd<TKey, TValue>(this Dictionary<TKey, TValue> dictionary, TKey key, TValue value)
{
if (dictionary.ContainsKey(key))
{
return false;
}
dictionary.Add(key, value);
return true;
}
}
}

namespace System.Threading.Tasks
{
internal static class AsyncEnumerableExtensions
{
public static IEnumerable<T> ToBlockingEnumerable<T>(this IAsyncEnumerable<T> source, CancellationToken cancellationToken = default)
{
IAsyncEnumerator<T> enumerator = source.GetAsyncEnumerator(cancellationToken);
// A ManualResetEventSlim variant that lets us reuse the same
// awaiter callback allocation across the entire enumeration.
ManualResetEventWithAwaiterSupport? mres = null;

try
{
while (true)
{
#pragma warning disable CA2012 // Use ValueTasks correctly
ValueTask<bool> moveNextTask = enumerator.MoveNextAsync();
#pragma warning restore CA2012 // Use ValueTasks correctly

if (!moveNextTask.IsCompleted)
{
#pragma warning disable CA2000 // Dispose objects before losing scope
(mres ??= new ManualResetEventWithAwaiterSupport()).Wait(moveNextTask.ConfigureAwait(false).GetAwaiter());
#pragma warning restore CA2000 // Dispose objects before losing scope
Debug.Assert(moveNextTask.IsCompleted);
}

#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits
if (!moveNextTask.Result)
{
yield break;
}
#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits

yield return enumerator.Current;
}
}
finally
{
ValueTask disposeTask = enumerator.DisposeAsync();

if (!disposeTask.IsCompleted)
{
#pragma warning disable CA2000 // Dispose objects before losing scope
(mres ?? new ManualResetEventWithAwaiterSupport()).Wait(disposeTask.ConfigureAwait(false).GetAwaiter());
#pragma warning restore CA2000 // Dispose objects before losing scope
Debug.Assert(disposeTask.IsCompleted);
}

#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits
disposeTask.GetAwaiter().GetResult();
#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits
}
}

private sealed class ManualResetEventWithAwaiterSupport : ManualResetEventSlim
{
private readonly Action _onCompleted;

public ManualResetEventWithAwaiterSupport()
{
_onCompleted = Set;
}

public void Wait<TAwaiter>(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion
{
awaiter.UnsafeOnCompleted(_onCompleted);
Wait();
Reset();
}
}
}
}

#pragma warning restore IDE0130 // Namespace does not match folder structure

#endif
10 changes: 5 additions & 5 deletions src/DeltaLake/Kernel/Arrow/Extensions/ArrowContextExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using Apache.Arrow;
using Apache.Arrow.C;
using Apache.Arrow.Types;
using DeltaLake.Extensions;
using DeltaLake.Kernel.Interop;
using DeltaLake.Kernel.State;

Expand Down Expand Up @@ -146,8 +146,8 @@ private static unsafe Schema AddPartitionColumnsToSchema(Apache.Arrow.Schema ori
for (int i = 0; i < partitionsPtr->Len; i++)
{
#pragma warning disable CS8600, CS8604 // If Kernel sends us back null pointers, we are in trouble anyway
string colName = Marshal.PtrToStringUTF8((IntPtr)partitionsPtr->ColNames[i]);
string colValue = Marshal.PtrToStringUTF8((IntPtr)partitionsPtr->ColValues[i]);
string colName = MarshalExtensions.PtrToStringUTF8((IntPtr)partitionsPtr->ColNames[i]);
string colValue = MarshalExtensions.PtrToStringUTF8((IntPtr)partitionsPtr->ColValues[i]);
IArrowType dataType = DeterminePartitionColumnType(colName, colValue);
#pragma warning restore CS8600, CS8604

Expand Down Expand Up @@ -179,8 +179,8 @@ private static unsafe RecordBatch AddPartitionColumnsToRecordBatch(RecordBatch r
StringArray.Builder columnBuilder = new();

#pragma warning disable CS8600
string colName = Marshal.PtrToStringUTF8((IntPtr)partitionsPtr->ColNames[i]);
string colValue = Marshal.PtrToStringUTF8((IntPtr)partitionsPtr->ColValues[i]);
string colName = MarshalExtensions.PtrToStringUTF8((IntPtr)partitionsPtr->ColNames[i]);
string colValue = MarshalExtensions.PtrToStringUTF8((IntPtr)partitionsPtr->ColValues[i]);
#pragma warning restore CS8600

Field field = new(colName, StringType.Default, nullable: true);
Expand Down
14 changes: 7 additions & 7 deletions src/DeltaLake/Kernel/Arrow/Handlers/ArrowFFIInterOpHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ KernelBoolSlice selectionVector
)
{
#pragma warning disable CS8600
string tableRoot = Marshal.PtrToStringUTF8((IntPtr)context->TableRoot)?.TrimEnd('/');
string parquetAbsolutePath = $"{tableRoot}/{Marshal.PtrToStringUTF8((IntPtr)path.ptr, (int)path.len)}";
string tableRoot = MarshalExtensions.PtrToStringUTF8((IntPtr)context->TableRoot)?.TrimEnd('/');
string parquetAbsolutePath = $"{tableRoot}/{MarshalExtensions.PtrToStringUTF8((IntPtr)path.ptr, (int)path.len)}";
#pragma warning restore CS8600

(GCHandle parquetAbsolutePathHandle, IntPtr gcPinnedParquetAbsolutePathPtr) = parquetAbsolutePath.ToPinnedSBytePointer();
Expand Down Expand Up @@ -146,7 +146,7 @@ private static unsafe ParquetStringPartitions ParseParquetStringPartitions(
for (int i = 0; i < partitionCols->Len; i++)
{
#pragma warning disable CS8600
string colName = Marshal.PtrToStringUTF8((IntPtr)partitionCols->Cols[i]);
string colName = MarshalExtensions.PtrToStringUTF8((IntPtr)partitionCols->Cols[i]);
#pragma warning restore CS8600

// The Kernel can currently only report String values back as
Expand All @@ -166,13 +166,13 @@ private static unsafe ParquetStringPartitions ParseParquetStringPartitions(
},
Marshal.GetFunctionPointerForDelegate<AllocateStringFn>(StringAllocatorCallbacks.AllocateString)
);
string colVal = colValPtr != null ? Marshal.PtrToStringUTF8((IntPtr)colValPtr) : String.Empty;
string colVal = colValPtr != null ? MarshalExtensions.PtrToStringUTF8((IntPtr)colValPtr) : String.Empty;
#pragma warning restore CS1024, CS8629, CS8600

if (!string.IsNullOrEmpty(colName) && !string.IsNullOrEmpty(colVal))
{
colNames.Add(colName);
colValues.Add(colVal);
colValues.Add(colVal!);
}
else
{
Expand All @@ -185,8 +185,8 @@ private static unsafe ParquetStringPartitions ParseParquetStringPartitions(

for (int i = 0; i < colNames.Count; i++)
{
colNamesPtr[i] = (byte*)Marshal.StringToCoTaskMemUTF8(colNames[i]);
colValuesPtr[i] = (byte*)Marshal.StringToCoTaskMemUTF8(colValues[i]);
colNamesPtr[i] = (byte*)MarshalExtensions.StringToCoTaskMemUTF8(colNames[i]);
colValuesPtr[i] = (byte*)MarshalExtensions.StringToCoTaskMemUTF8(colValues[i]);
}

return new ParquetStringPartitions
Expand Down
Loading
Loading