Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[C#] Use Arrow scalars #45095

Open
Platob opened this issue Dec 22, 2024 · 0 comments
Open

[C#] Use Arrow scalars #45095

Platob opened this issue Dec 22, 2024 · 0 comments

Comments

@Platob
Copy link
Contributor

Platob commented Dec 22, 2024

Describe the enhancement requested

Hello,

I would like use more user friendly methods to serialize/deserialize c# objects creating scalars, specifically for nested structures

But i have doubts if this is a good start implementation or completly wrong, any advice or correction ?

Br
Nick

Currently it looks like

Interfaces:

using Apache.Arrow.Types;

namespace Apache.Arrow.Scalars
{
    public interface Scalar
    {
        IArrowType DataType { get; }

        void Validate();

        void ValidateFull();
    }

    public interface Scalar<TArrowType> : Scalar where TArrowType : IArrowType
    {

    }

    public interface Scalar<TArrowType, TValue> : Scalar<TArrowType> where TArrowType : IArrowType
    {
        TValue Value { get; }
    }

    public interface PrimitiveScalar : Scalar
    {

    }

    public interface PrimitiveScalar<TArrowType> : Scalar<TArrowType>, PrimitiveScalar where TArrowType : IArrowType
    {

    }

    public interface PrimitiveScalar<TArrowType, TValue> : Scalar<TArrowType, TValue>, PrimitiveScalar<TArrowType> where TArrowType : IArrowType
    {

    }

    public interface NumericScalar : PrimitiveScalar
    {

    }

    public interface NumericScalar<TArrowType, TValue> : PrimitiveScalar<TArrowType, TValue>, NumericScalar where TArrowType : IArrowType
    {

    }

    public interface BinaryScalarBase : PrimitiveScalar
    {

    }

    public interface BinaryScalarBase<TArrowType> : PrimitiveScalar<TArrowType>, BinaryScalarBase where TArrowType : IArrowType
    {

    }

    public interface BinaryScalarBase<TArrowType, TValue> : PrimitiveScalar<TArrowType, TValue>, BinaryScalarBase<TArrowType> where TArrowType : IArrowType
    {

    }

    public interface TemporalScalar : PrimitiveScalar
    {

    }

    public interface TemporalScalar<TArrowType> : PrimitiveScalar<TArrowType>, TemporalScalar where TArrowType : IArrowType
    {

    }

    public interface TemporalScalar<TArrowType, TValue> : PrimitiveScalar<TArrowType, TValue>, TemporalScalar<TArrowType> where TArrowType : IArrowType
    {

    }

    public interface ListScalarBase : Scalar
    {

    }

    public interface ListScalarBase<TArrowType> : Scalar<TArrowType>, ListScalarBase where TArrowType : IArrowType
    {

    }

    public interface ListScalarBase<TArrowType, TValue> : Scalar<TArrowType, TValue>, ListScalarBase<TArrowType> where TArrowType : IArrowType
    {

    }
}

Scalar structs:

using System;
using Apache.Arrow.Types;

namespace Apache.Arrow.Scalars
{
    public readonly struct NullScalar : Scalar
    {
        public IArrowType DataType => NullType.Default;

        public void Validate()
        {

        }

        public void ValidateFull()
        {

        }
    }

    public readonly struct BooleanScalar : PrimitiveScalar<BooleanType, bool>
    {
        public IArrowType DataType => BooleanType.Default;

        public bool Value { get; }

        public BooleanScalar(bool value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Boolean)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(bool) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int8Scalar : NumericScalar<Int8Type, sbyte>
    {
        public IArrowType DataType => Int8Type.Default;

        public sbyte Value { get; }

        public Int8Scalar(sbyte value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int8)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(sbyte) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt8Scalar : NumericScalar<UInt8Type, byte>
    {
        public IArrowType DataType => UInt8Type.Default;

        public byte Value { get; }

        public UInt8Scalar(byte value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt8)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(byte) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int16Scalar : NumericScalar<Int16Type, short>
    {
        public IArrowType DataType => Int16Type.Default;

        public short Value { get; }

        public Int16Scalar(short value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int16)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(short) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt16Scalar : NumericScalar<UInt16Type, ushort>
    {
        public IArrowType DataType => UInt16Type.Default;

        public ushort Value { get; }

        public UInt16Scalar(ushort value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt16)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(ushort) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int32Scalar : NumericScalar<Int32Type, int>
    {
        public IArrowType DataType => Int32Type.Default;

        public int Value { get; }

        public Int32Scalar(int value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(int) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt32Scalar : NumericScalar<UInt32Type, uint>
    {
        public IArrowType DataType => UInt32Type.Default;

        public uint Value { get; }

        public UInt32Scalar(uint value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(uint) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int64Scalar : NumericScalar<Int64Type, long>
    {
        public IArrowType DataType => Int64Type.Default;

        public long Value { get; }

        public Int64Scalar(long value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt64Scalar : NumericScalar<UInt64Type, ulong>
    {
        public IArrowType DataType => UInt64Type.Default;

        public ulong Value { get; }

        public UInt64Scalar(ulong value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(ulong) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

# if NET6_0_OR_GREATER
    public readonly struct HalfFloatScalar : NumericScalar<HalfFloatType, Half>
    {
        public IArrowType DataType => HalfFloatType.Default;

        public Half Value { get; }

        public HalfFloatScalar(Half value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.HalfFloat)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(float) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }
# endif

    public readonly struct FloatScalar : NumericScalar<FloatType, float>
    {
        public IArrowType DataType => FloatType.Default;

        public float Value { get; }

        public FloatScalar(float value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Float)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(float) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct DoubleScalar : NumericScalar<DoubleType, double>
    {
        public IArrowType DataType => DoubleType.Default;

        public double Value { get; }

        public DoubleScalar(double value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Double)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(double) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Binary
    public readonly struct BinaryScalar : BinaryScalarBase<BinaryType, byte[]>
    {
        public IArrowType DataType => BinaryType.Default;

        public byte[] Value { get; }

        public BinaryScalar(byte[] value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Binary)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(byte[]) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // String
    public readonly struct StringScalar : BinaryScalarBase<StringType, string>
    {
        public IArrowType DataType => StringType.Default;

        public string Value { get; }

        public StringScalar(string value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (Value == null)
                throw new InvalidOperationException("Invalid csharp null value for string scalar");

            if (DataType.TypeId != ArrowTypeId.String)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(string) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // FixedSizedBinary
    public readonly struct FixedSizedBinaryScalar : BinaryScalarBase<FixedSizeBinaryType, byte[]>
    {
        public IArrowType DataType { get; }

        public byte[] Value { get; }

        public FixedSizedBinaryScalar(byte[] value, FixedSizeBinaryType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.FixedSizedBinary)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(byte[]) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Date32
    public readonly struct Date32Scalar : TemporalScalar<Date32Type, DateTime>
    {
        public IArrowType DataType => Date32Type.Default;

        public DateTime Value { get; }

        public Date32Scalar(DateTime value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Date32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(int) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Date64
    public readonly struct Date64Scalar : TemporalScalar<Date64Type, DateTime>
    {
        public IArrowType DataType => Date64Type.Default;

        public DateTime Value { get; }

        public Date64Scalar(DateTime value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Date64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Timestamp
    public readonly struct TimestampScalar : TemporalScalar<TimestampType, DateTime>
    {
        public IArrowType DataType { get; }

        public DateTime Value { get; }

        public TimestampScalar(DateTime value, TimestampType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Timestamp)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Time32
    public readonly struct Time32Scalar : TemporalScalar<Time32Type, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public Time32Scalar(TimeSpan value, Time32Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Time32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(int) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Time64
    public readonly struct Time64Scalar : TemporalScalar<Time64Type, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public Time64Scalar(TimeSpan value, Time64Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Time64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Interval
    public readonly struct IntervalScalar : TemporalScalar<IntervalType, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public IntervalScalar(TimeSpan value, IntervalType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Interval)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal128
    public readonly struct Decimal128Scalar : PrimitiveScalar<Decimal128Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal128Scalar(decimal value, Decimal128Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal128)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal256
    public readonly struct Decimal256Scalar : PrimitiveScalar<Decimal256Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal256Scalar(decimal value, Decimal256Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal256)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Duration
    public readonly struct DurationScalar : TemporalScalar<DurationType, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public DurationScalar(TimeSpan value, DurationType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Duration)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal32
    public readonly struct Decimal32Scalar : PrimitiveScalar<Decimal32Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal32Scalar(decimal value, Decimal32Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal64
    public readonly struct Decimal64Scalar : PrimitiveScalar<Decimal64Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal64Scalar(decimal value, Decimal64Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct ListScalar : ListScalarBase<ListType>
    {
        public IArrowType DataType => Values.Data.DataType;

        public IArrowArray Values { get; }

        public ListScalar(IArrowArray values)
        {
            Values = values;
        }

        public void Validate()
        {

        }

        public void ValidateFull()
        {

        }
    }

    public readonly struct StructScalar : Scalar<StructType>
    {
        public IArrowType DataType { get; }

        public Scalar[] Values { get; }

        public StructScalar(Scalar[] values, StructType dataType)
        {
            DataType = dataType;
            Values = values;
        }

        public void Validate()
        {
            foreach (var value in Values)
            {
                value.Validate();
            }
        }

        public void ValidateFull()
        {
            foreach (var value in Values)
            {
                value.ValidateFull();
            }
        }
    }
}

Component(s)

C#

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Development

No branches or pull requests

1 participant