From 5b0f0d00abc8233a4bce97ca2be949462a78657c Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Fri, 17 Dec 2021 12:07:23 -0500 Subject: [PATCH 1/6] Try creating module instead of overloading type getproperty --- src/FlatBuffers/FlatBuffers.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/FlatBuffers/FlatBuffers.jl b/src/FlatBuffers/FlatBuffers.jl index 187631b6..8826202f 100644 --- a/src/FlatBuffers/FlatBuffers.jl +++ b/src/FlatBuffers/FlatBuffers.jl @@ -121,7 +121,7 @@ macro scopedenum(T, syms...) push!(defs.args, :(const $(esc(sym)) = $(esc(typename))($i))) end end - mod = Symbol(typename, "Module") + mod = Symbol(typename, "s") syms = Tuple(Base.values(namemap)) blk = quote module $(esc(mod)) @@ -135,7 +135,6 @@ macro scopedenum(T, syms...) if isdefined(Base.Enums, :namemap) Base.Enums.namemap(::Type{$(esc(typename))}) = $(esc(namemap)) end - Base.getproperty(::Type{$(esc(typename))}, sym::Symbol) = sym in $syms ? getfield($(esc(mod)), sym) : getfield($(esc(typename)), sym) Base.typemin(x::Type{$(esc(typename))}) = $(esc(typename))($lo) Base.typemax(x::Type{$(esc(typename))}) = $(esc(typename))($hi) let insts = (Any[ $(esc(typename))(v) for v in $values ]...,) @@ -144,6 +143,8 @@ macro scopedenum(T, syms...) FlatBuffers.basetype(::$(esc(typename))) = $(basetype) FlatBuffers.basetype(::Type{$(esc(typename))}) = $(basetype) $defs + #Base.getproperty(::Type{$(esc(typename))}, sym::Symbol) = sym in $syms ? getfield($(esc(mod)), sym) : getfield($(esc(typename)), sym) + export $(syms...) end end push!(blk.args, :nothing) From da5746d6b2ea5a75049bf3723f134622a1b14b30 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Fri, 17 Dec 2021 12:15:16 -0500 Subject: [PATCH 2/6] pluralize nicely for strings ending in "s" --- src/FlatBuffers/FlatBuffers.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/FlatBuffers/FlatBuffers.jl b/src/FlatBuffers/FlatBuffers.jl index 8826202f..3df6b022 100644 --- a/src/FlatBuffers/FlatBuffers.jl +++ b/src/FlatBuffers/FlatBuffers.jl @@ -121,7 +121,8 @@ macro scopedenum(T, syms...) push!(defs.args, :(const $(esc(sym)) = $(esc(typename))($i))) end end - mod = Symbol(typename, "s") + pluralized = typename * (last(string(typename)) == 's' ? "es" : "s") + mod = Symbol(pluralized) syms = Tuple(Base.values(namemap)) blk = quote module $(esc(mod)) From 49c3708c410020dc3f4f3d7feb5f7ac2b4f59ebd Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Fri, 17 Dec 2021 12:16:23 -0500 Subject: [PATCH 3/6] pluralize nicely for strings ending in "s" --- src/FlatBuffers/FlatBuffers.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FlatBuffers/FlatBuffers.jl b/src/FlatBuffers/FlatBuffers.jl index 3df6b022..95314f40 100644 --- a/src/FlatBuffers/FlatBuffers.jl +++ b/src/FlatBuffers/FlatBuffers.jl @@ -121,8 +121,8 @@ macro scopedenum(T, syms...) push!(defs.args, :(const $(esc(sym)) = $(esc(typename))($i))) end end - pluralized = typename * (last(string(typename)) == 's' ? "es" : "s") - mod = Symbol(pluralized) + _typename_str = string(typename) + mod = Symbol(_typename_str, last(_typename_str) == 's' ? "es" : "s") syms = Tuple(Base.values(namemap)) blk = quote module $(esc(mod)) From e78212120c02bd758b966b5acf01ae5cfe84fc48 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Fri, 17 Dec 2021 12:23:36 -0500 Subject: [PATCH 4/6] Apply renaming from `@scopedenum` throughout the Package Example: - `Meta.UnionMode.Sparse` => `Meta.UnionModes.Sparse` --- src/arraytypes/arraytypes.jl | 8 ++--- src/arraytypes/unions.jl | 8 ++--- src/eltypes.jl | 70 ++++++++++++++++++------------------ src/metadata/File.jl | 4 +-- src/metadata/Message.jl | 6 ++-- src/metadata/Schema.jl | 14 ++++---- src/table.jl | 12 +++---- src/write.jl | 8 ++--- test/arrowjson.jl | 28 +++++++-------- test/runtests.jl | 2 +- test/testtables.jl | 14 ++++---- 11 files changed, 87 insertions(+), 87 deletions(-) diff --git a/src/arraytypes/arraytypes.jl b/src/arraytypes/arraytypes.jl index c6aba0bd..3b3273e3 100644 --- a/src/arraytypes/arraytypes.jl +++ b/src/arraytypes/arraytypes.jl @@ -35,13 +35,13 @@ function toarrowvector(x, i=1, de=Dict{Int64, Any}(), ded=DictEncoding[], meta=g @debug 3 x A = arrowvector(x, i, 0, 0, de, ded, meta; compression=compression, kw...) if compression isa LZ4FrameCompressor - A = compress(Meta.CompressionType.LZ4_FRAME, compression, A) + A = compress(Meta.CompressionTypes.LZ4_FRAME, compression, A) elseif compression isa Vector{LZ4FrameCompressor} - A = compress(Meta.CompressionType.LZ4_FRAME, compression[Threads.threadid()], A) + A = compress(Meta.CompressionTypes.LZ4_FRAME, compression[Threads.threadid()], A) elseif compression isa ZstdCompressor - A = compress(Meta.CompressionType.ZSTD, compression, A) + A = compress(Meta.CompressionTypes.ZSTD, compression, A) elseif compression isa Vector{ZstdCompressor} - A = compress(Meta.CompressionType.ZSTD, compression[Threads.threadid()], A) + A = compress(Meta.CompressionTypes.ZSTD, compression[Threads.threadid()], A) end @debug 2 "converted top-level column to arrow format: $(typeof(A))" @debug 3 A diff --git a/src/arraytypes/unions.jl b/src/arraytypes/unions.jl index 045d7ad7..72cb7c43 100644 --- a/src/arraytypes/unions.jl +++ b/src/arraytypes/unions.jl @@ -17,7 +17,7 @@ # Union arrays # need a custom representation of Union types since arrow unions # are ordered, and possibly indirected via separate typeIds array -# here, T is Meta.UnionMode.Dense or Meta.UnionMode.Sparse, +# here, T is Meta.UnionModes.Dense or Meta.UnionModes.Sparse, # typeIds is a NTuple{N, Int32}, and U is a Tuple{...} of the # unioned types struct UnionT{T, typeIds, U} @@ -93,7 +93,7 @@ end # convenience wrappers for signaling that an array shoudld be written # as with dense/sparse union arrow buffers -struct DenseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionMode.Dense, nothing, U}} +struct DenseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionModes.Dense, nothing, U}} itr::T end @@ -118,7 +118,7 @@ function todense(::Type{UnionT{T, typeIds, U}}, x) where {T, typeIds, U} return types, offsets, data end -struct SparseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionMode.Sparse, nothing, U}} +struct SparseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionModes.Sparse, nothing, U}} itr::T end @@ -214,7 +214,7 @@ arrowvector(::UnionKind, x::Union{DenseUnion, SparseUnion}, i, nl, fi, de, ded, function arrowvector(::UnionKind, x, i, nl, fi, de, ded, meta; kw...) UT = eltype(x) - if unionmode(UT) == Meta.UnionMode.Dense + if unionmode(UT) == Meta.UnionModes.Dense x = x isa DenseUnionVector ? x.itr : x typeids, offsets, data = todense(UT, x) data2 = map(y -> arrowvector(y[2], i, nl + 1, y[1], de, ded, nothing; kw...), enumerate(data)) diff --git a/src/eltypes.jl b/src/eltypes.jl index 2969bc2f..b201aba0 100644 --- a/src/eltypes.jl +++ b/src/eltypes.jl @@ -109,18 +109,18 @@ end # primitive types function juliaeltype(f::Meta.Field, fp::Meta.FloatingPoint, convert) - if fp.precision == Meta.Precision.HALF + if fp.precision == Meta.Precisions.HALF Float16 - elseif fp.precision == Meta.Precision.SINGLE + elseif fp.precision == Meta.Precisions.SINGLE Float32 - elseif fp.precision == Meta.Precision.DOUBLE + elseif fp.precision == Meta.Precisions.DOUBLE Float64 end end function arrowtype(b, ::Type{T}) where {T <: AbstractFloat} Meta.floatingPointStart(b) - Meta.floatingPointAddPrecision(b, T === Float16 ? Meta.Precision.HALF : T === Float32 ? Meta.Precision.SINGLE : Meta.Precision.DOUBLE) + Meta.floatingPointAddPrecision(b, T === Float16 ? Meta.Precisions.HALF : T === Float32 ? Meta.Precisions.SINGLE : Meta.Precisions.DOUBLE) return Meta.FloatingPoint, Meta.floatingPointEnd(b), nothing end @@ -175,18 +175,18 @@ struct Date{U, T} <: ArrowTimeType x::T end -const DATE = Date{Meta.DateUnit.DAY, Int32} +const DATE = Date{Meta.DateUnits.DAY, Int32} Base.zero(::Type{Date{U, T}}) where {U, T} = Date{U, T}(T(0)) storagetype(::Type{Date{U, T}}) where {U, T} = T -bitwidth(x::Meta.DateUnit) = x == Meta.DateUnit.DAY ? Int32 : Int64 -Date{Meta.DateUnit.DAY}(days) = DATE(Int32(days)) -Date{Meta.DateUnit.MILLISECOND}(ms) = Date{Meta.DateUnit.MILLISECOND, Int64}(Int64(ms)) +bitwidth(x::Meta.DateUnit) = x == Meta.DateUnits.DAY ? Int32 : Int64 +Date{Meta.DateUnits.DAY}(days) = DATE(Int32(days)) +Date{Meta.DateUnits.MILLISECOND}(ms) = Date{Meta.DateUnits.MILLISECOND, Int64}(Int64(ms)) juliaeltype(f::Meta.Field, x::Meta.Date, convert) = Date{x.unit, bitwidth(x.unit)} finaljuliatype(::Type{DATE}) = Dates.Date Base.convert(::Type{Dates.Date}, x::DATE) = Dates.Date(Dates.UTD(Int64(x.x + UNIX_EPOCH_DATE))) -finaljuliatype(::Type{Date{Meta.DateUnit.MILLISECOND, Int64}}) = Dates.DateTime -Base.convert(::Type{Dates.DateTime}, x::Date{Meta.DateUnit.MILLISECOND, Int64}) = Dates.DateTime(Dates.UTM(Int64(x.x + UNIX_EPOCH_DATETIME))) +finaljuliatype(::Type{Date{Meta.DateUnits.MILLISECOND, Int64}}) = Dates.DateTime +Base.convert(::Type{Dates.DateTime}, x::Date{Meta.DateUnits.MILLISECOND, Int64}) = Dates.DateTime(Dates.UTM(Int64(x.x + UNIX_EPOCH_DATETIME))) function arrowtype(b, ::Type{Date{U, T}}) where {U, T} Meta.dateStart(b) @@ -198,7 +198,7 @@ const UNIX_EPOCH_DATE = Dates.value(Dates.Date(1970)) Base.convert(::Type{DATE}, x::Dates.Date) = DATE(Int32(Dates.value(x) - UNIX_EPOCH_DATE)) const UNIX_EPOCH_DATETIME = Dates.value(Dates.DateTime(1970)) -Base.convert(::Type{Date{Meta.DateUnit.MILLISECOND, Int64}}, x::Dates.DateTime) = Date{Meta.DateUnit.MILLISECOND, Int64}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME)) +Base.convert(::Type{Date{Meta.DateUnits.MILLISECOND, Int64}}, x::Dates.DateTime) = Date{Meta.DateUnits.MILLISECOND, Int64}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME)) ArrowTypes.ArrowType(::Type{Dates.Date}) = DATE ArrowTypes.toarrow(x::Dates.Date) = convert(DATE, x) @@ -213,16 +213,16 @@ struct Time{U, T} <: ArrowTimeType end Base.zero(::Type{Time{U, T}}) where {U, T} = Time{U, T}(T(0)) -const TIME = Time{Meta.TimeUnit.NANOSECOND, Int64} +const TIME = Time{Meta.TimeUnits.NANOSECOND, Int64} -bitwidth(x::Meta.TimeUnit) = x == Meta.TimeUnit.SECOND || x == Meta.TimeUnit.MILLISECOND ? Int32 : Int64 +bitwidth(x::Meta.TimeUnit) = x == Meta.TimeUnits.SECOND || x == Meta.TimeUnits.MILLISECOND ? Int32 : Int64 Time{U}(x) where {U <: Meta.TimeUnit} = Time{U, bitwidth(U)}(bitwidth(U)(x)) storagetype(::Type{Time{U, T}}) where {U, T} = T juliaeltype(f::Meta.Field, x::Meta.Time, convert) = Time{x.unit, bitwidth(x.unit)} finaljuliatype(::Type{<:Time}) = Dates.Time -periodtype(U::Meta.TimeUnit) = U === Meta.TimeUnit.SECOND ? Dates.Second : - U === Meta.TimeUnit.MILLISECOND ? Dates.Millisecond : - U === Meta.TimeUnit.MICROSECOND ? Dates.Microsecond : Dates.Nanosecond +periodtype(U::Meta.TimeUnit) = U === Meta.TimeUnits.SECOND ? Dates.Second : + U === Meta.TimeUnits.MILLISECOND ? Dates.Millisecond : + U === Meta.TimeUnits.MICROSECOND ? Dates.Microsecond : Dates.Nanosecond Base.convert(::Type{Dates.Time}, x::Time{U, T}) where {U, T} = Dates.Time(Dates.Nanosecond(Dates.tons(periodtype(U)(x.x)))) function arrowtype(b, ::Type{Time{U, T}}) where {U, T} @@ -252,7 +252,7 @@ function juliaeltype(f::Meta.Field, x::Meta.Timestamp, convert) return Timestamp{x.unit, x.timezone === nothing ? nothing : Symbol(x.timezone)} end -const DATETIME = Timestamp{Meta.TimeUnit.MILLISECOND, nothing} +const DATETIME = Timestamp{Meta.TimeUnits.MILLISECOND, nothing} finaljuliatype(::Type{Timestamp{U, TZ}}) where {U, TZ} = ZonedDateTime finaljuliatype(::Type{Timestamp{U, nothing}}) where {U} = DateTime @@ -261,19 +261,19 @@ finaljuliatype(::Type{Timestamp{U, nothing}}) where {U} = DateTime @warn "automatically converting Arrow.Timestamp with precision = $U to `$T` which only supports millisecond precision; conversion may be lossy; to avoid converting, pass `Arrow.Table(source; convert=false)" maxlog=1 _id=hash((:warntimestamp, U, T)) function Base.convert(::Type{ZonedDateTime}, x::Timestamp{U, TZ}) where {U, TZ} - (U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) && warntimestamp(U, ZonedDateTime) + (U === Meta.TimeUnits.MICROSECOND || U == Meta.TimeUnits.NANOSECOND) && warntimestamp(U, ZonedDateTime) return ZonedDateTime(Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME))), TimeZone(String(TZ))) end function Base.convert(::Type{DateTime}, x::Timestamp{U, nothing}) where {U} - (U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) && warntimestamp(U, DateTime) + (U === Meta.TimeUnits.MICROSECOND || U == Meta.TimeUnits.NANOSECOND) && warntimestamp(U, DateTime) return Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME))) end -Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND, TZ}}, x::ZonedDateTime) where {TZ} = - Timestamp{Meta.TimeUnit.MILLISECOND, TZ}(Int64(Dates.value(DateTime(x)) - UNIX_EPOCH_DATETIME)) -Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND, nothing}}, x::DateTime) = - Timestamp{Meta.TimeUnit.MILLISECOND, nothing}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME)) +Base.convert(::Type{Timestamp{Meta.TimeUnits.MILLISECOND, TZ}}, x::ZonedDateTime) where {TZ} = + Timestamp{Meta.TimeUnits.MILLISECOND, TZ}(Int64(Dates.value(DateTime(x)) - UNIX_EPOCH_DATETIME)) +Base.convert(::Type{Timestamp{Meta.TimeUnits.MILLISECOND, nothing}}, x::DateTime) = + Timestamp{Meta.TimeUnits.MILLISECOND, nothing}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME)) function arrowtype(b, ::Type{Timestamp{U, TZ}}) where {U, TZ} tz = TZ !== nothing ? FlatBuffers.createstring!(b, String(TZ)) : FlatBuffers.UOffsetT(0) @@ -289,11 +289,11 @@ const DATETIME_SYMBOL = Symbol("JuliaLang.DateTime") ArrowTypes.arrowname(::Type{Dates.DateTime}) = DATETIME_SYMBOL ArrowTypes.JuliaType(::Val{DATETIME_SYMBOL}, S) = Dates.DateTime ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::Timestamp) = convert(Dates.DateTime, x) -ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::Date{Meta.DateUnit.MILLISECOND, Int64}) = convert(Dates.DateTime, x) +ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::Date{Meta.DateUnits.MILLISECOND, Int64}) = convert(Dates.DateTime, x) ArrowTypes.default(::Type{Dates.DateTime}) = Dates.DateTime(1,1,1,1,1,1) ArrowTypes.ArrowType(::Type{ZonedDateTime}) = Timestamp -ArrowTypes.toarrow(x::ZonedDateTime) = convert(Timestamp{Meta.TimeUnit.MILLISECOND, Symbol(x.timezone)}, x) +ArrowTypes.toarrow(x::ZonedDateTime) = convert(Timestamp{Meta.TimeUnits.MILLISECOND, Symbol(x.timezone)}, x) const ZONEDDATETIME_SYMBOL = Symbol("JuliaLang.ZonedDateTime") ArrowTypes.arrowname(::Type{ZonedDateTime}) = ZONEDDATETIME_SYMBOL ArrowTypes.JuliaType(::Val{ZONEDDATETIME_SYMBOL}, S) = ZonedDateTime @@ -310,15 +310,15 @@ scan each element to check each timezone. `Arrow.ToTimestamp` provides a "bypass first element of the `AbstractVector{ZonedDateTime}`, which in turn allows `Arrow.write` to avoid costly checking/conversion and can encode the `ZonedDateTime` as `Arrow.Timestamp` directly. """ -struct ToTimestamp{A, TZ} <: AbstractVector{Timestamp{Meta.TimeUnit.MILLISECOND, TZ}} +struct ToTimestamp{A, TZ} <: AbstractVector{Timestamp{Meta.TimeUnits.MILLISECOND, TZ}} data::A # AbstractVector{ZonedDateTime} end ToTimestamp(x::A) where {A <: AbstractVector{ZonedDateTime}} = ToTimestamp{A, Symbol(x[1].timezone)}(x) Base.IndexStyle(::Type{<:ToTimestamp}) = Base.IndexLinear() Base.size(x::ToTimestamp) = (length(x.data),) -Base.eltype(::ToTimestamp{A, TZ}) where {A, TZ} = Timestamp{Meta.TimeUnit.MILLISECOND, TZ} -Base.getindex(x::ToTimestamp{A, TZ}, i::Int) where {A, TZ} = convert(Timestamp{Meta.TimeUnit.MILLISECOND, TZ}, getindex(x.data, i)) +Base.eltype(::ToTimestamp{A, TZ}) where {A, TZ} = Timestamp{Meta.TimeUnits.MILLISECOND, TZ} +Base.getindex(x::ToTimestamp{A, TZ}, i::Int) where {A, TZ} = convert(Timestamp{Meta.TimeUnits.MILLISECOND, TZ}, getindex(x.data, i)) struct Interval{U, T} <: ArrowTimeType x::T @@ -326,9 +326,9 @@ end Base.zero(::Type{Interval{U, T}}) where {U, T} = Interval{U, T}(T(0)) -bitwidth(x::Meta.IntervalUnit) = x == Meta.IntervalUnit.YEAR_MONTH ? Int32 : Int64 -Interval{Meta.IntervalUnit.YEAR_MONTH}(x) = Interval{Meta.IntervalUnit.YEAR_MONTH, Int32}(Int32(x)) -Interval{Meta.IntervalUnit.DAY_TIME}(x) = Interval{Meta.IntervalUnit.DAY_TIME, Int64}(Int64(x)) +bitwidth(x::Meta.IntervalUnit) = x == Meta.IntervalUnits.YEAR_MONTH ? Int32 : Int64 +Interval{Meta.IntervalUnits.YEAR_MONTH}(x) = Interval{Meta.IntervalUnits.YEAR_MONTH, Int32}(Int32(x)) +Interval{Meta.IntervalUnits.DAY_TIME}(x) = Interval{Meta.IntervalUnits.DAY_TIME, Int64}(Int64(x)) function juliaeltype(f::Meta.Field, x::Meta.Interval, convert) return Interval{x.unit, bitwidth(x.unit)} @@ -359,10 +359,10 @@ function arrowtype(b, ::Type{Duration{U}}) where {U} return Meta.Duration, Meta.durationEnd(b), nothing end -arrowperiodtype(P) = Meta.TimeUnit.SECOND -arrowperiodtype(::Type{Dates.Millisecond}) = Meta.TimeUnit.MILLISECOND -arrowperiodtype(::Type{Dates.Microsecond}) = Meta.TimeUnit.MICROSECOND -arrowperiodtype(::Type{Dates.Nanosecond}) = Meta.TimeUnit.NANOSECOND +arrowperiodtype(P) = Meta.TimeUnits.SECOND +arrowperiodtype(::Type{Dates.Millisecond}) = Meta.TimeUnits.MILLISECOND +arrowperiodtype(::Type{Dates.Microsecond}) = Meta.TimeUnits.MICROSECOND +arrowperiodtype(::Type{Dates.Nanosecond}) = Meta.TimeUnits.NANOSECOND Base.convert(::Type{Duration{U}}, x::Dates.Period) where {U} = Duration{U}(Dates.value(periodtype(U)(x))) diff --git a/src/metadata/File.jl b/src/metadata/File.jl index 8c1622ce..6ad33ba6 100644 --- a/src/metadata/File.jl +++ b/src/metadata/File.jl @@ -25,7 +25,7 @@ function Base.getproperty(x::Footer, field::Symbol) if field === :version o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), MetadataVersion) - return MetadataVersion.V1 + return MetadataVersions.V1 elseif field === :schema o = FlatBuffers.offset(x, 6) if o != 0 @@ -87,4 +87,4 @@ function createBlock(b::FlatBuffers.Builder, offset::Int64, metadatalength::Int3 prepend!(b, metadatalength) prepend!(b, offset) return FlatBuffers.offset(b) -end \ No newline at end of file +end diff --git a/src/metadata/Message.jl b/src/metadata/Message.jl index 4fe6253b..6af3a6e1 100644 --- a/src/metadata/Message.jl +++ b/src/metadata/Message.jl @@ -54,11 +54,11 @@ function Base.getproperty(x::BodyCompression, field::Symbol) if field === :codec o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), CompressionType) - return CompressionType.LZ4_FRAME + return CompressionTypes.LZ4_FRAME elseif field === :method o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), BodyCompressionMethod) - return BodyCompressionMethod.BUFFER + return BodyCompressionMethods.BUFFER end return nothing end @@ -199,4 +199,4 @@ messageAddHeader(b::FlatBuffers.Builder, header::FlatBuffers.UOffsetT) = FlatBuf messageAddBodyLength(b::FlatBuffers.Builder, bodyLength::Int64) = FlatBuffers.prependslot!(b, 3, bodyLength, 0) messageAddCustomMetadata(b::FlatBuffers.Builder, meta::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 4, meta, 0) messageStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4) -messageEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) \ No newline at end of file +messageEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) diff --git a/src/metadata/Schema.jl b/src/metadata/Schema.jl index 9f83bef5..d09b6480 100644 --- a/src/metadata/Schema.jl +++ b/src/metadata/Schema.jl @@ -108,7 +108,7 @@ function Base.getproperty(x::Union, field::Symbol) if field === :mode o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), UnionMode) - return UnionMode.Sparse + return UnionModes.Sparse elseif field === :typeIds o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.Array{Int32}(x, o) @@ -159,7 +159,7 @@ function Base.getproperty(x::FloatingPoint, field::Symbol) if field === :precision o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Precision) - return Precision.HALF + return Precisions.HALF end return nothing end @@ -280,7 +280,7 @@ function Base.getproperty(x::Date, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), DateUnit) - return DateUnit.MILLISECOND + return DateUnits.MILLISECOND end return nothing end @@ -302,7 +302,7 @@ function Base.getproperty(x::Time, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit) - return TimeUnit.MILLISECOND + return TimeUnits.MILLISECOND elseif field === :bitWidth o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) @@ -327,7 +327,7 @@ function Base.getproperty(x::Timestamp, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit) - return TimeUnit.SECOND + return TimeUnits.SECOND elseif field === :timezone o = FlatBuffers.offset(x, 6) o != 0 && return String(x, o + FlatBuffers.pos(x)) @@ -353,7 +353,7 @@ function Base.getproperty(x::Interval, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), IntervalUnit) - return IntervalUnit.YEAR_MONTH + return IntervalUnits.YEAR_MONTH end return nothing end @@ -373,7 +373,7 @@ function Base.getproperty(x::Duration, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit) - return TimeUnit.MILLISECOND + return TimeUnits.MILLISECOND end return nothing end diff --git a/src/table.jl b/src/table.jl index d3a31976..88d65b16 100644 --- a/src/table.jl +++ b/src/table.jl @@ -173,9 +173,9 @@ function Base.iterate(x::Stream, (pos, id)=(x.pos, 1)) end if compression !== nothing - if compression.codec == Flatbuf.CompressionType.ZSTD + if compression.codec == Flatbuf.CompressionTypes.ZSTD x.compression[] = :zstd - elseif compression.codec == Flatbuf.CompressionType.LZ4_FRAME + elseif compression.codec == Flatbuf.CompressionTypes.LZ4_FRAME x.compression[] = :lz4 else throw(ArgumentError("unsupported compression codec: $(compression.codec)")) @@ -485,9 +485,9 @@ function uncompress(ptr::Ptr{UInt8}, buffer, compression) len = unsafe_load(convert(Ptr{Int64}, ptr)) ptr += 8 # skip past uncompressed length as Int64 encodedbytes = unsafe_wrap(Array, ptr, buffer.length - 8) - if compression.codec === Meta.CompressionType.LZ4_FRAME + if compression.codec === Meta.CompressionTypes.LZ4_FRAME decodedbytes = transcode(LZ4FrameDecompressor, encodedbytes) - elseif compression.codec === Meta.CompressionType.ZSTD + elseif compression.codec === Meta.CompressionTypes.ZSTD decodedbytes = transcode(ZstdDecompressor, encodedbytes) else error("unsupported compression type when reading arrow buffers: $(typeof(compression.codec))") @@ -590,7 +590,7 @@ function build(f::Meta.Field, L::Meta.Union, batch, rb, de, nodeidx, bufferidx, buffer = rb.buffers[bufferidx] bytes, typeIds = reinterp(UInt8, batch, buffer, rb.compression) bufferidx += 1 - if L.mode == Meta.UnionMode.Dense + if L.mode == Meta.UnionModes.Dense buffer = rb.buffers[bufferidx] bytes2, offsets = reinterp(Int32, batch, buffer, rb.compression) bufferidx += 1 @@ -605,7 +605,7 @@ function build(f::Meta.Field, L::Meta.Union, batch, rb, de, nodeidx, bufferidx, meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) UT = UnionT(f, convert) - if L.mode == Meta.UnionMode.Dense + if L.mode == Meta.UnionModes.Dense B = DenseUnion{T, UT, typeof(data)}(bytes, bytes2, typeIds, offsets, data, meta) else B = SparseUnion{T, UT, typeof(data)}(bytes, typeIds, data, meta) diff --git a/src/write.jl b/src/write.jl index 5020b321..1fd28aac 100644 --- a/src/write.jl +++ b/src/write.jl @@ -164,7 +164,7 @@ function write(io, source, writetofile, largelists, compress, denseunions, dicte dicts = FlatBuffers.UOffsetT(0) end Meta.footerStart(b) - Meta.footerAddVersion(b, Meta.MetadataVersion.V4) + Meta.footerAddVersion(b, Meta.MetadataVersions.V4) Meta.footerAddSchema(b, schfoot) Meta.footerAddDictionaries(b, dicts) Meta.footerAddRecordBatches(b, recordbatches) @@ -271,7 +271,7 @@ end function makemessage(b, headerType, header, columns=nothing, bodylen=0) # write the message flatbuffer object Meta.messageStart(b) - Meta.messageAddVersion(b, Meta.MetadataVersion.V5) + Meta.messageAddVersion(b, Meta.MetadataVersions.V5) Meta.messageAddHeaderType(b, headerType) Meta.messageAddHeader(b, header) Meta.messageAddBodyLength(b, Int64(bodylen)) @@ -312,7 +312,7 @@ function makeschema(b, sch::Tables.Schema{names}, columns) where {names} end # write schema object Meta.schemaStart(b) - Meta.schemaAddEndianness(b, Meta.Endianness.Little) + Meta.schemaAddEndianness(b, Meta.Endiannesses.Little) Meta.schemaAddFields(b, fields) Meta.schemaAddCustomMetadata(b, meta) return Meta.schemaEnd(b) @@ -443,7 +443,7 @@ function makerecordbatch(b, sch::Tables.Schema{names, types}, columns, alignment if compress !== nothing Meta.bodyCompressionStart(b) Meta.bodyCompressionAddCodec(b, compress) - Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethod.BUFFER) + Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethods.BUFFER) compression = Meta.bodyCompressionEnd(b) else compression = FlatBuffers.UOffsetT(0) diff --git a/test/arrowjson.jl b/test/arrowjson.jl index 586ab932..0c0c8e10 100644 --- a/test/arrowjson.jl +++ b/test/arrowjson.jl @@ -80,14 +80,14 @@ mutable struct Timestamp <: Type end Timestamp() = Timestamp("", "", nothing) -unit(U) = U == Arrow.Meta.TimeUnit.SECOND ? "SECOND" : - U == Arrow.Meta.TimeUnit.MILLISECOND ? "MILLISECOND" : - U == Arrow.Meta.TimeUnit.MICROSECOND ? "MICROSECOND" : "NANOSECOND" +unit(U) = U == Arrow.Meta.TimeUnits.SECOND ? "SECOND" : + U == Arrow.Meta.TimeUnits.MILLISECOND ? "MILLISECOND" : + U == Arrow.Meta.TimeUnits.MICROSECOND ? "MICROSECOND" : "NANOSECOND" Type(::Base.Type{Arrow.Timestamp{U, TZ}}) where {U, TZ} = Timestamp("timestamp", unit(U), TZ === nothing ? nothing : String(TZ)) StructTypes.StructType(::Base.Type{Timestamp}) = StructTypes.Mutable() -unitT(u) = u == "SECOND" ? Arrow.Meta.TimeUnit.SECOND : - u == "MILLISECOND" ? Arrow.Meta.TimeUnit.MILLISECOND : - u == "MICROSECOND" ? Arrow.Meta.TimeUnit.MICROSECOND : Arrow.Meta.TimeUnit.NANOSECOND +unitT(u) = u == "SECOND" ? Arrow.Meta.TimeUnits.SECOND : + u == "MILLISECOND" ? Arrow.Meta.TimeUnits.MILLISECOND : + u == "MICROSECOND" ? Arrow.Meta.TimeUnits.MICROSECOND : Arrow.Meta.TimeUnits.NANOSECOND juliatype(f, x::Timestamp) = Arrow.Timestamp{unitT(x.unit), x.timezone === nothing ? nothing : Symbol(x.timezone)} struct Duration <: Type @@ -104,9 +104,9 @@ struct Date <: Type unit::String end -Type(::Base.Type{Arrow.Date{U, T}}) where {U, T} = Date("date", U == Arrow.Meta.DateUnit.DAY ? "DAY" : "MILLISECOND") +Type(::Base.Type{Arrow.Date{U, T}}) where {U, T} = Date("date", U == Arrow.Meta.DateUnits.DAY ? "DAY" : "MILLISECOND") StructTypes.StructType(::Base.Type{Date}) = StructTypes.Struct() -juliatype(f, x::Date) = Arrow.Date{x.unit == "DAY" ? Arrow.Meta.DateUnit.DAY : Arrow.Meta.DateUnit.MILLISECOND, x.unit == "DAY" ? Int32 : Int64} +juliatype(f, x::Date) = Arrow.Date{x.unit == "DAY" ? Arrow.Meta.DateUnits.DAY : Arrow.Meta.DateUnits.MILLISECOND, x.unit == "DAY" ? Int32 : Int64} struct Time <: Type name::String @@ -123,9 +123,9 @@ struct Interval <: Type unit::String end -Type(::Base.Type{Arrow.Interval{U, T}}) where {U, T} = Interval("interval", U == Arrow.Meta.IntervalUnit.YEAR_MONTH ? "YEAR_MONTH" : "DAY_TIME") +Type(::Base.Type{Arrow.Interval{U, T}}) where {U, T} = Interval("interval", U == Arrow.Meta.IntervalUnits.YEAR_MONTH ? "YEAR_MONTH" : "DAY_TIME") StructTypes.StructType(::Base.Type{Interval}) = StructTypes.Struct() -juliatype(f, x::Interval) = Arrow.Interval{x.unit == "YEAR_MONTH" ? Arrow.Meta.IntervalUnit.YEAR_MONTH : Arrow.Meta.IntervalUnit.DAY_TIME, x.unit == "YEAR_MONTH" ? Int32 : Int64} +juliatype(f, x::Interval) = Arrow.Interval{x.unit == "YEAR_MONTH" ? Arrow.Meta.IntervalUnits.YEAR_MONTH : Arrow.Meta.IntervalUnits.DAY_TIME, x.unit == "YEAR_MONTH" ? Int32 : Int64} struct UnionT <: Type name::String @@ -133,10 +133,10 @@ struct UnionT <: Type typIds::Vector{Int64} end -Type(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} = UnionT("union", T == Arrow.Meta.UnionMode.Dense ? "DENSE" : "SPARSE", collect(typeIds)) +Type(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} = UnionT("union", T == Arrow.Meta.UnionModes.Dense ? "DENSE" : "SPARSE", collect(typeIds)) children(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} = Field[Field("", fieldtype(U, i), nothing) for i = 1:fieldcount(U)] StructTypes.StructType(::Base.Type{UnionT}) = StructTypes.Struct() -juliatype(f, x::UnionT) = Arrow.UnionT{x.mode == "DENSE" ? Arrow.Meta.UnionMode.DENSE : Arrow.Meta.UnionMode.SPARSE, Tuple(x.typeIds), Tuple{(juliatype(y) for y in f.children)...}} +juliatype(f, x::UnionT) = Arrow.UnionT{x.mode == "DENSE" ? Arrow.Meta.UnionModes.DENSE : Arrow.Meta.UnionModes.SPARSE, Tuple(x.typeIds), Tuple{(juliatype(y) for y in f.children)...}} struct List <: Type name::String @@ -381,7 +381,7 @@ function FieldData(nm, ::Base.Type{T}, col, dictencodings) where {T} U = eltype(S) tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) : Arrow.typeids(S) TYPE_ID = [x === missing ? 0 : tids[Arrow.isatypeid(x, U)] for x in col] - if Arrow.unionmode(S) == Arrow.Meta.UnionMode.Dense + if Arrow.unionmode(S) == Arrow.Meta.UnionModes.Dense offs = zeros(Int32, fieldcount(U)) OFFSET = Int32[] for x in col @@ -502,7 +502,7 @@ function Base.getindex(x::ArrowArray{T}, i::Base.Int) where {T} U = eltype(S) tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) : Arrow.typeids(S) typeid = tids[x.fielddata.TYPE_ID[i]] - if Arrow.unionmode(S) == Arrow.Meta.UnionMode.DENSE + if Arrow.unionmode(S) == Arrow.Meta.UnionModes.DENSE off = x.fielddata.OFFSET[i] return ArrowArray(x.field.children[typeid+1], x.fielddata.children[typeid+1], x.dictionaries)[off] else diff --git a/test/runtests.jl b/test/runtests.jl index 0098aba5..786f72b0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -332,7 +332,7 @@ tbl = Arrow.Table(Arrow.tobuffer(t)) # 166 t = ( - col1=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.NANOSECOND, nothing})], + col1=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.NANOSECOND, nothing})], ) tbl = Arrow.Table(Arrow.tobuffer(t)) @test_logs (:warn, r"automatically converting Arrow.Timestamp with precision = NANOSECOND") begin diff --git a/test/testtables.jl b/test/testtables.jl index 686f3364..4288b367 100644 --- a/test/testtables.jl +++ b/test/testtables.jl @@ -54,12 +54,12 @@ testtables = [ "arrow date/time types", ( col14=[zero(Arrow.Decimal{Int32(2), Int32(2), Int128}), zero(Arrow.Decimal{Int32(2), Int32(2), Int128}), zero(Arrow.Decimal{Int32(2), Int32(2), Int128}), missing], - col15=[zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}), zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}), zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}), missing], - col16=[zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}), missing], - col17=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}), missing], - col18=[zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}), missing], - col19=[zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), missing], - col20=[zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND, Int64}), zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND, Int64}), zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND, Int64}), missing], + col15=[zero(Arrow.Date{Arrow.Meta.DateUnits.DAY, Int32}), zero(Arrow.Date{Arrow.Meta.DateUnits.DAY, Int32}), zero(Arrow.Date{Arrow.Meta.DateUnits.DAY, Int32}), missing], + col16=[zero(Arrow.Time{Arrow.Meta.TimeUnits.SECOND, Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnits.SECOND, Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnits.SECOND, Int32}), missing], + col17=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.SECOND, nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.SECOND, nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.SECOND, nothing}), missing], + col18=[zero(Arrow.Interval{Arrow.Meta.IntervalUnits.YEAR_MONTH, Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnits.YEAR_MONTH, Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnits.YEAR_MONTH, Int32}), missing], + col19=[zero(Arrow.Duration{Arrow.Meta.TimeUnits.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnits.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnits.SECOND}), missing], + col20=[zero(Arrow.Date{Arrow.Meta.DateUnits.MILLISECOND, Int64}), zero(Arrow.Date{Arrow.Meta.DateUnits.MILLISECOND, Int64}), zero(Arrow.Date{Arrow.Meta.DateUnits.MILLISECOND, Int64}), missing], ), NamedTuple(), (convert=false,), @@ -254,4 +254,4 @@ function testtable(nm, t, writekw, readkw, extratests) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) return -end \ No newline at end of file +end From 2be7d606f46d9f70abd560469f11198df9c0be7e Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Fri, 17 Dec 2021 12:29:28 -0500 Subject: [PATCH 5/6] Return the module since it's now meant to be user-visible: ```julia julia> Arrow.FlatBuffers.@scopedenum MyEnum X=1 Y=2 Main.MyEnums ``` --- src/FlatBuffers/FlatBuffers.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/FlatBuffers/FlatBuffers.jl b/src/FlatBuffers/FlatBuffers.jl index 95314f40..cf1d5fd8 100644 --- a/src/FlatBuffers/FlatBuffers.jl +++ b/src/FlatBuffers/FlatBuffers.jl @@ -151,6 +151,8 @@ macro scopedenum(T, syms...) push!(blk.args, :nothing) blk.head = :toplevel push!(blk.args, :(using .$mod)) + # Return the newly created module, since it's (now) meant to be user visible. + push!(blk.args, esc(mod)) return blk end From 0a08d4eb3eb5a36dd72bdf8825dfda4ca4214e76 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Fri, 17 Dec 2021 12:30:34 -0500 Subject: [PATCH 6/6] fix missing test case --- test/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 786f72b0..51710a74 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -342,8 +342,8 @@ end # 95; Arrow.ToTimestamp x = [ZonedDateTime(Dates.DateTime(2020), tz"Europe/Paris")] c = Arrow.ToTimestamp(x) -@test eltype(c) == Arrow.Timestamp{Arrow.Flatbuf.TimeUnitModule.MILLISECOND, Symbol("Europe/Paris")} -@test c[1] == Arrow.Timestamp{Arrow.Flatbuf.TimeUnitModule.MILLISECOND, Symbol("Europe/Paris")}(1577836800000) +@test eltype(c) == Arrow.Timestamp{Arrow.Flatbuf.TimeUnits.MILLISECOND, Symbol("Europe/Paris")} +@test c[1] == Arrow.Timestamp{Arrow.Flatbuf.TimeUnits.MILLISECOND, Symbol("Europe/Paris")}(1577836800000) # 158 # arrow ipc stream generated from pyarrow with no record batches