diff --git a/docs/source/status.rst b/docs/source/status.rst index 9af2fd1921e..43803b919d6 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -85,7 +85,7 @@ Data Types +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | List | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Large List | ✓ | ✓ | ✓ | | | ✓ | ✓ | | +| Large List | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | List View | ✓ | | ✓ | | ✓ | | | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ diff --git a/js/src/Arrow.dom.ts b/js/src/Arrow.dom.ts index cdb4171162f..c221c3cefb5 100644 --- a/js/src/Arrow.dom.ts +++ b/js/src/Arrow.dom.ts @@ -54,7 +54,7 @@ export { Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Decimal, - List, + List, LargeList, Struct, StructRow, Union, DenseUnion, SparseUnion, Dictionary, @@ -89,7 +89,7 @@ export { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder, IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder, - ListBuilder, + ListBuilder, LargeListBuilder, MapBuilder, NullBuilder, StructBuilder, diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts index 6251a9e7771..6a1c25e046f 100644 --- a/js/src/Arrow.ts +++ b/js/src/Arrow.ts @@ -43,7 +43,7 @@ export { Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Decimal, - List, + List, LargeList, Struct, Union, DenseUnion, SparseUnion, Dictionary, @@ -82,6 +82,7 @@ export { LargeUtf8Builder } from './builder/largeutf8.js'; export { BinaryBuilder } from './builder/binary.js'; export { LargeBinaryBuilder } from './builder/largebinary.js'; export { ListBuilder } from './builder/list.js'; +export { LargeListBuilder } from './builder/largelist.js'; export { FixedSizeListBuilder } from './builder/fixedsizelist.js'; export { MapBuilder } from './builder/map.js'; export { StructBuilder } from './builder/struct.js'; diff --git a/js/src/builder.ts b/js/src/builder.ts index 1880db3818c..ce276db5f18 100644 --- a/js/src/builder.ts +++ b/js/src/builder.ts @@ -22,7 +22,7 @@ import { DataType, strideForType, Float, Int, Decimal, FixedSizeBinary, Date_, Time, Timestamp, Interval, Duration, - Utf8, LargeUtf8, Binary, LargeBinary, List, Map_, + Utf8, LargeUtf8, Binary, LargeBinary, List, LargeList, Map_, } from './type.js'; import { createIsValidFunction } from './builder/valid.js'; import { BufferBuilder, BitmapBufferBuilder, DataBufferBuilder, OffsetsBufferBuilder } from './builder/buffer.js'; @@ -285,7 +285,7 @@ export abstract class Builder { if (typeIds = _typeIds?.flush(length)) { // Unions, DenseUnions valueOffsets = _offsets?.flush(length); - } else if (valueOffsets = _offsets?.flush(length)) { // Variable-width primitives (Binary, LargeBinary, Utf8, LargeUtf8), and Lists + } else if (valueOffsets = _offsets?.flush(length)) { // Variable-width primitives (Binary, LargeBinary, Utf8, LargeUtf8), and Lists and LargeLists data = _values?.flush(_offsets.last()); } else { // Fixed-width primitives (Int, Float, Decimal, Time, Timestamp, Duration and Interval) data = _values?.flush(length); @@ -352,7 +352,7 @@ export abstract class FixedWidthBuilder extends Builder { +export abstract class VariableWidthBuilder extends Builder { protected _pendingLength = 0; protected _offsets: OffsetsBufferBuilder; protected _pending: Map | undefined; diff --git a/js/src/builder/buffer.ts b/js/src/builder/buffer.ts index ad1c06b0d9f..f6defdaacac 100644 --- a/js/src/builder/buffer.ts +++ b/js/src/builder/buffer.ts @@ -137,7 +137,10 @@ export class OffsetsBufferBuilder extends DataBufferBuilder< if (offset < index++ && offset >= 0) { buffer.fill(buffer[offset], offset, index); } - buffer[index] = buffer[index - 1] + value; + buffer[index] = + this.BYTES_PER_ELEMENT > 4 + ? buffer[index - 1] + BigInt(value) + : buffer[index - 1] + value; return this; } public flush(length = this.length - 1) { diff --git a/js/src/builder/largelist.ts b/js/src/builder/largelist.ts new file mode 100644 index 00000000000..c2da75422a6 --- /dev/null +++ b/js/src/builder/largelist.ts @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Field } from '../schema.js'; +import { DataType, LargeList } from '../type.js'; +import { OffsetsBufferBuilder } from './buffer.js'; +import { Builder, BuilderOptions, VariableWidthBuilder } from '../builder.js'; + +/** @ignore */ +export class LargeListBuilder extends VariableWidthBuilder, TNull> { + protected _offsets: OffsetsBufferBuilder>; + constructor(opts: BuilderOptions, TNull>) { + super(opts); + this._offsets = new OffsetsBufferBuilder(opts.type); + } + public addChild(child: Builder, name = '0') { + if (this.numChildren > 0) { + throw new Error('ListBuilder can only have one child.'); + } + this.children[this.numChildren] = child; + this.type = new LargeList(new Field(name, child.type, true)); + return this.numChildren - 1; + } + protected _flushPending(pending: Map) { + const offsets = this._offsets; + const [child] = this.children; + for (const [index, value] of pending) { + if (typeof value === 'undefined') { + offsets.set(index, BigInt(0)); + } else { + const v = value as T['TValue']; + const n = v.length; + const start = Number(offsets.set(index, n).buffer[index]); + for (let i = -1; ++i < n;) { + child.set(start + i, v[i]); + } + } + } + } +} diff --git a/js/src/data.ts b/js/src/data.ts index 6f879250885..1eed0e89892 100644 --- a/js/src/data.ts +++ b/js/src/data.ts @@ -253,7 +253,8 @@ export class Data { import { Dictionary, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, Float, Int, Date_, @@ -374,6 +375,13 @@ class MakeDataVisitor extends Visitor { const { ['length']: length = valueOffsets.length - 1, ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0 } = props; return new Data(type, offset, length, nullCount, [valueOffsets, undefined, nullBitmap], [child]); } + public visitLargeList(props: LargeListDataProps) { + const { ['type']: type, ['offset']: offset = 0, ['child']: child } = props; + const nullBitmap = toUint8Array(props['nullBitmap']); + const valueOffsets = toBigInt64Array(props['valueOffsets']); + const { ['length']: length = valueOffsets.length - 1, ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0 } = props; + return new Data(type, offset, length, nullCount, [valueOffsets, undefined, nullBitmap], [child]); + } public visitStruct(props: StructDataProps) { const { ['type']: type, ['offset']: offset = 0, ['children']: children = [] } = props; const nullBitmap = toUint8Array(props['nullBitmap']); @@ -456,6 +464,7 @@ interface LargeBinaryDataProps extends DataProps_ { va interface Utf8DataProps extends DataProps_ { valueOffsets: ValueOffsetsBuffer; data?: DataBuffer } interface LargeUtf8DataProps extends DataProps_ { valueOffsets: LargeValueOffsetsBuffer | ValueOffsetsBuffer; data?: DataBuffer } interface ListDataProps extends DataProps_ { valueOffsets: ValueOffsetsBuffer; child: Data } +interface LargeListDataProps extends DataProps_ { valueOffsets: LargeValueOffsetsBuffer | ValueOffsetsBuffer; child: Data } interface FixedSizeListDataProps extends DataProps_ { child: Data } interface StructDataProps extends DataProps_ { children: Data[] } interface Map_DataProps extends DataProps_ { valueOffsets: ValueOffsetsBuffer; child: Data } @@ -481,6 +490,7 @@ export type DataProps = ( T extends Utf8 /* */ ? Utf8DataProps : T extends LargeUtf8 /* */ ? LargeUtf8DataProps : T extends List /* */ ? ListDataProps : + T extends LargeList /* */ ? LargeListDataProps : T extends FixedSizeList /* */ ? FixedSizeListDataProps : T extends Struct /* */ ? StructDataProps : T extends Map_ /* */ ? Map_DataProps : @@ -509,6 +519,7 @@ export function makeData(props: LargeBinaryDataProps): export function makeData(props: Utf8DataProps): Data; export function makeData(props: LargeUtf8DataProps): Data; export function makeData(props: ListDataProps): Data; +export function makeData(props: LargeListDataProps): Data; export function makeData(props: FixedSizeListDataProps): Data; export function makeData(props: StructDataProps): Data; export function makeData(props: Map_DataProps): Data; diff --git a/js/src/enum.ts b/js/src/enum.ts index e4284e42774..23a10a77af8 100644 --- a/js/src/enum.ts +++ b/js/src/enum.ts @@ -70,6 +70,7 @@ export enum Type { Duration = 18, /** Measure of elapsed time in either seconds, milliseconds, microseconds or nanoseconds */ LargeBinary = 19, /** Large variable-length bytes (no guarantee of UTF8-ness) */ LargeUtf8 = 20, /** Large variable-length string as List */ + LargeList = 21, /** A Large list of some logical data type */ Dictionary = -1, /** Dictionary aka Category type */ Int8 = -2, diff --git a/js/src/interfaces.ts b/js/src/interfaces.ts index c4119a8bd28..d03937bfdda 100644 --- a/js/src/interfaces.ts +++ b/js/src/interfaces.ts @@ -36,6 +36,7 @@ import type { Utf8Builder } from './builder/utf8.js'; import type { LargeUtf8Builder } from './builder/largeutf8.js'; import type { BinaryBuilder } from './builder/binary.js'; import type { LargeBinaryBuilder } from './builder/largebinary.js'; +import type { LargeListBuilder } from './builder/largelist.js'; import type { ListBuilder } from './builder/list.js'; import type { FixedSizeListBuilder } from './builder/fixedsizelist.js'; import type { MapBuilder } from './builder/map.js'; @@ -240,6 +241,7 @@ export type TypeToDataType = { [Type.DurationNanosecond]: type.DurationNanosecond; [Type.Map]: type.Map_; [Type.List]: type.List; + [Type.LargeList]: type.LargeList; [Type.Struct]: type.Struct; [Type.Dictionary]: type.Dictionary; [Type.FixedSizeList]: type.FixedSizeList; @@ -295,6 +297,7 @@ type TypeToBuilder = { [Type.DurationNanosecond]: DurationNanosecondBuilder; [Type.Map]: MapBuilder; [Type.List]: ListBuilder; + [Type.LargeList]: LargeListBuilder; [Type.Struct]: StructBuilder; [Type.Dictionary]: DictionaryBuilder; [Type.FixedSizeList]: FixedSizeListBuilder; @@ -350,6 +353,7 @@ type DataTypeToBuilder = { [Type.DurationNanosecond]: T extends type.DurationNanosecond ? DurationNanosecondBuilder : never; [Type.Map]: T extends type.Map_ ? MapBuilder : never; [Type.List]: T extends type.List ? ListBuilder : never; + [Type.LargeList]: T extends type.LargeList ? LargeListBuilder : never; [Type.Struct]: T extends type.Struct ? StructBuilder : never; [Type.Dictionary]: T extends type.Dictionary ? DictionaryBuilder : never; [Type.FixedSizeList]: T extends type.FixedSizeList ? FixedSizeListBuilder : never; diff --git a/js/src/ipc/metadata/json.ts b/js/src/ipc/metadata/json.ts index 8dc81ced3ff..a980f9b56ba 100644 --- a/js/src/ipc/metadata/json.ts +++ b/js/src/ipc/metadata/json.ts @@ -21,7 +21,7 @@ import { Schema, Field } from '../../schema.js'; import { DataType, Dictionary, TimeBitWidth, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, - List, FixedSizeList, Map_, Struct, Union, + List, LargeList, FixedSizeList, Map_, Struct, Union, Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, Duration, } from '../../type.js'; @@ -154,6 +154,7 @@ function typeFromJSON(f: any, children?: Field[]): DataType { case 'largeutf8': return new LargeUtf8(); case 'bool': return new Bool(); case 'list': return new List((children || [])[0]); + case 'largelist': return new LargeList((children || [])[0]); case 'struct': return new Struct(children || []); case 'struct_': return new Struct(children || []); } diff --git a/js/src/ipc/metadata/message.ts b/js/src/ipc/metadata/message.ts index 552c4d846e8..b409592c3c5 100644 --- a/js/src/ipc/metadata/message.ts +++ b/js/src/ipc/metadata/message.ts @@ -57,7 +57,7 @@ import ByteBuffer = flatbuffers.ByteBuffer; import { DataType, Dictionary, TimeBitWidth, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, - List, FixedSizeList, Map_, Struct, Union, + List, LargeList, FixedSizeList, Map_, Struct, Union, Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, Duration, } from '../../type.js'; @@ -437,6 +437,7 @@ function decodeFieldType(f: _Field, children?: Field[]): DataType { case Type['LargeUtf8']: return new LargeUtf8(); case Type['Bool']: return new Bool(); case Type['List']: return new List((children || [])[0]); + case Type['LargeList']: return new LargeList((children || [])[0]); case Type['Struct_']: return new Struct(children || []); } diff --git a/js/src/type.ts b/js/src/type.ts index ae3aefa0259..7d527e81b41 100644 --- a/js/src/type.ts +++ b/js/src/type.ts @@ -69,6 +69,7 @@ export abstract class DataType { cons /** @ignore */ export interface List extends DataType { TArray: Array; + TOffsetArray: Int32Array; TValue: Vector; + OffsetArrayType: TypedArrayConstructor; } /** @ignore */ @@ -520,6 +523,31 @@ export class List extends DataType extends DataType { + TArray: Array; + TOffsetArray: BigInt64Array; + TValue: Vector; + OffsetArrayType: BigIntArrayConstructor; +} + +/** @ignore */ +export class LargeList extends DataType { + constructor(child: Field) { + super(Type.LargeList); + this.children = [child]; + } + public declare readonly children: Field[]; + public toString() { return `LargeList<${this.valueType}>`; } + public get valueType(): T { return this.children[0].type as T; } + public get valueField(): Field { return this.children[0] as Field; } + public get ArrayType(): T['ArrayType'] { return this.valueType.ArrayType; } + protected static [Symbol.toStringTag] = ((proto: LargeList) => { + (proto).children = null; + return proto[Symbol.toStringTag] = 'LargeList'; + })(LargeList.prototype); +} + /** @ignore */ export interface Struct extends DataType { TArray: Array>; diff --git a/js/src/visitor.ts b/js/src/visitor.ts index 2fb5e7e14bc..b5fbe753c28 100644 --- a/js/src/visitor.ts +++ b/js/src/visitor.ts @@ -45,6 +45,7 @@ export abstract class Visitor { public visitTime(_node: any, ..._args: any[]): any { return null; } public visitDecimal(_node: any, ..._args: any[]): any { return null; } public visitList(_node: any, ..._args: any[]): any { return null; } + public visitLargeList(_node: any, ..._args: any[]): any { return null; } public visitStruct(_node: any, ..._args: any[]): any { return null; } public visitUnion(_node: any, ..._args: any[]): any { return null; } public visitDictionary(_node: any, ..._args: any[]): any { return null; } @@ -110,6 +111,7 @@ function getVisitFnByTypeId(visitor: Visitor, dtype: Type, throwIfNotFound = tru case Type.TimeNanosecond: fn = visitor.visitTimeNanosecond || visitor.visitTime; break; case Type.Decimal: fn = visitor.visitDecimal; break; case Type.List: fn = visitor.visitList; break; + case Type.LargeList: fn = visitor.visitLargeList; break; case Type.Struct: fn = visitor.visitStruct; break; case Type.Union: fn = visitor.visitUnion; break; case Type.DenseUnion: fn = visitor.visitDenseUnion || visitor.visitUnion; break; @@ -203,6 +205,7 @@ function inferDType(type: T): Type { return Type.Duration; case Type.Map: return Type.Map; case Type.List: return Type.List; + case Type.LargeList: return Type.LargeList; case Type.Struct: return Type.Struct; case Type.Union: switch ((type as any as Union).mode) { @@ -254,6 +257,7 @@ export interface Visitor { visitTimeNanosecond?(node: any, ...args: any[]): any; visitDecimal(node: any, ...args: any[]): any; visitList(node: any, ...args: any[]): any; + visitLargeList(node: any, ...args: any[]): any; visitStruct(node: any, ...args: any[]): any; visitUnion(node: any, ...args: any[]): any; visitDenseUnion?(node: any, ...args: any[]): any; diff --git a/js/src/visitor/builderctor.ts b/js/src/visitor/builderctor.ts index 5b3758c4e0c..b32b4deffd2 100644 --- a/js/src/visitor/builderctor.ts +++ b/js/src/visitor/builderctor.ts @@ -34,6 +34,7 @@ import { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder } fro import { DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder } from '../builder/duration.js'; import { IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder } from '../builder/int.js'; import { ListBuilder } from '../builder/list.js'; +import { LargeListBuilder } from '../builder/largelist.js'; import { MapBuilder } from '../builder/map.js'; import { NullBuilder } from '../builder/null.js'; import { StructBuilder } from '../builder/struct.js'; @@ -88,6 +89,7 @@ export class GetBuilderCtor extends Visitor { public visitTimeNanosecond() { return TimeNanosecondBuilder; } public visitDecimal() { return DecimalBuilder; } public visitList() { return ListBuilder; } + public visitLargeList() { return LargeListBuilder; } public visitStruct() { return StructBuilder; } public visitUnion() { return UnionBuilder; } public visitDenseUnion() { return DenseUnionBuilder; } diff --git a/js/src/visitor/get.ts b/js/src/visitor/get.ts index 3ab3bcb68c3..69c81cb1591 100644 --- a/js/src/visitor/get.ts +++ b/js/src/visitor/get.ts @@ -28,7 +28,8 @@ import { uint16ToFloat64 } from '../util/math.js'; import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum.js'; import { DataType, Dictionary, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, Float, Float16, Float32, Float64, Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Date_, DateDay, DateMillisecond, @@ -80,6 +81,7 @@ export interface GetVisitor extends Visitor { visitTimeNanosecond(data: Data, index: number): T['TValue'] | null; visitDecimal(data: Data, index: number): T['TValue'] | null; visitList(data: Data, index: number): T['TValue'] | null; + visitLargeList(data: Data, index: number): T['TValue'] | null; visitStruct(data: Data, index: number): T['TValue'] | null; visitUnion(data: Data, index: number): T['TValue'] | null; visitDenseUnion(data: Data, index: number): T['TValue'] | null; @@ -227,6 +229,15 @@ const getList = (data: Data, index: number): T['TValue'] => { return new Vector([slice]) as T['TValue']; }; +/** @ignore */ +const getLargeList = (data: Data, index: number): T['TValue'] => { + const { valueOffsets, stride, children } = data; + const { [index * stride]: begin, [index * stride + 1]: end } = valueOffsets; + const child: Data = children[0]; + const slice = child.slice(Number(begin), Number(end - begin)); + return new Vector([slice]) as T['TValue']; +}; + /** @ignore */ const getMap = (data: Data, index: number): T['TValue'] => { const { valueOffsets, children } = data; @@ -350,6 +361,7 @@ GetVisitor.prototype.visitTimeMicrosecond = wrapGet(getTimeMicrosecond); GetVisitor.prototype.visitTimeNanosecond = wrapGet(getTimeNanosecond); GetVisitor.prototype.visitDecimal = wrapGet(getDecimal); GetVisitor.prototype.visitList = wrapGet(getList); +GetVisitor.prototype.visitLargeList = wrapGet(getLargeList); GetVisitor.prototype.visitStruct = wrapGet(getStruct); GetVisitor.prototype.visitUnion = wrapGet(getUnion); GetVisitor.prototype.visitDenseUnion = wrapGet(getDenseUnion); diff --git a/js/src/visitor/indexof.ts b/js/src/visitor/indexof.ts index 1e1cb87a984..ce3e16f2978 100644 --- a/js/src/visitor/indexof.ts +++ b/js/src/visitor/indexof.ts @@ -24,7 +24,8 @@ import { getBool, BitIterator } from '../util/bit.js'; import { createElementComparator } from '../util/vector.js'; import { DataType, Dictionary, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, Float, Float16, Float32, Float64, Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Date_, DateDay, DateMillisecond, @@ -76,6 +77,7 @@ export interface IndexOfVisitor extends Visitor { visitTimeNanosecond(data: Data, value: T['TValue'] | null, index?: number): number; visitDecimal(data: Data, value: T['TValue'] | null, index?: number): number; visitList(data: Data, value: T['TValue'] | null, index?: number): number; + visitLargeList(data: Data, value: T['TValue'] | null, index?: number): number; visitStruct(data: Data, value: T['TValue'] | null, index?: number): number; visitUnion(data: Data, value: T['TValue'] | null, index?: number): number; visitDenseUnion(data: Data, value: T['TValue'] | null, index?: number): number; @@ -193,6 +195,7 @@ IndexOfVisitor.prototype.visitTimeMicrosecond = indexOfValue; IndexOfVisitor.prototype.visitTimeNanosecond = indexOfValue; IndexOfVisitor.prototype.visitDecimal = indexOfValue; IndexOfVisitor.prototype.visitList = indexOfValue; +IndexOfVisitor.prototype.visitLargeList = indexOfValue; IndexOfVisitor.prototype.visitStruct = indexOfValue; IndexOfVisitor.prototype.visitUnion = indexOfValue; IndexOfVisitor.prototype.visitDenseUnion = indexOfUnion; diff --git a/js/src/visitor/iterator.ts b/js/src/visitor/iterator.ts index bf7e9d1591b..dbf720d9a4b 100644 --- a/js/src/visitor/iterator.ts +++ b/js/src/visitor/iterator.ts @@ -21,7 +21,8 @@ import { Type, Precision } from '../enum.js'; import { TypeToDataType } from '../interfaces.js'; import { DataType, Dictionary, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, Float, Float16, Float32, Float64, Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Date_, DateDay, DateMillisecond, @@ -74,6 +75,7 @@ export interface IteratorVisitor extends Visitor { visitTimeNanosecond(vector: Vector): IterableIterator; visitDecimal(vector: Vector): IterableIterator; visitList(vector: Vector): IterableIterator; + visitLargeList(vector: Vector): IterableIterator; visitStruct(vector: Vector): IterableIterator; visitUnion(vector: Vector): IterableIterator; visitDenseUnion(vector: Vector): IterableIterator; @@ -179,6 +181,7 @@ IteratorVisitor.prototype.visitTimeMicrosecond = vectorIterator; IteratorVisitor.prototype.visitTimeNanosecond = vectorIterator; IteratorVisitor.prototype.visitDecimal = vectorIterator; IteratorVisitor.prototype.visitList = vectorIterator; +IteratorVisitor.prototype.visitLargeList = vectorIterator; IteratorVisitor.prototype.visitStruct = vectorIterator; IteratorVisitor.prototype.visitUnion = vectorIterator; IteratorVisitor.prototype.visitDenseUnion = vectorIterator; diff --git a/js/src/visitor/jsontypeassembler.ts b/js/src/visitor/jsontypeassembler.ts index 823b1dea104..9647c42c21a 100644 --- a/js/src/visitor/jsontypeassembler.ts +++ b/js/src/visitor/jsontypeassembler.ts @@ -75,6 +75,9 @@ export class JSONTypeAssembler extends Visitor { public visitList({ typeId }: T) { return { 'name': ArrowType[typeId].toLowerCase() }; } + public visitLargeList({ typeId }: T) { + return { 'name': ArrowType[typeId].toLowerCase() }; + } public visitStruct({ typeId }: T) { return { 'name': ArrowType[typeId].toLowerCase() }; } diff --git a/js/src/visitor/jsonvectorassembler.ts b/js/src/visitor/jsonvectorassembler.ts index 88699d8f168..3081e3a17f4 100644 --- a/js/src/visitor/jsonvectorassembler.ts +++ b/js/src/visitor/jsonvectorassembler.ts @@ -27,7 +27,8 @@ import { BitIterator, getBit, getBool } from '../util/bit.js'; import { DataType, Float, Int, Date_, Interval, Time, Timestamp, Union, Duration, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, IntArray, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, IntArray, } from '../type.js'; /** @ignore */ @@ -51,6 +52,7 @@ export interface JSONVectorAssembler extends Visitor { visitTime(data: Data): { DATA: number[] }; visitDecimal(data: Data): { DATA: string[] }; visitList(data: Data): { children: any[]; OFFSET: number[] }; + visitLargeList(data: Data): { children: any[]; OFFSET: bigint[] }; visitStruct(data: Data): { children: any[] }; visitUnion(data: Data): { children: any[]; TYPE_ID: number[] }; visitInterval(data: Data): { DATA: number[] }; @@ -140,6 +142,12 @@ export class JSONVectorAssembler extends Visitor { 'children': this.visitMany(data.type.children, data.children) }; } + public visitLargeList(data: Data) { + return { + 'OFFSET': [...data.valueOffsets], + 'children': this.visitMany(data.type.children, data.children) + }; + } public visitStruct(data: Data) { return { 'children': this.visitMany(data.type.children, data.children) diff --git a/js/src/visitor/set.ts b/js/src/visitor/set.ts index 5dc42283c36..4548c096698 100644 --- a/js/src/visitor/set.ts +++ b/js/src/visitor/set.ts @@ -26,7 +26,8 @@ import { float64ToUint16 } from '../util/math.js'; import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum.js'; import { DataType, Dictionary, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, Float, Float16, Float32, Float64, Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Date_, DateDay, DateMillisecond, @@ -78,6 +79,7 @@ export interface SetVisitor extends Visitor { visitTimeNanosecond(data: Data, index: number, value: T['TValue']): void; visitDecimal(data: Data, index: number, value: T['TValue']): void; visitList(data: Data, index: number, value: T['TValue']): void; + visitLargeList(data: Data, index: number, value: T['TValue']): void; visitStruct(data: Data, index: number, value: T['TValue']): void; visitUnion(data: Data, index: number, value: T['TValue']): void; visitDenseUnion(data: Data, index: number, value: T['TValue']): void; @@ -234,6 +236,22 @@ const setList = (data: Data, index: number, value: T['TValue' } }; +/** @ignore */ +const setLargeList = (data: Data, index: number, value: T['TValue']): void => { + const values = data.children[0]; + const valueOffsets = data.valueOffsets; + const set = instance.getVisitFn(values); + if (Array.isArray(value)) { + for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) { + set(values, Number(itr++), value[++idx]); + } + } else { + for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) { + set(values, Number(itr++), value.get(++idx)); + } + } +}; + /** @ignore */ const setMap = (data: Data, index: number, value: T['TValue']) => { const values = data.children[0]; @@ -386,6 +404,7 @@ SetVisitor.prototype.visitTimeMicrosecond = wrapSet(setTimeMicrosecond); SetVisitor.prototype.visitTimeNanosecond = wrapSet(setTimeNanosecond); SetVisitor.prototype.visitDecimal = wrapSet(setDecimal); SetVisitor.prototype.visitList = wrapSet(setList); +SetVisitor.prototype.visitLargeList = wrapSet(setLargeList); SetVisitor.prototype.visitStruct = wrapSet(setStruct); SetVisitor.prototype.visitUnion = wrapSet(setUnion); SetVisitor.prototype.visitDenseUnion = wrapSet(setDenseUnion); diff --git a/js/src/visitor/typeassembler.ts b/js/src/visitor/typeassembler.ts index 169f3627a40..ac2c0a927a8 100644 --- a/js/src/visitor/typeassembler.ts +++ b/js/src/visitor/typeassembler.ts @@ -36,6 +36,7 @@ import { Timestamp } from '../fb/timestamp.js'; import { Interval } from '../fb/interval.js'; import { Duration } from '../fb/duration.js'; import { List } from '../fb/list.js'; +import { LargeList } from '../fb/large-list.js'; import { Struct_ as Struct } from '../fb/struct-.js'; import { Union } from '../fb/union.js'; import { DictionaryEncoding } from '../fb/dictionary-encoding.js'; @@ -129,6 +130,10 @@ export class TypeAssembler extends Visitor { List.startList(b); return List.endList(b); } + public visitLargeList(_node: T, b: Builder) { + LargeList.startLargeList(b); + return LargeList.endLargeList(b); + } public visitStruct(_node: T, b: Builder) { Struct.startStruct_(b); return Struct.endStruct_(b); diff --git a/js/src/visitor/typecomparator.ts b/js/src/visitor/typecomparator.ts index a113f2ea31e..5999d761d19 100644 --- a/js/src/visitor/typecomparator.ts +++ b/js/src/visitor/typecomparator.ts @@ -21,7 +21,8 @@ import { Visitor } from '../visitor.js'; import { Schema, Field } from '../schema.js'; import { DataType, TypeMap, Dictionary, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, Float, Float16, Float32, Float64, Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Date_, DateDay, DateMillisecond, @@ -72,6 +73,7 @@ export interface TypeComparator extends Visitor { visitTimeNanosecond(type: T, other?: DataType | null): other is T; visitDecimal(type: T, other?: DataType | null): other is T; visitList(type: T, other?: DataType | null): other is T; + visitLargeList(type: T, other?: DataType | null): other is T; visitStruct(type: T, other?: DataType | null): other is T; visitUnion(type: T, other?: DataType | null): other is T; visitDenseUnion(type: T, other?: DataType | null): other is T; @@ -176,6 +178,14 @@ function compareList(type: T, other?: DataType | null): other is ); } +function compareLargeList(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.children.length === other.children.length && + instance.compareManyFields(type.children, other.children) + ); +} + function compareStruct(type: T, other?: DataType | null): other is T { return (type === other) || ( compareConstructor(type, other) && @@ -270,6 +280,7 @@ TypeComparator.prototype.visitTimeMicrosecond = compareTime; TypeComparator.prototype.visitTimeNanosecond = compareTime; TypeComparator.prototype.visitDecimal = compareAny; TypeComparator.prototype.visitList = compareList; +TypeComparator.prototype.visitLargeList = compareLargeList; TypeComparator.prototype.visitStruct = compareStruct; TypeComparator.prototype.visitUnion = compareUnion; TypeComparator.prototype.visitDenseUnion = compareUnion; diff --git a/js/src/visitor/typector.ts b/js/src/visitor/typector.ts index a781b5fb14f..5cbc29aed61 100644 --- a/js/src/visitor/typector.ts +++ b/js/src/visitor/typector.ts @@ -68,6 +68,7 @@ export class GetDataTypeConstructor extends Visitor { public visitTimeNanosecond() { return type.TimeNanosecond; } public visitDecimal() { return type.Decimal; } public visitList() { return type.List; } + public visitLargeList() { return type.LargeList; } public visitStruct() { return type.Struct; } public visitUnion() { return type.Union; } public visitDenseUnion() { return type.DenseUnion; } diff --git a/js/src/visitor/vectorassembler.ts b/js/src/visitor/vectorassembler.ts index 7dc3695582d..af785cc4348 100644 --- a/js/src/visitor/vectorassembler.ts +++ b/js/src/visitor/vectorassembler.ts @@ -27,7 +27,8 @@ import { BufferRegion, FieldNode } from '../ipc/metadata/message.js'; import { DataType, Dictionary, Float, Int, Date_, Interval, Time, Timestamp, Union, Duration, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, } from '../type.js'; import { bigIntToNumber } from '../util/bigint.js'; @@ -51,6 +52,7 @@ export interface VectorAssembler extends Visitor { visitTime(data: Data): this; visitDecimal(data: Data): this; visitList(data: Data): this; + visitLargeList(data: Data): this; visitStruct(data: Data): this; visitUnion(data: Data): this; visitInterval(data: Data): this; @@ -216,11 +218,12 @@ function assembleFlatListVector(this: VectorAssembler, data: Data) { +function assembleListVector(this: VectorAssembler, data: Data) { const { length, valueOffsets } = data; // If we have valueOffsets (MapVector, ListVector), push that buffer first if (valueOffsets) { - const { [0]: begin, [length]: end } = valueOffsets; + const begin = bigIntToNumber(valueOffsets[0]); + const end = bigIntToNumber(valueOffsets[length]); addBuffer.call(this, rebaseValueOffsets(-begin, length + 1, valueOffsets)); // Then insert the List's values child return this.visit(data.children[0].slice(begin, end - begin)); @@ -247,6 +250,7 @@ VectorAssembler.prototype.visitTimestamp = assembleFlatVector; VectorAssembler.prototype.visitTime = assembleFlatVector; VectorAssembler.prototype.visitDecimal = assembleFlatVector; VectorAssembler.prototype.visitList = assembleListVector; +VectorAssembler.prototype.visitLargeList = assembleListVector; VectorAssembler.prototype.visitStruct = assembleNestedVector; VectorAssembler.prototype.visitUnion = assembleUnion; VectorAssembler.prototype.visitInterval = assembleFlatVector; diff --git a/js/src/visitor/vectorloader.ts b/js/src/visitor/vectorloader.ts index c9c016d6b46..8928bcfe1bd 100644 --- a/js/src/visitor/vectorloader.ts +++ b/js/src/visitor/vectorloader.ts @@ -98,6 +98,9 @@ export class VectorLoader extends Visitor { public visitList(type: T, { length, nullCount } = this.nextFieldNode()) { return makeData({ type, length, nullCount, nullBitmap: this.readNullBitmap(type, nullCount), valueOffsets: this.readOffsets(type), 'child': this.visit(type.children[0]) }); } + public visitLargeList(type: T, { length, nullCount } = this.nextFieldNode()) { + return makeData({ type, length, nullCount, nullBitmap: this.readNullBitmap(type, nullCount), valueOffsets: this.readOffsets(type), 'child': this.visit(type.children[0]) }); + } public visitStruct(type: T, { length, nullCount } = this.nextFieldNode()) { return makeData({ type, length, nullCount, nullBitmap: this.readNullBitmap(type, nullCount), children: this.visitMany(type.children) }); } diff --git a/js/test/data/tables.ts b/js/test/data/tables.ts index 89cf93eab58..904b91a04ce 100644 --- a/js/test/data/tables.ts +++ b/js/test/data/tables.ts @@ -22,7 +22,7 @@ import * as generate from '../generate-test-data.js'; import { Schema, Field, Dictionary } from 'apache-arrow'; -const listVectorGeneratorNames = ['list', 'fixedSizeList']; +const listVectorGeneratorNames = ['list', 'largeList', 'fixedSizeList']; const nestedVectorGeneratorNames = ['struct', 'denseUnion', 'sparseUnion', 'map']; const dictionaryKeyGeneratorNames = ['int8', 'int16', 'int32', 'uint8', 'uint16', 'uint32']; const valueVectorGeneratorNames = [ diff --git a/js/test/generate-test-data.ts b/js/test/generate-test-data.ts index 8e6e47de836..73ceb208031 100644 --- a/js/test/generate-test-data.ts +++ b/js/test/generate-test-data.ts @@ -29,7 +29,7 @@ import { Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Decimal, - List, + List, LargeList, Struct, Union, DenseUnion, SparseUnion, Dictionary, @@ -63,6 +63,7 @@ interface TestDataVectorGenerator extends Visitor { visit(type: T, length?: number, nullCount?: number): GeneratedVector; visit(type: T, length?: number, nullCount?: number): GeneratedVector; visit(type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector; + visit(type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector; visit(type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector; visit(type: T, length?: number, nullCount?: number, dictionary?: Vector): GeneratedVector; visit(type: T, length?: number, nullCount?: number, children?: Vector[]): GeneratedVector; @@ -86,6 +87,7 @@ interface TestDataVectorGenerator extends Visitor { visitTime: typeof generateTime; visitDecimal: typeof generateDecimal; visitList: typeof generateList; + visitLargeList: typeof generateLargeList; visitStruct: typeof generateStruct; visitUnion: typeof generateUnion; visitDictionary: typeof generateDictionary; @@ -113,6 +115,7 @@ TestDataVectorGenerator.prototype.visitTimestamp = generateTimestamp; TestDataVectorGenerator.prototype.visitTime = generateTime; TestDataVectorGenerator.prototype.visitDecimal = generateDecimal; TestDataVectorGenerator.prototype.visitList = generateList; +TestDataVectorGenerator.prototype.visitLargeList = generateLargeList; TestDataVectorGenerator.prototype.visitStruct = generateStruct; TestDataVectorGenerator.prototype.visitUnion = generateUnion; TestDataVectorGenerator.prototype.visitDictionary = generateDictionary; @@ -236,6 +239,7 @@ export const timeMicrosecond = (length = 100, nullCount = Math.trunc(length * 0. export const timeNanosecond = (length = 100, nullCount = Math.trunc(length * 0.2)) => vectorGenerator.visit(new TimeNanosecond(), length, nullCount); export const decimal = (length = 100, nullCount = Math.trunc(length * 0.2), scale = 2, precision = 9, bitWidth = 128) => vectorGenerator.visit(new Decimal(scale, precision, bitWidth), length, nullCount); export const list = (length = 100, nullCount = Math.trunc(length * 0.2), child = defaultListChild) => vectorGenerator.visit(new List(child), length, nullCount); +export const largeList = (length = 100, nullCount = Math.trunc(length * 0.2), child = defaultListChild) => vectorGenerator.visit(new LargeList(child), length, nullCount); export const struct = (length = 100, nullCount = Math.trunc(length * 0.2), children: Field[] = defaultStructChildren()) => vectorGenerator.visit(new Struct(children), length, nullCount); export const denseUnion = (length = 100, nullCount = Math.trunc(length * 0.2), children: Field[] = defaultUnionChildren()) => vectorGenerator.visit(new DenseUnion(children.map((f) => f.typeId), children), length, nullCount); export const sparseUnion = (length = 100, nullCount = Math.trunc(length * 0.2), children: Field[] = defaultUnionChildren()) => vectorGenerator.visit(new SparseUnion(children.map((f) => f.typeId), children), length, nullCount); @@ -250,7 +254,7 @@ export const fixedSizeList = (length = 100, nullCount = Math.trunc(length * 0.2) export const map = (length = 100, nullCount = Math.trunc(length * 0.2), child: Field> = defaultMapChild()) => vectorGenerator.visit(new Map_(child), length, nullCount); export const vecs = { - null_, bool, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, dateDay, dateMillisecond, timestampSecond, timestampMillisecond, timestampMicrosecond, timestampNanosecond, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, decimal, list, struct, denseUnion, sparseUnion, dictionary, intervalDayTime, intervalYearMonth, fixedSizeList, map, durationSecond, durationMillisecond, durationMicrosecond, durationNanosecond + null_, bool, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, dateDay, dateMillisecond, timestampSecond, timestampMillisecond, timestampMicrosecond, timestampNanosecond, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, decimal, list, largeList, struct, denseUnion, sparseUnion, dictionary, intervalDayTime, intervalYearMonth, fixedSizeList, map, durationSecond, durationMillisecond, durationMicrosecond, durationNanosecond } as { [k: string]: (...args: any[]) => any }; function generateNull(this: TestDataVectorGenerator, type: T, length = 100): GeneratedVector { @@ -494,6 +498,21 @@ function generateList(this: TestDataVectorGenerator, type: T, le return { values, vector: new Vector([makeData({ type, length, nullCount, nullBitmap, valueOffsets, child: childVec.data[0] })]) }; } +function generateLargeList(this: TestDataVectorGenerator, type: T, length = 100, nullCount = Math.trunc(length * 0.2), child = this.visit(type.children[0].type, length * 3, nullCount * 3)): GeneratedVector { + const childVec = child.vector; + const nullBitmap = createBitmap(length, nullCount); + const stride = childVec.length / (length - nullCount); + const valueOffsets = createVariableWidthOffsets32(length, nullBitmap, stride, stride); + const values = memoize(() => { + const childValues = child.values(); + const values: (T['valueType'] | null)[] = [...valueOffsets.slice(1)] + .map((offset, i) => isValid(nullBitmap, i) ? offset : null) + .map((o, i) => o == null ? null : childValues.slice(valueOffsets[i], o)); + return values; + }); + return { values, vector: new Vector([makeData({ type, length, nullCount, nullBitmap, valueOffsets, child: childVec.data[0] })]) }; +} + function generateFixedSizeList(this: TestDataVectorGenerator, type: T, length = 100, nullCount = Math.trunc(length * 0.2), child = this.visit(type.children[0].type, length * type.listSize, nullCount * type.listSize)): GeneratedVector { const nullBitmap = createBitmap(length, nullCount); const values = memoize(() => { diff --git a/js/test/unit/builders/builder-tests.ts b/js/test/unit/builders/builder-tests.ts index 4d1be9b225b..ad9df651649 100644 --- a/js/test/unit/builders/builder-tests.ts +++ b/js/test/unit/builders/builder-tests.ts @@ -60,6 +60,7 @@ describe('Generated Test Data', () => { describe('TimeNanosecondBuilder', () => { validateBuilder(generate.timeNanosecond); }); describe('DecimalBuilder', () => { validateBuilder(generate.decimal); }); describe('ListBuilder', () => { validateBuilder(generate.list); }); + describe('LargeListBuilder', () => { validateBuilder(generate.largeList); }); describe('StructBuilder', () => { validateBuilder(generate.struct); }); describe('DenseUnionBuilder', () => { validateBuilder(generate.denseUnion); }); describe('SparseUnionBuilder', () => { validateBuilder(generate.sparseUnion); }); diff --git a/js/test/unit/generated-data-tests.ts b/js/test/unit/generated-data-tests.ts index 1e26e74730a..f9e09383b86 100644 --- a/js/test/unit/generated-data-tests.ts +++ b/js/test/unit/generated-data-tests.ts @@ -54,6 +54,7 @@ describe('Generated Test Data', () => { describe('TimeNanosecond', () => { validateVector(generate.timeNanosecond()); }); describe('Decimal', () => { validateVector(generate.decimal()); }); describe('List', () => { validateVector(generate.list()); }); + describe('LargeList', () => { validateVector(generate.largeList()); }); describe('Struct', () => { validateVector(generate.struct()); }); describe('DenseUnion', () => { validateVector(generate.denseUnion()); }); describe('SparseUnion', () => { validateVector(generate.sparseUnion()); }); diff --git a/js/test/unit/visitor-tests.ts b/js/test/unit/visitor-tests.ts index 6ecb6cca33e..1ff29272f8c 100644 --- a/js/test/unit/visitor-tests.ts +++ b/js/test/unit/visitor-tests.ts @@ -18,7 +18,8 @@ import { Field, Visitor, DataType, Dictionary, - Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, + List, LargeList, FixedSizeList, Map_, Struct, Float, Float16, Float32, Float64, Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Date_, DateDay, DateMillisecond, @@ -45,6 +46,7 @@ class BasicVisitor extends Visitor { public visitTime(type: T) { return (this.type = type); } public visitDecimal(type: T) { return (this.type = type); } public visitList(type: T) { return (this.type = type); } + public visitLargeList(type: T) { return (this.type = type); } public visitStruct(type: T) { return (this.type = type); } public visitUnion(type: T) { return (this.type = type); } public visitDictionary(type: T) { return (this.type = type); } @@ -86,6 +88,7 @@ class FeatureVisitor extends Visitor { public visitTimeNanosecond(type: T) { return (this.type = type); } public visitDecimal(type: T) { return (this.type = type); } public visitList(type: T) { return (this.type = type); } + public visitLargeList(type: T) { return (this.type = type); } public visitStruct(type: T) { return (this.type = type); } public visitDenseUnion(type: T) { return (this.type = type); } public visitSparseUnion(type: T) { return (this.type = type); } @@ -117,6 +120,7 @@ describe('Visitor', () => { test(`visits Time types`, () => validateBasicVisitor(new Time(0, 64))); test(`visits Decimal types`, () => validateBasicVisitor(new Decimal(2, 9, 128))); test(`visits List types`, () => validateBasicVisitor(new List(null as any))); + test(`visits Large List types`, () => validateBasicVisitor(new LargeList(null as any))); test(`visits Struct types`, () => validateBasicVisitor(new Struct([] as any[]))); test(`visits Union types`, () => validateBasicVisitor(new Union(0, [] as any[], [] as any[]))); test(`visits Dictionary types`, () => validateBasicVisitor(new Dictionary(null as any, null as any))); @@ -166,6 +170,7 @@ describe('Visitor', () => { test(`visits TimeNanosecond types`, () => validateFeatureVisitor(new TimeNanosecond())); test(`visits Decimal types`, () => validateFeatureVisitor(new Decimal(2, 9, 128))); test(`visits List types`, () => validateFeatureVisitor(new List(null as any))); + test(`visits Large List types`, () => validateFeatureVisitor(new LargeList(null as any))); test(`visits Struct types`, () => validateFeatureVisitor(new Struct([] as any[]))); test(`visits DenseUnion types`, () => validateFeatureVisitor(new DenseUnion([] as any[], [] as any[]))); test(`visits SparseUnion types`, () => validateFeatureVisitor(new SparseUnion([] as any[], [] as any[])));