GH-39454: [JS] Support LargeList

docs/source/status.rst

@@ -85,7 +85,7 @@ Data Types
 +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
 | List              | ✓     | ✓     | ✓     | ✓          |  ✓    |  ✓    | ✓     |       |
 +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
-| Large List        | ✓     | ✓     | ✓     |            |       |  ✓    | ✓     |       |
+| Large List        | ✓     | ✓     | ✓     | ✓          |       |  ✓    | ✓     |       |
 +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
 | List View         | ✓     |       | ✓     |            |   ✓   |       |       |       |
 +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+

js/src/Arrow.dom.ts

@@ -54,7 +54,7 @@ export {
     Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond,
     Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond,
     Decimal,
-    List,
+    List, LargeList,
     Struct, StructRow,
     Union, DenseUnion, SparseUnion,
     Dictionary,
@@ -89,7 +89,7 @@ export {
     IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder,
     DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder,
     IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder,
-    ListBuilder,
+    ListBuilder, LargeListBuilder,
     MapBuilder,
     NullBuilder,
     StructBuilder,

js/src/Arrow.ts

@@ -43,7 +43,7 @@ export {
     Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond,
     Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond,
     Decimal,
-    List,
+    List, LargeList,
     Struct,
     Union, DenseUnion, SparseUnion,
     Dictionary,
@@ -82,6 +82,7 @@ export { LargeUtf8Builder } from './builder/largeutf8.js';
 export { BinaryBuilder } from './builder/binary.js';
 export { LargeBinaryBuilder } from './builder/largebinary.js';
 export { ListBuilder } from './builder/list.js';
+export { LargeListBuilder } from './builder/largelist.js';
 export { FixedSizeListBuilder } from './builder/fixedsizelist.js';
 export { MapBuilder } from './builder/map.js';
 export { StructBuilder } from './builder/struct.js';

js/src/builder.ts

@@ -22,7 +22,7 @@ import {
     DataType, strideForType,
     Float, Int, Decimal, FixedSizeBinary,
     Date_, Time, Timestamp, Interval, Duration,
-    Utf8, LargeUtf8, Binary, LargeBinary, List, Map_,
+    Utf8, LargeUtf8, Binary, LargeBinary, List, LargeList, Map_,
 } from './type.js';
 import { createIsValidFunction } from './builder/valid.js';
 import { BufferBuilder, BitmapBufferBuilder, DataBufferBuilder, OffsetsBufferBuilder } from './builder/buffer.js';
@@ -285,7 +285,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
 
         if (typeIds = _typeIds?.flush(length)) { // Unions, DenseUnions
             valueOffsets = _offsets?.flush(length);
-        } else if (valueOffsets = _offsets?.flush(length)) { // Variable-width primitives (Binary, LargeBinary, Utf8, LargeUtf8), and Lists
+        } else if (valueOffsets = _offsets?.flush(length)) { // Variable-width primitives (Binary, LargeBinary, Utf8, LargeUtf8), and Lists and LargeLists
             data = _values?.flush(_offsets.last());
         } else { // Fixed-width primitives (Int, Float, Decimal, Time, Timestamp, Duration and Interval)
             data = _values?.flush(length);
@@ -352,7 +352,7 @@ export abstract class FixedWidthBuilder<T extends Int | Float | FixedSizeBinary
 }
 
 /** @ignore */
-export abstract class VariableWidthBuilder<T extends Binary | LargeBinary | Utf8 | LargeUtf8 | List | Map_, TNull = any> extends Builder<T, TNull> {
+export abstract class VariableWidthBuilder<T extends Binary | LargeBinary | Utf8 | LargeUtf8 | List | LargeList | Map_, TNull = any> extends Builder<T, TNull> {
     protected _pendingLength = 0;
     protected _offsets: OffsetsBufferBuilder<T>;
     protected _pending: Map<number, any> | undefined;

js/src/builder/buffer.ts

@@ -137,7 +137,10 @@ export class OffsetsBufferBuilder<T extends DataType> extends DataBufferBuilder<
         if (offset < index++ && offset >= 0) {
             buffer.fill(buffer[offset], offset, index);
         }
-        buffer[index] = buffer[index - 1] + value;
+        buffer[index] =
+            this.BYTES_PER_ELEMENT > 4
+                ? buffer[index - 1] + BigInt(value)
+                : buffer[index - 1] + value;
         return this;
     }
     public flush(length = this.length - 1) {

js/src/builder/largelist.ts

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Field } from '../schema.js';
+import { DataType, LargeList } from '../type.js';
+import { OffsetsBufferBuilder } from './buffer.js';
+import { Builder, BuilderOptions, VariableWidthBuilder } from '../builder.js';
+
+/** @ignore */
+export class LargeListBuilder<T extends DataType = any, TNull = any> extends VariableWidthBuilder<LargeList<T>, TNull> {
+    protected _offsets: OffsetsBufferBuilder<LargeList<T>>;
+    constructor(opts: BuilderOptions<LargeList<T>, TNull>) {
+        super(opts);
+        this._offsets = new OffsetsBufferBuilder(opts.type);
+    }
+    public addChild(child: Builder<T>, name = '0') {
+        if (this.numChildren > 0) {
+            throw new Error('ListBuilder can only have one child.');
+        }
+        this.children[this.numChildren] = child;
+        this.type = new LargeList(new Field(name, child.type, true));
+        return this.numChildren - 1;
+    }
+    protected _flushPending(pending: Map<number, T['TValue'] | undefined>) {
+        const offsets = this._offsets;
+        const [child] = this.children;
+        for (const [index, value] of pending) {
+            if (typeof value === 'undefined') {
+                offsets.set(index, BigInt(0));
+            } else {
+                const v = value as T['TValue'];
+                const n = v.length;
+                const start = Number(offsets.set(index, n).buffer[index]);
+                for (let i = -1; ++i < n;) {
+                    child.set(start + i, v[i]);
+                }
+            }
+        }
+    }
+}

js/src/data.ts

@@ -253,7 +253,8 @@ export class Data<T extends DataType = DataType> {
 
 import {
     Dictionary,
-    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct,
+    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary,
+    List, LargeList, FixedSizeList, Map_, Struct,
     Float,
     Int,
     Date_,
@@ -374,6 +375,13 @@ class MakeDataVisitor extends Visitor {
         const { ['length']: length = valueOffsets.length - 1, ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0 } = props;
         return new Data(type, offset, length, nullCount, [valueOffsets, undefined, nullBitmap], [child]);
     }
+    public visitLargeList<T extends LargeList>(props: LargeListDataProps<T>) {
+        const { ['type']: type, ['offset']: offset = 0, ['child']: child } = props;
+        const nullBitmap = toUint8Array(props['nullBitmap']);
+        const valueOffsets = toBigInt64Array(props['valueOffsets']);
+        const { ['length']: length = valueOffsets.length - 1, ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0 } = props;
+        return new Data(type, offset, length, nullCount, [valueOffsets, undefined, nullBitmap], [child]);
+    }
     public visitStruct<T extends Struct>(props: StructDataProps<T>) {
         const { ['type']: type, ['offset']: offset = 0, ['children']: children = [] } = props;
         const nullBitmap = toUint8Array(props['nullBitmap']);
@@ -456,6 +464,7 @@ interface LargeBinaryDataProps<T extends LargeBinary> extends DataProps_<T> { va
 interface Utf8DataProps<T extends Utf8> extends DataProps_<T> { valueOffsets: ValueOffsetsBuffer; data?: DataBuffer<T> }
 interface LargeUtf8DataProps<T extends LargeUtf8> extends DataProps_<T> { valueOffsets: LargeValueOffsetsBuffer | ValueOffsetsBuffer; data?: DataBuffer<T> }
 interface ListDataProps<T extends List> extends DataProps_<T> { valueOffsets: ValueOffsetsBuffer; child: Data<T['valueType']> }
+interface LargeListDataProps<T extends LargeList> extends DataProps_<T> { valueOffsets: LargeValueOffsetsBuffer | ValueOffsetsBuffer; child: Data<T['valueType']> }
 interface FixedSizeListDataProps<T extends FixedSizeList> extends DataProps_<T> { child: Data<T['valueType']> }
 interface StructDataProps<T extends Struct> extends DataProps_<T> { children: Data[] }
 interface Map_DataProps<T extends Map_> extends DataProps_<T> { valueOffsets: ValueOffsetsBuffer; child: Data }
@@ -481,6 +490,7 @@ export type DataProps<T extends DataType> = (
     T extends Utf8 /*            */ ? Utf8DataProps<T> :
     T extends LargeUtf8 /*       */ ? LargeUtf8DataProps<T> :
     T extends List /*            */ ? ListDataProps<T> :
+    T extends LargeList /*       */ ? LargeListDataProps<T> :
     T extends FixedSizeList /*   */ ? FixedSizeListDataProps<T> :
     T extends Struct /*          */ ? StructDataProps<T> :
     T extends Map_ /*            */ ? Map_DataProps<T> :
@@ -509,6 +519,7 @@ export function makeData<T extends LargeBinary>(props: LargeBinaryDataProps<T>):
 export function makeData<T extends Utf8>(props: Utf8DataProps<T>): Data<T>;
 export function makeData<T extends LargeUtf8>(props: LargeUtf8DataProps<T>): Data<T>;
 export function makeData<T extends List>(props: ListDataProps<T>): Data<T>;
+export function makeData<T extends LargeList>(props: LargeListDataProps<T>): Data<T>;
 export function makeData<T extends FixedSizeList>(props: FixedSizeListDataProps<T>): Data<T>;
 export function makeData<T extends Struct>(props: StructDataProps<T>): Data<T>;
 export function makeData<T extends Map_>(props: Map_DataProps<T>): Data<T>;

js/src/enum.ts

@@ -70,6 +70,7 @@ export enum Type {
     Duration = 18, /** Measure of elapsed time in either seconds, milliseconds, microseconds or nanoseconds */
     LargeBinary = 19, /** Large variable-length bytes (no guarantee of UTF8-ness) */
     LargeUtf8 = 20, /** Large variable-length string as List<Char> */
+    LargeList = 21, /** A Large list of some logical data type */
 
     Dictionary = -1, /** Dictionary aka Category type */
     Int8 = -2,

js/src/interfaces.ts

@@ -36,6 +36,7 @@ import type { Utf8Builder } from './builder/utf8.js';
 import type { LargeUtf8Builder } from './builder/largeutf8.js';
 import type { BinaryBuilder } from './builder/binary.js';
 import type { LargeBinaryBuilder } from './builder/largebinary.js';
+import type { LargeListBuilder } from './builder/largelist.js';
 import type { ListBuilder } from './builder/list.js';
 import type { FixedSizeListBuilder } from './builder/fixedsizelist.js';
 import type { MapBuilder } from './builder/map.js';
@@ -240,6 +241,7 @@ export type TypeToDataType<T extends Type> = {
     [Type.DurationNanosecond]: type.DurationNanosecond;
     [Type.Map]: type.Map_;
     [Type.List]: type.List;
+    [Type.LargeList]: type.LargeList;
     [Type.Struct]: type.Struct;
     [Type.Dictionary]: type.Dictionary;
     [Type.FixedSizeList]: type.FixedSizeList;
@@ -295,6 +297,7 @@ type TypeToBuilder<T extends Type = any, TNull = any> = {
     [Type.DurationNanosecond]: DurationNanosecondBuilder<TNull>;
     [Type.Map]: MapBuilder<any, any, TNull>;
     [Type.List]: ListBuilder<any, TNull>;
+    [Type.LargeList]: LargeListBuilder<any, TNull>;
     [Type.Struct]: StructBuilder<any, TNull>;
     [Type.Dictionary]: DictionaryBuilder<any, TNull>;
     [Type.FixedSizeList]: FixedSizeListBuilder<any, TNull>;
@@ -350,6 +353,7 @@ type DataTypeToBuilder<T extends DataType = any, TNull = any> = {
     [Type.DurationNanosecond]: T extends type.DurationNanosecond ? DurationNanosecondBuilder<TNull> : never;
     [Type.Map]: T extends type.Map_ ? MapBuilder<T['keyType'], T['valueType'], TNull> : never;
     [Type.List]: T extends type.List ? ListBuilder<T['valueType'], TNull> : never;
+    [Type.LargeList]: T extends type.LargeList ? LargeListBuilder<T['valueType'], TNull> : never;
     [Type.Struct]: T extends type.Struct ? StructBuilder<T['dataTypes'], TNull> : never;
     [Type.Dictionary]: T extends type.Dictionary ? DictionaryBuilder<T, TNull> : never;
     [Type.FixedSizeList]: T extends type.FixedSizeList ? FixedSizeListBuilder<T['valueType'], TNull> : never;

js/src/ipc/metadata/json.ts

@@ -21,7 +21,7 @@ import { Schema, Field } from '../../schema.js';
 import {
     DataType, Dictionary, TimeBitWidth,
     Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary,
-    List, FixedSizeList, Map_, Struct, Union,
+    List, LargeList, FixedSizeList, Map_, Struct, Union,
     Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, Duration,
 } from '../../type.js';
 
@@ -154,6 +154,7 @@ function typeFromJSON(f: any, children?: Field[]): DataType<any> {
         case 'largeutf8': return new LargeUtf8();
         case 'bool': return new Bool();
         case 'list': return new List((children || [])[0]);
+        case 'largelist': return new LargeList((children || [])[0]);
         case 'struct': return new Struct(children || []);
         case 'struct_': return new Struct(children || []);
     }

js/src/ipc/metadata/message.ts

@@ -57,7 +57,7 @@ import ByteBuffer = flatbuffers.ByteBuffer;
 import {
     DataType, Dictionary, TimeBitWidth,
     Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary,
-    List, FixedSizeList, Map_, Struct, Union,
+    List, LargeList, FixedSizeList, Map_, Struct, Union,
     Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, Duration,
 } from '../../type.js';
 
@@ -437,6 +437,7 @@ function decodeFieldType(f: _Field, children?: Field[]): DataType<any> {
         case Type['LargeUtf8']: return new LargeUtf8();
         case Type['Bool']: return new Bool();
         case Type['List']: return new List((children || [])[0]);
+        case Type['LargeList']: return new LargeList((children || [])[0]);
         case Type['Struct_']: return new Struct(children || []);
     }
 

js/src/type.ts

@@ -69,6 +69,7 @@ export abstract class DataType<TType extends Type = Type, TChildren extends Type
     /** @nocollapse */ static isInterval(x: any): x is Interval_ { return x?.typeId === Type.Interval; }
     /** @nocollapse */ static isDuration(x: any): x is Duration { return x?.typeId === Type.Duration; }
     /** @nocollapse */ static isList(x: any): x is List { return x?.typeId === Type.List; }
+    /** @nocollapse */ static isLargeList(x: any): x is LargeList { return x?.typeId === Type.LargeList; }
     /** @nocollapse */ static isStruct(x: any): x is Struct { return x?.typeId === Type.Struct; }
     /** @nocollapse */ static isUnion(x: any): x is Union_ { return x?.typeId === Type.Union; }
     /** @nocollapse */ static isFixedSizeBinary(x: any): x is FixedSizeBinary { return x?.typeId === Type.FixedSizeBinary; }
@@ -500,7 +501,9 @@ export class DurationNanosecond extends Duration<Type.DurationNanosecond> { cons
 /** @ignore */
 export interface List<T extends DataType = any> extends DataType<Type.List, { [0]: T }> {
     TArray: Array<T>;
+    TOffsetArray: Int32Array;
     TValue: Vector<T>;
+    OffsetArrayType: TypedArrayConstructor<Int32Array>;
 }
 
 /** @ignore */
@@ -520,6 +523,31 @@ export class List<T extends DataType = any> extends DataType<Type.List, { [0]: T
     })(List.prototype);
 }
 
+/** @ignore */
+export interface LargeList<T extends DataType = any> extends DataType<Type.LargeList, { [0]: T }> {
+    TArray: Array<T>;
+    TOffsetArray: BigInt64Array;
+    TValue: Vector<T>;
+    OffsetArrayType: BigIntArrayConstructor<BigInt64Array>;
+}
+
+/** @ignore */
+export class LargeList<T extends DataType = any> extends DataType<Type.LargeList, { [0]: T }> {
+    constructor(child: Field<T>) {
+        super(Type.LargeList);
+        this.children = [child];
+    }
+    public declare readonly children: Field<T>[];
+    public toString() { return `LargeList<${this.valueType}>`; }
+    public get valueType(): T { return this.children[0].type as T; }
+    public get valueField(): Field<T> { return this.children[0] as Field<T>; }
+    public get ArrayType(): T['ArrayType'] { return this.valueType.ArrayType; }
+    protected static [Symbol.toStringTag] = ((proto: LargeList) => {
+        (<any>proto).children = null;
+        return proto[Symbol.toStringTag] = 'LargeList';
+    })(LargeList.prototype);
+}
+
 /** @ignore */
 export interface Struct<T extends TypeMap = any> extends DataType<Type.Struct, T> {
     TArray: Array<StructRowProxy<T>>;

js/src/visitor.ts

@@ -45,6 +45,7 @@ export abstract class Visitor {
     public visitTime(_node: any, ..._args: any[]): any { return null; }
     public visitDecimal(_node: any, ..._args: any[]): any { return null; }
     public visitList(_node: any, ..._args: any[]): any { return null; }
+    public visitLargeList(_node: any, ..._args: any[]): any { return null; }
     public visitStruct(_node: any, ..._args: any[]): any { return null; }
     public visitUnion(_node: any, ..._args: any[]): any { return null; }
     public visitDictionary(_node: any, ..._args: any[]): any { return null; }
@@ -110,6 +111,7 @@ function getVisitFnByTypeId(visitor: Visitor, dtype: Type, throwIfNotFound = tru
         case Type.TimeNanosecond: fn = visitor.visitTimeNanosecond || visitor.visitTime; break;
         case Type.Decimal: fn = visitor.visitDecimal; break;
         case Type.List: fn = visitor.visitList; break;
+        case Type.LargeList: fn = visitor.visitLargeList; break;
         case Type.Struct: fn = visitor.visitStruct; break;
         case Type.Union: fn = visitor.visitUnion; break;
         case Type.DenseUnion: fn = visitor.visitDenseUnion || visitor.visitUnion; break;
@@ -203,6 +205,7 @@ function inferDType<T extends DataType>(type: T): Type {
             return Type.Duration;
         case Type.Map: return Type.Map;
         case Type.List: return Type.List;
+        case Type.LargeList: return Type.LargeList;
         case Type.Struct: return Type.Struct;
         case Type.Union:
             switch ((type as any as Union).mode) {
@@ -254,6 +257,7 @@ export interface Visitor {
     visitTimeNanosecond?(node: any, ...args: any[]): any;
     visitDecimal(node: any, ...args: any[]): any;
     visitList(node: any, ...args: any[]): any;
+    visitLargeList(node: any, ...args: any[]): any;
     visitStruct(node: any, ...args: any[]): any;
     visitUnion(node: any, ...args: any[]): any;
     visitDenseUnion?(node: any, ...args: any[]): any;

js/src/visitor/builderctor.ts

@@ -34,6 +34,7 @@ import { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder } fro
 import { DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder } from '../builder/duration.js';
 import { IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder } from '../builder/int.js';
 import { ListBuilder } from '../builder/list.js';
+import { LargeListBuilder } from '../builder/largelist.js';
 import { MapBuilder } from '../builder/map.js';
 import { NullBuilder } from '../builder/null.js';
 import { StructBuilder } from '../builder/struct.js';
@@ -88,6 +89,7 @@ export class GetBuilderCtor extends Visitor {
     public visitTimeNanosecond() { return TimeNanosecondBuilder; }
     public visitDecimal() { return DecimalBuilder; }
     public visitList() { return ListBuilder; }
+    public visitLargeList() { return LargeListBuilder; }
     public visitStruct() { return StructBuilder; }
     public visitUnion() { return UnionBuilder; }
     public visitDenseUnion() { return DenseUnionBuilder; }