diff --git a/js/src/data.ts b/js/src/data.ts index 632f9c9b968c7..cc689ffc15d46 100644 --- a/js/src/data.ts +++ b/js/src/data.ts @@ -19,7 +19,7 @@ import { popcnt_bit_range } from './util/bit'; import { VectorLike, Vector } from './vector'; import { VectorType, TypedArray, TypedArrayConstructor, Dictionary } from './type'; import { Int, Bool, FlatListType, List, FixedSizeList, Struct, Map_ } from './type'; -import { DataType, FlatType, ListType, NestedType, DenseUnion, SparseUnion } from './type'; +import { DataType, FlatType, ListType, NestedType, SingleNestedType, DenseUnion, SparseUnion } from './type'; export function toTypedArray(ArrayType: TypedArrayConstructor, values?: T | ArrayLike | Iterable | null): T { if (!ArrayType && ArrayBuffer.isView(values)) { return values; } @@ -46,7 +46,7 @@ export interface DataTypes { /* [Type.Struct]*/ 13: NestedData; /* [Type.Union]*/ 14: UnionData; /* [Type.FixedSizeBinary]*/ 15: FlatData; -/* [Type.FixedSizeList]*/ 16: ListData>; +/* [Type.FixedSizeList]*/ 16: SingleNestedData>; /* [Type.Map]*/ 17: NestedData; /* [Type.DenseUnion]*/ DenseUnion: DenseUnionData; /*[Type.SparseUnion]*/ SparseUnion: SparseUnionData; @@ -195,15 +195,21 @@ export class NestedData extends BaseData { } } -export class ListData extends NestedData { - public /* [VectorType.OFFSET]:*/ 0: Int32Array; - public /*[VectorType.VALIDITY]:*/ 2: Uint8Array; +export class SingleNestedData extends NestedData { protected _valuesData: Data; public get values() { return this._valuesData; } - public get valueOffsets() { return this[VectorType.OFFSET]; } - constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable, valueChildData: Data, offset?: number, nullCount?: number) { + constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueChildData: Data, offset?: number, nullCount?: number) { super(type, length, nullBitmap, [valueChildData], offset, nullCount); this._valuesData = valueChildData; + } +} + +export class ListData extends SingleNestedData { + public /* [VectorType.OFFSET]:*/ 0: Int32Array; + public /*[VectorType.VALIDITY]:*/ 2: Uint8Array; + public get valueOffsets() { return this[VectorType.OFFSET]; } + constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable, valueChildData: Data, offset?: number, nullCount?: number) { + super(type, length, nullBitmap, valueChildData, offset, nullCount); this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets); } public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { diff --git a/js/src/ipc/reader/vector.ts b/js/src/ipc/reader/vector.ts index c659294522f12..809069c6d9864 100644 --- a/js/src/ipc/reader/vector.ts +++ b/js/src/ipc/reader/vector.ts @@ -20,7 +20,7 @@ import { RecordBatch } from '../../recordbatch'; import { TypeVisitor } from '../../visitor'; import { FlatType, NestedType, ListType } from '../../type'; import { Message, FieldMetadata, BufferMetadata } from '../metadata'; -import { FlatData, ListData, NestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data'; +import { FlatData, ListData, NestedData, SingleNestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data'; import { Schema, Field, Dictionary, @@ -89,7 +89,7 @@ export abstract class TypeDataLoader extends TypeVisitor { public visitStruct (type: Struct) { return this.visitNestedType(type); } public visitUnion (type: Union) { return this.visitUnionType(type); } public visitFixedSizeBinary(type: FixedSizeBinary) { return this.visitFlatType(type); } - public visitFixedSizeList (type: FixedSizeList) { return this.visitListType(type); } + public visitFixedSizeList (type: FixedSizeList) { return this.visitFixedSizeListType(type); } public visitMap (type: Map_) { return this.visitNestedType(type); } public visitDictionary (type: Dictionary) { return new DictionaryData(type, this.dictionaries.get(type.id)!, this.visit(type.indicies)); @@ -117,6 +117,9 @@ export abstract class TypeDataLoader extends TypeVisitor { protected visitListType(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) { return new ListData(type, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children![0].type), 0, nullCount); } + protected visitFixedSizeListType(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) { + return new SingleNestedData(type, length, this.readNullBitmap(type, nullCount), this.visit(type.children![0].type), 0, nullCount); + } protected visitNestedType(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) { return new NestedData(type, length, this.readNullBitmap(type, nullCount), this.visitFields(type.children), 0, nullCount); } diff --git a/js/src/type.ts b/js/src/type.ts index 95d71ca3914b7..4b317ed7fab5e 100644 --- a/js/src/type.ts +++ b/js/src/type.ts @@ -97,8 +97,9 @@ export type PrimitiveType = NumericType | FixedSizeType; export type FlatListType = Utf8 | Binary; // <-- these types have `offset`, `data`, and `validity` buffers export type FlatType = Bool | PrimitiveType | FlatListType; // <-- these types have `data` and `validity` buffers -export type ListType = List | FixedSizeList; // <-- these types have `offset` and `validity` buffers +export type ListType = List; // <-- these types have `offset` and `validity` buffers export type NestedType = Map_ | Struct | List | FixedSizeList | Union; // <-- these types have `validity` buffer and nested childData +export type SingleNestedType = List | FixedSizeList; // <-- these are nested types that can only have a single child /** * * diff --git a/js/src/vector.ts b/js/src/vector.ts index b2fc206cf6d1c..33b13ad739d1c 100644 --- a/js/src/vector.ts +++ b/js/src/vector.ts @@ -336,7 +336,7 @@ export class ListVector extends ListVectorBase
  • { +export class FixedSizeListVector extends Vector { constructor(data: Data, view: View = new FixedSizeListView(data)) { super(data, view); } diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts index 2185841ee3843..0ea5a93befb73 100644 --- a/js/src/vector/list.ts +++ b/js/src/vector/list.ts @@ -29,14 +29,13 @@ export const decodeUtf8 = ((decoder) => decoder.decode.bind(decoder) as (input?: ArrayBufferLike | ArrayBufferView) => string )(new TextDecoder('utf-8')); -export abstract class ListViewBase implements View { +export abstract class ListViewBase implements View { public length: number; public values: T['TArray']; public valueOffsets?: Int32Array; constructor(data: Data) { this.length = data.length; this.values = data.values; - this.valueOffsets = data.valueOffsets; } public clone(data: Data): this { return new ( this.constructor)(data) as this; @@ -64,7 +63,15 @@ export abstract class ListViewBase implemen protected abstract setList(values: T['TArray'], index: number, value: T['TValue'], valueOffsets?: Int32Array): void; } -export class ListView extends ListViewBase> { +export abstract class VariableListViewBase extends ListViewBase { + constructor(data: Data) { + super(data) + this.length = data.length; + this.valueOffsets = data.valueOffsets; + } +} + +export class ListView extends VariableListViewBase> { constructor(data: Data>) { super(data); this.values = createVector(data.values); @@ -101,7 +108,7 @@ export class FixedSizeListView extends ListViewBase { +export class BinaryView extends VariableListViewBase { protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) { return values.subarray(valueOffsets[index], valueOffsets[index + 1]); } @@ -111,7 +118,7 @@ export class BinaryView extends ListViewBase { } } -export class Utf8View extends ListViewBase { +export class Utf8View extends VariableListViewBase { protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) { return decodeUtf8(values.subarray(valueOffsets[index], valueOffsets[index + 1])); }