Skip to content

Commit

Permalink
Support TypeScript 5.6 and 5.7
Browse files Browse the repository at this point in the history 
TypeScript 5.7 made TypeArray objects generic over an `ArrayBufferLike`
source. We only use `ArrayBuffer` internally, so isdeally we would be
explicit about the generic parameter in our return types. However, older
versions of TS will fail with an explicit generic parameter (i.e.,
`Uint8Array<ArrayBuffer>`).

The biggest issue is a mismatch between `ndarray` types and our own.
This change vendors the types for `ndarray`, to align them with our own.
  • Loading branch information
@manzt
manzt committed Dec 3, 2024
1 parent 7756cdc commit aa1cff6
Show file tree
Hide file tree
Showing 8 changed files with 124 additions and 39 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"@svitejs/changesets-changelog-github-compact": "^1.2.0",
"@types/node": "^22.10.1",
"publint": "^0.2.12",
"typescript": "5.6.3",
"typescript": "5.7.2",
"vitest": "^2.1.8"
},
"packageManager": "pnpm@9.7.0"
Expand Down
4 changes: 3 additions & 1 deletion packages/core/src/codecs/vlen-utf8.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ export class VLenUTF8 {
for (let i = 0; i < data.length; i++) {
let item_length = view.getUint32(pos, true);
pos += 4;
data[i] = decoder.decode(bytes.buffer.slice(pos, pos + item_length));
// @ts-expect-error - we know this is an ArrayBuffer, TS just isn't smart enough to know it's not a SharedArrayBuffer
let buffer: ArrayBuffer = bytes.buffer;
data[i] = decoder.decode(buffer.slice(pos, pos + item_length));
pos += item_length;
}
return { data, shape: this.#shape, stride: this.#strides };
Expand Down
43 changes: 22 additions & 21 deletions packages/indexing/__tests__/setter.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ function to_c<D extends DataType>({ data, shape, stride }: Chunk<D>): Chunk<D> {
let size = shape.reduce((a, b) => a * b, 1);
// @ts-expect-error - We know constructor exists on TypedArray
let out = ndarray(new data.constructor(size), shape);
// @ts-expect-error - ndarray types are a mismatch with ours but this operation is safe
assign(out, ndarray(data, shape, stride));
return out;
}
Expand All @@ -59,7 +60,7 @@ describe("setter", () => {
1, 1, 1, 1,
1, 1, 1, 1,
1, 1, 1, 1,

1, 1, 1, 1,
1, 1, 1, 1,
1, 1, 1, 1,
Expand All @@ -79,7 +80,7 @@ describe("setter", () => {
1, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,

0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
Expand All @@ -91,7 +92,7 @@ describe("setter", () => {
1, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,

0, 0, 0, 0,
0, 2, 0, 0,
0, 0, 0, 0,
Expand All @@ -103,7 +104,7 @@ describe("setter", () => {
1, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,

0, 0, 0, 0,
0, 2, 0, 0,
0, 0, 0, 3,
Expand All @@ -115,7 +116,7 @@ describe("setter", () => {
1, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,

0, 0, 0, 0,
0, 2, 0, 0,
0, 0, 4, 3,
Expand All @@ -136,7 +137,7 @@ describe("setter", () => {
1, 0, 0, 0,
1, 0, 0, 0,
0, 0, 0, 0,

1, 0, 0, 0,
1, 0, 0, 0,
0, 0, 0, 0,
Expand All @@ -150,7 +151,7 @@ describe("setter", () => {
2, 2, 2, 2,
2, 2, 2, 2,
2, 2, 2, 2,

1, 0, 0, 0,
1, 0, 0, 0,
0, 0, 0, 0,
Expand Down Expand Up @@ -190,7 +191,7 @@ describe("setter", () => {
2, 2, 2, 2,
2, 2, 2, 2,
2, 2, 2, 2,

1, 0, 0, 0,
1, 0, 0, 0,
0, 0, 0, 0,
Expand Down Expand Up @@ -222,7 +223,7 @@ describe("setter", () => {
1, 1, 0, 0,
1, 1, 0, 0,
0, 0, 0, 0,

1, 1, 0, 0,
1, 1, 0, 0,
0, 0, 0, 0,
Expand Down Expand Up @@ -254,7 +255,7 @@ describe("setter", () => {
2, 0, 2, 0,
0, 0, 0, 0,
2, 0, 2, 0,

2, 0, 2, 0,
0, 0, 0, 0,
2, 0, 2, 0,
Expand All @@ -274,7 +275,7 @@ describe("setter", () => {
2, 0, 2, 0,
0, 0, 0, 0,
2, 0, 2, 0,

2, 0, 2, 0,
0, 0, 0, 0,
2, 0, 2, 0,
Expand Down Expand Up @@ -318,7 +319,7 @@ describe("setter", () => {
0, 2, 0, 0,
0, 0, 0, 0,
0, 2, 0, 0,

0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
Expand All @@ -334,7 +335,7 @@ describe("setter", () => {
0, 2, 0, 0,
0, 0, 0, 0,
0, 2, 0, 0,

0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
Expand Down Expand Up @@ -378,7 +379,7 @@ describe("setter", () => {
1, 1, 0, 0,
1, 1, 0, 0,
0, 0, 0, 0,

1, 1, 0, 0,
1, 1, 0, 0,
0, 0, 0, 0,
Expand Down Expand Up @@ -439,14 +440,14 @@ describe("setter", () => {
setter.set_from_chunk(dest, src, mapping);
// biome-ignore format: the array should not be formatted
expect(to_c(dest).data).toStrictEqual(new Float32Array([
0, 2, 0, 0,
0, 0, 0, 0,
0, 2, 0, 0,
0, 2, 0, 0,
0, 0, 0, 0,
0, 2, 0, 0,

0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
]));
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
]));
});

it("set_from_chunk - dest=C order, src=F order", async () => {
Expand Down
1 change: 0 additions & 1 deletion packages/ndarray/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
}
},
"dependencies": {
"@types/ndarray": "^1.0.11",
"@types/ndarray-ops": "^1.2.4",
"@zarrita/core": "workspace:^",
"@zarrita/indexing": "workspace:^",
Expand Down
89 changes: 89 additions & 0 deletions packages/ndarray/src/missing-types.d.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
declare module "ndarray" {
declare function ndarray<D extends ndarray.Data = ndarray.Data<number>>(
data: D,
shape?: number[],
stride?: number[],
offset?: number,
): ndarray.NdArray<D>;

declare namespace ndarray {
interface NdArray<D extends Data = Data<number>> {
data: D;
shape: number[];
stride: number[];
offset: number;
dtype: DataType<D>;
size: number;
order: number[];
dimension: number;
get(...args: number[]): Value<D>;
set(...args: number[]): Value<D>;
index(...args: number[]): Value<D>;
lo(...args: number[]): NdArray<D>;
hi(...args: number[]): NdArray<D>;
step(...args: number[]): NdArray<D>;
transpose(...args: number[]): NdArray<D>;
pick(...args: Array<number | null>): NdArray<D>;
T: NdArray<D>;
}

interface GenericArray<T> {
get(idx: number): T;
set(idx: number, value: T): void;
length: number;
}

// biome-ignore lint/suspicious/noExplicitAny: not our library
type Data<T = any> = T extends number
? GenericArray<T> | T[] | TypedArray
: T extends bigint
? GenericArray<T> | T[] | BigInt64Array | BigUint64Array
: GenericArray<T> | T[];

type TypedArray =
| Int8Array
| Int16Array
| Int32Array
| Uint8Array
| Uint8ClampedArray
| Uint16Array
| Uint32Array
| Float32Array
| Float64Array;

type Value<D extends Data> = D extends
| GenericArray<infer T>
// biome-ignore lint/suspicious/noRedeclare: not our library
| Record<number, infer T>
? T
: never;

type DataType<D extends Data = Data> = D extends Int8Array
? "int8"
: D extends Int16Array
? "int16"
: D extends Int32Array
? "int32"
: D extends Uint8Array
? "uint8"
: D extends Uint8ClampedArray
? "uint8_clamped"
: D extends Uint16Array
? "uint16"
: D extends Uint32Array
? "uint32"
: D extends Float32Array
? "float32"
: D extends Float64Array
? "float64"
: D extends BigInt64Array
? "bigint64"
: D extends BigUint64Array
? "biguint64"
: D extends GenericArray<unknown>
? "generic"
: "array";
}

export = ndarray;
}
4 changes: 3 additions & 1 deletion packages/typedarray/__tests__/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,9 @@ describe("ByteStringArray", () => {
let data = new TextEncoder().encode(
"Hello\x00world!\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
);
let arr = new ByteStringArray(2, data.buffer, 3, 4);
// @ts-expect-error - we know this is an ArrayBuffer, TS just isn't smart enough to know it's not a SharedArrayBuffer
let buffer: ArrayBuffer = data.buffer;
let arr = new ByteStringArray(2, buffer, 3, 4);
expect({
length: arr.length,
BYTES_PER_ELEMENT: arr.BYTES_PER_ELEMENT,
Expand Down
3 changes: 3 additions & 0 deletions packages/typedarray/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export class BoolArray {
}

get buffer(): ArrayBuffer {
// @ts-expect-error - we know this is an ArrayBuffer (and not SharedArrayBuffer)
return this.#bytes.buffer;
}

Expand Down Expand Up @@ -106,6 +107,7 @@ export class ByteStringArray {
}

get buffer(): ArrayBuffer {
// @ts-expect-error - we know this is an ArrayBuffer (and not SharedArrayBuffer)
return this._data.buffer;
}

Expand Down Expand Up @@ -198,6 +200,7 @@ export class UnicodeStringArray {
}

get buffer(): ArrayBuffer {
// @ts-expect-error - we know this is an ArrayBuffer (and not SharedArrayBuffer)
return this.#data.buffer;
}

Expand Down
17 changes: 3 additions & 14 deletions pnpm-lock.yaml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit aa1cff6

Please sign in to comment.