From c581f201aa3012ff6824c8e4507fc925dd83b008 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Mon, 2 Dec 2024 13:34:20 -0800 Subject: [PATCH] fix(gzip): Use Compression Streams API in place of pako (#674) This is expected to be faster and also reduces the bundle size. --- build_tools/update-conditions.ts | 4 ++-- .../parcel-project-built/package-lock.json | 5 ++-- .../parcel-project-source/package-lock.json | 16 ++----------- .../vite/vite-project-built/package-lock.json | 5 ++-- .../vite-project-source/package-lock.json | 15 ++---------- .../webpack-project-built/package-lock.json | 5 ++-- .../webpack-project-source/package-lock.json | 16 ++----------- package-lock.json | 15 +----------- package.json | 10 +------- src/async_computation/decode_gzip.ts | 23 ------------------- src/async_computation/decode_gzip_request.ts | 20 ---------------- src/datasource/boss/async_computation.ts | 1 - .../enabled_async_computation_modules.ts | 1 - src/datasource/graphene/async_computation.ts | 1 - src/datasource/n5/async_computation.ts | 1 - src/datasource/n5/backend.ts | 9 ++------ src/datasource/nifti/async_computation.ts | 1 - src/datasource/nifti/backend.ts | 14 +++-------- .../precomputed/async_computation.ts | 1 - src/datasource/precomputed/backend.ts | 21 +++-------------- src/datasource/zarr/async_computation.ts | 1 - src/datasource/zarr/codec/gzip/decode.ts | 13 ++++------- .../backend_chunk_decoders/bossNpz.ts | 12 ++-------- .../backend_chunk_decoders/ndstoreNpz.ts | 12 ++-------- src/util/gzip.ts | 9 +++++++- 25 files changed, 40 insertions(+), 191 deletions(-) delete mode 100644 src/async_computation/decode_gzip.ts delete mode 100644 src/async_computation/decode_gzip_request.ts delete mode 100644 src/datasource/nifti/async_computation.ts diff --git a/build_tools/update-conditions.ts b/build_tools/update-conditions.ts index 6581bf11c..873829e16 100644 --- a/build_tools/update-conditions.ts +++ b/build_tools/update-conditions.ts @@ -124,8 +124,8 @@ packageJson.imports = imports; packageJson.exports = { ".": "./src/main_module.ts", - "./*.js": "./src/*.ts", - "./*": "./src/*", + "./unstable/*.js": "./src/*.ts", + "./unstable/*": "./src/*", }; const tempPackageJsonPath = packageJsonPath + ".tmp"; diff --git a/examples/parcel/parcel-project-built/package-lock.json b/examples/parcel/parcel-project-built/package-lock.json index 84b1c7eb7..8b7cb6c2c 100644 --- a/examples/parcel/parcel-project-built/package-lock.json +++ b/examples/parcel/parcel-project-built/package-lock.json @@ -28,11 +28,10 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.1", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.10 <21 || >=21.2" + "node": ">=20.11 <21 || >=21.2" } }, "node_modules/@babel/code-frame": { diff --git a/examples/parcel/parcel-project-source/package-lock.json b/examples/parcel/parcel-project-source/package-lock.json index d48589d7e..a365a3d15 100644 --- a/examples/parcel/parcel-project-source/package-lock.json +++ b/examples/parcel/parcel-project-source/package-lock.json @@ -18,7 +18,6 @@ } }, "../../..": { - "name": "neuroglancer", "version": "2.40.1", "license": "Apache-2.0", "dependencies": { @@ -28,15 +27,13 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.1", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "devDependencies": { "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", "@types/lodash-es": "^4.17.12", "@types/node": "^20.14.12", - "@types/pako": "^2.0.3", "@types/yargs": "^17.0.32", "@typescript-eslint/eslint-plugin": "^7.17.0", "@typescript-eslint/parser": "^7.17.0", @@ -69,7 +66,7 @@ "webpack-merge": "^6.0.1" }, "engines": { - "node": ">=20.10 <21 || >=21.2" + "node": ">=20.11 <21 || >=21.2" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -1291,11 +1288,6 @@ "@types/node": "*" } }, - "../../../node_modules/@types/pako": { - "version": "2.0.3", - "dev": true, - "license": "MIT" - }, "../../../node_modules/@types/parse-json": { "version": "4.0.2", "dev": true, @@ -9458,10 +9450,6 @@ "node": ">= 14" } }, - "../../../node_modules/pako": { - "version": "2.1.0", - "license": "(MIT AND Zlib)" - }, "../../../node_modules/param-case": { "version": "3.0.4", "dev": true, diff --git a/examples/vite/vite-project-built/package-lock.json b/examples/vite/vite-project-built/package-lock.json index 4c9640951..426618e87 100644 --- a/examples/vite/vite-project-built/package-lock.json +++ b/examples/vite/vite-project-built/package-lock.json @@ -26,11 +26,10 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.1", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.10 <21 || >=21.2" + "node": ">=20.11 <21 || >=21.2" } }, "node_modules/@esbuild/aix-ppc64": { diff --git a/examples/vite/vite-project-source/package-lock.json b/examples/vite/vite-project-source/package-lock.json index 7427d782d..92a35cca7 100644 --- a/examples/vite/vite-project-source/package-lock.json +++ b/examples/vite/vite-project-source/package-lock.json @@ -25,15 +25,13 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.1", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "devDependencies": { "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", "@types/lodash-es": "^4.17.12", "@types/node": "^20.14.12", - "@types/pako": "^2.0.3", "@types/yargs": "^17.0.32", "@typescript-eslint/eslint-plugin": "^7.17.0", "@typescript-eslint/parser": "^7.17.0", @@ -66,7 +64,7 @@ "webpack-merge": "^6.0.1" }, "engines": { - "node": ">=20.10 <21 || >=21.2" + "node": ">=20.11 <21 || >=21.2" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -752,11 +750,6 @@ "@types/node": "*" } }, - "../../../node_modules/@types/pako": { - "version": "2.0.3", - "dev": true, - "license": "MIT" - }, "../../../node_modules/@types/parse-json": { "version": "4.0.2", "dev": true, @@ -8554,10 +8547,6 @@ "node": ">= 14" } }, - "../../../node_modules/pako": { - "version": "2.1.0", - "license": "(MIT AND Zlib)" - }, "../../../node_modules/param-case": { "version": "3.0.4", "dev": true, diff --git a/examples/webpack/webpack-project-built/package-lock.json b/examples/webpack/webpack-project-built/package-lock.json index 557bec6cc..281acb7b1 100644 --- a/examples/webpack/webpack-project-built/package-lock.json +++ b/examples/webpack/webpack-project-built/package-lock.json @@ -32,11 +32,10 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.1", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.10 <21 || >=21.2" + "node": ">=20.11 <21 || >=21.2" } }, "node_modules/@discoveryjs/json-ext": { diff --git a/examples/webpack/webpack-project-source/package-lock.json b/examples/webpack/webpack-project-source/package-lock.json index 307be7727..2c9d8f5e5 100644 --- a/examples/webpack/webpack-project-source/package-lock.json +++ b/examples/webpack/webpack-project-source/package-lock.json @@ -23,7 +23,6 @@ } }, "../../..": { - "name": "neuroglancer", "version": "2.40.1", "license": "Apache-2.0", "dependencies": { @@ -33,15 +32,13 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.1", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "devDependencies": { "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", "@types/lodash-es": "^4.17.12", "@types/node": "^20.14.12", - "@types/pako": "^2.0.3", "@types/yargs": "^17.0.32", "@typescript-eslint/eslint-plugin": "^7.17.0", "@typescript-eslint/parser": "^7.17.0", @@ -74,7 +71,7 @@ "webpack-merge": "^6.0.1" }, "engines": { - "node": ">=20.10 <21 || >=21.2" + "node": ">=20.11 <21 || >=21.2" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -1296,11 +1293,6 @@ "@types/node": "*" } }, - "../../../node_modules/@types/pako": { - "version": "2.0.3", - "dev": true, - "license": "MIT" - }, "../../../node_modules/@types/parse-json": { "version": "4.0.2", "dev": true, @@ -9464,10 +9456,6 @@ "node": ">= 14" } }, - "../../../node_modules/pako": { - "version": "2.1.0", - "license": "(MIT AND Zlib)" - }, "../../../node_modules/param-case": { "version": "3.0.4", "dev": true, diff --git a/package-lock.json b/package-lock.json index a44c39f48..bb11614bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,15 +15,13 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.2", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "devDependencies": { "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", "@types/lodash-es": "^4.17.12", "@types/node": "^20.14.12", - "@types/pako": "^2.0.3", "@types/yargs": "^17.0.32", "@typescript-eslint/eslint-plugin": "^7.17.0", "@typescript-eslint/parser": "^7.17.0", @@ -1666,12 +1664,6 @@ "@types/node": "*" } }, - "node_modules/@types/pako": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.3.tgz", - "integrity": "sha512-bq0hMV9opAcrmE0Byyo0fY3Ew4tgOevJmQ9grUhpXQhYfyLJ1Kqg3P33JT5fdbT2AjeAjR51zqqVjAL/HMkx7Q==", - "dev": true - }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -10421,11 +10413,6 @@ "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==" }, - "node_modules/pako": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", - "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" - }, "node_modules/param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", diff --git a/package.json b/package.json index 3ac3f3e7b..81747aaa2 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,6 @@ "@types/gl-matrix": "^2.4.5", "@types/lodash-es": "^4.17.12", "@types/node": "^20.14.12", - "@types/pako": "^2.0.3", "@types/yargs": "^17.0.32", "@typescript-eslint/eslint-plugin": "^7.17.0", "@typescript-eslint/parser": "^7.17.0", @@ -83,8 +82,7 @@ "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", "nifti-reader-js": "^0.6.8", - "numcodecs": "^0.3.2", - "pako": "^2.1.0" + "numcodecs": "^0.3.2" }, "overrides": { "@puppeteer/browsers": ">=2" @@ -265,12 +263,6 @@ "neuroglancer/datasource/nifti:disabled": "./src/datasource/nifti/backend.ts", "default": "./src/datasource/nifti/backend.ts" }, - "#datasource/nifti/async_computation": { - "neuroglancer/datasource/nifti:enabled": "./src/datasource/nifti/async_computation.ts", - "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/nifti:disabled": "./src/datasource/nifti/async_computation.ts", - "default": "./src/datasource/nifti/async_computation.ts" - }, "#datasource/nifti/register_default": { "neuroglancer/datasource/nifti:enabled": "./src/datasource/nifti/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", diff --git a/src/async_computation/decode_gzip.ts b/src/async_computation/decode_gzip.ts deleted file mode 100644 index 68eeabd70..000000000 --- a/src/async_computation/decode_gzip.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * @license - * Copyright 2019 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { decodeGzip } from "#src/async_computation/decode_gzip_request.js"; -import { registerAsyncComputation } from "#src/async_computation/handler.js"; - -registerAsyncComputation(decodeGzip, async (data: Uint8Array) => { - const result = (await import("pako")).inflate(data); - return { value: result, transfer: [result.buffer] }; -}); diff --git a/src/async_computation/decode_gzip_request.ts b/src/async_computation/decode_gzip_request.ts deleted file mode 100644 index 91dddb6b8..000000000 --- a/src/async_computation/decode_gzip_request.ts +++ /dev/null @@ -1,20 +0,0 @@ -/** - * @license - * Copyright 2019 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { asyncComputation } from "#src/async_computation/index.js"; - -export const decodeGzip = - asyncComputation<(data: Uint8Array) => Uint8Array>("decodeGzip"); diff --git a/src/datasource/boss/async_computation.ts b/src/datasource/boss/async_computation.ts index 69e1442c3..6ffe0e5ec 100644 --- a/src/datasource/boss/async_computation.ts +++ b/src/datasource/boss/async_computation.ts @@ -1,2 +1 @@ import "#src/async_computation/decode_jpeg.js"; -import "#src/async_computation/decode_gzip.js"; diff --git a/src/datasource/enabled_async_computation_modules.ts b/src/datasource/enabled_async_computation_modules.ts index 94f0d6043..2607da3fc 100644 --- a/src/datasource/enabled_async_computation_modules.ts +++ b/src/datasource/enabled_async_computation_modules.ts @@ -5,7 +5,6 @@ import "#datasource/deepzoom/async_computation"; import "#datasource/dvid/async_computation"; import "#datasource/graphene/async_computation"; import "#datasource/n5/async_computation"; -import "#datasource/nifti/async_computation"; import "#datasource/obj/async_computation"; import "#datasource/precomputed/async_computation"; import "#datasource/render/async_computation"; diff --git a/src/datasource/graphene/async_computation.ts b/src/datasource/graphene/async_computation.ts index 69e1442c3..6ffe0e5ec 100644 --- a/src/datasource/graphene/async_computation.ts +++ b/src/datasource/graphene/async_computation.ts @@ -1,2 +1 @@ import "#src/async_computation/decode_jpeg.js"; -import "#src/async_computation/decode_gzip.js"; diff --git a/src/datasource/n5/async_computation.ts b/src/datasource/n5/async_computation.ts index df029623c..db56b860f 100644 --- a/src/datasource/n5/async_computation.ts +++ b/src/datasource/n5/async_computation.ts @@ -1,3 +1,2 @@ -import "#src/async_computation/decode_gzip.js"; import "#src/async_computation/decode_blosc.js"; import "#src/async_computation/decode_zstd.js"; diff --git a/src/datasource/n5/backend.ts b/src/datasource/n5/backend.ts index 4f51ce2c6..ef3faee19 100644 --- a/src/datasource/n5/backend.ts +++ b/src/datasource/n5/backend.ts @@ -15,7 +15,6 @@ */ import { decodeBlosc } from "#src/async_computation/decode_blosc_request.js"; -import { decodeGzip } from "#src/async_computation/decode_gzip_request.js"; import { decodeZstd } from "#src/async_computation/decode_zstd_request.js"; import { requestAsyncComputation } from "#src/async_computation/request.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; @@ -29,6 +28,7 @@ import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import { VolumeChunkSource } from "#src/sliceview/volume/backend.js"; import type { CancellationToken } from "#src/util/cancellation.js"; import { Endianness } from "#src/util/endian.js"; +import { decodeGzip } from "#src/util/gzip.js"; import { isNotFoundError, responseArrayBuffer, @@ -62,12 +62,7 @@ async function decodeChunk( let buffer = new Uint8Array(response, offset); switch (encoding) { case VolumeChunkEncoding.GZIP: - buffer = await requestAsyncComputation( - decodeGzip, - cancellationToken, - [buffer.buffer], - buffer, - ); + buffer = new Uint8Array(await decodeGzip(buffer)); break; case VolumeChunkEncoding.BLOSC: buffer = await requestAsyncComputation( diff --git a/src/datasource/nifti/async_computation.ts b/src/datasource/nifti/async_computation.ts deleted file mode 100644 index 9f856a9ef..000000000 --- a/src/datasource/nifti/async_computation.ts +++ /dev/null @@ -1 +0,0 @@ -import "#src/async_computation/decode_gzip.js"; diff --git a/src/datasource/nifti/backend.ts b/src/datasource/nifti/backend.ts index e3670324e..0b8192e8a 100644 --- a/src/datasource/nifti/backend.ts +++ b/src/datasource/nifti/backend.ts @@ -16,8 +16,6 @@ import type { NIFTI2 } from "nifti-reader-js"; import { isCompressed, NIFTI1, readHeader, readImage } from "nifti-reader-js"; -import { decodeGzip } from "#src/async_computation/decode_gzip_request.js"; -import { requestAsyncComputation } from "#src/async_computation/request.js"; import type { ChunkManager } from "#src/chunk_manager/backend.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; import { ChunkPriorityTier } from "#src/chunk_manager/base.js"; @@ -44,6 +42,7 @@ import { translationRotationScaleZReflectionToMat4, vec3, } from "#src/util/geom.js"; +import { decodeGzip } from "#src/util/gzip.js"; import * as matrix from "#src/util/matrix.js"; import type { SpecialProtocolCredentials, @@ -59,17 +58,10 @@ export class NiftiFileData { async function decodeNiftiFile( buffer: ArrayBuffer, - cancellationToken: CancellationToken, + _cancellationToken: CancellationToken, ) { if (isCompressed(buffer)) { - buffer = ( - await requestAsyncComputation( - decodeGzip, - cancellationToken, - [buffer], - new Uint8Array(buffer), - ) - ).buffer; + buffer = await decodeGzip(buffer); } const data = new NiftiFileData(); data.uncompressedData = buffer; diff --git a/src/datasource/precomputed/async_computation.ts b/src/datasource/precomputed/async_computation.ts index e6b6457a4..16f6033b9 100644 --- a/src/datasource/precomputed/async_computation.ts +++ b/src/datasource/precomputed/async_computation.ts @@ -1,5 +1,4 @@ import "#src/async_computation/decode_jpeg.js"; import "#src/async_computation/decode_jxl.js"; -import "#src/async_computation/decode_gzip.js"; import "#src/async_computation/decode_compresso.js"; import "#src/async_computation/decode_png.js"; diff --git a/src/datasource/precomputed/backend.ts b/src/datasource/precomputed/backend.ts index ac308f0db..4ae0bacf1 100644 --- a/src/datasource/precomputed/backend.ts +++ b/src/datasource/precomputed/backend.ts @@ -30,8 +30,6 @@ import { annotationTypeHandlers, annotationTypes, } from "#src/annotation/index.js"; -import { decodeGzip } from "#src/async_computation/decode_gzip_request.js"; -import { requestAsyncComputation } from "#src/async_computation/request.js"; import type { Chunk, ChunkManager } from "#src/chunk_manager/backend.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; import { GenericSharedDataSource } from "#src/chunk_manager/generic_file_source.js"; @@ -84,6 +82,7 @@ import type { CancellationToken } from "#src/util/cancellation.js"; import type { Borrowed } from "#src/util/disposable.js"; import { convertEndian32, Endianness } from "#src/util/endian.js"; import { vec3 } from "#src/util/geom.js"; +import { decodeGzip } from "#src/util/gzip.js"; import { murmurHash3_x86_128Hash64Bits } from "#src/util/hash.js"; import { isNotFoundError, @@ -219,14 +218,7 @@ function getMinishardIndexDataSource( cancellationToken, ); if (sharding.minishardIndexEncoding === DataEncoding.GZIP) { - minishardIndexResponse = ( - await requestAsyncComputation( - decodeGzip, - cancellationToken, - [minishardIndexResponse], - new Uint8Array(minishardIndexResponse), - ) - ).buffer; + minishardIndexResponse = await decodeGzip(minishardIndexResponse); } if (minishardIndexResponse.byteLength % 24 !== 0) { throw new Error( @@ -352,14 +344,7 @@ async function getShardedData( cancellationToken, ); if (minishardIndexSource.sharding.dataEncoding === DataEncoding.GZIP) { - data = ( - await requestAsyncComputation( - decodeGzip, - cancellationToken, - [data], - new Uint8Array(data), - ) - ).buffer; + data = await decodeGzip(data); } return { data, diff --git a/src/datasource/zarr/async_computation.ts b/src/datasource/zarr/async_computation.ts index df029623c..db56b860f 100644 --- a/src/datasource/zarr/async_computation.ts +++ b/src/datasource/zarr/async_computation.ts @@ -1,3 +1,2 @@ -import "#src/async_computation/decode_gzip.js"; import "#src/async_computation/decode_blosc.js"; import "#src/async_computation/decode_zstd.js"; diff --git a/src/datasource/zarr/codec/gzip/decode.ts b/src/datasource/zarr/codec/gzip/decode.ts index 3be3ef77e..8003868eb 100644 --- a/src/datasource/zarr/codec/gzip/decode.ts +++ b/src/datasource/zarr/codec/gzip/decode.ts @@ -14,27 +14,22 @@ * limitations under the License. */ -import { decodeGzip } from "#src/async_computation/decode_gzip_request.js"; -import { requestAsyncComputation } from "#src/async_computation/request.js"; import { registerCodec } from "#src/datasource/zarr/codec/decode.js"; import type { Configuration } from "#src/datasource/zarr/codec/gzip/resolve.js"; import { CodecKind } from "#src/datasource/zarr/codec/index.js"; import type { CancellationToken } from "#src/util/cancellation.js"; +import { decodeGzip } from "#src/util/gzip.js"; registerCodec({ name: "gzip", kind: CodecKind.bytesToBytes, - decode( + async decode( configuration: Configuration, encoded: Uint8Array, cancellationToken: CancellationToken, ): Promise { configuration; - return requestAsyncComputation( - decodeGzip, - cancellationToken, - [encoded.buffer], - encoded, - ); + cancellationToken; + return new Uint8Array(await decodeGzip(encoded)); }, }); diff --git a/src/sliceview/backend_chunk_decoders/bossNpz.ts b/src/sliceview/backend_chunk_decoders/bossNpz.ts index 2ee975c7b..e8f056ba0 100644 --- a/src/sliceview/backend_chunk_decoders/bossNpz.ts +++ b/src/sliceview/backend_chunk_decoders/bossNpz.ts @@ -22,13 +22,12 @@ * (each corresponding to a different variable) in NPY binary format. */ -import { decodeGzip } from "#src/async_computation/decode_gzip_request.js"; -import { requestAsyncComputation } from "#src/async_computation/request.js"; import { postProcessRawData } from "#src/sliceview/backend_chunk_decoders/postprocess.js"; import { DataType } from "#src/sliceview/base.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import type { CancellationToken } from "#src/util/cancellation.js"; import { vec3Key } from "#src/util/geom.js"; +import { decodeGzip } from "#src/util/gzip.js"; import { parseNpy } from "#src/util/npy.js"; export async function decodeBossNpzChunk( @@ -36,14 +35,7 @@ export async function decodeBossNpzChunk( cancellationToken: CancellationToken, response: ArrayBuffer, ) { - const parseResult = parseNpy( - await requestAsyncComputation( - decodeGzip, - cancellationToken, - [response], - new Uint8Array(response), - ), - ); + const parseResult = parseNpy(new Uint8Array(await decodeGzip(response))); const chunkDataSize = chunk.chunkDataSize!; const source = chunk.source!; const { shape } = parseResult; diff --git a/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts b/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts index 6f7164d62..a4b4fa215 100644 --- a/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts +++ b/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts @@ -22,13 +22,12 @@ * (each corresponding to a different variable) in NPY binary format. */ -import { decodeGzip } from "#src/async_computation/decode_gzip_request.js"; -import { requestAsyncComputation } from "#src/async_computation/request.js"; import { postProcessRawData } from "#src/sliceview/backend_chunk_decoders/postprocess.js"; import { DataType } from "#src/sliceview/base.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import { arraysEqual } from "#src/util/array.js"; import type { CancellationToken } from "#src/util/cancellation.js"; +import { decodeGzip } from "#src/util/gzip.js"; import { parseNpy } from "#src/util/npy.js"; export async function decodeNdstoreNpzChunk( @@ -36,14 +35,7 @@ export async function decodeNdstoreNpzChunk( cancellationToken: CancellationToken, response: ArrayBuffer, ) { - const parseResult = parseNpy( - await requestAsyncComputation( - decodeGzip, - cancellationToken, - [response], - new Uint8Array(response), - ), - ); + const parseResult = parseNpy(new Uint8Array(await decodeGzip(response))); const chunkDataSize = chunk.chunkDataSize!; const source = chunk.source!; const { shape } = parseResult; diff --git a/src/util/gzip.ts b/src/util/gzip.ts index 35ab3cfad..118e03703 100644 --- a/src/util/gzip.ts +++ b/src/util/gzip.ts @@ -22,6 +22,13 @@ export function isGzipFormat(data: ArrayBufferView) { return view.length > 2 && view[0] === 0x1f && view[1] === 0x8b; } +export async function decodeGzip(data: ArrayBuffer | ArrayBufferView) { + const decompressedStream = new Response(data).body!.pipeThrough( + new DecompressionStream("gzip"), + ); + return await new Response(decompressedStream).arrayBuffer(); +} + /** * Decompress `data` if it is in gzip format, otherwise just return it. */ @@ -33,7 +40,7 @@ export async function maybeDecompressGzip(data: ArrayBuffer | ArrayBufferView) { byteView = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } if (isGzipFormat(byteView)) { - return (await import("pako")).inflate(byteView); + return new Uint8Array(await decodeGzip(byteView)); } return byteView; }