From 17d81565bedbf5a2403f059ed829eddd9263e086 Mon Sep 17 00:00:00 2001 From: Daria Terekhova Date: Tue, 11 Jul 2023 12:57:03 +0300 Subject: [PATCH 1/6] Warning fix --- modules/i3s/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/i3s/src/index.ts b/modules/i3s/src/index.ts index c50f71ed3f..78ea82b813 100644 --- a/modules/i3s/src/index.ts +++ b/modules/i3s/src/index.ts @@ -33,6 +33,7 @@ export type { OperationalLayer, TextureSetDefinitionFormats } from './types'; +export type {FileProvider} from './lib/parsers/parse-zip/file-provider'; export {COORDINATE_SYSTEM} from './lib/parsers/constants'; @@ -44,4 +45,4 @@ export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader'; export {I3SNodePageLoader} from './i3s-node-page-loader'; export {ArcGisWebSceneLoader} from './arcgis-webscene-loader'; export {parseZipLocalFileHeader} from './lib/parsers/parse-zip/local-file-header'; -export {FileProvider} from './lib/parsers/parse-zip/file-provider'; +export {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk'; From 8ba55eb8034e69650050207bafdf0c487ade4218 Mon Sep 17 00:00:00 2001 From: Daria Terekhova Date: Tue, 11 Jul 2023 12:57:03 +0300 Subject: [PATCH 2/6] Logic via FileHandler added Bigint logic implementation --- modules/i3s/src/i3s-slpk-loader.ts | 10 ++- .../src/lib/parsers/parse-slpk/parse-slpk.ts | 49 ++----------- .../parsers/parse-slpk/search-from-the-end.ts | 26 +++++++ .../lib/parsers/parse-slpk/slpk-archieve.ts | 21 +++--- .../parsers/parse-zip/buffer-file-provider.ts | 55 --------------- .../lib/parsers/parse-zip/cd-file-header.ts | 34 ++++----- .../parse-zip/data-view-file-provider.ts | 70 +++++++++++++++++++ .../lib/parsers/parse-zip/file-provider.ts | 16 +++-- .../parsers/parse-zip/local-file-header.ts | 24 ++++--- modules/i3s/test/cd-file-header.spec.js | 4 +- modules/i3s/test/local-file-header.spec.js | 4 +- .../i3s-server/controllers/slpk-controller.js | 15 ++-- modules/tile-converter/src/index.ts | 1 + .../helpers/file-handle-provider.ts | 48 +++++++++---- .../src/slpk-extractor/helpers/fs-promises.ts | 44 ++++++++++++ .../src/slpk-extractor/slpk-extractor.ts | 20 +++--- 16 files changed, 259 insertions(+), 182 deletions(-) create mode 100644 modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts delete mode 100644 modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts create mode 100644 modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts create mode 100644 modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts diff --git a/modules/i3s/src/i3s-slpk-loader.ts b/modules/i3s/src/i3s-slpk-loader.ts index 63216c431f..fe5fbfd62a 100644 --- a/modules/i3s/src/i3s-slpk-loader.ts +++ b/modules/i3s/src/i3s-slpk-loader.ts @@ -1,5 +1,6 @@ import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils'; -import {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk'; +import {parseSLPK as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk'; +import {DataViewFileProvider} from './lib/parsers/parse-zip/data-view-file-provider'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. @@ -12,6 +13,13 @@ export type SLPKLoaderOptions = LoaderOptions & { }; }; +async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { + return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile( + options.slpk?.path ?? '', + options.slpk?.pathMode + ); +} + /** * Loader for SLPK - Scene Layer Package */ diff --git a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts index 4ef1aa7cf6..4a9ac8d8c6 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts @@ -1,45 +1,13 @@ -import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader'; -import {DataViewFileProvider} from '../parse-zip/buffer-file-provider'; import {parseZipCDFileHeader} from '../parse-zip/cd-file-header'; +import {FileProvider} from '../parse-zip/file-provider'; import {parseZipLocalFileHeader} from '../parse-zip/local-file-header'; +import {searchFromTheEnd} from './search-from-the-end'; import {SLPKArchive} from './slpk-archieve'; -/** - * Returns one byte from the provided buffer at the provided position - * @param offset - position where to read - * @param buffer - buffer to read - * @returns one byte from the provided buffer at the provided position - */ -const getByteAt = (offset: number, buffer: DataView): number => { - return buffer.getUint8(buffer.byteOffset + offset); -}; - -export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { - const archive = new DataView(data); +export const parseSLPK = async (fileProvider: FileProvider): Promise => { const cdFileHeaderSignature = [80, 75, 1, 2]; - const searchWindow = [ - getByteAt(archive.byteLength - 1, archive), - getByteAt(archive.byteLength - 2, archive), - getByteAt(archive.byteLength - 3, archive), - undefined - ]; - - let hashCDOffset = 0; - - // looking for the last record in the central directory - for (let i = archive.byteLength - 4; i > -1; i--) { - searchWindow[3] = searchWindow[2]; - searchWindow[2] = searchWindow[1]; - searchWindow[1] = searchWindow[0]; - searchWindow[0] = getByteAt(i, archive); - if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) { - hashCDOffset = i; - break; - } - } - - const fileProvider = new DataViewFileProvider(archive); + const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature); const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider); @@ -56,7 +24,7 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = } const fileDataOffset = localFileHeader.fileDataOffset; - const hashFile = archive.buffer.slice( + const hashFile = await fileProvider.slice( fileDataOffset, fileDataOffset + localFileHeader.compressedSize ); @@ -65,8 +33,5 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = throw new Error('No hash file in slpk'); } - return await new SLPKArchive(data, hashFile).getFile( - options.slpk?.path ?? '', - options.slpk?.pathMode - ); -} + return new SLPKArchive(fileProvider, hashFile); +}; diff --git a/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts new file mode 100644 index 0000000000..7996a4a16d --- /dev/null +++ b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts @@ -0,0 +1,26 @@ +import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider'; + +export const searchFromTheEnd = async (file: FileProvider, target: number[]): Promise => { + const searchWindow = [ + await file.getUint8(file.length - 1n), + await file.getUint8(file.length - 2n), + await file.getUint8(file.length - 3n), + undefined + ]; + + let targetOffset = 0n; + + // looking for the last record in the central directory + for (let i = file.length - 4n; i > -1; i--) { + searchWindow[3] = searchWindow[2]; + searchWindow[2] = searchWindow[1]; + searchWindow[1] = searchWindow[0]; + searchWindow[0] = await file.getUint8(i); + if (searchWindow.every((val, index) => val === target[index])) { + targetOffset = i; + break; + } + } + + return targetOffset; +}; diff --git a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts index f689dccda2..e588e8dd09 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts @@ -1,7 +1,7 @@ import md5 from 'md5'; import {parseZipLocalFileHeader} from '../parse-zip/local-file-header'; -import {DataViewFileProvider} from '../parse-zip/buffer-file-provider'; import {GZipCompression} from '@loaders.gl/compression'; +import {FileProvider} from '../parse-zip/file-provider'; /** Element of hash array */ type HashElement = { @@ -12,7 +12,7 @@ type HashElement = { /** * File offset in the archive */ - offset: number; + offset: bigint; }; /** Description of real paths for different file types */ @@ -55,10 +55,10 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [ * Class for handling information about slpk file */ export class SLPKArchive { - slpkArchive: DataView; - hashArray: {hash: Buffer; offset: number}[]; - constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) { - this.slpkArchive = new DataView(slpkArchiveBuffer); + slpkArchive: FileProvider; + hashArray: {hash: Buffer; offset: bigint}[]; + constructor(slpkArchive: FileProvider, hashFile: ArrayBuffer) { + this.slpkArchive = slpkArchive; this.hashArray = this.parseHashFile(hashFile); } @@ -77,7 +77,7 @@ export class SLPKArchive { hashFileBuffer.byteOffset + i + 24 ) ); - const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true); + const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true); hashArray.push({ hash: Buffer.from( hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16) @@ -155,15 +155,12 @@ export class SLPKArchive { return undefined; } - const localFileHeader = await parseZipLocalFileHeader( - this.slpkArchive.byteOffset + fileInfo?.offset, - new DataViewFileProvider(this.slpkArchive) - ); + const localFileHeader = await parseZipLocalFileHeader(fileInfo?.offset, this.slpkArchive); if (!localFileHeader) { return undefined; } - const compressedFile = this.slpkArchive.buffer.slice( + const compressedFile = this.slpkArchive.slice( localFileHeader.fileDataOffset, localFileHeader.fileDataOffset + localFileHeader.compressedSize ); diff --git a/modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts deleted file mode 100644 index 4f1d048c85..0000000000 --- a/modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts +++ /dev/null @@ -1,55 +0,0 @@ -import {FileProvider} from './file-provider'; - -/** - * Provides file data using DataView - */ -export class DataViewFileProvider implements FileProvider { - /** - * The DataView from which data is provided - */ - private file: DataView; - - constructor(file: DataView) { - this.file = file; - } - - /** - * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. - * @param offset The offset, in bytes, from the start of the file where to read the data. - */ - getUint8(offset: number): Promise { - return Promise.resolve(this.file.getUint8(offset)); - } - - /** - * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file. - * @param offset The offset, in bytes, from the start of the file where to read the data. - */ - getUint16(offset: number): Promise { - return Promise.resolve(this.file.getUint16(offset, true)); - } - - /** - * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. - * @param offset The offset, in bytes, from the start of the file where to read the data. - */ - getUint32(offset: number): Promise { - return Promise.resolve(this.file.getUint32(offset, true)); - } - - /** - * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. - * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. - * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. - */ - slice(startOffset: number, endOffset: number): Promise { - return Promise.resolve(this.file.buffer.slice(startOffset, endOffset)); - } - - /** - * the length (in bytes) of the data. - */ - get length() { - return this.file.byteLength; - } -} diff --git a/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts b/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts index ba9f4083a1..8e7e1a0fc2 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts @@ -14,9 +14,9 @@ export type ZipCDFileHeader = { /** File name */ fileName: string; /** Extra field offset */ - extraOffset: number; + extraOffset: bigint; /** Relative offset of local file header */ - localHeaderOffset: number; + localHeaderOffset: bigint; }; /** @@ -26,16 +26,16 @@ export type ZipCDFileHeader = { * @returns Info from the header */ export const parseZipCDFileHeader = async ( - headerOffset: number, + headerOffset: bigint, buffer: FileProvider ): Promise => { const offsets = { - CD_COMPRESSED_SIZE_OFFSET: 20, - CD_UNCOMPRESSED_SIZE_OFFSET: 24, - CD_FILE_NAME_LENGTH_OFFSET: 28, - CD_EXTRA_FIELD_LENGTH_OFFSET: 30, - CD_LOCAL_HEADER_OFFSET_OFFSET: 42, - CD_FILE_NAME_OFFSET: 46 + CD_COMPRESSED_SIZE_OFFSET: 20n, + CD_UNCOMPRESSED_SIZE_OFFSET: 24n, + CD_FILE_NAME_LENGTH_OFFSET: 28n, + CD_EXTRA_FIELD_LENGTH_OFFSET: 30n, + CD_LOCAL_HEADER_OFFSET_OFFSET: 42n, + CD_FILE_NAME_OFFSET: 46n }; const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET); @@ -49,29 +49,29 @@ export const parseZipCDFileHeader = async ( const fileName = new TextDecoder().decode( await buffer.slice( headerOffset + offsets.CD_FILE_NAME_OFFSET, - headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength + headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength) ) ); - const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength; + const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength); const oldFormatOffset = await buffer.getUint32( headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET ); - let fileDataOffset = oldFormatOffset; - if (fileDataOffset === 0xffffffff) { - let offsetInZip64Data = 4; + let fileDataOffset = BigInt(oldFormatOffset); + if (fileDataOffset === BigInt(0xffffffff)) { + let offsetInZip64Data = 4n; // looking for info that might be also be in zip64 extra field if (compressedSize === 0xffffffff) { - offsetInZip64Data += 8; + offsetInZip64Data += 8n; } if (uncompressedSize === 0xffffffff) { - offsetInZip64Data += 8; + offsetInZip64Data += 8n; } // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb - fileDataOffset = await buffer.getUint32(extraOffset + offsetInZip64Data); // setting it to the one from zip64 + fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 } const localHeaderOffset = fileDataOffset; diff --git a/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts new file mode 100644 index 0000000000..66bc6bda7c --- /dev/null +++ b/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts @@ -0,0 +1,70 @@ +import {FileProvider} from './file-provider'; + +const toNumber = (bigint: bigint) => { + if (bigint > Number.MAX_SAFE_INTEGER) { + throw new Error('Offset is out of bounds'); + } + return Number(bigint); +}; + +/** + * Provides file data using DataView + */ +export class DataViewFileProvider implements FileProvider { + /** + * The DataView from which data is provided + */ + private file: DataView; + + constructor(file: DataView) { + this.file = file; + } + + /** + * Gets an unsigned 8-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + getUint8(offset: bigint): Promise { + return Promise.resolve(this.file.getUint8(toNumber(offset))); + } + + /** + * Gets an unsigned 16-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + getUint16(offset: bigint): Promise { + return Promise.resolve(this.file.getUint16(toNumber(offset), true)); + } + + /** + * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + getUint32(offset: bigint): Promise { + return Promise.resolve(this.file.getUint32(toNumber(offset), true)); + } + + /** + * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + getBigUint64(offset: bigint): Promise { + return Promise.resolve(this.file.getBigUint64(toNumber(offset), true)); + } + + /** + * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. + * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. + * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. + */ + slice(startOffset: bigint, endOffset: bigint): Promise { + return Promise.resolve(this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset))); + } + + /** + * the length (in bytes) of the data. + */ + get length() { + return BigInt(this.file.byteLength); + } +} diff --git a/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts index 4d0cebf9c7..656c75ad00 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts @@ -6,29 +6,35 @@ export interface FileProvider { * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getUint8(offset: number): Promise; + getUint8(offset: bigint): Promise; /** * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getUint16(offset: number): Promise; + getUint16(offset: bigint): Promise; /** * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the file of the view where to read the data. */ - getUint32(offset: number): Promise; + getUint32(offset: bigint): Promise; + + /** + * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * @param offset The offset, in byte, from the file of the view where to read the data. + */ + getBigUint64(offset: bigint): Promise; /** * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. */ - slice(startOffset: number, endOffset: number): Promise; + slice(startOffset: bigint, endOffset: bigint): Promise; /** * the length (in bytes) of the data. */ - length: number; + length: bigint; } diff --git a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts index 8e47b0f8c9..1528a8623f 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts @@ -12,16 +12,16 @@ export type ZipLocalFileHeader = { /** Extra field length */ extraFieldLength: number; /** Offset of the file data */ - fileDataOffset: number; + fileDataOffset: bigint; /** Compressed size */ - compressedSize: number; + compressedSize: bigint; }; const offsets = { - COMPRESSED_SIZE_OFFSET: 18, - FILE_NAME_LENGTH_OFFSET: 26, - EXTRA_FIELD_LENGTH_OFFSET: 28, - FILE_NAME_OFFSET: 30 + COMPRESSED_SIZE_OFFSET: 18n, + FILE_NAME_LENGTH_OFFSET: 26n, + EXTRA_FIELD_LENGTH_OFFSET: 28n, + FILE_NAME_OFFSET: 30n }; const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]); @@ -33,10 +33,10 @@ const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]); * @returns Info from the header */ export const parseZipLocalFileHeader = async ( - headerOffset: number, + headerOffset: bigint, buffer: FileProvider ): Promise => { - if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4)).compare(signature) !== 0) { + if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(signature) !== 0) { return Promise.resolve(undefined); } @@ -45,15 +45,17 @@ export const parseZipLocalFileHeader = async ( const fileName = new TextDecoder().decode( await buffer.slice( headerOffset + offsets.FILE_NAME_OFFSET, - headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength) ) ); const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET); const fileDataOffset = - headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength; + headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength); - const compressedSize = await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET); + const compressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET) + ); // add zip 64 logic return { fileNameLength, diff --git a/modules/i3s/test/cd-file-header.spec.js b/modules/i3s/test/cd-file-header.spec.js index dc5921333d..314e4e3f62 100644 --- a/modules/i3s/test/cd-file-header.spec.js +++ b/modules/i3s/test/cd-file-header.spec.js @@ -1,11 +1,11 @@ import test from 'tape-promise/tape'; import {DATA_ARRAY} from './data/test.zip.js'; import {parseZipCDFileHeader} from '../src/lib/parsers/parse-zip/cd-file-header.js'; -import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/buffer-file-provider.js'; +import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/data-view-file-provider.js'; test('SLPKLoader#central directory file header parse', async (t) => { const cdFileHeader = await parseZipCDFileHeader( - 78, + 78n, new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)) ); t.deepEqual(cdFileHeader.compressedSize, 39); diff --git a/modules/i3s/test/local-file-header.spec.js b/modules/i3s/test/local-file-header.spec.js index 1a4c5bba79..2df38d1815 100644 --- a/modules/i3s/test/local-file-header.spec.js +++ b/modules/i3s/test/local-file-header.spec.js @@ -1,11 +1,11 @@ import test from 'tape-promise/tape'; import {DATA_ARRAY} from './data/test.zip.js'; import {parseZipLocalFileHeader} from '../src/lib/parsers/parse-zip/local-file-header.js'; -import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/buffer-file-provider.js'; +import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/data-view-file-provider.js'; test('SLPKLoader#local file header parse', async (t) => { const localFileHeader = await parseZipLocalFileHeader( - 0, + 0n, new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)) ); t.deepEqual(localFileHeader?.compressedSize, 39); diff --git a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js index 590574dd29..6b2e9317da 100644 --- a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js +++ b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js @@ -1,12 +1,12 @@ require('@loaders.gl/polyfills'); -const {fetchFile, parse} = require('@loaders.gl/core'); -const {SLPKLoader} = require('@loaders.gl/i3s'); +const {parseSLPK} = require('@loaders.gl/i3s'); const path = require('path'); +const {FileHandleProvider} = require('@loaders.gl/tile-converter'); let slpkArchive; const loadArchive = async (fullLayerPath) => { - slpkArchive = await (await fetchFile(fullLayerPath)).arrayBuffer(); + slpkArchive = await parseSLPK(await FileHandleProvider.from(fullLayerPath)); }; const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef @@ -19,14 +19,7 @@ async function getFileByUrl(url) { let uncompressedFile; if (trimmedPath) { try { - uncompressedFile = Buffer.from( - await parse(slpkArchive, SLPKLoader, { - slpk: { - path: trimmedPath[1], - pathMode: 'http' - } - }) - ); + uncompressedFile = Buffer.from(await slpkArchive.getFile(trimmedPath[1], 'http')); } catch (e) {} } return uncompressedFile; diff --git a/modules/tile-converter/src/index.ts b/modules/tile-converter/src/index.ts index c2b412ee13..3deecad0e4 100644 --- a/modules/tile-converter/src/index.ts +++ b/modules/tile-converter/src/index.ts @@ -1,2 +1,3 @@ export {default as I3SConverter} from './i3s-converter/i3s-converter'; export {default as Tiles3DConverter} from './3d-tiles-converter/3d-tiles-converter'; +export {FileHandleProvider} from './slpk-extractor/helpers/file-handle-provider'; diff --git a/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts b/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts index 777a7ef942..201384d8a3 100644 --- a/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts +++ b/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts @@ -1,5 +1,5 @@ import {FileProvider} from '@loaders.gl/i3s'; -import {promises as fsPromises, PathLike} from 'fs'; +import {FileHandle} from './fs-promises'; /** * Provides file data using node fs library @@ -9,22 +9,22 @@ export class FileHandleProvider implements FileProvider { * Returns a new copy of FileHandleProvider * @param path The path to the file in file system */ - static async from(path: PathLike): Promise { - const fileDescriptor = await fsPromises.open(path); - return new FileHandleProvider(fileDescriptor, (await fileDescriptor.stat()).size); + static async from(path: string): Promise { + const fileDescriptor = await FileHandle.open(path); + return new FileHandleProvider(fileDescriptor, fileDescriptor.stat.size); } /** * The FileHandle from which data is provided */ - private fileDescriptor: fsPromises.FileHandle; + private fileDescriptor: FileHandle; /** * The file length in bytes */ - private size: number; + private size: bigint; - private constructor(fileDescriptor: fsPromises.FileHandle, size: number) { + private constructor(fileDescriptor: FileHandle, size: bigint) { this.fileDescriptor = fileDescriptor; this.size = size; } @@ -33,7 +33,7 @@ export class FileHandleProvider implements FileProvider { * Gets an unsigned 8-bit integer (unsigned byte) at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint8(offset: number): Promise { + async getUint8(offset: bigint): Promise { const val = new Uint8Array( (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer ).at(0); @@ -47,7 +47,7 @@ export class FileHandleProvider implements FileProvider { * Gets an unsigned 16-bit integer (unsigned byte) at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint16(offset: number): Promise { + async getUint16(offset: bigint): Promise { const val = new Uint16Array( (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer ).at(0); @@ -61,7 +61,7 @@ export class FileHandleProvider implements FileProvider { * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint32(offset: number): Promise { + async getUint32(offset: bigint): Promise { const val = new Uint32Array( (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer ).at(0); @@ -71,21 +71,39 @@ export class FileHandleProvider implements FileProvider { return val; } + /** + * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + async getBigUint64(offset: bigint): Promise { + const val = new BigInt64Array( + (await this.fileDescriptor.read(Buffer.alloc(8), 0, 4, offset)).buffer.buffer + ).at(0); + if (val === undefined) { + throw new Error('something went wrong'); + } + return val; + } + /** * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. - * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. + * @param startOffsset The offset, in byte, from the start of the file where to start reading the data. * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. */ - async slice(startOffset: number, endOffset: number): Promise { - const length = endOffset - startOffset; - return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffset)).buffer + async slice(startOffsset: bigint, endOffset: bigint): Promise { + const bigLength = endOffset - startOffsset; + if (bigLength > Number.MAX_SAFE_INTEGER) { + throw new Error('too big slice'); + } + const length = Number(bigLength); + return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer .buffer; } /** * the length (in bytes) of the data. */ - get length(): number { + get length(): bigint { return this.size; } } diff --git a/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts b/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts new file mode 100644 index 0000000000..d2e53fd865 --- /dev/null +++ b/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts @@ -0,0 +1,44 @@ +import {read, open, stat, BigIntStats} from 'fs'; + +export type FileReadResult = { + bytesRead: number; + buffer: Buffer; +}; + +export class FileHandle { + private fileDescriptor: number; + private stats: BigIntStats; + private constructor(fileDescriptor: number, stats: BigIntStats) { + this.fileDescriptor = fileDescriptor; + this.stats = stats; + } + + static open = async (path: string): Promise => { + const [fd, stats] = await Promise.all([ + new Promise((s) => { + open(path, undefined, undefined, (_err, fd) => s(fd)); + }), + new Promise((s) => { + stat(path, {bigint: true}, (_err, stats) => s(stats)); + }) + ]); + return new FileHandle(fd, stats); + }; + + read = ( + buffer: Buffer, + offset: number, + length: number, + position: number | bigint + ): Promise => { + return new Promise((s) => { + read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) => + s({bytesRead, buffer}) + ); + }); + }; + + get stat(): BigIntStats { + return this.stats; + } +} diff --git a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts index 897a3a48bb..6266585862 100644 --- a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts +++ b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts @@ -3,13 +3,11 @@ import {isBrowser} from '@loaders.gl/core'; import {BROWSER_ERROR_MESSAGE} from '../constants'; import {FileHandleProvider} from './helpers/file-handle-provider'; import {parseZipLocalFileHeader} from '@loaders.gl/i3s'; +import {promises as fsPromises, existsSync} from 'fs'; import {path} from '@loaders.gl/loader-utils'; import {GZipCompression} from '@loaders.gl/compression'; -import {writeFile} from '../lib/utils/file-utils'; +// import { writeFile } from '../lib/utils/file-utils'; -/** - * names of files that should be changed to index - */ const indexNames = [ '3dSceneLayer.json.gz', '3dNodeIndexDocument.json.gz', @@ -27,9 +25,9 @@ type File = { /** * Converter from slpk to i3s */ -export default class SLPKExtractor { +export default class SLPKConverter { /** - * extract slpk to i3s + * Convert slpk to i3s * @param options * @param options.inputUrl the url to read SLPK file * @param options.outputPath the output filename @@ -43,7 +41,7 @@ export default class SLPKExtractor { const provider = await FileHandleProvider.from(inputUrl); - let localHeader = await parseZipLocalFileHeader(0, provider); + let localHeader = await parseZipLocalFileHeader(0n, provider); while (localHeader) { await this.writeFile( await this.unGzip({ @@ -68,6 +66,7 @@ export default class SLPKExtractor { * Defines file name and path for i3s format * @param fileName initial file name and path */ + private correctIndexNames(fileName: string): string | null { if (indexNames.includes(path.filename(path.join('/', fileName)))) { return path.join(path.dirname(fileName), 'index.json.gz'); @@ -85,6 +84,7 @@ export default class SLPKExtractor { const compression = new GZipCompression(); const decompressedData = await compression.decompress(file.data); + return {data: decompressedData, name: (file.name ?? '').slice(0, -3)}; } return Promise.resolve(file); @@ -96,7 +96,9 @@ export default class SLPKExtractor { } const finalPath = path.join(outputPath, options.name); const dirName = path.dirname(finalPath); - const fileName = path.filename(finalPath); - await writeFile(dirName, options.data, fileName); + if (!existsSync(dirName)) { + await fsPromises.mkdir(dirName, {recursive: true}); + } + await fsPromises.writeFile(finalPath, Buffer.from(options.data)); } } From 3337ae2a69b864785b7bce27292a1bf65e917c1b Mon Sep 17 00:00:00 2001 From: Daria Terekhova Date: Wed, 12 Jul 2023 14:23:42 +0300 Subject: [PATCH 3/6] ZIP64 support added --- .../lib/parsers/parse-zip/cd-file-header.ts | 32 ++++++++-------- .../parsers/parse-zip/local-file-header.ts | 38 +++++++++++++++---- modules/tile-converter/src/i3s-server/app.js | 2 +- .../src/slpk-extractor/slpk-extractor.ts | 9 ++--- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts b/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts index 8e7e1a0fc2..b5febad67a 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts @@ -6,9 +6,9 @@ import {FileProvider} from './file-provider'; */ export type ZipCDFileHeader = { /** Compressed size */ - compressedSize: number; + compressedSize: bigint; /** Uncompressed size */ - uncompressedSize: number; + uncompressedSize: bigint; /** File name length */ fileNameLength: number; /** File name */ @@ -38,10 +38,12 @@ export const parseZipCDFileHeader = async ( CD_FILE_NAME_OFFSET: 46n }; - const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET); + let compressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET) + ); - const uncompressedSize = await buffer.getUint32( - headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET + let uncompressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET) ); const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET); @@ -60,17 +62,17 @@ export const parseZipCDFileHeader = async ( ); let fileDataOffset = BigInt(oldFormatOffset); + let offsetInZip64Data = 4n; + // looking for info that might be also be in zip64 extra field + if (uncompressedSize === BigInt(0xffffffff)) { + uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } + if (compressedSize === BigInt(0xffffffff)) { + compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } if (fileDataOffset === BigInt(0xffffffff)) { - let offsetInZip64Data = 4n; - // looking for info that might be also be in zip64 extra field - if (compressedSize === 0xffffffff) { - offsetInZip64Data += 8n; - } - if (uncompressedSize === 0xffffffff) { - offsetInZip64Data += 8n; - } - - // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 } const localHeaderOffset = fileDataOffset; diff --git a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts index 1528a8623f..c25bede77e 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts @@ -19,6 +19,7 @@ export type ZipLocalFileHeader = { const offsets = { COMPRESSED_SIZE_OFFSET: 18n, + UNCOMPRESSED_SIZE_OFFSET: 22n, FILE_NAME_LENGTH_OFFSET: 26n, EXTRA_FIELD_LENGTH_OFFSET: 28n, FILE_NAME_OFFSET: 30n @@ -42,21 +43,44 @@ export const parseZipLocalFileHeader = async ( const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET); - const fileName = new TextDecoder().decode( - await buffer.slice( - headerOffset + offsets.FILE_NAME_OFFSET, - headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength) + const fileName = new TextDecoder() + .decode( + await buffer.slice( + headerOffset + offsets.FILE_NAME_OFFSET, + headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength) + ) ) - ); + .split('\\') + .join('/'); const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET); - const fileDataOffset = + let fileDataOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength); - const compressedSize = BigInt( + let compressedSize = BigInt( await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET) ); // add zip 64 logic + let uncompressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.UNCOMPRESSED_SIZE_OFFSET) + ); // add zip 64 logic + + const extraOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength); + + let offsetInZip64Data = 4n; + // looking for info that might be also be in zip64 extra field + if (uncompressedSize === BigInt(0xffffffff)) { + uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } + if (compressedSize === BigInt(0xffffffff)) { + compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } + if (fileDataOffset === BigInt(0xffffffff)) { + fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 + } + return { fileNameLength, fileName, diff --git a/modules/tile-converter/src/i3s-server/app.js b/modules/tile-converter/src/i3s-server/app.js index aa65065d01..8ae6c97691 100644 --- a/modules/tile-converter/src/i3s-server/app.js +++ b/modules/tile-converter/src/i3s-server/app.js @@ -4,7 +4,6 @@ const logger = require('morgan'); const cors = require('cors'); const indexRouter = require('./routes/index'); -const {sceneServerRouter, router} = require('./routes/slpk-router'); const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef const app = express(); @@ -16,6 +15,7 @@ app.use(express.static(path.join(__dirname, 'public'))); app.use(cors()); if (/\.slpk$/.test(I3S_LAYER_PATH)) { + const {sceneServerRouter, router} = require('./routes/slpk-router'); app.use('/SceneServer/layers/0', router); app.use('/SceneServer', sceneServerRouter); } else { diff --git a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts index 6266585862..46dcca5ffb 100644 --- a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts +++ b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts @@ -3,10 +3,9 @@ import {isBrowser} from '@loaders.gl/core'; import {BROWSER_ERROR_MESSAGE} from '../constants'; import {FileHandleProvider} from './helpers/file-handle-provider'; import {parseZipLocalFileHeader} from '@loaders.gl/i3s'; -import {promises as fsPromises, existsSync} from 'fs'; import {path} from '@loaders.gl/loader-utils'; import {GZipCompression} from '@loaders.gl/compression'; -// import { writeFile } from '../lib/utils/file-utils'; +import {writeFile} from '../lib/utils/file-utils'; const indexNames = [ '3dSceneLayer.json.gz', @@ -96,9 +95,7 @@ export default class SLPKConverter { } const finalPath = path.join(outputPath, options.name); const dirName = path.dirname(finalPath); - if (!existsSync(dirName)) { - await fsPromises.mkdir(dirName, {recursive: true}); - } - await fsPromises.writeFile(finalPath, Buffer.from(options.data)); + const fileName = path.filename(finalPath); + await writeFile(dirName, options.data, fileName); } } From 2ca05a49cd8205385cef3b30321d31c6ea57b92e Mon Sep 17 00:00:00 2001 From: Daria Terekhova Date: Tue, 18 Jul 2023 22:19:13 +0300 Subject: [PATCH 4/6] Docs and tests added --- modules/i3s/src/i3s-slpk-loader.ts | 24 +++++++--- .../src/lib/parsers/parse-slpk/parse-slpk.ts | 9 +++- .../parsers/parse-slpk/search-from-the-end.ts | 12 ++++- .../parse-zip/data-view-file-provider.ts | 43 +++++++++-------- .../lib/parsers/parse-zip/file-provider.ts | 2 +- .../parsers/parse-zip/local-file-header.ts | 2 +- .../src/lib/parsers/parse-zip/signature.ts | 2 + modules/i3s/test/index.js | 4 ++ modules/i3s/test/search-from-the-end.spec.js | 15 ++++++ .../{ => zip-utils}/cd-file-header.spec.js | 10 ++-- .../test/zip-utils/data-view-provider.spec.js | 34 ++++++++++++++ .../{ => zip-utils}/local-file-header.spec.js | 8 ++-- .../helpers/file-handle-provider.ts | 8 ++-- .../src/slpk-extractor/helpers/fs-promises.ts | 22 +++++++++ .../src/slpk-extractor/slpk-extractor.ts | 2 +- modules/tile-converter/test/index.js | 2 + .../file-handle-provider.spec.js | 47 +++++++++++++++++++ .../test/slpk-extractor/file-handle.spec.js | 20 ++++++++ 18 files changed, 219 insertions(+), 47 deletions(-) create mode 100644 modules/i3s/src/lib/parsers/parse-zip/signature.ts create mode 100644 modules/i3s/test/search-from-the-end.spec.js rename modules/i3s/test/{ => zip-utils}/cd-file-header.spec.js (51%) create mode 100644 modules/i3s/test/zip-utils/data-view-provider.spec.js rename modules/i3s/test/{ => zip-utils}/local-file-header.spec.js (50%) create mode 100644 modules/tile-converter/test/slpk-extractor/file-handle-provider.spec.js create mode 100644 modules/tile-converter/test/slpk-extractor/file-handle.spec.js diff --git a/modules/i3s/src/i3s-slpk-loader.ts b/modules/i3s/src/i3s-slpk-loader.ts index fe5fbfd62a..e2bdf39865 100644 --- a/modules/i3s/src/i3s-slpk-loader.ts +++ b/modules/i3s/src/i3s-slpk-loader.ts @@ -6,20 +6,16 @@ import {DataViewFileProvider} from './lib/parsers/parse-zip/data-view-file-provi // @ts-ignore TS2304: Cannot find name '__VERSION__'. const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; +/** options to load data from SLPK */ export type SLPKLoaderOptions = LoaderOptions & { slpk?: { + /** path inside the slpk archive */ path?: string; + /** mode of the path */ pathMode?: 'http' | 'raw'; }; }; -async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { - return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile( - options.slpk?.path ?? '', - options.slpk?.pathMode - ); -} - /** * Loader for SLPK - Scene Layer Package */ @@ -33,3 +29,17 @@ export const SLPKLoader: LoaderWithParser = { extensions: ['slpk'], options: {} }; + +/** + * returns a single file from the slpk archive + * @param data slpk archive data + * @param options options + * @returns requested file + */ + +async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { + return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile( + options.slpk?.path ?? '', + options.slpk?.pathMode + ); +} diff --git a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts index 4a9ac8d8c6..c36ddcb507 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts @@ -1,11 +1,18 @@ import {parseZipCDFileHeader} from '../parse-zip/cd-file-header'; import {FileProvider} from '../parse-zip/file-provider'; import {parseZipLocalFileHeader} from '../parse-zip/local-file-header'; +import {ZipSignature} from '../parse-zip/signature'; import {searchFromTheEnd} from './search-from-the-end'; import {SLPKArchive} from './slpk-archieve'; +/** + * Creates slpk file handler from raw file + * @param fileProvider raw file data + * @returns slpk file handler + */ + export const parseSLPK = async (fileProvider: FileProvider): Promise => { - const cdFileHeaderSignature = [80, 75, 1, 2]; + const cdFileHeaderSignature: ZipSignature = [0x50, 0x4b, 0x01, 0x02]; const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature); diff --git a/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts index 7996a4a16d..b370cdb51b 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts @@ -1,6 +1,16 @@ import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider'; +import {ZipSignature} from '../parse-zip/signature'; -export const searchFromTheEnd = async (file: FileProvider, target: number[]): Promise => { +/** + * looking for the last occurrence of the provided + * @param file + * @param target + * @returns + */ +export const searchFromTheEnd = async ( + file: FileProvider, + target: ZipSignature +): Promise => { const searchWindow = [ await file.getUint8(file.length - 1n), await file.getUint8(file.length - 2n), diff --git a/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts index 66bc6bda7c..c0a18508f1 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts @@ -1,5 +1,10 @@ import {FileProvider} from './file-provider'; +/** + * Checks if bigint can be converted to number and convert it if possible + * @param bigint bigint to be converted + * @returns number + */ const toNumber = (bigint: bigint) => { if (bigint > Number.MAX_SAFE_INTEGER) { throw new Error('Offset is out of bounds'); @@ -7,13 +12,9 @@ const toNumber = (bigint: bigint) => { return Number(bigint); }; -/** - * Provides file data using DataView - */ +/** Provides file data using DataView */ export class DataViewFileProvider implements FileProvider { - /** - * The DataView from which data is provided - */ + /** The DataView from which data is provided */ private file: DataView; constructor(file: DataView) { @@ -21,35 +22,35 @@ export class DataViewFileProvider implements FileProvider { } /** - * Gets an unsigned 8-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getUint8(offset: bigint): Promise { - return Promise.resolve(this.file.getUint8(toNumber(offset))); + async getUint8(offset: bigint): Promise { + return this.file.getUint8(toNumber(offset)); } /** - * Gets an unsigned 16-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 16-bit intege at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getUint16(offset: bigint): Promise { - return Promise.resolve(this.file.getUint16(toNumber(offset), true)); + async getUint16(offset: bigint): Promise { + return this.file.getUint16(toNumber(offset), true); } /** - * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getUint32(offset: bigint): Promise { - return Promise.resolve(this.file.getUint32(toNumber(offset), true)); + async getUint32(offset: bigint): Promise { + return this.file.getUint32(toNumber(offset), true); } /** * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getBigUint64(offset: bigint): Promise { - return Promise.resolve(this.file.getBigUint64(toNumber(offset), true)); + async getBigUint64(offset: bigint): Promise { + return this.file.getBigUint64(toNumber(offset), true); } /** @@ -57,13 +58,11 @@ export class DataViewFileProvider implements FileProvider { * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. */ - slice(startOffset: bigint, endOffset: bigint): Promise { - return Promise.resolve(this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset))); + async slice(startOffset: bigint, endOffset: bigint): Promise { + return this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset)); } - /** - * the length (in bytes) of the data. - */ + /** the length (in bytes) of the data. */ get length() { return BigInt(this.file.byteLength); } diff --git a/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts index 656c75ad00..5f5d2e0455 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts @@ -21,7 +21,7 @@ export interface FileProvider { getUint32(offset: bigint): Promise; /** - * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in byte, from the file of the view where to read the data. */ getBigUint64(offset: bigint): Promise; diff --git a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts index c25bede77e..9b3833d1e6 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts @@ -25,7 +25,7 @@ const offsets = { FILE_NAME_OFFSET: 30n }; -const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]); +export const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]); /** * Parses local file header of zip file diff --git a/modules/i3s/src/lib/parsers/parse-zip/signature.ts b/modules/i3s/src/lib/parsers/parse-zip/signature.ts new file mode 100644 index 0000000000..60b6c486b8 --- /dev/null +++ b/modules/i3s/src/lib/parsers/parse-zip/signature.ts @@ -0,0 +1,2 @@ +/** Description of zip signature type */ +export type ZipSignature = [number, number, number, number]; diff --git a/modules/i3s/test/index.js b/modules/i3s/test/index.js index 04a0e240b1..eec73ed793 100644 --- a/modules/i3s/test/index.js +++ b/modules/i3s/test/index.js @@ -8,3 +8,7 @@ import './i3s-attribute-loader.spec'; import './i3s-content-loader.spec'; import './i3s-building-scene-layer-loader.spec'; import './arcgis-webscene-loader.spec'; +import './zip-utils/data-view-provider.spec'; +import './zip-utils/cd-file-header.spec'; +import './zip-utils/local-file-header.spec'; +import './search-from-the-end.spec'; diff --git a/modules/i3s/test/search-from-the-end.spec.js b/modules/i3s/test/search-from-the-end.spec.js new file mode 100644 index 0000000000..20e886d95d --- /dev/null +++ b/modules/i3s/test/search-from-the-end.spec.js @@ -0,0 +1,15 @@ +import test from 'tape-promise/tape'; +import {DATA_ARRAY} from './data/test.zip.js'; +import {searchFromTheEnd} from '../src/lib/parsers/parse-slpk/search-from-the-end'; +import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/data-view-file-provider'; + +test('SLPKLoader#searchFromTheEnd', async (t) => { + t.equals( + await searchFromTheEnd( + new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)), + [0x50, 0x4b, 0x03, 0x04] + ), + 0n + ); + t.end(); +}); diff --git a/modules/i3s/test/cd-file-header.spec.js b/modules/i3s/test/zip-utils/cd-file-header.spec.js similarity index 51% rename from modules/i3s/test/cd-file-header.spec.js rename to modules/i3s/test/zip-utils/cd-file-header.spec.js index 314e4e3f62..c06945164a 100644 --- a/modules/i3s/test/cd-file-header.spec.js +++ b/modules/i3s/test/zip-utils/cd-file-header.spec.js @@ -1,16 +1,16 @@ import test from 'tape-promise/tape'; -import {DATA_ARRAY} from './data/test.zip.js'; -import {parseZipCDFileHeader} from '../src/lib/parsers/parse-zip/cd-file-header.js'; -import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/data-view-file-provider.js'; +import {DATA_ARRAY} from '../data/test.zip.js'; +import {parseZipCDFileHeader} from '../../src/lib/parsers/parse-zip/cd-file-header'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; test('SLPKLoader#central directory file header parse', async (t) => { const cdFileHeader = await parseZipCDFileHeader( 78n, new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)) ); - t.deepEqual(cdFileHeader.compressedSize, 39); + t.deepEqual(cdFileHeader.compressedSize, 39n); t.deepEqual(cdFileHeader.fileNameLength, 9); t.deepEqual(cdFileHeader.fileName, 'test.json'); - t.deepEqual(cdFileHeader.localHeaderOffset, 0); + t.deepEqual(cdFileHeader.localHeaderOffset, 0n); t.end(); }); diff --git a/modules/i3s/test/zip-utils/data-view-provider.spec.js b/modules/i3s/test/zip-utils/data-view-provider.spec.js new file mode 100644 index 0000000000..8739a5b094 --- /dev/null +++ b/modules/i3s/test/zip-utils/data-view-provider.spec.js @@ -0,0 +1,34 @@ +import test from 'tape-promise/tape'; +import {DATA_ARRAY} from '../data/test.zip.js'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; +import {signature} from '../../src/lib/parsers/parse-zip/local-file-header'; + +test('DataViewFileProvider#slice', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(Buffer.from(await provider.slice(0n, 4n)).compare(signature), 0); + t.end(); +}); + +test('DataViewFileProvider#getUint8', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getUint8(0n), 80); + t.end(); +}); + +test('DataViewFileProvider#local file header parse', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getUint16(0n), 19280); + t.end(); +}); + +test('DataViewFileProvider#local file header parse', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getUint32(0n), 67324752); + t.end(); +}); + +test('DataViewFileProvider#local file header parse', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getBigUint64(0n), 563035920091984n); + t.end(); +}); diff --git a/modules/i3s/test/local-file-header.spec.js b/modules/i3s/test/zip-utils/local-file-header.spec.js similarity index 50% rename from modules/i3s/test/local-file-header.spec.js rename to modules/i3s/test/zip-utils/local-file-header.spec.js index 2df38d1815..d3ba69ff38 100644 --- a/modules/i3s/test/local-file-header.spec.js +++ b/modules/i3s/test/zip-utils/local-file-header.spec.js @@ -1,14 +1,14 @@ import test from 'tape-promise/tape'; -import {DATA_ARRAY} from './data/test.zip.js'; -import {parseZipLocalFileHeader} from '../src/lib/parsers/parse-zip/local-file-header.js'; -import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/data-view-file-provider.js'; +import {DATA_ARRAY} from '../data/test.zip.js'; +import {parseZipLocalFileHeader} from '../../src/lib/parsers/parse-zip/local-file-header'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; test('SLPKLoader#local file header parse', async (t) => { const localFileHeader = await parseZipLocalFileHeader( 0n, new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)) ); - t.deepEqual(localFileHeader?.compressedSize, 39); + t.deepEqual(localFileHeader?.compressedSize, 39n); t.deepEqual(localFileHeader?.fileNameLength, 9); t.end(); }); diff --git a/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts b/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts index 201384d8a3..95c9de64a9 100644 --- a/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts +++ b/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts @@ -30,7 +30,7 @@ export class FileHandleProvider implements FileProvider { } /** - * Gets an unsigned 8-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ async getUint8(offset: bigint): Promise { @@ -44,7 +44,7 @@ export class FileHandleProvider implements FileProvider { } /** - * Gets an unsigned 16-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ async getUint16(offset: bigint): Promise { @@ -58,7 +58,7 @@ export class FileHandleProvider implements FileProvider { } /** - * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ async getUint32(offset: bigint): Promise { @@ -72,7 +72,7 @@ export class FileHandleProvider implements FileProvider { } /** - * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ async getBigUint64(offset: bigint): Promise { diff --git a/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts b/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts index d2e53fd865..785f583a11 100644 --- a/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts +++ b/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts @@ -1,10 +1,14 @@ import {read, open, stat, BigIntStats} from 'fs'; +/** file reading result */ export type FileReadResult = { + /** amount of the bytes read */ bytesRead: number; + /** the buffer filled with data from file*/ buffer: Buffer; }; +/** Object handling file info */ export class FileHandle { private fileDescriptor: number; private stats: BigIntStats; @@ -12,6 +16,12 @@ export class FileHandle { this.fileDescriptor = fileDescriptor; this.stats = stats; } + /** + * Opens a `FileHandle`. + * + * @param path path to the file + * @return Fulfills with a {FileHandle} object. + */ static open = async (path: string): Promise => { const [fd, stats] = await Promise.all([ @@ -25,6 +35,18 @@ export class FileHandle { return new FileHandle(fd, stats); }; + /** + * Reads data from the file and stores that in the given buffer. + * + * If the file is not modified concurrently, the end-of-file is reached when the + * number of bytes read is zero. + * @param buffer A buffer that will be filled with the file data read. + * @param offset The location in the buffer at which to start filling. + * @param length The number of bytes to read. + * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an + * integer, the current file position will remain unchanged. + * @return Fulfills upon success with a FileReadResult object + */ read = ( buffer: Buffer, offset: number, diff --git a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts index 46dcca5ffb..f19bd12174 100644 --- a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts +++ b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts @@ -26,7 +26,7 @@ type File = { */ export default class SLPKConverter { /** - * Convert slpk to i3s + * Extract slpk to i3s * @param options * @param options.inputUrl the url to read SLPK file * @param options.outputPath the output filename diff --git a/modules/tile-converter/test/index.js b/modules/tile-converter/test/index.js index aa5222a129..d70c66b0ac 100644 --- a/modules/tile-converter/test/index.js +++ b/modules/tile-converter/test/index.js @@ -13,6 +13,8 @@ import './i3s-converter/helpers/preprocess-3d-tiles.spec'; import './i3s-converter/i3s-converter.spec'; import './slpk-extractor/slpk-extractor.spec'; +import './slpk-extractor/file-handle-provider.spec'; +import './slpk-extractor/file-handle.spec'; import './utils/cli-utils.spec'; import './3d-tiles-converter/helpers/b3dm-converter.spec'; diff --git a/modules/tile-converter/test/slpk-extractor/file-handle-provider.spec.js b/modules/tile-converter/test/slpk-extractor/file-handle-provider.spec.js new file mode 100644 index 0000000000..1ccf3592c6 --- /dev/null +++ b/modules/tile-converter/test/slpk-extractor/file-handle-provider.spec.js @@ -0,0 +1,47 @@ +import test from 'tape-promise/tape'; +import {signature} from '../../../i3s/src/lib/parsers/parse-zip/local-file-header'; +import {FileHandleProvider} from '@loaders.gl/tile-converter'; +import {isBrowser} from '@loaders.gl/core'; + +const SLPKUrl = 'modules/i3s/test/data/DA12_subset.slpk'; + +test('FileHandleProvider#slice', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(Buffer.from(await provider.slice(0n, 4n)).compare(signature), 0); + } + t.end(); +}); + +test('FileHandleProvider#getUint8', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getUint8(0n), 80); + t.end(); + } + t.end(); +}); + +test('FileHandleProvider#local file header parse', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getUint16(0n), 19280); + } + t.end(); +}); + +test('FileHandleProvider#local file header parse', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getUint32(0n), 67324752); + } + t.end(); +}); + +test('FileHandleProvider#local file header parse', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getBigUint64(0n), 67324752n); + } + t.end(); +}); diff --git a/modules/tile-converter/test/slpk-extractor/file-handle.spec.js b/modules/tile-converter/test/slpk-extractor/file-handle.spec.js new file mode 100644 index 0000000000..15edda5d74 --- /dev/null +++ b/modules/tile-converter/test/slpk-extractor/file-handle.spec.js @@ -0,0 +1,20 @@ +import test from 'tape-promise/tape'; +import {isBrowser} from '@loaders.gl/core'; +import {FileHandle} from '../../src/slpk-extractor/helpers/fs-promises'; +import {promises as fsPromises} from 'fs'; + +const SLPKUrl = 'modules/i3s/test/data/DA12_subset.slpk'; + +test('FileHandle#open and read', async (t) => { + if (!isBrowser) { + const provider = await FileHandle.open(SLPKUrl); + const fsHandler = await fsPromises.open(SLPKUrl); + t.equals( + (await provider.read(Buffer.alloc(4), 0, 4, 1)).buffer.compare( + (await fsHandler.read(Buffer.alloc(4), 0, 4, 1)).buffer + ), + 0 + ); + } + t.end(); +}); From 2a4e08e843c4c41597035bfc8df9134163e25300 Mon Sep 17 00:00:00 2001 From: Daria Terekhova Date: Thu, 20 Jul 2023 13:02:32 +0300 Subject: [PATCH 5/6] Review fixes --- modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts | 3 +-- modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts | 4 +++- modules/i3s/src/lib/parsers/parse-zip/signature.ts | 2 -- ...{data-view-provider.spec.js => data-view-provider.spec.ts} | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 modules/i3s/src/lib/parsers/parse-zip/signature.ts rename modules/i3s/test/zip-utils/{data-view-provider.spec.js => data-view-provider.spec.ts} (96%) diff --git a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts index c36ddcb507..e99ee17705 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts @@ -1,8 +1,7 @@ import {parseZipCDFileHeader} from '../parse-zip/cd-file-header'; import {FileProvider} from '../parse-zip/file-provider'; import {parseZipLocalFileHeader} from '../parse-zip/local-file-header'; -import {ZipSignature} from '../parse-zip/signature'; -import {searchFromTheEnd} from './search-from-the-end'; +import {ZipSignature, searchFromTheEnd} from './search-from-the-end'; import {SLPKArchive} from './slpk-archieve'; /** diff --git a/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts index b370cdb51b..55ba569159 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts @@ -1,5 +1,7 @@ import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider'; -import {ZipSignature} from '../parse-zip/signature'; + +/** Description of zip signature type */ +export type ZipSignature = [number, number, number, number]; /** * looking for the last occurrence of the provided diff --git a/modules/i3s/src/lib/parsers/parse-zip/signature.ts b/modules/i3s/src/lib/parsers/parse-zip/signature.ts deleted file mode 100644 index 60b6c486b8..0000000000 --- a/modules/i3s/src/lib/parsers/parse-zip/signature.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Description of zip signature type */ -export type ZipSignature = [number, number, number, number]; diff --git a/modules/i3s/test/zip-utils/data-view-provider.spec.js b/modules/i3s/test/zip-utils/data-view-provider.spec.ts similarity index 96% rename from modules/i3s/test/zip-utils/data-view-provider.spec.js rename to modules/i3s/test/zip-utils/data-view-provider.spec.ts index 8739a5b094..82bae226c1 100644 --- a/modules/i3s/test/zip-utils/data-view-provider.spec.js +++ b/modules/i3s/test/zip-utils/data-view-provider.spec.ts @@ -1,7 +1,7 @@ import test from 'tape-promise/tape'; import {DATA_ARRAY} from '../data/test.zip.js'; -import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; -import {signature} from '../../src/lib/parsers/parse-zip/local-file-header'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider.js'; +import {signature} from '../../src/lib/parsers/parse-zip/local-file-header.js'; test('DataViewFileProvider#slice', async (t) => { const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); From 4b9cedadfb8e92813e605babec7e95ab200fa636 Mon Sep 17 00:00:00 2001 From: Daria Terekhova Date: Thu, 20 Jul 2023 15:24:42 +0300 Subject: [PATCH 6/6] Test fix --- modules/i3s/test/zip-utils/data-view-provider.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/i3s/test/zip-utils/data-view-provider.spec.ts b/modules/i3s/test/zip-utils/data-view-provider.spec.ts index 82bae226c1..8739a5b094 100644 --- a/modules/i3s/test/zip-utils/data-view-provider.spec.ts +++ b/modules/i3s/test/zip-utils/data-view-provider.spec.ts @@ -1,7 +1,7 @@ import test from 'tape-promise/tape'; import {DATA_ARRAY} from '../data/test.zip.js'; -import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider.js'; -import {signature} from '../../src/lib/parsers/parse-zip/local-file-header.js'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; +import {signature} from '../../src/lib/parsers/parse-zip/local-file-header'; test('DataViewFileProvider#slice', async (t) => { const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer));