diff --git a/modules/i3s/src/i3s-slpk-loader.ts b/modules/i3s/src/i3s-slpk-loader.ts index 63216c431f..e2bdf39865 100644 --- a/modules/i3s/src/i3s-slpk-loader.ts +++ b/modules/i3s/src/i3s-slpk-loader.ts @@ -1,13 +1,17 @@ import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils'; -import {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk'; +import {parseSLPK as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk'; +import {DataViewFileProvider} from './lib/parsers/parse-zip/data-view-file-provider'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; +/** options to load data from SLPK */ export type SLPKLoaderOptions = LoaderOptions & { slpk?: { + /** path inside the slpk archive */ path?: string; + /** mode of the path */ pathMode?: 'http' | 'raw'; }; }; @@ -25,3 +29,17 @@ export const SLPKLoader: LoaderWithParser = { extensions: ['slpk'], options: {} }; + +/** + * returns a single file from the slpk archive + * @param data slpk archive data + * @param options options + * @returns requested file + */ + +async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { + return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile( + options.slpk?.path ?? '', + options.slpk?.pathMode + ); +} diff --git a/modules/i3s/src/index.ts b/modules/i3s/src/index.ts index c50f71ed3f..78ea82b813 100644 --- a/modules/i3s/src/index.ts +++ b/modules/i3s/src/index.ts @@ -33,6 +33,7 @@ export type { OperationalLayer, TextureSetDefinitionFormats } from './types'; +export type {FileProvider} from './lib/parsers/parse-zip/file-provider'; export {COORDINATE_SYSTEM} from './lib/parsers/constants'; @@ -44,4 +45,4 @@ export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader'; export {I3SNodePageLoader} from './i3s-node-page-loader'; export {ArcGisWebSceneLoader} from './arcgis-webscene-loader'; export {parseZipLocalFileHeader} from './lib/parsers/parse-zip/local-file-header'; -export {FileProvider} from './lib/parsers/parse-zip/file-provider'; +export {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk'; diff --git a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts index 4ef1aa7cf6..e99ee17705 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts @@ -1,45 +1,19 @@ -import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader'; -import {DataViewFileProvider} from '../parse-zip/buffer-file-provider'; import {parseZipCDFileHeader} from '../parse-zip/cd-file-header'; +import {FileProvider} from '../parse-zip/file-provider'; import {parseZipLocalFileHeader} from '../parse-zip/local-file-header'; +import {ZipSignature, searchFromTheEnd} from './search-from-the-end'; import {SLPKArchive} from './slpk-archieve'; /** - * Returns one byte from the provided buffer at the provided position - * @param offset - position where to read - * @param buffer - buffer to read - * @returns one byte from the provided buffer at the provided position + * Creates slpk file handler from raw file + * @param fileProvider raw file data + * @returns slpk file handler */ -const getByteAt = (offset: number, buffer: DataView): number => { - return buffer.getUint8(buffer.byteOffset + offset); -}; - -export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { - const archive = new DataView(data); - const cdFileHeaderSignature = [80, 75, 1, 2]; - const searchWindow = [ - getByteAt(archive.byteLength - 1, archive), - getByteAt(archive.byteLength - 2, archive), - getByteAt(archive.byteLength - 3, archive), - undefined - ]; +export const parseSLPK = async (fileProvider: FileProvider): Promise => { + const cdFileHeaderSignature: ZipSignature = [0x50, 0x4b, 0x01, 0x02]; - let hashCDOffset = 0; - - // looking for the last record in the central directory - for (let i = archive.byteLength - 4; i > -1; i--) { - searchWindow[3] = searchWindow[2]; - searchWindow[2] = searchWindow[1]; - searchWindow[1] = searchWindow[0]; - searchWindow[0] = getByteAt(i, archive); - if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) { - hashCDOffset = i; - break; - } - } - - const fileProvider = new DataViewFileProvider(archive); + const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature); const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider); @@ -56,7 +30,7 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = } const fileDataOffset = localFileHeader.fileDataOffset; - const hashFile = archive.buffer.slice( + const hashFile = await fileProvider.slice( fileDataOffset, fileDataOffset + localFileHeader.compressedSize ); @@ -65,8 +39,5 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = throw new Error('No hash file in slpk'); } - return await new SLPKArchive(data, hashFile).getFile( - options.slpk?.path ?? '', - options.slpk?.pathMode - ); -} + return new SLPKArchive(fileProvider, hashFile); +}; diff --git a/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts new file mode 100644 index 0000000000..55ba569159 --- /dev/null +++ b/modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts @@ -0,0 +1,38 @@ +import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider'; + +/** Description of zip signature type */ +export type ZipSignature = [number, number, number, number]; + +/** + * looking for the last occurrence of the provided + * @param file + * @param target + * @returns + */ +export const searchFromTheEnd = async ( + file: FileProvider, + target: ZipSignature +): Promise => { + const searchWindow = [ + await file.getUint8(file.length - 1n), + await file.getUint8(file.length - 2n), + await file.getUint8(file.length - 3n), + undefined + ]; + + let targetOffset = 0n; + + // looking for the last record in the central directory + for (let i = file.length - 4n; i > -1; i--) { + searchWindow[3] = searchWindow[2]; + searchWindow[2] = searchWindow[1]; + searchWindow[1] = searchWindow[0]; + searchWindow[0] = await file.getUint8(i); + if (searchWindow.every((val, index) => val === target[index])) { + targetOffset = i; + break; + } + } + + return targetOffset; +}; diff --git a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts index f689dccda2..e588e8dd09 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts @@ -1,7 +1,7 @@ import md5 from 'md5'; import {parseZipLocalFileHeader} from '../parse-zip/local-file-header'; -import {DataViewFileProvider} from '../parse-zip/buffer-file-provider'; import {GZipCompression} from '@loaders.gl/compression'; +import {FileProvider} from '../parse-zip/file-provider'; /** Element of hash array */ type HashElement = { @@ -12,7 +12,7 @@ type HashElement = { /** * File offset in the archive */ - offset: number; + offset: bigint; }; /** Description of real paths for different file types */ @@ -55,10 +55,10 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [ * Class for handling information about slpk file */ export class SLPKArchive { - slpkArchive: DataView; - hashArray: {hash: Buffer; offset: number}[]; - constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) { - this.slpkArchive = new DataView(slpkArchiveBuffer); + slpkArchive: FileProvider; + hashArray: {hash: Buffer; offset: bigint}[]; + constructor(slpkArchive: FileProvider, hashFile: ArrayBuffer) { + this.slpkArchive = slpkArchive; this.hashArray = this.parseHashFile(hashFile); } @@ -77,7 +77,7 @@ export class SLPKArchive { hashFileBuffer.byteOffset + i + 24 ) ); - const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true); + const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true); hashArray.push({ hash: Buffer.from( hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16) @@ -155,15 +155,12 @@ export class SLPKArchive { return undefined; } - const localFileHeader = await parseZipLocalFileHeader( - this.slpkArchive.byteOffset + fileInfo?.offset, - new DataViewFileProvider(this.slpkArchive) - ); + const localFileHeader = await parseZipLocalFileHeader(fileInfo?.offset, this.slpkArchive); if (!localFileHeader) { return undefined; } - const compressedFile = this.slpkArchive.buffer.slice( + const compressedFile = this.slpkArchive.slice( localFileHeader.fileDataOffset, localFileHeader.fileDataOffset + localFileHeader.compressedSize ); diff --git a/modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts deleted file mode 100644 index 4f1d048c85..0000000000 --- a/modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts +++ /dev/null @@ -1,55 +0,0 @@ -import {FileProvider} from './file-provider'; - -/** - * Provides file data using DataView - */ -export class DataViewFileProvider implements FileProvider { - /** - * The DataView from which data is provided - */ - private file: DataView; - - constructor(file: DataView) { - this.file = file; - } - - /** - * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. - * @param offset The offset, in bytes, from the start of the file where to read the data. - */ - getUint8(offset: number): Promise { - return Promise.resolve(this.file.getUint8(offset)); - } - - /** - * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file. - * @param offset The offset, in bytes, from the start of the file where to read the data. - */ - getUint16(offset: number): Promise { - return Promise.resolve(this.file.getUint16(offset, true)); - } - - /** - * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. - * @param offset The offset, in bytes, from the start of the file where to read the data. - */ - getUint32(offset: number): Promise { - return Promise.resolve(this.file.getUint32(offset, true)); - } - - /** - * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. - * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. - * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. - */ - slice(startOffset: number, endOffset: number): Promise { - return Promise.resolve(this.file.buffer.slice(startOffset, endOffset)); - } - - /** - * the length (in bytes) of the data. - */ - get length() { - return this.file.byteLength; - } -} diff --git a/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts b/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts index ba9f4083a1..b5febad67a 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts @@ -6,17 +6,17 @@ import {FileProvider} from './file-provider'; */ export type ZipCDFileHeader = { /** Compressed size */ - compressedSize: number; + compressedSize: bigint; /** Uncompressed size */ - uncompressedSize: number; + uncompressedSize: bigint; /** File name length */ fileNameLength: number; /** File name */ fileName: string; /** Extra field offset */ - extraOffset: number; + extraOffset: bigint; /** Relative offset of local file header */ - localHeaderOffset: number; + localHeaderOffset: bigint; }; /** @@ -26,22 +26,24 @@ export type ZipCDFileHeader = { * @returns Info from the header */ export const parseZipCDFileHeader = async ( - headerOffset: number, + headerOffset: bigint, buffer: FileProvider ): Promise => { const offsets = { - CD_COMPRESSED_SIZE_OFFSET: 20, - CD_UNCOMPRESSED_SIZE_OFFSET: 24, - CD_FILE_NAME_LENGTH_OFFSET: 28, - CD_EXTRA_FIELD_LENGTH_OFFSET: 30, - CD_LOCAL_HEADER_OFFSET_OFFSET: 42, - CD_FILE_NAME_OFFSET: 46 + CD_COMPRESSED_SIZE_OFFSET: 20n, + CD_UNCOMPRESSED_SIZE_OFFSET: 24n, + CD_FILE_NAME_LENGTH_OFFSET: 28n, + CD_EXTRA_FIELD_LENGTH_OFFSET: 30n, + CD_LOCAL_HEADER_OFFSET_OFFSET: 42n, + CD_FILE_NAME_OFFSET: 46n }; - const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET); + let compressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET) + ); - const uncompressedSize = await buffer.getUint32( - headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET + let uncompressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET) ); const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET); @@ -49,29 +51,29 @@ export const parseZipCDFileHeader = async ( const fileName = new TextDecoder().decode( await buffer.slice( headerOffset + offsets.CD_FILE_NAME_OFFSET, - headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength + headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength) ) ); - const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength; + const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength); const oldFormatOffset = await buffer.getUint32( headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET ); - let fileDataOffset = oldFormatOffset; - if (fileDataOffset === 0xffffffff) { - let offsetInZip64Data = 4; - // looking for info that might be also be in zip64 extra field - if (compressedSize === 0xffffffff) { - offsetInZip64Data += 8; - } - if (uncompressedSize === 0xffffffff) { - offsetInZip64Data += 8; - } - - // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb - fileDataOffset = await buffer.getUint32(extraOffset + offsetInZip64Data); // setting it to the one from zip64 + let fileDataOffset = BigInt(oldFormatOffset); + let offsetInZip64Data = 4n; + // looking for info that might be also be in zip64 extra field + if (uncompressedSize === BigInt(0xffffffff)) { + uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } + if (compressedSize === BigInt(0xffffffff)) { + compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } + if (fileDataOffset === BigInt(0xffffffff)) { + fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 } const localHeaderOffset = fileDataOffset; diff --git a/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts new file mode 100644 index 0000000000..c0a18508f1 --- /dev/null +++ b/modules/i3s/src/lib/parsers/parse-zip/data-view-file-provider.ts @@ -0,0 +1,69 @@ +import {FileProvider} from './file-provider'; + +/** + * Checks if bigint can be converted to number and convert it if possible + * @param bigint bigint to be converted + * @returns number + */ +const toNumber = (bigint: bigint) => { + if (bigint > Number.MAX_SAFE_INTEGER) { + throw new Error('Offset is out of bounds'); + } + return Number(bigint); +}; + +/** Provides file data using DataView */ +export class DataViewFileProvider implements FileProvider { + /** The DataView from which data is provided */ + private file: DataView; + + constructor(file: DataView) { + this.file = file; + } + + /** + * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + async getUint8(offset: bigint): Promise { + return this.file.getUint8(toNumber(offset)); + } + + /** + * Gets an unsigned 16-bit intege at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + async getUint16(offset: bigint): Promise { + return this.file.getUint16(toNumber(offset), true); + } + + /** + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + async getUint32(offset: bigint): Promise { + return this.file.getUint32(toNumber(offset), true); + } + + /** + * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + async getBigUint64(offset: bigint): Promise { + return this.file.getBigUint64(toNumber(offset), true); + } + + /** + * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. + * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. + * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. + */ + async slice(startOffset: bigint, endOffset: bigint): Promise { + return this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset)); + } + + /** the length (in bytes) of the data. */ + get length() { + return BigInt(this.file.byteLength); + } +} diff --git a/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts b/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts index 4d0cebf9c7..5f5d2e0455 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/file-provider.ts @@ -6,29 +6,35 @@ export interface FileProvider { * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getUint8(offset: number): Promise; + getUint8(offset: bigint): Promise; /** * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - getUint16(offset: number): Promise; + getUint16(offset: bigint): Promise; /** * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the file of the view where to read the data. */ - getUint32(offset: number): Promise; + getUint32(offset: bigint): Promise; + + /** + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. + * @param offset The offset, in byte, from the file of the view where to read the data. + */ + getBigUint64(offset: bigint): Promise; /** * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. */ - slice(startOffset: number, endOffset: number): Promise; + slice(startOffset: bigint, endOffset: bigint): Promise; /** * the length (in bytes) of the data. */ - length: number; + length: bigint; } diff --git a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts index 8e47b0f8c9..9b3833d1e6 100644 --- a/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts +++ b/modules/i3s/src/lib/parsers/parse-zip/local-file-header.ts @@ -12,19 +12,20 @@ export type ZipLocalFileHeader = { /** Extra field length */ extraFieldLength: number; /** Offset of the file data */ - fileDataOffset: number; + fileDataOffset: bigint; /** Compressed size */ - compressedSize: number; + compressedSize: bigint; }; const offsets = { - COMPRESSED_SIZE_OFFSET: 18, - FILE_NAME_LENGTH_OFFSET: 26, - EXTRA_FIELD_LENGTH_OFFSET: 28, - FILE_NAME_OFFSET: 30 + COMPRESSED_SIZE_OFFSET: 18n, + UNCOMPRESSED_SIZE_OFFSET: 22n, + FILE_NAME_LENGTH_OFFSET: 26n, + EXTRA_FIELD_LENGTH_OFFSET: 28n, + FILE_NAME_OFFSET: 30n }; -const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]); +export const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]); /** * Parses local file header of zip file @@ -33,27 +34,52 @@ const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]); * @returns Info from the header */ export const parseZipLocalFileHeader = async ( - headerOffset: number, + headerOffset: bigint, buffer: FileProvider ): Promise => { - if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4)).compare(signature) !== 0) { + if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(signature) !== 0) { return Promise.resolve(undefined); } const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET); - const fileName = new TextDecoder().decode( - await buffer.slice( - headerOffset + offsets.FILE_NAME_OFFSET, - headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + const fileName = new TextDecoder() + .decode( + await buffer.slice( + headerOffset + offsets.FILE_NAME_OFFSET, + headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength) + ) ) - ); + .split('\\') + .join('/'); const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET); - const fileDataOffset = - headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength; + let fileDataOffset = + headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength); - const compressedSize = await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET); + let compressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET) + ); // add zip 64 logic + + let uncompressedSize = BigInt( + await buffer.getUint32(headerOffset + offsets.UNCOMPRESSED_SIZE_OFFSET) + ); // add zip 64 logic + + const extraOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength); + + let offsetInZip64Data = 4n; + // looking for info that might be also be in zip64 extra field + if (uncompressedSize === BigInt(0xffffffff)) { + uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } + if (compressedSize === BigInt(0xffffffff)) { + compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + offsetInZip64Data += 8n; + } + if (fileDataOffset === BigInt(0xffffffff)) { + fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 + } return { fileNameLength, diff --git a/modules/i3s/test/index.js b/modules/i3s/test/index.js index 04a0e240b1..eec73ed793 100644 --- a/modules/i3s/test/index.js +++ b/modules/i3s/test/index.js @@ -8,3 +8,7 @@ import './i3s-attribute-loader.spec'; import './i3s-content-loader.spec'; import './i3s-building-scene-layer-loader.spec'; import './arcgis-webscene-loader.spec'; +import './zip-utils/data-view-provider.spec'; +import './zip-utils/cd-file-header.spec'; +import './zip-utils/local-file-header.spec'; +import './search-from-the-end.spec'; diff --git a/modules/i3s/test/local-file-header.spec.js b/modules/i3s/test/local-file-header.spec.js deleted file mode 100644 index 1a4c5bba79..0000000000 --- a/modules/i3s/test/local-file-header.spec.js +++ /dev/null @@ -1,14 +0,0 @@ -import test from 'tape-promise/tape'; -import {DATA_ARRAY} from './data/test.zip.js'; -import {parseZipLocalFileHeader} from '../src/lib/parsers/parse-zip/local-file-header.js'; -import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/buffer-file-provider.js'; - -test('SLPKLoader#local file header parse', async (t) => { - const localFileHeader = await parseZipLocalFileHeader( - 0, - new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)) - ); - t.deepEqual(localFileHeader?.compressedSize, 39); - t.deepEqual(localFileHeader?.fileNameLength, 9); - t.end(); -}); diff --git a/modules/i3s/test/search-from-the-end.spec.js b/modules/i3s/test/search-from-the-end.spec.js new file mode 100644 index 0000000000..20e886d95d --- /dev/null +++ b/modules/i3s/test/search-from-the-end.spec.js @@ -0,0 +1,15 @@ +import test from 'tape-promise/tape'; +import {DATA_ARRAY} from './data/test.zip.js'; +import {searchFromTheEnd} from '../src/lib/parsers/parse-slpk/search-from-the-end'; +import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/data-view-file-provider'; + +test('SLPKLoader#searchFromTheEnd', async (t) => { + t.equals( + await searchFromTheEnd( + new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)), + [0x50, 0x4b, 0x03, 0x04] + ), + 0n + ); + t.end(); +}); diff --git a/modules/i3s/test/cd-file-header.spec.js b/modules/i3s/test/zip-utils/cd-file-header.spec.js similarity index 50% rename from modules/i3s/test/cd-file-header.spec.js rename to modules/i3s/test/zip-utils/cd-file-header.spec.js index dc5921333d..c06945164a 100644 --- a/modules/i3s/test/cd-file-header.spec.js +++ b/modules/i3s/test/zip-utils/cd-file-header.spec.js @@ -1,16 +1,16 @@ import test from 'tape-promise/tape'; -import {DATA_ARRAY} from './data/test.zip.js'; -import {parseZipCDFileHeader} from '../src/lib/parsers/parse-zip/cd-file-header.js'; -import {DataViewFileProvider} from '../src/lib/parsers/parse-zip/buffer-file-provider.js'; +import {DATA_ARRAY} from '../data/test.zip.js'; +import {parseZipCDFileHeader} from '../../src/lib/parsers/parse-zip/cd-file-header'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; test('SLPKLoader#central directory file header parse', async (t) => { const cdFileHeader = await parseZipCDFileHeader( - 78, + 78n, new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)) ); - t.deepEqual(cdFileHeader.compressedSize, 39); + t.deepEqual(cdFileHeader.compressedSize, 39n); t.deepEqual(cdFileHeader.fileNameLength, 9); t.deepEqual(cdFileHeader.fileName, 'test.json'); - t.deepEqual(cdFileHeader.localHeaderOffset, 0); + t.deepEqual(cdFileHeader.localHeaderOffset, 0n); t.end(); }); diff --git a/modules/i3s/test/zip-utils/data-view-provider.spec.ts b/modules/i3s/test/zip-utils/data-view-provider.spec.ts new file mode 100644 index 0000000000..8739a5b094 --- /dev/null +++ b/modules/i3s/test/zip-utils/data-view-provider.spec.ts @@ -0,0 +1,34 @@ +import test from 'tape-promise/tape'; +import {DATA_ARRAY} from '../data/test.zip.js'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; +import {signature} from '../../src/lib/parsers/parse-zip/local-file-header'; + +test('DataViewFileProvider#slice', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(Buffer.from(await provider.slice(0n, 4n)).compare(signature), 0); + t.end(); +}); + +test('DataViewFileProvider#getUint8', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getUint8(0n), 80); + t.end(); +}); + +test('DataViewFileProvider#local file header parse', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getUint16(0n), 19280); + t.end(); +}); + +test('DataViewFileProvider#local file header parse', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getUint32(0n), 67324752); + t.end(); +}); + +test('DataViewFileProvider#local file header parse', async (t) => { + const provider = new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)); + t.equals(await provider.getBigUint64(0n), 563035920091984n); + t.end(); +}); diff --git a/modules/i3s/test/zip-utils/local-file-header.spec.js b/modules/i3s/test/zip-utils/local-file-header.spec.js new file mode 100644 index 0000000000..d3ba69ff38 --- /dev/null +++ b/modules/i3s/test/zip-utils/local-file-header.spec.js @@ -0,0 +1,14 @@ +import test from 'tape-promise/tape'; +import {DATA_ARRAY} from '../data/test.zip.js'; +import {parseZipLocalFileHeader} from '../../src/lib/parsers/parse-zip/local-file-header'; +import {DataViewFileProvider} from '../../src/lib/parsers/parse-zip/data-view-file-provider'; + +test('SLPKLoader#local file header parse', async (t) => { + const localFileHeader = await parseZipLocalFileHeader( + 0n, + new DataViewFileProvider(new DataView(DATA_ARRAY.buffer)) + ); + t.deepEqual(localFileHeader?.compressedSize, 39n); + t.deepEqual(localFileHeader?.fileNameLength, 9); + t.end(); +}); diff --git a/modules/tile-converter/src/i3s-server/app.js b/modules/tile-converter/src/i3s-server/app.js index aa65065d01..8ae6c97691 100644 --- a/modules/tile-converter/src/i3s-server/app.js +++ b/modules/tile-converter/src/i3s-server/app.js @@ -4,7 +4,6 @@ const logger = require('morgan'); const cors = require('cors'); const indexRouter = require('./routes/index'); -const {sceneServerRouter, router} = require('./routes/slpk-router'); const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef const app = express(); @@ -16,6 +15,7 @@ app.use(express.static(path.join(__dirname, 'public'))); app.use(cors()); if (/\.slpk$/.test(I3S_LAYER_PATH)) { + const {sceneServerRouter, router} = require('./routes/slpk-router'); app.use('/SceneServer/layers/0', router); app.use('/SceneServer', sceneServerRouter); } else { diff --git a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js index 590574dd29..6b2e9317da 100644 --- a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js +++ b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.js @@ -1,12 +1,12 @@ require('@loaders.gl/polyfills'); -const {fetchFile, parse} = require('@loaders.gl/core'); -const {SLPKLoader} = require('@loaders.gl/i3s'); +const {parseSLPK} = require('@loaders.gl/i3s'); const path = require('path'); +const {FileHandleProvider} = require('@loaders.gl/tile-converter'); let slpkArchive; const loadArchive = async (fullLayerPath) => { - slpkArchive = await (await fetchFile(fullLayerPath)).arrayBuffer(); + slpkArchive = await parseSLPK(await FileHandleProvider.from(fullLayerPath)); }; const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef @@ -19,14 +19,7 @@ async function getFileByUrl(url) { let uncompressedFile; if (trimmedPath) { try { - uncompressedFile = Buffer.from( - await parse(slpkArchive, SLPKLoader, { - slpk: { - path: trimmedPath[1], - pathMode: 'http' - } - }) - ); + uncompressedFile = Buffer.from(await slpkArchive.getFile(trimmedPath[1], 'http')); } catch (e) {} } return uncompressedFile; diff --git a/modules/tile-converter/src/index.ts b/modules/tile-converter/src/index.ts index c2b412ee13..3deecad0e4 100644 --- a/modules/tile-converter/src/index.ts +++ b/modules/tile-converter/src/index.ts @@ -1,2 +1,3 @@ export {default as I3SConverter} from './i3s-converter/i3s-converter'; export {default as Tiles3DConverter} from './3d-tiles-converter/3d-tiles-converter'; +export {FileHandleProvider} from './slpk-extractor/helpers/file-handle-provider'; diff --git a/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts b/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts index 777a7ef942..95c9de64a9 100644 --- a/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts +++ b/modules/tile-converter/src/slpk-extractor/helpers/file-handle-provider.ts @@ -1,5 +1,5 @@ import {FileProvider} from '@loaders.gl/i3s'; -import {promises as fsPromises, PathLike} from 'fs'; +import {FileHandle} from './fs-promises'; /** * Provides file data using node fs library @@ -9,31 +9,31 @@ export class FileHandleProvider implements FileProvider { * Returns a new copy of FileHandleProvider * @param path The path to the file in file system */ - static async from(path: PathLike): Promise { - const fileDescriptor = await fsPromises.open(path); - return new FileHandleProvider(fileDescriptor, (await fileDescriptor.stat()).size); + static async from(path: string): Promise { + const fileDescriptor = await FileHandle.open(path); + return new FileHandleProvider(fileDescriptor, fileDescriptor.stat.size); } /** * The FileHandle from which data is provided */ - private fileDescriptor: fsPromises.FileHandle; + private fileDescriptor: FileHandle; /** * The file length in bytes */ - private size: number; + private size: bigint; - private constructor(fileDescriptor: fsPromises.FileHandle, size: number) { + private constructor(fileDescriptor: FileHandle, size: bigint) { this.fileDescriptor = fileDescriptor; this.size = size; } /** - * Gets an unsigned 8-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint8(offset: number): Promise { + async getUint8(offset: bigint): Promise { const val = new Uint8Array( (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer ).at(0); @@ -44,10 +44,10 @@ export class FileHandleProvider implements FileProvider { } /** - * Gets an unsigned 16-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint16(offset: number): Promise { + async getUint16(offset: bigint): Promise { const val = new Uint16Array( (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer ).at(0); @@ -58,10 +58,10 @@ export class FileHandleProvider implements FileProvider { } /** - * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file. + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint32(offset: number): Promise { + async getUint32(offset: bigint): Promise { const val = new Uint32Array( (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer ).at(0); @@ -71,21 +71,39 @@ export class FileHandleProvider implements FileProvider { return val; } + /** + * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. + * @param offset The offset, in bytes, from the start of the file where to read the data. + */ + async getBigUint64(offset: bigint): Promise { + const val = new BigInt64Array( + (await this.fileDescriptor.read(Buffer.alloc(8), 0, 4, offset)).buffer.buffer + ).at(0); + if (val === undefined) { + throw new Error('something went wrong'); + } + return val; + } + /** * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. - * @param startOffset The offset, in bytes, from the start of the file where to start reading the data. + * @param startOffsset The offset, in byte, from the start of the file where to start reading the data. * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. */ - async slice(startOffset: number, endOffset: number): Promise { - const length = endOffset - startOffset; - return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffset)).buffer + async slice(startOffsset: bigint, endOffset: bigint): Promise { + const bigLength = endOffset - startOffsset; + if (bigLength > Number.MAX_SAFE_INTEGER) { + throw new Error('too big slice'); + } + const length = Number(bigLength); + return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer .buffer; } /** * the length (in bytes) of the data. */ - get length(): number { + get length(): bigint { return this.size; } } diff --git a/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts b/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts new file mode 100644 index 0000000000..785f583a11 --- /dev/null +++ b/modules/tile-converter/src/slpk-extractor/helpers/fs-promises.ts @@ -0,0 +1,66 @@ +import {read, open, stat, BigIntStats} from 'fs'; + +/** file reading result */ +export type FileReadResult = { + /** amount of the bytes read */ + bytesRead: number; + /** the buffer filled with data from file*/ + buffer: Buffer; +}; + +/** Object handling file info */ +export class FileHandle { + private fileDescriptor: number; + private stats: BigIntStats; + private constructor(fileDescriptor: number, stats: BigIntStats) { + this.fileDescriptor = fileDescriptor; + this.stats = stats; + } + /** + * Opens a `FileHandle`. + * + * @param path path to the file + * @return Fulfills with a {FileHandle} object. + */ + + static open = async (path: string): Promise => { + const [fd, stats] = await Promise.all([ + new Promise((s) => { + open(path, undefined, undefined, (_err, fd) => s(fd)); + }), + new Promise((s) => { + stat(path, {bigint: true}, (_err, stats) => s(stats)); + }) + ]); + return new FileHandle(fd, stats); + }; + + /** + * Reads data from the file and stores that in the given buffer. + * + * If the file is not modified concurrently, the end-of-file is reached when the + * number of bytes read is zero. + * @param buffer A buffer that will be filled with the file data read. + * @param offset The location in the buffer at which to start filling. + * @param length The number of bytes to read. + * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an + * integer, the current file position will remain unchanged. + * @return Fulfills upon success with a FileReadResult object + */ + read = ( + buffer: Buffer, + offset: number, + length: number, + position: number | bigint + ): Promise => { + return new Promise((s) => { + read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) => + s({bytesRead, buffer}) + ); + }); + }; + + get stat(): BigIntStats { + return this.stats; + } +} diff --git a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts index 897a3a48bb..f19bd12174 100644 --- a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts +++ b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts @@ -7,9 +7,6 @@ import {path} from '@loaders.gl/loader-utils'; import {GZipCompression} from '@loaders.gl/compression'; import {writeFile} from '../lib/utils/file-utils'; -/** - * names of files that should be changed to index - */ const indexNames = [ '3dSceneLayer.json.gz', '3dNodeIndexDocument.json.gz', @@ -27,9 +24,9 @@ type File = { /** * Converter from slpk to i3s */ -export default class SLPKExtractor { +export default class SLPKConverter { /** - * extract slpk to i3s + * Extract slpk to i3s * @param options * @param options.inputUrl the url to read SLPK file * @param options.outputPath the output filename @@ -43,7 +40,7 @@ export default class SLPKExtractor { const provider = await FileHandleProvider.from(inputUrl); - let localHeader = await parseZipLocalFileHeader(0, provider); + let localHeader = await parseZipLocalFileHeader(0n, provider); while (localHeader) { await this.writeFile( await this.unGzip({ @@ -68,6 +65,7 @@ export default class SLPKExtractor { * Defines file name and path for i3s format * @param fileName initial file name and path */ + private correctIndexNames(fileName: string): string | null { if (indexNames.includes(path.filename(path.join('/', fileName)))) { return path.join(path.dirname(fileName), 'index.json.gz'); @@ -85,6 +83,7 @@ export default class SLPKExtractor { const compression = new GZipCompression(); const decompressedData = await compression.decompress(file.data); + return {data: decompressedData, name: (file.name ?? '').slice(0, -3)}; } return Promise.resolve(file); diff --git a/modules/tile-converter/test/index.js b/modules/tile-converter/test/index.js index aa5222a129..d70c66b0ac 100644 --- a/modules/tile-converter/test/index.js +++ b/modules/tile-converter/test/index.js @@ -13,6 +13,8 @@ import './i3s-converter/helpers/preprocess-3d-tiles.spec'; import './i3s-converter/i3s-converter.spec'; import './slpk-extractor/slpk-extractor.spec'; +import './slpk-extractor/file-handle-provider.spec'; +import './slpk-extractor/file-handle.spec'; import './utils/cli-utils.spec'; import './3d-tiles-converter/helpers/b3dm-converter.spec'; diff --git a/modules/tile-converter/test/slpk-extractor/file-handle-provider.spec.js b/modules/tile-converter/test/slpk-extractor/file-handle-provider.spec.js new file mode 100644 index 0000000000..1ccf3592c6 --- /dev/null +++ b/modules/tile-converter/test/slpk-extractor/file-handle-provider.spec.js @@ -0,0 +1,47 @@ +import test from 'tape-promise/tape'; +import {signature} from '../../../i3s/src/lib/parsers/parse-zip/local-file-header'; +import {FileHandleProvider} from '@loaders.gl/tile-converter'; +import {isBrowser} from '@loaders.gl/core'; + +const SLPKUrl = 'modules/i3s/test/data/DA12_subset.slpk'; + +test('FileHandleProvider#slice', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(Buffer.from(await provider.slice(0n, 4n)).compare(signature), 0); + } + t.end(); +}); + +test('FileHandleProvider#getUint8', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getUint8(0n), 80); + t.end(); + } + t.end(); +}); + +test('FileHandleProvider#local file header parse', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getUint16(0n), 19280); + } + t.end(); +}); + +test('FileHandleProvider#local file header parse', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getUint32(0n), 67324752); + } + t.end(); +}); + +test('FileHandleProvider#local file header parse', async (t) => { + if (!isBrowser) { + const provider = await FileHandleProvider.from(SLPKUrl); + t.equals(await provider.getBigUint64(0n), 67324752n); + } + t.end(); +}); diff --git a/modules/tile-converter/test/slpk-extractor/file-handle.spec.js b/modules/tile-converter/test/slpk-extractor/file-handle.spec.js new file mode 100644 index 0000000000..15edda5d74 --- /dev/null +++ b/modules/tile-converter/test/slpk-extractor/file-handle.spec.js @@ -0,0 +1,20 @@ +import test from 'tape-promise/tape'; +import {isBrowser} from '@loaders.gl/core'; +import {FileHandle} from '../../src/slpk-extractor/helpers/fs-promises'; +import {promises as fsPromises} from 'fs'; + +const SLPKUrl = 'modules/i3s/test/data/DA12_subset.slpk'; + +test('FileHandle#open and read', async (t) => { + if (!isBrowser) { + const provider = await FileHandle.open(SLPKUrl); + const fsHandler = await fsPromises.open(SLPKUrl); + t.equals( + (await provider.read(Buffer.alloc(4), 0, 4, 1)).buffer.compare( + (await fsHandler.read(Buffer.alloc(4), 0, 4, 1)).buffer + ), + 0 + ); + } + t.end(); +});