chore(tile-converter): Support for SLPKs larger than 2 Gb

modules/i3s/src/i3s-slpk-loader.ts

@@ -1,5 +1,6 @@
 import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
-import {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';
+import {parseSLPK as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk';
+import {DataViewFileProvider} from './lib/parsers/parse-zip/data-view-file-provider';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -12,6 +13,13 @@ export type SLPKLoaderOptions = LoaderOptions & {
   };
 };
 
+async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
+  return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile(
+    options.slpk?.path ?? '',
+    options.slpk?.pathMode
+  );
+}
+
 /**
  * Loader for SLPK - Scene Layer Package
  */

modules/i3s/src/index.ts

@@ -33,6 +33,7 @@ export type {
   OperationalLayer,
   TextureSetDefinitionFormats
 } from './types';
+export type {FileProvider} from './lib/parsers/parse-zip/file-provider';
 
 export {COORDINATE_SYSTEM} from './lib/parsers/constants';
 
@@ -44,4 +45,4 @@ export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader';
 export {I3SNodePageLoader} from './i3s-node-page-loader';
 export {ArcGisWebSceneLoader} from './arcgis-webscene-loader';
 export {parseZipLocalFileHeader} from './lib/parsers/parse-zip/local-file-header';
-export {FileProvider} from './lib/parsers/parse-zip/file-provider';
+export {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';

modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -1,45 +1,13 @@
-import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
-import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
 import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
+import {FileProvider} from '../parse-zip/file-provider';
 import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {searchFromTheEnd} from './search-from-the-end';
 import {SLPKArchive} from './slpk-archieve';
 
-/**
- * Returns one byte from the provided buffer at the provided position
- * @param offset - position where to read
- * @param buffer - buffer to read
- * @returns one byte from the provided buffer at the provided position
- */
-const getByteAt = (offset: number, buffer: DataView): number => {
-  return buffer.getUint8(buffer.byteOffset + offset);
-};
-
-export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
-  const archive = new DataView(data);
+export const parseSLPK = async (fileProvider: FileProvider): Promise<SLPKArchive> => {
   const cdFileHeaderSignature = [80, 75, 1, 2];
 
-  const searchWindow = [
-    getByteAt(archive.byteLength - 1, archive),
-    getByteAt(archive.byteLength - 2, archive),
-    getByteAt(archive.byteLength - 3, archive),
-    undefined
-  ];
-
-  let hashCDOffset = 0;
-
-  // looking for the last record in the central directory
-  for (let i = archive.byteLength - 4; i > -1; i--) {
-    searchWindow[3] = searchWindow[2];
-    searchWindow[2] = searchWindow[1];
-    searchWindow[1] = searchWindow[0];
-    searchWindow[0] = getByteAt(i, archive);
-    if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
-      hashCDOffset = i;
-      break;
-    }
-  }
-
-  const fileProvider = new DataViewFileProvider(archive);
+  const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature);
 
   const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
 
@@ -56,7 +24,7 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
   }
 
   const fileDataOffset = localFileHeader.fileDataOffset;
-  const hashFile = archive.buffer.slice(
+  const hashFile = await fileProvider.slice(
     fileDataOffset,
     fileDataOffset + localFileHeader.compressedSize
   );
@@ -65,8 +33,5 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
     throw new Error('No hash file in slpk');
   }
 
-  return await new SLPKArchive(data, hashFile).getFile(
-    options.slpk?.path ?? '',
-    options.slpk?.pathMode
-  );
-}
+  return new SLPKArchive(fileProvider, hashFile);
+};

modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts

@@ -0,0 +1,26 @@
+import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider';
+
+export const searchFromTheEnd = async (file: FileProvider, target: number[]): Promise<bigint> => {
+  const searchWindow = [
+    await file.getUint8(file.length - 1n),
+    await file.getUint8(file.length - 2n),
+    await file.getUint8(file.length - 3n),
+    undefined
+  ];
+
+  let targetOffset = 0n;
+
+  // looking for the last record in the central directory
+  for (let i = file.length - 4n; i > -1; i--) {
+    searchWindow[3] = searchWindow[2];
+    searchWindow[2] = searchWindow[1];
+    searchWindow[1] = searchWindow[0];
+    searchWindow[0] = await file.getUint8(i);
+    if (searchWindow.every((val, index) => val === target[index])) {
+      targetOffset = i;
+      break;
+    }
+  }
+
+  return targetOffset;
+};

modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -1,7 +1,7 @@
 import md5 from 'md5';
 import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
-import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
 import {GZipCompression} from '@loaders.gl/compression';
+import {FileProvider} from '../parse-zip/file-provider';
 
 /** Element of hash array */
 type HashElement = {
@@ -12,7 +12,7 @@ type HashElement = {
   /**
    * File offset in the archive
    */
-  offset: number;
+  offset: bigint;
 };
 
 /** Description of real paths for different file types */
@@ -55,10 +55,10 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
  * Class for handling information about slpk file
  */
 export class SLPKArchive {
-  slpkArchive: DataView;
-  hashArray: {hash: Buffer; offset: number}[];
-  constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) {
-    this.slpkArchive = new DataView(slpkArchiveBuffer);
+  slpkArchive: FileProvider;
+  hashArray: {hash: Buffer; offset: bigint}[];
+  constructor(slpkArchive: FileProvider, hashFile: ArrayBuffer) {
+    this.slpkArchive = slpkArchive;
     this.hashArray = this.parseHashFile(hashFile);
   }
 
@@ -77,7 +77,7 @@ export class SLPKArchive {
           hashFileBuffer.byteOffset + i + 24
         )
       );
-      const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true);
+      const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
       hashArray.push({
         hash: Buffer.from(
           hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
@@ -155,15 +155,12 @@ export class SLPKArchive {
       return undefined;
     }
 
-    const localFileHeader = await parseZipLocalFileHeader(
-      this.slpkArchive.byteOffset + fileInfo?.offset,
-      new DataViewFileProvider(this.slpkArchive)
-    );
+    const localFileHeader = await parseZipLocalFileHeader(fileInfo?.offset, this.slpkArchive);
     if (!localFileHeader) {
       return undefined;
     }
 
-    const compressedFile = this.slpkArchive.buffer.slice(
+    const compressedFile = this.slpkArchive.slice(
       localFileHeader.fileDataOffset,
       localFileHeader.fileDataOffset + localFileHeader.compressedSize
     );

modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts

@@ -6,17 +6,17 @@ import {FileProvider} from './file-provider';
  */
 export type ZipCDFileHeader = {
   /** Compressed size */
-  compressedSize: number;
+  compressedSize: bigint;
   /** Uncompressed size */
-  uncompressedSize: number;
+  uncompressedSize: bigint;
   /** File name length */
   fileNameLength: number;
   /** File name */
   fileName: string;
   /** Extra field offset */
-  extraOffset: number;
+  extraOffset: bigint;
   /** Relative offset of local file header */
-  localHeaderOffset: number;
+  localHeaderOffset: bigint;
 };
 
 /**
@@ -26,52 +26,54 @@ export type ZipCDFileHeader = {
  * @returns Info from the header
  */
 export const parseZipCDFileHeader = async (
-  headerOffset: number,
+  headerOffset: bigint,
   buffer: FileProvider
 ): Promise<ZipCDFileHeader> => {
   const offsets = {
-    CD_COMPRESSED_SIZE_OFFSET: 20,
-    CD_UNCOMPRESSED_SIZE_OFFSET: 24,
-    CD_FILE_NAME_LENGTH_OFFSET: 28,
-    CD_EXTRA_FIELD_LENGTH_OFFSET: 30,
-    CD_LOCAL_HEADER_OFFSET_OFFSET: 42,
-    CD_FILE_NAME_OFFSET: 46
+    CD_COMPRESSED_SIZE_OFFSET: 20n,
+    CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
+    CD_FILE_NAME_LENGTH_OFFSET: 28n,
+    CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
+    CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
+    CD_FILE_NAME_OFFSET: 46n
   };
 
-  const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET);
+  let compressedSize = BigInt(
+    await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET)
+  );
 
-  const uncompressedSize = await buffer.getUint32(
-    headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET
+  let uncompressedSize = BigInt(
+    await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET)
   );
 
   const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);
 
   const fileName = new TextDecoder().decode(
     await buffer.slice(
       headerOffset + offsets.CD_FILE_NAME_OFFSET,
-      headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
+      headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
     )
   );
 
-  const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
+  const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength);
 
   const oldFormatOffset = await buffer.getUint32(
     headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET
   );
 
-  let fileDataOffset = oldFormatOffset;
-  if (fileDataOffset === 0xffffffff) {
-    let offsetInZip64Data = 4;
-    // looking for info that might be also be in zip64 extra field
-    if (compressedSize === 0xffffffff) {
-      offsetInZip64Data += 8;
-    }
-    if (uncompressedSize === 0xffffffff) {
-      offsetInZip64Data += 8;
-    }
-
-    // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb
-    fileDataOffset = await buffer.getUint32(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+  let fileDataOffset = BigInt(oldFormatOffset);
+  let offsetInZip64Data = 4n;
+  // looking for info that might be also be in zip64 extra field
+  if (uncompressedSize === BigInt(0xffffffff)) {
+    uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (compressedSize === BigInt(0xffffffff)) {
+    compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (fileDataOffset === BigInt(0xffffffff)) {
+    fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
   }
   const localHeaderOffset = fileDataOffset;