chore(tile-converter): Support for SLPKs larger than 2 Gb

modules/i3s/src/i3s-slpk-loader.ts

@@ -1,13 +1,17 @@
 import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
-import {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';
+import {parseSLPK as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk';
+import {DataViewFileProvider} from './lib/parsers/parse-zip/data-view-file-provider';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+/** options to load data from SLPK */
 export type SLPKLoaderOptions = LoaderOptions & {
   slpk?: {
+    /** path inside the slpk archive */
     path?: string;
+    /** mode of the path */
     pathMode?: 'http' | 'raw';
   };
 };
@@ -25,3 +29,17 @@ export const SLPKLoader: LoaderWithParser<Buffer, never, SLPKLoaderOptions> = {
   extensions: ['slpk'],
   options: {}
 };
+
+/**
+ * returns a single file from the slpk archive
+ * @param data slpk archive data
+ * @param options options
+ * @returns requested file
+ */
+
+async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
+  return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile(
+    options.slpk?.path ?? '',
+    options.slpk?.pathMode
+  );
+}

modules/i3s/src/index.ts

@@ -33,6 +33,7 @@ export type {
   OperationalLayer,
   TextureSetDefinitionFormats
 } from './types';
+export type {FileProvider} from './lib/parsers/parse-zip/file-provider';
 
 export {COORDINATE_SYSTEM} from './lib/parsers/constants';
 
@@ -44,4 +45,4 @@ export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader';
 export {I3SNodePageLoader} from './i3s-node-page-loader';
 export {ArcGisWebSceneLoader} from './arcgis-webscene-loader';
 export {parseZipLocalFileHeader} from './lib/parsers/parse-zip/local-file-header';
-export {FileProvider} from './lib/parsers/parse-zip/file-provider';
+export {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';

modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -1,45 +1,19 @@
-import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
-import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
 import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
+import {FileProvider} from '../parse-zip/file-provider';
 import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
 import {SLPKArchive} from './slpk-archieve';
 
 /**
- * Returns one byte from the provided buffer at the provided position
- * @param offset - position where to read
- * @param buffer - buffer to read
- * @returns one byte from the provided buffer at the provided position
+ * Creates slpk file handler from raw file
+ * @param fileProvider raw file data
+ * @returns slpk file handler
  */
-const getByteAt = (offset: number, buffer: DataView): number => {
-  return buffer.getUint8(buffer.byteOffset + offset);
-};
-
-export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
-  const archive = new DataView(data);
-  const cdFileHeaderSignature = [80, 75, 1, 2];
 
-  const searchWindow = [
-    getByteAt(archive.byteLength - 1, archive),
-    getByteAt(archive.byteLength - 2, archive),
-    getByteAt(archive.byteLength - 3, archive),
-    undefined
-  ];
+export const parseSLPK = async (fileProvider: FileProvider): Promise<SLPKArchive> => {
+  const cdFileHeaderSignature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
 
-  let hashCDOffset = 0;
-
-  // looking for the last record in the central directory
-  for (let i = archive.byteLength - 4; i > -1; i--) {
-    searchWindow[3] = searchWindow[2];
-    searchWindow[2] = searchWindow[1];
-    searchWindow[1] = searchWindow[0];
-    searchWindow[0] = getByteAt(i, archive);
-    if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
-      hashCDOffset = i;
-      break;
-    }
-  }
-
-  const fileProvider = new DataViewFileProvider(archive);
+  const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature);
 
   const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
 
@@ -56,7 +30,7 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
   }
 
   const fileDataOffset = localFileHeader.fileDataOffset;
-  const hashFile = archive.buffer.slice(
+  const hashFile = await fileProvider.slice(
     fileDataOffset,
     fileDataOffset + localFileHeader.compressedSize
   );
@@ -65,8 +39,5 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
     throw new Error('No hash file in slpk');
   }
 
-  return await new SLPKArchive(data, hashFile).getFile(
-    options.slpk?.path ?? '',
-    options.slpk?.pathMode
-  );
-}
+  return new SLPKArchive(fileProvider, hashFile);
+};

modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts

@@ -0,0 +1,38 @@
+import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider';
+
+/** Description of zip signature type */
+export type ZipSignature = [number, number, number, number];
+
+/**
+ * looking for the last occurrence of the provided
+ * @param file
+ * @param target
+ * @returns
+ */
+export const searchFromTheEnd = async (
+  file: FileProvider,
+  target: ZipSignature
+): Promise<bigint> => {
+  const searchWindow = [
+    await file.getUint8(file.length - 1n),
+    await file.getUint8(file.length - 2n),
+    await file.getUint8(file.length - 3n),
+    undefined
+  ];
+
+  let targetOffset = 0n;
+
+  // looking for the last record in the central directory
+  for (let i = file.length - 4n; i > -1; i--) {
+    searchWindow[3] = searchWindow[2];
+    searchWindow[2] = searchWindow[1];
+    searchWindow[1] = searchWindow[0];
+    searchWindow[0] = await file.getUint8(i);
+    if (searchWindow.every((val, index) => val === target[index])) {
+      targetOffset = i;
+      break;
+    }
+  }
+
+  return targetOffset;
+};

modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -1,7 +1,7 @@
 import md5 from 'md5';
 import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
-import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
 import {GZipCompression} from '@loaders.gl/compression';
+import {FileProvider} from '../parse-zip/file-provider';
 
 /** Element of hash array */
 type HashElement = {
@@ -12,7 +12,7 @@ type HashElement = {
   /**
    * File offset in the archive
    */
-  offset: number;
+  offset: bigint;
 };
 
 /** Description of real paths for different file types */
@@ -55,10 +55,10 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
  * Class for handling information about slpk file
  */
 export class SLPKArchive {
-  slpkArchive: DataView;
-  hashArray: {hash: Buffer; offset: number}[];
-  constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) {
-    this.slpkArchive = new DataView(slpkArchiveBuffer);
+  slpkArchive: FileProvider;
+  hashArray: {hash: Buffer; offset: bigint}[];
+  constructor(slpkArchive: FileProvider, hashFile: ArrayBuffer) {
+    this.slpkArchive = slpkArchive;
     this.hashArray = this.parseHashFile(hashFile);
   }
 
@@ -77,7 +77,7 @@ export class SLPKArchive {
           hashFileBuffer.byteOffset + i + 24
         )
       );
-      const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true);
+      const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
       hashArray.push({
         hash: Buffer.from(
           hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
@@ -155,15 +155,12 @@ export class SLPKArchive {
       return undefined;
     }
 
-    const localFileHeader = await parseZipLocalFileHeader(
-      this.slpkArchive.byteOffset + fileInfo?.offset,
-      new DataViewFileProvider(this.slpkArchive)
-    );
+    const localFileHeader = await parseZipLocalFileHeader(fileInfo?.offset, this.slpkArchive);
     if (!localFileHeader) {
       return undefined;
     }
 
-    const compressedFile = this.slpkArchive.buffer.slice(
+    const compressedFile = this.slpkArchive.slice(
       localFileHeader.fileDataOffset,
       localFileHeader.fileDataOffset + localFileHeader.compressedSize
     );

modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts

@@ -6,17 +6,17 @@ import {FileProvider} from './file-provider';
  */
 export type ZipCDFileHeader = {
   /** Compressed size */
-  compressedSize: number;
+  compressedSize: bigint;
   /** Uncompressed size */
-  uncompressedSize: number;
+  uncompressedSize: bigint;
   /** File name length */
   fileNameLength: number;
   /** File name */
   fileName: string;
   /** Extra field offset */
-  extraOffset: number;
+  extraOffset: bigint;
   /** Relative offset of local file header */
-  localHeaderOffset: number;
+  localHeaderOffset: bigint;
 };
 
 /**
@@ -26,52 +26,54 @@ export type ZipCDFileHeader = {
  * @returns Info from the header
  */
 export const parseZipCDFileHeader = async (
-  headerOffset: number,
+  headerOffset: bigint,
   buffer: FileProvider
 ): Promise<ZipCDFileHeader> => {
   const offsets = {
-    CD_COMPRESSED_SIZE_OFFSET: 20,
-    CD_UNCOMPRESSED_SIZE_OFFSET: 24,
-    CD_FILE_NAME_LENGTH_OFFSET: 28,
-    CD_EXTRA_FIELD_LENGTH_OFFSET: 30,
-    CD_LOCAL_HEADER_OFFSET_OFFSET: 42,
-    CD_FILE_NAME_OFFSET: 46
+    CD_COMPRESSED_SIZE_OFFSET: 20n,
+    CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
+    CD_FILE_NAME_LENGTH_OFFSET: 28n,
+    CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
+    CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
+    CD_FILE_NAME_OFFSET: 46n
   };
 
-  const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET);
+  let compressedSize = BigInt(
+    await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET)
+  );
 
-  const uncompressedSize = await buffer.getUint32(
-    headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET
+  let uncompressedSize = BigInt(
+    await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET)
   );
 
   const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);
 
   const fileName = new TextDecoder().decode(
     await buffer.slice(
       headerOffset + offsets.CD_FILE_NAME_OFFSET,
-      headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
+      headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
     )
   );
 
-  const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
+  const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength);
 
   const oldFormatOffset = await buffer.getUint32(
     headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET
   );
 
-  let fileDataOffset = oldFormatOffset;
-  if (fileDataOffset === 0xffffffff) {
-    let offsetInZip64Data = 4;
-    // looking for info that might be also be in zip64 extra field
-    if (compressedSize === 0xffffffff) {
-      offsetInZip64Data += 8;
-    }
-    if (uncompressedSize === 0xffffffff) {
-      offsetInZip64Data += 8;
-    }
-
-    // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb
-    fileDataOffset = await buffer.getUint32(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+  let fileDataOffset = BigInt(oldFormatOffset);
+  let offsetInZip64Data = 4n;
+  // looking for info that might be also be in zip64 extra field
+  if (uncompressedSize === BigInt(0xffffffff)) {
+    uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (compressedSize === BigInt(0xffffffff)) {
+    compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (fileDataOffset === BigInt(0xffffffff)) {
+    fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
   }
   const localHeaderOffset = fileDataOffset;