Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(tile-converter): Support for SLPKs larger than 2 Gb #2547

Merged
merged 6 commits into from
Jul 20, 2023
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion modules/i3s/src/i3s-slpk-loader.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
import {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';
import {parseSLPK as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk';
import {DataViewFileProvider} from './lib/parsers/parse-zip/data-view-file-provider';

// __VERSION__ is injected by babel-plugin-version-inline
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';

/** options to load data from SLPK */
export type SLPKLoaderOptions = LoaderOptions & {
slpk?: {
/** path inside the slpk archive */
path?: string;
/** mode of the path */
pathMode?: 'http' | 'raw';
};
};
Expand All @@ -25,3 +29,17 @@ export const SLPKLoader: LoaderWithParser<Buffer, never, SLPKLoaderOptions> = {
extensions: ['slpk'],
options: {}
};

/**
* returns a single file from the slpk archive
* @param data slpk archive data
* @param options options
* @returns requested file
*/

async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile(
options.slpk?.path ?? '',
options.slpk?.pathMode
);
}
3 changes: 2 additions & 1 deletion modules/i3s/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export type {
OperationalLayer,
TextureSetDefinitionFormats
} from './types';
export type {FileProvider} from './lib/parsers/parse-zip/file-provider';

export {COORDINATE_SYSTEM} from './lib/parsers/constants';

Expand All @@ -44,4 +45,4 @@ export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader';
export {I3SNodePageLoader} from './i3s-node-page-loader';
export {ArcGisWebSceneLoader} from './arcgis-webscene-loader';
export {parseZipLocalFileHeader} from './lib/parsers/parse-zip/local-file-header';
export {FileProvider} from './lib/parsers/parse-zip/file-provider';
export {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';
52 changes: 12 additions & 40 deletions modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts
Original file line number Diff line number Diff line change
@@ -1,45 +1,20 @@
import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
import {FileProvider} from '../parse-zip/file-provider';
import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
import {ZipSignature} from '../parse-zip/signature';
import {searchFromTheEnd} from './search-from-the-end';
import {SLPKArchive} from './slpk-archieve';

/**
* Returns one byte from the provided buffer at the provided position
* @param offset - position where to read
* @param buffer - buffer to read
* @returns one byte from the provided buffer at the provided position
* Creates slpk file handler from raw file
* @param fileProvider raw file data
* @returns slpk file handler
*/
const getByteAt = (offset: number, buffer: DataView): number => {
return buffer.getUint8(buffer.byteOffset + offset);
};

export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
const archive = new DataView(data);
const cdFileHeaderSignature = [80, 75, 1, 2];

const searchWindow = [
getByteAt(archive.byteLength - 1, archive),
getByteAt(archive.byteLength - 2, archive),
getByteAt(archive.byteLength - 3, archive),
undefined
];
export const parseSLPK = async (fileProvider: FileProvider): Promise<SLPKArchive> => {
const cdFileHeaderSignature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];

let hashCDOffset = 0;

// looking for the last record in the central directory
for (let i = archive.byteLength - 4; i > -1; i--) {
searchWindow[3] = searchWindow[2];
searchWindow[2] = searchWindow[1];
searchWindow[1] = searchWindow[0];
searchWindow[0] = getByteAt(i, archive);
if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
hashCDOffset = i;
break;
}
}

const fileProvider = new DataViewFileProvider(archive);
const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature);

const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);

Expand All @@ -56,7 +31,7 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
}

const fileDataOffset = localFileHeader.fileDataOffset;
const hashFile = archive.buffer.slice(
const hashFile = await fileProvider.slice(
fileDataOffset,
fileDataOffset + localFileHeader.compressedSize
);
Expand All @@ -65,8 +40,5 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
throw new Error('No hash file in slpk');
}

return await new SLPKArchive(data, hashFile).getFile(
options.slpk?.path ?? '',
options.slpk?.pathMode
);
}
return new SLPKArchive(fileProvider, hashFile);
};
36 changes: 36 additions & 0 deletions modules/i3s/src/lib/parsers/parse-slpk/search-from-the-end.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider';
import {ZipSignature} from '../parse-zip/signature';

/**
* looking for the last occurrence of the provided
* @param file
* @param target
* @returns
*/
export const searchFromTheEnd = async (
file: FileProvider,
target: ZipSignature
): Promise<bigint> => {
const searchWindow = [
await file.getUint8(file.length - 1n),
await file.getUint8(file.length - 2n),
await file.getUint8(file.length - 3n),
undefined
];

let targetOffset = 0n;

// looking for the last record in the central directory
for (let i = file.length - 4n; i > -1; i--) {
searchWindow[3] = searchWindow[2];
searchWindow[2] = searchWindow[1];
searchWindow[1] = searchWindow[0];
searchWindow[0] = await file.getUint8(i);
if (searchWindow.every((val, index) => val === target[index])) {
targetOffset = i;
break;
}
}

return targetOffset;
};
21 changes: 9 additions & 12 deletions modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import md5 from 'md5';
import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
import {GZipCompression} from '@loaders.gl/compression';
import {FileProvider} from '../parse-zip/file-provider';

/** Element of hash array */
type HashElement = {
Expand All @@ -12,7 +12,7 @@ type HashElement = {
/**
* File offset in the archive
*/
offset: number;
offset: bigint;
};

/** Description of real paths for different file types */
Expand Down Expand Up @@ -55,10 +55,10 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
* Class for handling information about slpk file
*/
export class SLPKArchive {
slpkArchive: DataView;
hashArray: {hash: Buffer; offset: number}[];
constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) {
this.slpkArchive = new DataView(slpkArchiveBuffer);
slpkArchive: FileProvider;
hashArray: {hash: Buffer; offset: bigint}[];
constructor(slpkArchive: FileProvider, hashFile: ArrayBuffer) {
this.slpkArchive = slpkArchive;
this.hashArray = this.parseHashFile(hashFile);
}

Expand All @@ -77,7 +77,7 @@ export class SLPKArchive {
hashFileBuffer.byteOffset + i + 24
)
);
const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true);
const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
hashArray.push({
hash: Buffer.from(
hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
Expand Down Expand Up @@ -155,15 +155,12 @@ export class SLPKArchive {
return undefined;
}

const localFileHeader = await parseZipLocalFileHeader(
this.slpkArchive.byteOffset + fileInfo?.offset,
new DataViewFileProvider(this.slpkArchive)
);
const localFileHeader = await parseZipLocalFileHeader(fileInfo?.offset, this.slpkArchive);
if (!localFileHeader) {
return undefined;
}

const compressedFile = this.slpkArchive.buffer.slice(
const compressedFile = this.slpkArchive.slice(
localFileHeader.fileDataOffset,
localFileHeader.fileDataOffset + localFileHeader.compressedSize
);
Expand Down
55 changes: 0 additions & 55 deletions modules/i3s/src/lib/parsers/parse-zip/buffer-file-provider.ts

This file was deleted.

60 changes: 31 additions & 29 deletions modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ import {FileProvider} from './file-provider';
*/
export type ZipCDFileHeader = {
/** Compressed size */
compressedSize: number;
compressedSize: bigint;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we really need bigints
image

/** Uncompressed size */
uncompressedSize: number;
uncompressedSize: bigint;
/** File name length */
fileNameLength: number;
/** File name */
fileName: string;
/** Extra field offset */
extraOffset: number;
extraOffset: bigint;
/** Relative offset of local file header */
localHeaderOffset: number;
localHeaderOffset: bigint;
};

/**
Expand All @@ -26,52 +26,54 @@ export type ZipCDFileHeader = {
* @returns Info from the header
*/
export const parseZipCDFileHeader = async (
headerOffset: number,
headerOffset: bigint,
buffer: FileProvider
): Promise<ZipCDFileHeader> => {
const offsets = {
CD_COMPRESSED_SIZE_OFFSET: 20,
CD_UNCOMPRESSED_SIZE_OFFSET: 24,
CD_FILE_NAME_LENGTH_OFFSET: 28,
CD_EXTRA_FIELD_LENGTH_OFFSET: 30,
CD_LOCAL_HEADER_OFFSET_OFFSET: 42,
CD_FILE_NAME_OFFSET: 46
CD_COMPRESSED_SIZE_OFFSET: 20n,
CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
CD_FILE_NAME_LENGTH_OFFSET: 28n,
CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
CD_FILE_NAME_OFFSET: 46n
};

const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET);
let compressedSize = BigInt(
await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET)
);

const uncompressedSize = await buffer.getUint32(
headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET
let uncompressedSize = BigInt(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please test BigInts on Safari / iOS. It should be supported now but better find out early

await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET)
);

const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);

const fileName = new TextDecoder().decode(
await buffer.slice(
headerOffset + offsets.CD_FILE_NAME_OFFSET,
headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
)
);

const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength);

const oldFormatOffset = await buffer.getUint32(
headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET
);

let fileDataOffset = oldFormatOffset;
if (fileDataOffset === 0xffffffff) {
let offsetInZip64Data = 4;
// looking for info that might be also be in zip64 extra field
if (compressedSize === 0xffffffff) {
offsetInZip64Data += 8;
}
if (uncompressedSize === 0xffffffff) {
offsetInZip64Data += 8;
}

// getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb
fileDataOffset = await buffer.getUint32(extraOffset + offsetInZip64Data); // setting it to the one from zip64
let fileDataOffset = BigInt(oldFormatOffset);
let offsetInZip64Data = 4n;
// looking for info that might be also be in zip64 extra field
if (uncompressedSize === BigInt(0xffffffff)) {
uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
offsetInZip64Data += 8n;
}
if (compressedSize === BigInt(0xffffffff)) {
compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
offsetInZip64Data += 8n;
}
if (fileDataOffset === BigInt(0xffffffff)) {
fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
}
const localHeaderOffset = fileDataOffset;

Expand Down
Loading