Skip to content

Commit

Permalink
chore(tile-converter): create SLPK hash during serve (#2565)
Browse files Browse the repository at this point in the history
  • Loading branch information
dariaterekhova-actionengine authored Jul 26, 2023
1 parent 0db5012 commit 5df9cb0
Show file tree
Hide file tree
Showing 15 changed files with 271 additions and 87 deletions.
104 changes: 79 additions & 25 deletions modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts
Original file line number Diff line number Diff line change
@@ -1,43 +1,97 @@
import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
import md5 from 'md5';
import {parseZipCDFileHeader, signature as cdHeaderSignature} from '../parse-zip/cd-file-header';
import {parseEoCDRecord} from '../parse-zip/end-of-central-directory';
import {FileProvider} from '../parse-zip/file-provider';
import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
import {SLPKArchive} from './slpk-archieve';
import {searchFromTheEnd} from '../parse-zip/search-from-the-end';
import {HashElement, SLPKArchive, compareHashes} from './slpk-archieve';

/**
* Creates slpk file handler from raw file
* @param fileProvider raw file data
* @param cb is called with information message during parsing
* @returns slpk file handler
*/

export const parseSLPK = async (fileProvider: FileProvider): Promise<SLPKArchive> => {
const cdFileHeaderSignature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];

const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature);
export const parseSLPK = async (
fileProvider: FileProvider,
cb?: (msg: string) => void
): Promise<SLPKArchive> => {
const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature);

const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);

if (cdFileHeader.fileName !== '@specialIndexFileHASH128@') {
throw new Error('No hash file in slpk');
}
let hashData: HashElement[];
if (cdFileHeader?.fileName !== '@specialIndexFileHASH128@') {
cb?.('SLPK doesnt contain hash file');
hashData = await generateHashInfo(fileProvider);
cb?.('hash info has been composed according to central directory records');
} else {
cb?.('SLPK contains hash file');
const localFileHeader = await parseZipLocalFileHeader(
cdFileHeader.localHeaderOffset,
fileProvider
);
if (!localFileHeader) {
throw new Error('corrupted SLPK');
}

const localFileHeader = await parseZipLocalFileHeader(
cdFileHeader.localHeaderOffset,
fileProvider
);
if (!localFileHeader) {
throw new Error('No hash file in slpk');
const fileDataOffset = localFileHeader.fileDataOffset;
const hashFile = await fileProvider.slice(
fileDataOffset,
fileDataOffset + localFileHeader.compressedSize
);

hashData = parseHashFile(hashFile);
}

const fileDataOffset = localFileHeader.fileDataOffset;
const hashFile = await fileProvider.slice(
fileDataOffset,
fileDataOffset + localFileHeader.compressedSize
);
return new SLPKArchive(fileProvider, hashData);
};

if (!hashFile) {
throw new Error('No hash file in slpk');
/**
* generates hash info from central directory
* @param fileProvider - provider of the archive
* @returns ready to use hash info
*/
const generateHashInfo = async (fileProvider: FileProvider): Promise<HashElement[]> => {
const {cdStartOffset} = await parseEoCDRecord(fileProvider);
let cdHeader = await parseZipCDFileHeader(cdStartOffset, fileProvider);
const hashInfo: HashElement[] = [];
while (cdHeader) {
hashInfo.push({
hash: Buffer.from(md5(cdHeader.fileName.split('\\').join('/').toLocaleLowerCase()), 'hex'),
offset: cdHeader.localHeaderOffset
});
cdHeader = await parseZipCDFileHeader(
cdHeader.extraOffset + BigInt(cdHeader.extraFieldLength),
fileProvider
);
}
hashInfo.sort((a, b) => compareHashes(a.hash, b.hash));
return hashInfo;
};

return new SLPKArchive(fileProvider, hashFile);
/**
* Reads hash file from buffer and returns it in ready-to-use form
* @param hashFile - bufer containing hash file
* @returns Array containing file info
*/
const parseHashFile = (hashFile: ArrayBuffer): HashElement[] => {
const hashFileBuffer = Buffer.from(hashFile);
const hashArray: HashElement[] = [];
for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
const offsetBuffer = new DataView(
hashFileBuffer.buffer.slice(
hashFileBuffer.byteOffset + i + 16,
hashFileBuffer.byteOffset + i + 24
)
);
const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
hashArray.push({
hash: Buffer.from(
hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
),
offset
});
}
return hashArray;
};
90 changes: 54 additions & 36 deletions modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,33 @@ import {GZipCompression} from '@loaders.gl/compression';
import {FileProvider} from '../parse-zip/file-provider';

/** Element of hash array */
type HashElement = {
/**
* File name hash
*/
export type HashElement = {
/** File name hash */
hash: Buffer;
/**
* File offset in the archive
*/
/** File offset in the archive */
offset: bigint;
};

/**
* Comparing md5 hashes according to https://github.com/Esri/i3s-spec/blob/master/docs/2.0/slpk_hashtable.pcsl.md step 5
* @param hash1 hash to compare
* @param hash2 hash to compare
* @returns -1 if hash1 < hash2, 0 of hash1 === hash2, 1 if hash1 > hash2
*/
export const compareHashes = (hash1: Buffer, hash2: Buffer): number => {
const h1 = new BigUint64Array(hash1.buffer, hash1.byteOffset, 2);
const h2 = new BigUint64Array(hash2.buffer, hash2.byteOffset, 2);

const diff = h1[0] === h2[0] ? h1[1] - h2[1] : h1[0] - h2[0];

if (diff < 0n) {
return -1;
} else if (diff === 0n) {
return 0;
}
return 1;
};

/** Description of real paths for different file types */
const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
{
Expand Down Expand Up @@ -55,38 +71,35 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
* Class for handling information about slpk file
*/
export class SLPKArchive {
slpkArchive: FileProvider;
hashArray: {hash: Buffer; offset: bigint}[];
constructor(slpkArchive: FileProvider, hashFile: ArrayBuffer) {
private slpkArchive: FileProvider;
private hashArray: HashElement[];
constructor(slpkArchive: FileProvider, hashFile: HashElement[]) {
this.slpkArchive = slpkArchive;
this.hashArray = this.parseHashFile(hashFile);
this.hashArray = hashFile;
}

/**
* Reads hash file from buffer and returns it in ready-to-use form
* @param hashFile - bufer containing hash file
* @returns Array containing file info
* Binary search in the hash info
* @param hashToSearch hash that we need to find
* @returns required hash element or undefined if not found
*/
private parseHashFile(hashFile: ArrayBuffer): HashElement[] {
const hashFileBuffer = Buffer.from(hashFile);
const hashArray: HashElement[] = [];
for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
const offsetBuffer = new DataView(
hashFileBuffer.buffer.slice(
hashFileBuffer.byteOffset + i + 16,
hashFileBuffer.byteOffset + i + 24
)
);
const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
hashArray.push({
hash: Buffer.from(
hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
),
offset
});
private findBin = (hashToSearch: Buffer): HashElement | undefined => {
let lowerBorder = 0;
let upperBorder = this.hashArray.length;

while (upperBorder - lowerBorder > 1) {
const middle = lowerBorder + Math.floor((upperBorder - lowerBorder) / 2);
const value = compareHashes(this.hashArray[middle].hash, hashToSearch);
if (value === 0) {
return this.hashArray[middle];
} else if (value < 0) {
lowerBorder = middle;
} else {
upperBorder = middle;
}
}
return hashArray;
}
return undefined;
};

/**
* Returns file with the given path from slpk archive
Expand Down Expand Up @@ -130,7 +143,12 @@ export class SLPKArchive {
* @returns buffer with the file data
*/
private async getDataByPath(path: string): Promise<ArrayBuffer | undefined> {
const data = await this.getFileBytes(path);
// sometimes paths are not in lower case when hash file is created,
// so first we're looking for lower case file name and then for original one
let data = await this.getFileBytes(path.toLocaleLowerCase());
if (!data) {
data = await this.getFileBytes(path);
}
if (!data) {
return undefined;
}
Expand All @@ -150,12 +168,12 @@ export class SLPKArchive {
*/
private async getFileBytes(path: string): Promise<ArrayBuffer | undefined> {
const nameHash = Buffer.from(md5(path), 'hex');
const fileInfo = this.hashArray.find((val) => Buffer.compare(val.hash, nameHash) === 0);
const fileInfo = this.findBin(nameHash); // implement binary search
if (!fileInfo) {
return undefined;
}

const localFileHeader = await parseZipLocalFileHeader(fileInfo?.offset, this.slpkArchive);
const localFileHeader = await parseZipLocalFileHeader(fileInfo.offset, this.slpkArchive);
if (!localFileHeader) {
return undefined;
}
Expand Down
36 changes: 27 additions & 9 deletions modules/i3s/src/lib/parsers/parse-zip/cd-file-header.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {FileProvider} from './file-provider';
import {ZipSignature} from './search-from-the-end';

/**
* zip central directory file header info
Expand All @@ -9,6 +10,8 @@ export type ZipCDFileHeader = {
compressedSize: bigint;
/** Uncompressed size */
uncompressedSize: bigint;
/** Extra field size */
extraFieldLength: number;
/** File name length */
fileNameLength: number;
/** File name */
Expand All @@ -19,6 +22,17 @@ export type ZipCDFileHeader = {
localHeaderOffset: bigint;
};

const offsets = {
CD_COMPRESSED_SIZE_OFFSET: 20n,
CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
CD_FILE_NAME_LENGTH_OFFSET: 28n,
CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
CD_FILE_NAME_OFFSET: 46n
};

export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];

/**
* Parses central directory file header of zip file
* @param headerOffset - offset in the archive where header starts
Expand All @@ -28,15 +42,14 @@ export type ZipCDFileHeader = {
export const parseZipCDFileHeader = async (
headerOffset: bigint,
buffer: FileProvider
): Promise<ZipCDFileHeader> => {
const offsets = {
CD_COMPRESSED_SIZE_OFFSET: 20n,
CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
CD_FILE_NAME_LENGTH_OFFSET: 28n,
CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
CD_FILE_NAME_OFFSET: 46n
};
): Promise<ZipCDFileHeader | null> => {
if (
Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(
Buffer.from(signature)
) !== 0
) {
return null;
}

let compressedSize = BigInt(
await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET)
Expand All @@ -46,6 +59,10 @@ export const parseZipCDFileHeader = async (
await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET)
);

const extraFieldLength = await buffer.getUint16(
headerOffset + offsets.CD_EXTRA_FIELD_LENGTH_OFFSET
);

const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);

const fileName = new TextDecoder().decode(
Expand Down Expand Up @@ -80,6 +97,7 @@ export const parseZipCDFileHeader = async (
return {
compressedSize,
uncompressedSize,
extraFieldLength,
fileNameLength,
fileName,
extraOffset,
Expand Down
Loading

0 comments on commit 5df9cb0

Please sign in to comment.