diff --git a/modules/tile-converter/src/constants.ts b/modules/tile-converter/src/constants.ts index 3a0cfcc4bb..27021e7e1d 100644 --- a/modules/tile-converter/src/constants.ts +++ b/modules/tile-converter/src/constants.ts @@ -1,2 +1,3 @@ export const BROWSER_ERROR_MESSAGE = 'Tile converter does not work in browser, only in node js environment'; +export const DUMP_FILE_SUFFIX = '.dump.json'; diff --git a/modules/tile-converter/src/i3s-converter/i3s-converter.ts b/modules/tile-converter/src/i3s-converter/i3s-converter.ts index 93a8a39e9b..d8873a6545 100644 --- a/modules/tile-converter/src/i3s-converter/i3s-converter.ts +++ b/modules/tile-converter/src/i3s-converter/i3s-converter.ts @@ -59,6 +59,7 @@ import { GLTFPrimitiveModeString, I3SConvertedResources, PreprocessData, + ResourceType, SharedResourcesArrays } from './types'; import {WorkerFarm} from '@loaders.gl/worker-utils'; @@ -82,6 +83,7 @@ import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-t import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles'; import {Progress} from './helpers/progress'; import {addOneFile, composeHashFile} from '@loaders.gl/zip'; +import {ConversionDump, ConversionDumpOptions} from '../lib/utils/conversion-dump'; const ION_DEFAULT_TOKEN = process.env?.IonToken; const HARDCODED_NODES_PER_PAGE = 64; @@ -136,13 +138,14 @@ export default class I3SConverter { generateBoundingVolumes: boolean; layersHasTexture: boolean; workerSource: {[key: string]: string} = {}; - writeQueue: WriteQueue = new WriteQueue(); + writeQueue: WriteQueue = new WriteQueue(new ConversionDump()); compressList: string[] | null = null; preprocessData: PreprocessData = { meshTopologyTypes: new Set(), metadataClasses: new Set() }; progresses: Record = {}; + conversionDump: ConversionDump; constructor() { this.attributeMetadataInfo = new AttributeMetadataInfo(); @@ -165,6 +168,7 @@ export default class I3SConverter { this.generateBoundingVolumes = false; this.layersHasTexture = false; this.compressList = null; + this.conversionDump = new ConversionDump(); } /** @@ -228,6 +232,8 @@ export default class I3SConverter { analyze = false } = options; this.options = { + outputPath, + tilesetName, maxDepth, slpk, sevenZipExe, @@ -247,7 +253,7 @@ export default class I3SConverter { this.generateTextures = Boolean(generateTextures); this.generateBoundingVolumes = Boolean(generateBoundingVolumes); - this.writeQueue = new WriteQueue(); + this.writeQueue = new WriteQueue(this.conversionDump); this.writeQueue.startListening(); console.log('Loading egm file...'); // eslint-disable-line @@ -258,6 +264,9 @@ export default class I3SConverter { this.nodePages.useWriteFunction(writeFileForSlpk); } + //create a dump file with convertion options + await this.conversionDump.createDumpFile(options as ConversionDumpOptions); + try { const preloadOptions = await this._fetchPreloadOptions(); let tilesetUrl = inputUrl; @@ -554,6 +563,7 @@ export default class I3SConverter { * @param tilesetPath - Path to save file */ private async _createSlpk(tilesetPath: string): Promise { + await this.conversionDump.deleteDumpFile(); if (this.options.slpk) { const slpkTilesetPath = join(tilesetPath, 'SceneServer', 'layers', '0'); const slpkFileName = `${tilesetPath}.slpk`; @@ -618,6 +628,7 @@ export default class I3SConverter { if (sourceTile.id) { console.log(`[convert]: ${sourceTile.id}`); // eslint-disable-line } + const {parentNodes, transform} = traversalProps; let transformationMatrix: Matrix4 = transform.clone(); if (sourceTile.transform) { @@ -756,7 +767,13 @@ export default class I3SConverter { nodes.push(node); if (nodeInPage.mesh) { - await this._writeResources(resources, node.id); + //update a record in a dump file + if (sourceTile.id) { + await this.conversionDump.addNode(sourceTile.id, nodeInPage.index); + } + + //write resources + await this._writeResources(resources, node.id, sourceTile); } if (this.validate) { @@ -905,9 +922,15 @@ export default class I3SConverter { * @param resources.texture - texture image * @param resources.sharedResources - shared resource data object * @param resources.attributes - feature attributes + * @param nodePath - node path + * @param sourceTile - source tile (3DTile) * @return {Promise} */ - private async _writeResources(resources: I3SConvertedResources, nodePath: string): Promise { + private async _writeResources( + resources: I3SConvertedResources, + nodePath: string, + sourceTile: Tiles3DTileJSONPostprocessed + ): Promise { const { geometry: geometryBuffer, compressedGeometry, @@ -918,10 +941,36 @@ export default class I3SConverter { const childPath = join(this.layers0Path, 'nodes', nodePath); const slpkChildPath = join('nodes', nodePath); - await this._writeGeometries(geometryBuffer!, compressedGeometry!, childPath, slpkChildPath); - await this._writeShared(sharedResources, childPath, slpkChildPath, nodePath); - await this._writeTexture(texture, childPath, slpkChildPath); - await this._writeAttributes(attributes, childPath, slpkChildPath); + await this._writeGeometries( + geometryBuffer!, + compressedGeometry!, + childPath, + slpkChildPath, + sourceTile.id || '', + parseInt(nodePath) + ); + await this._writeShared( + sharedResources, + childPath, + slpkChildPath, + nodePath, + sourceTile.id || '', + parseInt(nodePath) + ); + await this._writeTexture( + texture, + childPath, + slpkChildPath, + sourceTile.id || '', + parseInt(nodePath) + ); + await this._writeAttributes( + attributes, + childPath, + slpkChildPath, + sourceTile.id || '', + parseInt(nodePath) + ); } /** @@ -930,37 +979,57 @@ export default class I3SConverter { * @param compressedGeometry - Uint8Array with compressed (draco) geometry * @param childPath - a child path to write resources * @param slpkChildPath - resource path inside *slpk file + * @param sourceId - source filename + * @param nodeId - nodeId of a converted node for the writing */ private async _writeGeometries( geometryBuffer: ArrayBuffer, compressedGeometry: Promise, childPath: string, - slpkChildPath: string + slpkChildPath: string, + sourceId: string, + nodeId: number ): Promise { + this.conversionDump.updateDoneStatus(sourceId, nodeId, ResourceType.GEOMETRY, false); + if (this.options.slpk) { const slpkGeometryPath = join(childPath, 'geometries'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/geometries/0.bin.gz`, + sourceId, + outputId: nodeId, + resourceType: ResourceType.GEOMETRY, writePromise: () => writeFileForSlpk(slpkGeometryPath, geometryBuffer, '0.bin') }); } else { const geometryPath = join(childPath, 'geometries/0/'); await this.writeQueue.enqueue({ + sourceId, + outputId: nodeId, + resourceType: ResourceType.GEOMETRY, writePromise: () => writeFile(geometryPath, geometryBuffer, 'index.bin') }); } if (this.options.draco) { + this.conversionDump.updateDoneStatus(sourceId, nodeId, ResourceType.DRACO_GEOMETRY, false); + if (this.options.slpk) { const slpkCompressedGeometryPath = join(childPath, 'geometries'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/geometries/1.bin.gz`, + sourceId, + outputId: nodeId, + resourceType: ResourceType.DRACO_GEOMETRY, writePromise: () => writeFileForSlpk(slpkCompressedGeometryPath, compressedGeometry, '1.bin') }); } else { const compressedGeometryPath = join(childPath, 'geometries/1/'); await this.writeQueue.enqueue({ + sourceId, + outputId: nodeId, + resourceType: ResourceType.DRACO_GEOMETRY, writePromise: () => writeFile(compressedGeometryPath, compressedGeometry, 'index.bin') }); } @@ -973,12 +1042,16 @@ export default class I3SConverter { * @param childPath - a child path to write resources * @param slpkChildPath - resource path inside *slpk file * @param nodePath - a node path + * @param sourceId - source filename + * @param nodeId - nodeId of a converted node for the writing */ private async _writeShared( sharedResources: SharedResourcesArrays | null, childPath: string, slpkChildPath: string, - nodePath: string + nodePath: string, + sourceId: string, + nodeId: number ): Promise { if (!sharedResources) { return; @@ -986,15 +1059,24 @@ export default class I3SConverter { sharedResources.nodePath = nodePath; const sharedData = transform(sharedResources, sharedResourcesTemplate()); const sharedDataStr = JSON.stringify(sharedData); + this.conversionDump.updateDoneStatus(sourceId, nodeId, ResourceType.SHARED, false); if (this.options.slpk) { const slpkSharedPath = join(childPath, 'shared'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/shared/sharedResource.json.gz`, + sourceId, + outputId: nodeId, + resourceType: ResourceType.SHARED, writePromise: () => writeFileForSlpk(slpkSharedPath, sharedDataStr, 'sharedResource.json') }); } else { const sharedPath = join(childPath, 'shared/'); - await this.writeQueue.enqueue({writePromise: () => writeFile(sharedPath, sharedDataStr)}); + await this.writeQueue.enqueue({ + sourceId, + outputId: nodeId, + resourceType: ResourceType.SHARED, + writePromise: () => writeFile(sharedPath, sharedDataStr) + }); } } @@ -1003,11 +1085,15 @@ export default class I3SConverter { * @param texture - the texture image * @param childPath - a child path to write resources * @param slpkChildPath - the resource path inside *slpk file + * @param sourceId - source filename + * @param nodeId - nodeId of a converted node for the writing */ private async _writeTexture( texture: GLTFImagePostprocessed, childPath: string, - slpkChildPath: string + slpkChildPath: string, + sourceId: string, + nodeId: number ): Promise { if (texture) { const format = this._getFormatByMimeType(texture?.mimeType); @@ -1018,7 +1104,21 @@ export default class I3SConverter { case 'jpg': case 'png': { formats.push({name: '0', format}); - await this.writeTextureFile(textureData, '0', format, childPath, slpkChildPath); + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/${format}`, + false + ); + await this.writeTextureFile( + textureData, + '0', + format, + childPath, + slpkChildPath, + sourceId, + nodeId + ); if (this.generateTextures) { formats.push({name: '1', format: 'ktx2'}); @@ -1041,7 +1141,22 @@ export default class I3SConverter { } ); - await this.writeTextureFile(ktx2TextureData, '1', 'ktx2', childPath, slpkChildPath); + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/ktx2`, + false + ); + + await this.writeTextureFile( + ktx2TextureData, + '1', + 'ktx2', + childPath, + slpkChildPath, + sourceId, + nodeId + ); } break; @@ -1049,17 +1164,39 @@ export default class I3SConverter { case 'ktx2': { formats.push({name: '1', format}); - await this.writeTextureFile(textureData, '1', format, childPath, slpkChildPath); + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/${format}`, + false + ); + await this.writeTextureFile( + textureData, + '1', + format, + childPath, + slpkChildPath, + sourceId, + nodeId + ); if (this.generateTextures) { formats.push({name: '0', format: 'jpg'}); const decodedFromKTX2TextureData = encode(texture.image!.data[0], ImageWriter); + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/jpg`, + false + ); await this.writeTextureFile( decodedFromKTX2TextureData, '0', 'jpg', childPath, - slpkChildPath + slpkChildPath, + sourceId, + nodeId ); } } @@ -1079,13 +1216,17 @@ export default class I3SConverter { * @param format * @param childPath * @param slpkChildPath + * @param sourceId + * @param nodeId */ private async writeTextureFile( textureData: Uint8Array | Promise, name: string, format: 'jpg' | 'png' | 'ktx2', childPath: string, - slpkChildPath: string + slpkChildPath: string, + sourceId: string, + nodeId: number ): Promise { if (this.options.slpk) { const slpkTexturePath = join(childPath, 'textures'); @@ -1093,12 +1234,18 @@ export default class I3SConverter { await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/textures/${name}.${format}`, + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.TEXTURE}/${format}`, writePromise: () => writeFileForSlpk(slpkTexturePath, textureData, `${name}.${format}`, compress) }); } else { const texturePath = join(childPath, `textures/${name}/`); await this.writeQueue.enqueue({ + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.TEXTURE}/${format}`, writePromise: () => writeFile(texturePath, textureData, `index.${format}`) }); } @@ -1109,11 +1256,15 @@ export default class I3SConverter { * @param attributes - feature attributes * @param childPath - a child path to write resources * @param slpkChildPath - the resource path inside *slpk file + * @param sourceId - source filename + * @param nodeId - nodeId of a converted node for the writing */ private async _writeAttributes( attributes: ArrayBuffer[] | null = [], childPath: string, - slpkChildPath: string + slpkChildPath: string, + sourceId: string, + nodeId: number ): Promise { if (attributes?.length && this.attributeMetadataInfo.attributeStorageInfo.length) { const minimumLength = @@ -1124,16 +1275,27 @@ export default class I3SConverter { for (let index = 0; index < minimumLength; index++) { const folderName = this.attributeMetadataInfo.attributeStorageInfo[index].key; const fileBuffer = new Uint8Array(attributes[index]); - + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.ATTRIBUTES}/${folderName}`, + false + ); if (this.options.slpk) { const slpkAttributesPath = join(childPath, 'attributes', folderName); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/attributes/${folderName}.bin.gz`, + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.ATTRIBUTES}/${folderName}`, writePromise: () => writeFileForSlpk(slpkAttributesPath, fileBuffer, '0.bin') }); } else { const attributesPath = join(childPath, `attributes/${folderName}/0`); await this.writeQueue.enqueue({ + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.ATTRIBUTES}/${folderName}`, writePromise: () => writeFile(attributesPath, fileBuffer, 'index.bin') }); } diff --git a/modules/tile-converter/src/i3s-converter/types.ts b/modules/tile-converter/src/i3s-converter/types.ts index d18998bf43..60a5e5bb15 100644 --- a/modules/tile-converter/src/i3s-converter/types.ts +++ b/modules/tile-converter/src/i3s-converter/types.ts @@ -241,3 +241,11 @@ export const AttributeType = { /** Integer data type name for feature attributes */ SHORT_INT_TYPE: 'Int32' } as const; + +export enum ResourceType { + ATTRIBUTES = 'ATTRIBUTES', + DRACO_GEOMETRY = 'DRACO_GEOMETRY', + GEOMETRY = 'GEOMETRY', + SHARED = 'SHARED', + TEXTURE = 'TEXTURE' +} diff --git a/modules/tile-converter/src/lib/utils/conversion-dump.ts b/modules/tile-converter/src/lib/utils/conversion-dump.ts new file mode 100644 index 0000000000..215075ace5 --- /dev/null +++ b/modules/tile-converter/src/lib/utils/conversion-dump.ts @@ -0,0 +1,198 @@ +import {DUMP_FILE_SUFFIX} from '../../constants'; +import {removeFile, writeFile} from './file-utils'; +import {join} from 'path'; + +export type ConversionDumpOptions = { + inputUrl: string; + outputPath: string; + tilesetName: string; + maxDepth: number; + slpk: boolean; + egmFilePath: string; + token: string; + draco: boolean; + mergeMaterials: boolean; + generateTextures: boolean; + generateBoundingVolumes: boolean; + metadataClass: string; + analyze: boolean; +}; + +type NodeDoneStatus = { + nodeId: number; + done: Record | boolean; +}; + +type TilesConverted = { + nodes: NodeDoneStatus[]; +}; + +export class ConversionDump { + /** Conversion options */ + private options?: ConversionDumpOptions; + /** Tiles conversion progress status map */ + tilesConverted: Record; + + constructor() { + this.tilesConverted = {}; + } + + /** + * Create a dump file with convertion options + * @param options - converter options + */ + async createDumpFile(options: ConversionDumpOptions): Promise { + const { + tilesetName, + slpk, + egmFilePath, + inputUrl, + outputPath, + draco = true, + maxDepth, + token, + generateTextures, + generateBoundingVolumes, + mergeMaterials = true, + metadataClass, + analyze = false + } = options; + this.options = { + tilesetName, + slpk, + egmFilePath, + inputUrl, + outputPath, + draco, + maxDepth, + token, + generateTextures, + generateBoundingVolumes, + mergeMaterials, + metadataClass, + analyze + }; + + try { + await writeFile( + options.outputPath, + JSON.stringify({options: this.options}), + `${options.tilesetName}${DUMP_FILE_SUFFIX}` + ); + } catch (error) { + console.log("Can't create dump file", error); + } + } + + /** + * Update conversion status in the dump file + */ + private async updateDumpFile(): Promise { + if (this.options?.outputPath && this.options.tilesetName) { + try { + await writeFile( + this.options.outputPath, + JSON.stringify({ + options: this.options, + tilesConverted: this.tilesConverted + }), + `${this.options.tilesetName}${DUMP_FILE_SUFFIX}` + ); + } catch (error) { + console.log("Can't update dump file", error); + } + } + } + + /** + * Delete a dump file + */ + async deleteDumpFile(): Promise { + if (this.options?.outputPath && this.options.tilesetName) { + await removeFile( + join(this.options.outputPath, `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`) + ); + } + } + + /** + * Get record from the tilesConverted Map + * @param fileName - source filename + * @returns existing object from the tilesConverted Map + */ + private getRecord(fileName: string) { + return this.tilesConverted[fileName]; + } + + /** + * Set a record for the dump file + * @param fileName - key - source filename + * @param object - value + */ + private setRecord(fileName: string, object: any) { + this.tilesConverted[fileName] = object; + } + + /** + * Add a node into the dump file for the source file record + * @param fileName - source filename + * @param nodeId - nodeId of the node + */ + async addNode(filename: string, nodeId: number) { + const {nodes} = this.getRecord(filename) || {nodes: []}; + nodes.push({nodeId, done: {}}); + if (nodes.length === 1) { + this.setRecord(filename, {nodes}); + } + await this.updateDumpFile(); + } + + /** + * Update done status object for the writing resources + * @param fileName - key - source filename + * @param nodeId - nodeId for the source filename + * @param resourceType - resource type to update status + * @param value - value + */ + updateDoneStatus(filename: string, nodeId: number, resourceType: string, value: boolean) { + const nodeDump = this.tilesConverted[filename]?.nodes.find( + (element) => element.nodeId === nodeId + ); + if (nodeDump) { + nodeDump.done[resourceType] = value; + } + } + + /** + * Update dump file according to writing results + * @param changedRecords - array of parameters ids for the written resources + * @param writeResults - array of writing resource files results + */ + async updateConvertedTilesDump( + changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[], + writeResults: PromiseSettledResult[] + ) { + for (let i = 0; i < changedRecords.length; i++) { + if (changedRecords[i] && 'value' in writeResults[i]) { + const {sourceId, resourceType, outputId} = changedRecords[i]; + if (!sourceId || !resourceType || !outputId) continue; + for (const node of this.tilesConverted[sourceId].nodes) { + if (typeof node.done !== 'boolean' && node.nodeId === outputId) { + node.done[resourceType] = true; + } + if (typeof node.done !== 'boolean') { + let done = false; + for (const key in node.done) { + done = node.done[key]; + if (!done) break; + } + if (done) { + node.done = true; + } + } + } + } + } + await this.updateDumpFile(); + } +} diff --git a/modules/tile-converter/src/lib/utils/write-queue.ts b/modules/tile-converter/src/lib/utils/write-queue.ts index 8a1719a6c3..f8b8268247 100644 --- a/modules/tile-converter/src/lib/utils/write-queue.ts +++ b/modules/tile-converter/src/lib/utils/write-queue.ts @@ -1,11 +1,15 @@ import {Queue} from './queue'; import process from 'process'; +import {ConversionDump} from './conversion-dump'; /** Memory limit size is based on testing */ const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB export type WriteQueueItem = { archiveKey?: string; + sourceId?: string; + outputId?: number; + resourceType?: string; /** * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises); * Arguments for this call are specified in writeQueue.enqueue call like this: @@ -27,13 +31,19 @@ export type WriteQueueItem = { export default class WriteQueue extends Queue { private intervalId?: NodeJS.Timeout; + private conversionDump: ConversionDump; public writePromise: Promise | null = null; public fileMap: {[key: string]: string} = {}; public listeningInterval: number; public writeConcurrency: number; - constructor(listeningInterval: number = 2000, writeConcurrency: number = 400) { + constructor( + conversionDump: ConversionDump, + listeningInterval: number = 2000, + writeConcurrency: number = 400 + ) { super(); + this.conversionDump = conversionDump; this.listeningInterval = listeningInterval; this.writeConcurrency = writeConcurrency; } @@ -81,18 +91,21 @@ export default class WriteQueue extends Queue { while (this.length) { const promises: Promise[] = []; const archiveKeys: (string | undefined)[] = []; + const changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[] = []; for (let i = 0; i < this.writeConcurrency; i++) { const item = this.dequeue(); if (!item) { break; } - const {archiveKey, writePromise} = item as WriteQueueItem; + const {archiveKey, sourceId, outputId, resourceType, writePromise} = item as WriteQueueItem; archiveKeys.push(archiveKey); + changedRecords.push({sourceId, outputId, resourceType}); const promise = writePromise(); promises.push(promise); } const writeResults = await Promise.allSettled(promises); this.updateFileMap(archiveKeys, writeResults); + await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults); } } diff --git a/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js b/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js index 35154d431c..3c4d167440 100644 --- a/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js +++ b/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js @@ -3,6 +3,7 @@ import {isBrowser} from '@loaders.gl/core'; import {NodeIndexDocument} from '../../../src/i3s-converter/helpers/node-index-document'; import I3SConverter from '../../../src/i3s-converter/i3s-converter'; import WriteQueue from '../../../src/lib/utils/write-queue'; +import {ConversionDump} from '../../../src/lib/utils/conversion-dump'; const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWriting: false}) => { const converter = new I3SConverter(); @@ -11,7 +12,7 @@ const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWrit instantNodeWriting }; converter.layers0Path = '.data/node-pages-test/layers/0'; - converter.writeQueue = new WriteQueue(); + converter.writeQueue = new WriteQueue(new ConversionDump()); return converter; }; diff --git a/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js b/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js index 8efc9f9a2a..4cd7127c82 100644 --- a/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js +++ b/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js @@ -3,6 +3,7 @@ import {default as NodePages} from '../../../src/i3s-converter/helpers/node-page import {isBrowser} from '@loaders.gl/core'; import I3SConverter from '../../../src/i3s-converter/i3s-converter'; import WriteQueue from '../../../src/lib/utils/write-queue'; +import {ConversionDump} from '../../../src/lib/utils/conversion-dump'; const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWriting: false}) => { const converter = new I3SConverter(); @@ -11,7 +12,7 @@ const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWrit instantNodeWriting }; converter.layers0Path = '.data/node-pages-test/layers/0'; - converter.writeQueue = new WriteQueue(); + converter.writeQueue = new WriteQueue(new ConversionDump()); return converter; }; diff --git a/modules/tile-converter/test/index.js b/modules/tile-converter/test/index.js index b1b323b1f5..20b7cd20f3 100644 --- a/modules/tile-converter/test/index.js +++ b/modules/tile-converter/test/index.js @@ -30,3 +30,5 @@ import './i3s-server/controllers/index-controller.spec'; import './i3s-server/controllers/slpk-controller.spec'; import './i3s-server/utils/create-scene-server.spec.ts'; import './i3s-server/utils/server-utils.spec'; + +import './lib/utils/conversion-dump.spec.ts'; diff --git a/modules/tile-converter/test/lib/utils/conversion-dump.spec.ts b/modules/tile-converter/test/lib/utils/conversion-dump.spec.ts new file mode 100644 index 0000000000..056703fbed --- /dev/null +++ b/modules/tile-converter/test/lib/utils/conversion-dump.spec.ts @@ -0,0 +1,144 @@ +import test from 'tape-promise/tape'; +import {ConversionDump, ConversionDumpOptions} from '../../../src/lib/utils/conversion-dump'; +import {join} from 'path'; +import {isFileExists, openJson} from '../../../src/lib/utils/file-utils'; +import {DUMP_FILE_SUFFIX} from '../../../src/constants'; +import {cleanUpPath} from '../../utils/file-utils'; + +test('tile-converter(i3s)#ConversionDump - Should create and delete conversion dump with options', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + inputUrl: 'testInputUrl', + outputPath: 'testPath', + tilesetName: 'testTileset', + maxDepth: 5, + slpk: true, + egmFilePath: 'testEGM', + token: 'testToken', + draco: true, + mergeMaterials: true, + generateTextures: true, + generateBoundingVolumes: true, + metadataClass: 'testMetadataClass', + analyze: true, + something: 'test' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + + let isDumpExists = await isFileExists( + join(testOptions.outputPath, `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}`) + ); + t.equal(isDumpExists, true); + + const {options} = await openJson( + testOptions.outputPath, + `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}` + ); + const {something, ...correctOptions} = testOptions; + t.deepEqual(options, correctOptions); + + await conversionDump.deleteDumpFile(); + isDumpExists = await isFileExists( + join(testOptions.outputPath, `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}`) + ); + t.equal(isDumpExists, false); + await cleanUpPath(testOptions.outputPath); + t.end(); +}); + +test('tile-converter(i3s)#ConversionDump - Add node to the dump', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + outputPath: 'testPath', + tilesetName: 'testTileset' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + + await conversionDump.addNode('1.glb', 1); + await conversionDump.addNode('1.glb', 2); + await conversionDump.addNode('2.glb', 3); + + const {tilesConverted} = await openJson( + testOptions.outputPath, + `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}` + ); + + t.deepEqual(tilesConverted, { + '1.glb': { + nodes: [ + {nodeId: 1, done: {}}, + {nodeId: 2, done: {}} + ] + }, + '2.glb': { + nodes: [{nodeId: 3, done: {}}] + } + }); + + await conversionDump.deleteDumpFile(); + await cleanUpPath(testOptions.outputPath); + t.end(); +}); + +test('tile-converter(i3s)#ConversionDump - update Done Status', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + outputPath: 'testPath', + tilesetName: 'testTileset' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + await conversionDump.addNode('1.glb', 1); + conversionDump.updateDoneStatus('1.glb', 1, 'testResource', false); + + t.deepEqual(conversionDump.tilesConverted, { + '1.glb': { + nodes: [{nodeId: 1, done: {testResource: false}}] + } + }); + + await conversionDump.deleteDumpFile(); + await cleanUpPath(testOptions.outputPath); + t.end(); +}); + +test('tile-converter(i3s)#ConversionDump - updateConvertedTilesDump', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + outputPath: 'testPath', + tilesetName: 'testTileset' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + await conversionDump.addNode('1.glb', 1); + conversionDump.updateDoneStatus('1.glb', 1, 'testResource', false); + + const promises: Promise[] = []; + promises.push(Promise.resolve('')); + const writeResults = await Promise.allSettled(promises); + const changedRecords = [{sourceId: '1.glb', outputId: 1, resourceType: 'testResource'}]; + await conversionDump.updateConvertedTilesDump(changedRecords, writeResults); + + t.deepEqual(conversionDump.tilesConverted, { + '1.glb': { + nodes: [{nodeId: 1, done: true}] + } + }); + + const {tilesConverted} = await openJson( + testOptions.outputPath, + `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}` + ); + + t.deepEqual(tilesConverted, { + '1.glb': { + nodes: [{nodeId: 1, done: true}] + } + }); + + await conversionDump.deleteDumpFile(); + await cleanUpPath(testOptions.outputPath); + t.end(); +});