From 8db523fb3366eb5432518a71d24f82cf436d3088 Mon Sep 17 00:00:00 2001 From: Maxim Kuznetsov Date: Wed, 17 Jan 2024 12:51:44 +0300 Subject: [PATCH 1/4] feat(tile-converter): add convertion status dump file --- modules/tile-converter/src/constants.ts | 1 + .../src/i3s-converter/helpers/dump-parser.ts | 23 ++ .../src/i3s-converter/i3s-converter.ts | 252 ++++++++++++++++-- .../tile-converter/src/i3s-converter/types.ts | 9 + .../src/lib/utils/write-queue.ts | 34 ++- 5 files changed, 298 insertions(+), 21 deletions(-) create mode 100644 modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts diff --git a/modules/tile-converter/src/constants.ts b/modules/tile-converter/src/constants.ts index 3a0cfcc4bb..27021e7e1d 100644 --- a/modules/tile-converter/src/constants.ts +++ b/modules/tile-converter/src/constants.ts @@ -1,2 +1,3 @@ export const BROWSER_ERROR_MESSAGE = 'Tile converter does not work in browser, only in node js environment'; +export const DUMP_FILE_SUFFIX = '.dump.json'; diff --git a/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts b/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts new file mode 100644 index 0000000000..fa40967588 --- /dev/null +++ b/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts @@ -0,0 +1,23 @@ +import {isMap} from 'util/types'; + +/** + * Convert dump map to object for stringifying to JSON + * @param tilesConverted - Map of tilesConverted with conversion status + * @returns tilesConverted Map converted to Object + */ +export const dumpToObject = (tilesConverted: Map) => { + const object = {}; + for (const [key, value] of tilesConverted) { + if (value && value.nodes) { + object[key] = {nodes: []}; + for (const node of value.nodes) { + if (isMap(node.done)) { + object[key].nodes.push({nodeId: node.nodeId, done: Object.fromEntries(node.done)}); + } else { + object[key].nodes.push({nodeId: node.nodeId, done: node.done}); + } + } + } + } + return object; +}; diff --git a/modules/tile-converter/src/i3s-converter/i3s-converter.ts b/modules/tile-converter/src/i3s-converter/i3s-converter.ts index 93a8a39e9b..a4989df41b 100644 --- a/modules/tile-converter/src/i3s-converter/i3s-converter.ts +++ b/modules/tile-converter/src/i3s-converter/i3s-converter.ts @@ -51,7 +51,7 @@ import {GEOMETRY_DEFINITION as geometryDefinitionTemlate} from './json-templates import {SHARED_RESOURCES as sharedResourcesTemplate} from './json-templates/shared-resources'; import {validateNodeBoundingVolumes} from './helpers/node-debug'; import {KTX2BasisWriterWorker} from '@loaders.gl/textures'; -import {FileHandleFile, LoaderWithParser} from '@loaders.gl/loader-utils'; +import {FileHandleFile, LoaderWithParser, path} from '@loaders.gl/loader-utils'; import {I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s'; import {ImageWriter} from '@loaders.gl/images'; import {GLTFImagePostprocessed} from '@loaders.gl/gltf'; @@ -59,11 +59,12 @@ import { GLTFPrimitiveModeString, I3SConvertedResources, PreprocessData, + ResourceType, SharedResourcesArrays } from './types'; import {WorkerFarm} from '@loaders.gl/worker-utils'; import WriteQueue from '../lib/utils/write-queue'; -import {BROWSER_ERROR_MESSAGE} from '../constants'; +import {BROWSER_ERROR_MESSAGE, DUMP_FILE_SUFFIX} from '../constants'; import { getAttributeTypesMapFromPropertyTable, getAttributeTypesMapFromSchema @@ -82,6 +83,7 @@ import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-t import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles'; import {Progress} from './helpers/progress'; import {addOneFile, composeHashFile} from '@loaders.gl/zip'; +import {dumpToObject} from './helpers/dump-parser'; const ION_DEFAULT_TOKEN = process.env?.IonToken; const HARDCODED_NODES_PER_PAGE = 64; @@ -143,6 +145,7 @@ export default class I3SConverter { metadataClasses: new Set() }; progresses: Record = {}; + tilesConverted: Map; constructor() { this.attributeMetadataInfo = new AttributeMetadataInfo(); @@ -165,6 +168,7 @@ export default class I3SConverter { this.generateBoundingVolumes = false; this.layersHasTexture = false; this.compressList = null; + this.tilesConverted = new Map(); } /** @@ -228,6 +232,8 @@ export default class I3SConverter { analyze = false } = options; this.options = { + outputPath, + tilesetName, maxDepth, slpk, sevenZipExe, @@ -249,6 +255,7 @@ export default class I3SConverter { this.writeQueue = new WriteQueue(); this.writeQueue.startListening(); + this.writeQueue.writeDumpFile = this.writeDumpFile.bind(this); console.log('Loading egm file...'); // eslint-disable-line this.geoidHeightModel = await load(egmFilePath, PGMLoader); @@ -258,6 +265,17 @@ export default class I3SConverter { this.nodePages.useWriteFunction(writeFileForSlpk); } + //create a dump file with convertion options + try { + writeFile( + options.outputPath, + JSON.stringify({options: this.options}), + `${options.tilesetName}${DUMP_FILE_SUFFIX}` + ); + } catch (error) { + console.log("Can't create dump file", error); + } + try { const preloadOptions = await this._fetchPreloadOptions(); let tilesetUrl = inputUrl; @@ -618,6 +636,13 @@ export default class I3SConverter { if (sourceTile.id) { console.log(`[convert]: ${sourceTile.id}`); // eslint-disable-line } + + //add a record to dump map + if (sourceTile.id) { + const fileName = path.filename(sourceTile.id); + this.tilesConverted.set(fileName, null); + } + const {parentNodes, transform} = traversalProps; let transformationMatrix: Matrix4 = transform.clone(); if (sourceTile.transform) { @@ -709,6 +734,17 @@ export default class I3SConverter { propertyTable ); + //update a record in dump map and file + if (sourceTile.id) { + const fileName = path.filename(sourceTile.id); + this.tilesConverted.set(fileName, { + nodes: resourcesData?.map((node) => { + return {nodeId: node.nodeId, done: new Map()}; + }) + }); + this.writeDumpFile(); + } + const nodes: NodeIndexDocument[] = []; const nodeIds: number[] = []; const nodesInPage: NodeInPage[] = []; @@ -756,7 +792,7 @@ export default class I3SConverter { nodes.push(node); if (nodeInPage.mesh) { - await this._writeResources(resources, node.id); + this._writeResources(resources, node.id, sourceTile); } if (this.validate) { @@ -907,7 +943,11 @@ export default class I3SConverter { * @param resources.attributes - feature attributes * @return {Promise} */ - private async _writeResources(resources: I3SConvertedResources, nodePath: string): Promise { + private async _writeResources( + resources: I3SConvertedResources, + nodePath: string, + sourceTile: Tiles3DTileJSONPostprocessed + ): Promise { const { geometry: geometryBuffer, compressedGeometry, @@ -917,11 +957,32 @@ export default class I3SConverter { } = resources; const childPath = join(this.layers0Path, 'nodes', nodePath); const slpkChildPath = join('nodes', nodePath); - - await this._writeGeometries(geometryBuffer!, compressedGeometry!, childPath, slpkChildPath); - await this._writeShared(sharedResources, childPath, slpkChildPath, nodePath); - await this._writeTexture(texture, childPath, slpkChildPath); - await this._writeAttributes(attributes, childPath, slpkChildPath); + const dumpTileRecord = this.tilesConverted.get(path.filename(sourceTile.id || '')); + + await this._writeGeometries( + geometryBuffer!, + compressedGeometry!, + childPath, + slpkChildPath, + dumpTileRecord, + resources.nodeId + ); + await this._writeShared( + sharedResources, + childPath, + slpkChildPath, + nodePath, + dumpTileRecord, + resources.nodeId + ); + await this._writeTexture(texture, childPath, slpkChildPath, dumpTileRecord, resources.nodeId); + await this._writeAttributes( + attributes, + childPath, + slpkChildPath, + dumpTileRecord, + resources.nodeId + ); } /** @@ -930,37 +991,74 @@ export default class I3SConverter { * @param compressedGeometry - Uint8Array with compressed (draco) geometry * @param childPath - a child path to write resources * @param slpkChildPath - resource path inside *slpk file + * @param dumpTileRecord - a record of the tilesConverted Map + * @param nodeId - nodeId of a converted node for the writing */ private async _writeGeometries( geometryBuffer: ArrayBuffer, compressedGeometry: Promise, childPath: string, - slpkChildPath: string + slpkChildPath: string, + dumpTileRecord: any, + nodeId?: number ): Promise { + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(ResourceType.GEOMETRY, false); + } + } + if (this.options.slpk) { const slpkGeometryPath = join(childPath, 'geometries'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/geometries/0.bin.gz`, + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.GEOMETRY + }, + writePromise: () => writeFileForSlpk(slpkGeometryPath, geometryBuffer, '0.bin') }); } else { const geometryPath = join(childPath, 'geometries/0/'); await this.writeQueue.enqueue({ + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.GEOMETRY + }, writePromise: () => writeFile(geometryPath, geometryBuffer, 'index.bin') }); } if (this.options.draco) { + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(ResourceType.DRACO_GEOMETRY, false); + } + } + if (this.options.slpk) { const slpkCompressedGeometryPath = join(childPath, 'geometries'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/geometries/1.bin.gz`, + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.DRACO_GEOMETRY + }, writePromise: () => writeFileForSlpk(slpkCompressedGeometryPath, compressedGeometry, '1.bin') }); } else { const compressedGeometryPath = join(childPath, 'geometries/1/'); await this.writeQueue.enqueue({ + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.DRACO_GEOMETRY + }, writePromise: () => writeFile(compressedGeometryPath, compressedGeometry, 'index.bin') }); } @@ -973,12 +1071,16 @@ export default class I3SConverter { * @param childPath - a child path to write resources * @param slpkChildPath - resource path inside *slpk file * @param nodePath - a node path + * @param dumpTileRecord - a record of the tilesConverted Map + * @param nodeId - nodeId of a converted node for the writing */ private async _writeShared( sharedResources: SharedResourcesArrays | null, childPath: string, slpkChildPath: string, - nodePath: string + nodePath: string, + dumpTileRecord: any, + nodeId?: number ): Promise { if (!sharedResources) { return; @@ -986,15 +1088,32 @@ export default class I3SConverter { sharedResources.nodePath = nodePath; const sharedData = transform(sharedResources, sharedResourcesTemplate()); const sharedDataStr = JSON.stringify(sharedData); + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(ResourceType.SHARED, false); + } + } if (this.options.slpk) { const slpkSharedPath = join(childPath, 'shared'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/shared/sharedResource.json.gz`, + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.SHARED + }, writePromise: () => writeFileForSlpk(slpkSharedPath, sharedDataStr, 'sharedResource.json') }); } else { const sharedPath = join(childPath, 'shared/'); - await this.writeQueue.enqueue({writePromise: () => writeFile(sharedPath, sharedDataStr)}); + await this.writeQueue.enqueue({ + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.SHARED + }, + writePromise: () => writeFile(sharedPath, sharedDataStr) + }); } } @@ -1003,22 +1122,40 @@ export default class I3SConverter { * @param texture - the texture image * @param childPath - a child path to write resources * @param slpkChildPath - the resource path inside *slpk file + * @param dumpTileRecord - a record of the tilesConverted Map + * @param nodeId - nodeId of a converted node for the writing */ private async _writeTexture( texture: GLTFImagePostprocessed, childPath: string, - slpkChildPath: string + slpkChildPath: string, + dumpTileRecord: any, + nodeId?: number ): Promise { if (texture) { const format = this._getFormatByMimeType(texture?.mimeType); const formats: TextureSetDefinitionFormats = []; const textureData = texture.bufferView!.data; + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(ResourceType.TEXTURE, false); + } + } + switch (format) { case 'jpg': case 'png': { formats.push({name: '0', format}); - await this.writeTextureFile(textureData, '0', format, childPath, slpkChildPath); + await this.writeTextureFile( + textureData, + '0', + format, + childPath, + slpkChildPath, + dumpTileRecord, + nodeId + ); if (this.generateTextures) { formats.push({name: '1', format: 'ktx2'}); @@ -1041,7 +1178,21 @@ export default class I3SConverter { } ); - await this.writeTextureFile(ktx2TextureData, '1', 'ktx2', childPath, slpkChildPath); + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(ResourceType.TEXTURE_KTX2, false); + } + } + + await this.writeTextureFile( + ktx2TextureData, + '1', + 'ktx2', + childPath, + slpkChildPath, + dumpTileRecord, + nodeId + ); } break; @@ -1049,7 +1200,15 @@ export default class I3SConverter { case 'ktx2': { formats.push({name: '1', format}); - await this.writeTextureFile(textureData, '1', format, childPath, slpkChildPath); + await this.writeTextureFile( + textureData, + '1', + format, + childPath, + slpkChildPath, + dumpTileRecord, + nodeId + ); if (this.generateTextures) { formats.push({name: '0', format: 'jpg'}); @@ -1059,7 +1218,9 @@ export default class I3SConverter { '0', 'jpg', childPath, - slpkChildPath + slpkChildPath, + dumpTileRecord, + nodeId ); } } @@ -1079,13 +1240,17 @@ export default class I3SConverter { * @param format * @param childPath * @param slpkChildPath + * @param dumpTileRecord + * @param nodeId */ private async writeTextureFile( textureData: Uint8Array | Promise, name: string, format: 'jpg' | 'png' | 'ktx2', childPath: string, - slpkChildPath: string + slpkChildPath: string, + dumpTileRecord: any, + nodeId?: number ): Promise { if (this.options.slpk) { const slpkTexturePath = join(childPath, 'textures'); @@ -1093,12 +1258,22 @@ export default class I3SConverter { await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/textures/${name}.${format}`, + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.TEXTURE + }, writePromise: () => writeFileForSlpk(slpkTexturePath, textureData, `${name}.${format}`, compress) }); } else { const texturePath = join(childPath, `textures/${name}/`); await this.writeQueue.enqueue({ + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: ResourceType.TEXTURE + }, writePromise: () => writeFile(texturePath, textureData, `index.${format}`) }); } @@ -1109,11 +1284,15 @@ export default class I3SConverter { * @param attributes - feature attributes * @param childPath - a child path to write resources * @param slpkChildPath - the resource path inside *slpk file + * @param dumpTileRecord - a record of the tilesConverted Map + * @param nodeId - nodeId of a converted node for the writing */ private async _writeAttributes( attributes: ArrayBuffer[] | null = [], childPath: string, - slpkChildPath: string + slpkChildPath: string, + dumpTileRecord: any, + nodeId?: number ): Promise { if (attributes?.length && this.attributeMetadataInfo.attributeStorageInfo.length) { const minimumLength = @@ -1124,16 +1303,30 @@ export default class I3SConverter { for (let index = 0; index < minimumLength; index++) { const folderName = this.attributeMetadataInfo.attributeStorageInfo[index].key; const fileBuffer = new Uint8Array(attributes[index]); - + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(`${ResourceType.ATTRIBUTES}/${folderName}`, false); + } + } if (this.options.slpk) { const slpkAttributesPath = join(childPath, 'attributes', folderName); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/attributes/${folderName}.bin.gz`, + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: `${ResourceType.ATTRIBUTES}/${folderName}` + }, writePromise: () => writeFileForSlpk(slpkAttributesPath, fileBuffer, '0.bin') }); } else { const attributesPath = join(childPath, `attributes/${folderName}/0`); await this.writeQueue.enqueue({ + convertedTileDumpMap: { + dumpTileRecord, + nodeId: nodeId, + resourceType: `${ResourceType.ATTRIBUTES}/${folderName}` + }, writePromise: () => writeFile(attributesPath, fileBuffer, 'index.bin') }); } @@ -1247,6 +1440,7 @@ export default class I3SConverter { console.log(`File(s) size: `, filesSize, ' bytes'); // eslint-disable-line no-undef, no-console console.log(`Percentage of tiles with "ADD" refinement type:`, addRefinementPercentage, '%'); // eslint-disable-line no-undef, no-console console.log(`------------------------------------------------`); // eslint-disable-line no-undef, no-console + removeFile(join(params.outputPath, `${params.tilesetName}${DUMP_FILE_SUFFIX}`)); } /** @@ -1306,4 +1500,22 @@ export default class I3SConverter { private isContentSupported(sourceTile: Tiles3DTileJSONPostprocessed): boolean { return ['b3dm', 'glTF', 'scenegraph'].includes(sourceTile.type || ''); } + + /** + * Write conversion status into dump file + */ + private async writeDumpFile(): Promise { + try { + await writeFile( + this.options.outputPath, + JSON.stringify({ + options: this.options, + tilesConverted: dumpToObject(this.tilesConverted) + }), + `${this.options.tilesetName}${DUMP_FILE_SUFFIX}` + ); + } catch (error) { + console.log("Can't update dump file", error); + } + } } diff --git a/modules/tile-converter/src/i3s-converter/types.ts b/modules/tile-converter/src/i3s-converter/types.ts index d18998bf43..2467e4394b 100644 --- a/modules/tile-converter/src/i3s-converter/types.ts +++ b/modules/tile-converter/src/i3s-converter/types.ts @@ -241,3 +241,12 @@ export const AttributeType = { /** Integer data type name for feature attributes */ SHORT_INT_TYPE: 'Int32' } as const; + +export enum ResourceType { + ATTRIBUTES = 'ATTRIBUTES', + DRACO_GEOMETRY = 'DRACO_GEOMETRY', + GEOMETRY = 'GEOMETRY', + SHARED = 'SHARED', + TEXTURE = 'TEXTURE', + TEXTURE_KTX2 = 'TEXTURE_KTX2' +} diff --git a/modules/tile-converter/src/lib/utils/write-queue.ts b/modules/tile-converter/src/lib/utils/write-queue.ts index 8a1719a6c3..410b6f8274 100644 --- a/modules/tile-converter/src/lib/utils/write-queue.ts +++ b/modules/tile-converter/src/lib/utils/write-queue.ts @@ -6,6 +6,7 @@ const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB export type WriteQueueItem = { archiveKey?: string; + convertedTileDumpMap?: any; /** * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises); * Arguments for this call are specified in writeQueue.enqueue call like this: @@ -28,6 +29,7 @@ export type WriteQueueItem = { export default class WriteQueue extends Queue { private intervalId?: NodeJS.Timeout; public writePromise: Promise | null = null; + public writeDumpFile: (() => void) | undefined; public fileMap: {[key: string]: string} = {}; public listeningInterval: number; public writeConcurrency: number; @@ -81,18 +83,21 @@ export default class WriteQueue extends Queue { while (this.length) { const promises: Promise[] = []; const archiveKeys: (string | undefined)[] = []; + const convertedTileDumpMaps: any[] = []; for (let i = 0; i < this.writeConcurrency; i++) { const item = this.dequeue(); if (!item) { break; } - const {archiveKey, writePromise} = item as WriteQueueItem; + const {archiveKey, convertedTileDumpMap, writePromise} = item as WriteQueueItem; archiveKeys.push(archiveKey); + convertedTileDumpMaps.push(convertedTileDumpMap); const promise = writePromise(); promises.push(promise); } const writeResults = await Promise.allSettled(promises); this.updateFileMap(archiveKeys, writeResults); + this.updateConvertedTilesMap(convertedTileDumpMaps, writeResults); } } @@ -107,4 +112,31 @@ export default class WriteQueue extends Queue { } } } + + private updateConvertedTilesMap( + convertedTileDumpMaps: any[], + writeResults: PromiseSettledResult[] + ) { + for (let i = 0; i < convertedTileDumpMaps.length; i++) { + if (convertedTileDumpMaps[i] && 'value' in writeResults[i]) { + const {dumpTileRecord} = convertedTileDumpMaps[i]; + let done = true; + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === convertedTileDumpMaps[i].nodeId) { + node.done.set(convertedTileDumpMaps[i].resourceType, true); + } + for (const [_, value] of node.done) { + done = value; + } + if (done) { + delete node.done; + node.done = true; + } + } + } + } + if (this.writeDumpFile) { + this.writeDumpFile(); + } + } } From c3d4062d0a9107d466c59fe0733b896216a8740e Mon Sep 17 00:00:00 2001 From: Maxim Kuznetsov Date: Thu, 18 Jan 2024 17:41:34 +0300 Subject: [PATCH 2/4] code complete --- .../src/i3s-converter/helpers/dump-parser.ts | 4 +- .../src/i3s-converter/i3s-converter.ts | 66 +++++++++++-------- .../tile-converter/src/i3s-converter/types.ts | 3 +- .../src/lib/utils/write-queue.ts | 18 +++-- .../i3s-converter/helpers/dump-parser.spec.ts | 42 ++++++++++++ modules/tile-converter/test/index.js | 1 + 6 files changed, 95 insertions(+), 39 deletions(-) create mode 100644 modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts diff --git a/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts b/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts index fa40967588..2651d348db 100644 --- a/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts +++ b/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts @@ -5,10 +5,10 @@ import {isMap} from 'util/types'; * @param tilesConverted - Map of tilesConverted with conversion status * @returns tilesConverted Map converted to Object */ -export const dumpToObject = (tilesConverted: Map) => { +export const dumpTilesToObject = (tilesConverted: Map) => { const object = {}; for (const [key, value] of tilesConverted) { - if (value && value.nodes) { + if (value?.nodes) { object[key] = {nodes: []}; for (const node of value.nodes) { if (isMap(node.done)) { diff --git a/modules/tile-converter/src/i3s-converter/i3s-converter.ts b/modules/tile-converter/src/i3s-converter/i3s-converter.ts index a4989df41b..a7d858e772 100644 --- a/modules/tile-converter/src/i3s-converter/i3s-converter.ts +++ b/modules/tile-converter/src/i3s-converter/i3s-converter.ts @@ -83,7 +83,7 @@ import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-t import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles'; import {Progress} from './helpers/progress'; import {addOneFile, composeHashFile} from '@loaders.gl/zip'; -import {dumpToObject} from './helpers/dump-parser'; +import {dumpTilesToObject} from './helpers/dump-parser'; const ION_DEFAULT_TOKEN = process.env?.IonToken; const HARDCODED_NODES_PER_PAGE = 64; @@ -734,17 +734,6 @@ export default class I3SConverter { propertyTable ); - //update a record in dump map and file - if (sourceTile.id) { - const fileName = path.filename(sourceTile.id); - this.tilesConverted.set(fileName, { - nodes: resourcesData?.map((node) => { - return {nodeId: node.nodeId, done: new Map()}; - }) - }); - this.writeDumpFile(); - } - const nodes: NodeIndexDocument[] = []; const nodeIds: number[] = []; const nodesInPage: NodeInPage[] = []; @@ -792,7 +781,19 @@ export default class I3SConverter { nodes.push(node); if (nodeInPage.mesh) { - this._writeResources(resources, node.id, sourceTile); + //update a record in a dump map file + const nodeId = parseInt(node.id); + if (!isNaN(nodeId) && sourceTile.id) { + const fileName = path.filename(sourceTile.id); + const {nodes} = this.tilesConverted.get(fileName) || {nodes: []}; + nodes.push({nodeId, done: new Map()}); + if (nodes.length === 1) { + this.tilesConverted.set(fileName, {nodes}); + } + this.writeDumpFile(); + } + //write resources + await this._writeResources(resources, node.id, sourceTile); } if (this.validate) { @@ -965,7 +966,7 @@ export default class I3SConverter { childPath, slpkChildPath, dumpTileRecord, - resources.nodeId + parseInt(nodePath) ); await this._writeShared( sharedResources, @@ -973,15 +974,15 @@ export default class I3SConverter { slpkChildPath, nodePath, dumpTileRecord, - resources.nodeId + parseInt(nodePath) ); - await this._writeTexture(texture, childPath, slpkChildPath, dumpTileRecord, resources.nodeId); + await this._writeTexture(texture, childPath, slpkChildPath, dumpTileRecord, parseInt(nodePath)); await this._writeAttributes( attributes, childPath, slpkChildPath, dumpTileRecord, - resources.nodeId + parseInt(nodePath) ); } @@ -1137,16 +1138,15 @@ export default class I3SConverter { const formats: TextureSetDefinitionFormats = []; const textureData = texture.bufferView!.data; - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(ResourceType.TEXTURE, false); - } - } - switch (format) { case 'jpg': case 'png': { formats.push({name: '0', format}); + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(`${ResourceType.TEXTURE}/${format}`, false); + } + } await this.writeTextureFile( textureData, '0', @@ -1180,7 +1180,7 @@ export default class I3SConverter { for (const node of dumpTileRecord.nodes) { if (node.nodeId === nodeId) { - node.done.set(ResourceType.TEXTURE_KTX2, false); + node.done.set(`${ResourceType.TEXTURE}/ktx2`, false); } } @@ -1200,6 +1200,11 @@ export default class I3SConverter { case 'ktx2': { formats.push({name: '1', format}); + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(`${ResourceType.TEXTURE}/${format}`, false); + } + } await this.writeTextureFile( textureData, '1', @@ -1213,6 +1218,11 @@ export default class I3SConverter { if (this.generateTextures) { formats.push({name: '0', format: 'jpg'}); const decodedFromKTX2TextureData = encode(texture.image!.data[0], ImageWriter); + for (const node of dumpTileRecord.nodes) { + if (node.nodeId === nodeId) { + node.done.set(`${ResourceType.TEXTURE}/jpg`, false); + } + } await this.writeTextureFile( decodedFromKTX2TextureData, '0', @@ -1261,7 +1271,7 @@ export default class I3SConverter { convertedTileDumpMap: { dumpTileRecord, nodeId: nodeId, - resourceType: ResourceType.TEXTURE + resourceType: `${ResourceType.TEXTURE}/${format}` }, writePromise: () => writeFileForSlpk(slpkTexturePath, textureData, `${name}.${format}`, compress) @@ -1272,7 +1282,7 @@ export default class I3SConverter { convertedTileDumpMap: { dumpTileRecord, nodeId: nodeId, - resourceType: ResourceType.TEXTURE + resourceType: `${ResourceType.TEXTURE}/${format}` }, writePromise: () => writeFile(texturePath, textureData, `index.${format}`) }); @@ -1506,11 +1516,11 @@ export default class I3SConverter { */ private async writeDumpFile(): Promise { try { - await writeFile( + writeFile( this.options.outputPath, JSON.stringify({ options: this.options, - tilesConverted: dumpToObject(this.tilesConverted) + tilesConverted: dumpTilesToObject(this.tilesConverted) }), `${this.options.tilesetName}${DUMP_FILE_SUFFIX}` ); diff --git a/modules/tile-converter/src/i3s-converter/types.ts b/modules/tile-converter/src/i3s-converter/types.ts index 2467e4394b..60a5e5bb15 100644 --- a/modules/tile-converter/src/i3s-converter/types.ts +++ b/modules/tile-converter/src/i3s-converter/types.ts @@ -247,6 +247,5 @@ export enum ResourceType { DRACO_GEOMETRY = 'DRACO_GEOMETRY', GEOMETRY = 'GEOMETRY', SHARED = 'SHARED', - TEXTURE = 'TEXTURE', - TEXTURE_KTX2 = 'TEXTURE_KTX2' + TEXTURE = 'TEXTURE' } diff --git a/modules/tile-converter/src/lib/utils/write-queue.ts b/modules/tile-converter/src/lib/utils/write-queue.ts index 410b6f8274..38165093c5 100644 --- a/modules/tile-converter/src/lib/utils/write-queue.ts +++ b/modules/tile-converter/src/lib/utils/write-queue.ts @@ -1,3 +1,4 @@ +import {isMap} from 'util/types'; import {Queue} from './queue'; import process from 'process'; @@ -120,17 +121,20 @@ export default class WriteQueue extends Queue { for (let i = 0; i < convertedTileDumpMaps.length; i++) { if (convertedTileDumpMaps[i] && 'value' in writeResults[i]) { const {dumpTileRecord} = convertedTileDumpMaps[i]; - let done = true; for (const node of dumpTileRecord.nodes) { if (node.nodeId === convertedTileDumpMaps[i].nodeId) { node.done.set(convertedTileDumpMaps[i].resourceType, true); } - for (const [_, value] of node.done) { - done = value; - } - if (done) { - delete node.done; - node.done = true; + if (isMap(node.done)) { + let done = false; + for (const [_, value] of node.done) { + done = value; + if (!done) break; + } + if (done) { + delete node.done; + node.done = true; + } } } } diff --git a/modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts b/modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts new file mode 100644 index 0000000000..29d30a9006 --- /dev/null +++ b/modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts @@ -0,0 +1,42 @@ +import test from 'tape-promise/tape'; +import {dumpTilesToObject} from '../../../src/i3s-converter/helpers/dump-parser'; + +test('tile-converter(i3s)#dumpTilesToObject - conversion from dump map to object', (t) => { + const inputMap = new Map(); + inputMap.set('file1', { + nodes: [{nodeId: 1, done: true}] + }); + inputMap.set('file2', { + nodes: [ + {nodeId: 2, done: new Map()}, + {nodeId: 3, done: new Map()} + ] + }); + const {nodes} = inputMap.get('file2'); + nodes[0].done.set('geometry', false); + nodes[0].done.set('texture', false); + nodes[0].done.set('shared', true); + nodes[1].done.set('geometry', true); + nodes[1].done.set('texture', true); + nodes[1].done.set('shared', false); + inputMap.set('file3', {nodes: []}); + inputMap.set('file4', {nodes: [{nodeId: 4, done: new Map()}]}); + + const expectedResult = { + file1: { + nodes: [{nodeId: 1, done: true}] + }, + file2: { + nodes: [ + {nodeId: 2, done: {geometry: false, texture: false, shared: true}}, + {nodeId: 3, done: {geometry: true, texture: true, shared: false}} + ] + }, + file3: {nodes: []}, + file4: {nodes: [{nodeId: 4, done: {}}]} + }; + + const result = dumpTilesToObject(inputMap); + t.deepEqual(result, expectedResult); + t.end(); +}); diff --git a/modules/tile-converter/test/index.js b/modules/tile-converter/test/index.js index b1b323b1f5..c97cce1a0a 100644 --- a/modules/tile-converter/test/index.js +++ b/modules/tile-converter/test/index.js @@ -13,6 +13,7 @@ import './i3s-converter/helpers/preprocess-3d-tiles.spec'; import './i3s-converter/helpers/geometry-attributes.spec'; import './i3s-converter/helpers/attribute-metadata-info.spec.ts'; import './i3s-converter/helpers/progress.spec.ts'; +import './i3s-converter/helpers/dump-parser.spec.ts'; import './i3s-converter/i3s-converter.spec'; import './slpk-extractor/slpk-extractor.spec'; From 2be25eaca3950d876bc1e1896bf073e681d0f2f5 Mon Sep 17 00:00:00 2001 From: Maxim Kuznetsov Date: Tue, 23 Jan 2024 11:37:10 +0300 Subject: [PATCH 3/4] created class ConversionDump --- .../src/i3s-converter/helpers/dump-parser.ts | 23 -- .../src/i3s-converter/i3s-converter.ts | 265 +++++++----------- .../src/lib/utils/conversion-dump.ts | 182 ++++++++++++ .../src/lib/utils/write-queue.ts | 53 +--- .../i3s-converter/helpers/dump-parser.spec.ts | 42 --- .../helpers/node-index-document.spec.js | 3 +- .../i3s-converter/helpers/node-pages.spec.js | 3 +- modules/tile-converter/test/index.js | 1 - 8 files changed, 303 insertions(+), 269 deletions(-) delete mode 100644 modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts create mode 100644 modules/tile-converter/src/lib/utils/conversion-dump.ts delete mode 100644 modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts diff --git a/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts b/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts deleted file mode 100644 index 2651d348db..0000000000 --- a/modules/tile-converter/src/i3s-converter/helpers/dump-parser.ts +++ /dev/null @@ -1,23 +0,0 @@ -import {isMap} from 'util/types'; - -/** - * Convert dump map to object for stringifying to JSON - * @param tilesConverted - Map of tilesConverted with conversion status - * @returns tilesConverted Map converted to Object - */ -export const dumpTilesToObject = (tilesConverted: Map) => { - const object = {}; - for (const [key, value] of tilesConverted) { - if (value?.nodes) { - object[key] = {nodes: []}; - for (const node of value.nodes) { - if (isMap(node.done)) { - object[key].nodes.push({nodeId: node.nodeId, done: Object.fromEntries(node.done)}); - } else { - object[key].nodes.push({nodeId: node.nodeId, done: node.done}); - } - } - } - } - return object; -}; diff --git a/modules/tile-converter/src/i3s-converter/i3s-converter.ts b/modules/tile-converter/src/i3s-converter/i3s-converter.ts index a7d858e772..9849f1959a 100644 --- a/modules/tile-converter/src/i3s-converter/i3s-converter.ts +++ b/modules/tile-converter/src/i3s-converter/i3s-converter.ts @@ -51,7 +51,7 @@ import {GEOMETRY_DEFINITION as geometryDefinitionTemlate} from './json-templates import {SHARED_RESOURCES as sharedResourcesTemplate} from './json-templates/shared-resources'; import {validateNodeBoundingVolumes} from './helpers/node-debug'; import {KTX2BasisWriterWorker} from '@loaders.gl/textures'; -import {FileHandleFile, LoaderWithParser, path} from '@loaders.gl/loader-utils'; +import {FileHandleFile, LoaderWithParser} from '@loaders.gl/loader-utils'; import {I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s'; import {ImageWriter} from '@loaders.gl/images'; import {GLTFImagePostprocessed} from '@loaders.gl/gltf'; @@ -64,7 +64,7 @@ import { } from './types'; import {WorkerFarm} from '@loaders.gl/worker-utils'; import WriteQueue from '../lib/utils/write-queue'; -import {BROWSER_ERROR_MESSAGE, DUMP_FILE_SUFFIX} from '../constants'; +import {BROWSER_ERROR_MESSAGE} from '../constants'; import { getAttributeTypesMapFromPropertyTable, getAttributeTypesMapFromSchema @@ -83,7 +83,7 @@ import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-t import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles'; import {Progress} from './helpers/progress'; import {addOneFile, composeHashFile} from '@loaders.gl/zip'; -import {dumpTilesToObject} from './helpers/dump-parser'; +import {ConversionDump, ConversionDumpOptions} from '../lib/utils/conversion-dump'; const ION_DEFAULT_TOKEN = process.env?.IonToken; const HARDCODED_NODES_PER_PAGE = 64; @@ -138,14 +138,14 @@ export default class I3SConverter { generateBoundingVolumes: boolean; layersHasTexture: boolean; workerSource: {[key: string]: string} = {}; - writeQueue: WriteQueue = new WriteQueue(); + writeQueue: WriteQueue = new WriteQueue(new ConversionDump()); compressList: string[] | null = null; preprocessData: PreprocessData = { meshTopologyTypes: new Set(), metadataClasses: new Set() }; progresses: Record = {}; - tilesConverted: Map; + conversionDump: ConversionDump; constructor() { this.attributeMetadataInfo = new AttributeMetadataInfo(); @@ -168,7 +168,7 @@ export default class I3SConverter { this.generateBoundingVolumes = false; this.layersHasTexture = false; this.compressList = null; - this.tilesConverted = new Map(); + this.conversionDump = new ConversionDump(); } /** @@ -253,9 +253,8 @@ export default class I3SConverter { this.generateTextures = Boolean(generateTextures); this.generateBoundingVolumes = Boolean(generateBoundingVolumes); - this.writeQueue = new WriteQueue(); + this.writeQueue = new WriteQueue(this.conversionDump); this.writeQueue.startListening(); - this.writeQueue.writeDumpFile = this.writeDumpFile.bind(this); console.log('Loading egm file...'); // eslint-disable-line this.geoidHeightModel = await load(egmFilePath, PGMLoader); @@ -266,15 +265,7 @@ export default class I3SConverter { } //create a dump file with convertion options - try { - writeFile( - options.outputPath, - JSON.stringify({options: this.options}), - `${options.tilesetName}${DUMP_FILE_SUFFIX}` - ); - } catch (error) { - console.log("Can't create dump file", error); - } + this.conversionDump.createDumpFile(options as ConversionDumpOptions); try { const preloadOptions = await this._fetchPreloadOptions(); @@ -637,12 +628,6 @@ export default class I3SConverter { console.log(`[convert]: ${sourceTile.id}`); // eslint-disable-line } - //add a record to dump map - if (sourceTile.id) { - const fileName = path.filename(sourceTile.id); - this.tilesConverted.set(fileName, null); - } - const {parentNodes, transform} = traversalProps; let transformationMatrix: Matrix4 = transform.clone(); if (sourceTile.transform) { @@ -781,16 +766,15 @@ export default class I3SConverter { nodes.push(node); if (nodeInPage.mesh) { - //update a record in a dump map file + //update a record in a dump file const nodeId = parseInt(node.id); if (!isNaN(nodeId) && sourceTile.id) { - const fileName = path.filename(sourceTile.id); - const {nodes} = this.tilesConverted.get(fileName) || {nodes: []}; - nodes.push({nodeId, done: new Map()}); + const {nodes} = this.conversionDump.getRecord(sourceTile.id) || {nodes: []}; + nodes.push({nodeId, done: {}}); if (nodes.length === 1) { - this.tilesConverted.set(fileName, {nodes}); + this.conversionDump.setRecord(sourceTile.id, {nodes}); } - this.writeDumpFile(); + this.conversionDump.updateDumpFile(); } //write resources await this._writeResources(resources, node.id, sourceTile); @@ -942,6 +926,8 @@ export default class I3SConverter { * @param resources.texture - texture image * @param resources.sharedResources - shared resource data object * @param resources.attributes - feature attributes + * @param nodePath - node path + * @param sourceTile - source tile (3DTile) * @return {Promise} */ private async _writeResources( @@ -958,14 +944,13 @@ export default class I3SConverter { } = resources; const childPath = join(this.layers0Path, 'nodes', nodePath); const slpkChildPath = join('nodes', nodePath); - const dumpTileRecord = this.tilesConverted.get(path.filename(sourceTile.id || '')); await this._writeGeometries( geometryBuffer!, compressedGeometry!, childPath, slpkChildPath, - dumpTileRecord, + sourceTile.id!, parseInt(nodePath) ); await this._writeShared( @@ -973,15 +958,15 @@ export default class I3SConverter { childPath, slpkChildPath, nodePath, - dumpTileRecord, + sourceTile.id!, parseInt(nodePath) ); - await this._writeTexture(texture, childPath, slpkChildPath, dumpTileRecord, parseInt(nodePath)); + await this._writeTexture(texture, childPath, slpkChildPath, sourceTile.id!, parseInt(nodePath)); await this._writeAttributes( attributes, childPath, slpkChildPath, - dumpTileRecord, + sourceTile.id!, parseInt(nodePath) ); } @@ -992,7 +977,7 @@ export default class I3SConverter { * @param compressedGeometry - Uint8Array with compressed (draco) geometry * @param childPath - a child path to write resources * @param slpkChildPath - resource path inside *slpk file - * @param dumpTileRecord - a record of the tilesConverted Map + * @param sourceId - source filename * @param nodeId - nodeId of a converted node for the writing */ private async _writeGeometries( @@ -1000,66 +985,49 @@ export default class I3SConverter { compressedGeometry: Promise, childPath: string, slpkChildPath: string, - dumpTileRecord: any, - nodeId?: number + sourceId: string, + nodeId: number ): Promise { - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(ResourceType.GEOMETRY, false); - } - } + this.conversionDump.updateDoneStatus(sourceId, nodeId, ResourceType.GEOMETRY, false); if (this.options.slpk) { const slpkGeometryPath = join(childPath, 'geometries'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/geometries/0.bin.gz`, - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: ResourceType.GEOMETRY - }, - + sourceId, + outputId: nodeId, + resourceType: ResourceType.GEOMETRY, writePromise: () => writeFileForSlpk(slpkGeometryPath, geometryBuffer, '0.bin') }); } else { const geometryPath = join(childPath, 'geometries/0/'); await this.writeQueue.enqueue({ - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: ResourceType.GEOMETRY - }, + sourceId, + outputId: nodeId, + resourceType: ResourceType.GEOMETRY, writePromise: () => writeFile(geometryPath, geometryBuffer, 'index.bin') }); } if (this.options.draco) { - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(ResourceType.DRACO_GEOMETRY, false); - } - } + this.conversionDump.updateDoneStatus(sourceId, nodeId, ResourceType.DRACO_GEOMETRY, false); if (this.options.slpk) { const slpkCompressedGeometryPath = join(childPath, 'geometries'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/geometries/1.bin.gz`, - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: ResourceType.DRACO_GEOMETRY - }, + sourceId, + outputId: nodeId, + resourceType: ResourceType.DRACO_GEOMETRY, writePromise: () => writeFileForSlpk(slpkCompressedGeometryPath, compressedGeometry, '1.bin') }); } else { const compressedGeometryPath = join(childPath, 'geometries/1/'); await this.writeQueue.enqueue({ - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: ResourceType.DRACO_GEOMETRY - }, + sourceId, + outputId: nodeId, + resourceType: ResourceType.DRACO_GEOMETRY, writePromise: () => writeFile(compressedGeometryPath, compressedGeometry, 'index.bin') }); } @@ -1072,7 +1040,7 @@ export default class I3SConverter { * @param childPath - a child path to write resources * @param slpkChildPath - resource path inside *slpk file * @param nodePath - a node path - * @param dumpTileRecord - a record of the tilesConverted Map + * @param sourceId - source filename * @param nodeId - nodeId of a converted node for the writing */ private async _writeShared( @@ -1080,8 +1048,8 @@ export default class I3SConverter { childPath: string, slpkChildPath: string, nodePath: string, - dumpTileRecord: any, - nodeId?: number + sourceId: string, + nodeId: number ): Promise { if (!sharedResources) { return; @@ -1089,30 +1057,22 @@ export default class I3SConverter { sharedResources.nodePath = nodePath; const sharedData = transform(sharedResources, sharedResourcesTemplate()); const sharedDataStr = JSON.stringify(sharedData); - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(ResourceType.SHARED, false); - } - } + this.conversionDump.updateDoneStatus(sourceId, nodeId, ResourceType.SHARED, false); if (this.options.slpk) { const slpkSharedPath = join(childPath, 'shared'); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/shared/sharedResource.json.gz`, - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: ResourceType.SHARED - }, + sourceId, + outputId: nodeId, + resourceType: ResourceType.SHARED, writePromise: () => writeFileForSlpk(slpkSharedPath, sharedDataStr, 'sharedResource.json') }); } else { const sharedPath = join(childPath, 'shared/'); await this.writeQueue.enqueue({ - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: ResourceType.SHARED - }, + sourceId, + outputId: nodeId, + resourceType: ResourceType.SHARED, writePromise: () => writeFile(sharedPath, sharedDataStr) }); } @@ -1123,15 +1083,15 @@ export default class I3SConverter { * @param texture - the texture image * @param childPath - a child path to write resources * @param slpkChildPath - the resource path inside *slpk file - * @param dumpTileRecord - a record of the tilesConverted Map + * @param sourceId - source filename * @param nodeId - nodeId of a converted node for the writing */ private async _writeTexture( texture: GLTFImagePostprocessed, childPath: string, slpkChildPath: string, - dumpTileRecord: any, - nodeId?: number + sourceId: string, + nodeId: number ): Promise { if (texture) { const format = this._getFormatByMimeType(texture?.mimeType); @@ -1142,18 +1102,19 @@ export default class I3SConverter { case 'jpg': case 'png': { formats.push({name: '0', format}); - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(`${ResourceType.TEXTURE}/${format}`, false); - } - } + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/${format}`, + false + ); await this.writeTextureFile( textureData, '0', format, childPath, slpkChildPath, - dumpTileRecord, + sourceId, nodeId ); @@ -1178,11 +1139,12 @@ export default class I3SConverter { } ); - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(`${ResourceType.TEXTURE}/ktx2`, false); - } - } + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/ktx2`, + false + ); await this.writeTextureFile( ktx2TextureData, @@ -1190,7 +1152,7 @@ export default class I3SConverter { 'ktx2', childPath, slpkChildPath, - dumpTileRecord, + sourceId, nodeId ); } @@ -1200,36 +1162,38 @@ export default class I3SConverter { case 'ktx2': { formats.push({name: '1', format}); - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(`${ResourceType.TEXTURE}/${format}`, false); - } - } + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/${format}`, + false + ); await this.writeTextureFile( textureData, '1', format, childPath, slpkChildPath, - dumpTileRecord, + sourceId, nodeId ); if (this.generateTextures) { formats.push({name: '0', format: 'jpg'}); const decodedFromKTX2TextureData = encode(texture.image!.data[0], ImageWriter); - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(`${ResourceType.TEXTURE}/jpg`, false); - } - } + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.TEXTURE}/jpg`, + false + ); await this.writeTextureFile( decodedFromKTX2TextureData, '0', 'jpg', childPath, slpkChildPath, - dumpTileRecord, + sourceId, nodeId ); } @@ -1250,7 +1214,7 @@ export default class I3SConverter { * @param format * @param childPath * @param slpkChildPath - * @param dumpTileRecord + * @param sourceId * @param nodeId */ private async writeTextureFile( @@ -1259,8 +1223,8 @@ export default class I3SConverter { format: 'jpg' | 'png' | 'ktx2', childPath: string, slpkChildPath: string, - dumpTileRecord: any, - nodeId?: number + sourceId: string, + nodeId: number ): Promise { if (this.options.slpk) { const slpkTexturePath = join(childPath, 'textures'); @@ -1268,22 +1232,18 @@ export default class I3SConverter { await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/textures/${name}.${format}`, - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: `${ResourceType.TEXTURE}/${format}` - }, + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.TEXTURE}/${format}`, writePromise: () => writeFileForSlpk(slpkTexturePath, textureData, `${name}.${format}`, compress) }); } else { const texturePath = join(childPath, `textures/${name}/`); await this.writeQueue.enqueue({ - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: `${ResourceType.TEXTURE}/${format}` - }, + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.TEXTURE}/${format}`, writePromise: () => writeFile(texturePath, textureData, `index.${format}`) }); } @@ -1294,15 +1254,15 @@ export default class I3SConverter { * @param attributes - feature attributes * @param childPath - a child path to write resources * @param slpkChildPath - the resource path inside *slpk file - * @param dumpTileRecord - a record of the tilesConverted Map + * @param sourceId - source filename * @param nodeId - nodeId of a converted node for the writing */ private async _writeAttributes( attributes: ArrayBuffer[] | null = [], childPath: string, slpkChildPath: string, - dumpTileRecord: any, - nodeId?: number + sourceId: string, + nodeId: number ): Promise { if (attributes?.length && this.attributeMetadataInfo.attributeStorageInfo.length) { const minimumLength = @@ -1313,30 +1273,27 @@ export default class I3SConverter { for (let index = 0; index < minimumLength; index++) { const folderName = this.attributeMetadataInfo.attributeStorageInfo[index].key; const fileBuffer = new Uint8Array(attributes[index]); - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === nodeId) { - node.done.set(`${ResourceType.ATTRIBUTES}/${folderName}`, false); - } - } + this.conversionDump.updateDoneStatus( + sourceId, + nodeId, + `${ResourceType.ATTRIBUTES}/${folderName}`, + false + ); if (this.options.slpk) { const slpkAttributesPath = join(childPath, 'attributes', folderName); await this.writeQueue.enqueue({ archiveKey: `${slpkChildPath}/attributes/${folderName}.bin.gz`, - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: `${ResourceType.ATTRIBUTES}/${folderName}` - }, + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.ATTRIBUTES}/${folderName}`, writePromise: () => writeFileForSlpk(slpkAttributesPath, fileBuffer, '0.bin') }); } else { const attributesPath = join(childPath, `attributes/${folderName}/0`); await this.writeQueue.enqueue({ - convertedTileDumpMap: { - dumpTileRecord, - nodeId: nodeId, - resourceType: `${ResourceType.ATTRIBUTES}/${folderName}` - }, + sourceId, + outputId: nodeId, + resourceType: `${ResourceType.ATTRIBUTES}/${folderName}`, writePromise: () => writeFile(attributesPath, fileBuffer, 'index.bin') }); } @@ -1450,7 +1407,7 @@ export default class I3SConverter { console.log(`File(s) size: `, filesSize, ' bytes'); // eslint-disable-line no-undef, no-console console.log(`Percentage of tiles with "ADD" refinement type:`, addRefinementPercentage, '%'); // eslint-disable-line no-undef, no-console console.log(`------------------------------------------------`); // eslint-disable-line no-undef, no-console - removeFile(join(params.outputPath, `${params.tilesetName}${DUMP_FILE_SUFFIX}`)); + this.conversionDump.deleteDumpFile(); } /** @@ -1510,22 +1467,4 @@ export default class I3SConverter { private isContentSupported(sourceTile: Tiles3DTileJSONPostprocessed): boolean { return ['b3dm', 'glTF', 'scenegraph'].includes(sourceTile.type || ''); } - - /** - * Write conversion status into dump file - */ - private async writeDumpFile(): Promise { - try { - writeFile( - this.options.outputPath, - JSON.stringify({ - options: this.options, - tilesConverted: dumpTilesToObject(this.tilesConverted) - }), - `${this.options.tilesetName}${DUMP_FILE_SUFFIX}` - ); - } catch (error) { - console.log("Can't update dump file", error); - } - } } diff --git a/modules/tile-converter/src/lib/utils/conversion-dump.ts b/modules/tile-converter/src/lib/utils/conversion-dump.ts new file mode 100644 index 0000000000..879237b41b --- /dev/null +++ b/modules/tile-converter/src/lib/utils/conversion-dump.ts @@ -0,0 +1,182 @@ +import {DUMP_FILE_SUFFIX} from '../../constants'; +import {removeFile, writeFile} from './file-utils'; +import {join} from 'path'; + +export type ConversionDumpOptions = { + inputUrl: string; + outputPath: string; + tilesetName: string; + maxDepth: number; + slpk: boolean; + egmFilePath: string; + token: string; + draco: boolean; + mergeMaterials: boolean; + generateTextures: boolean; + generateBoundingVolumes: boolean; + metadataClass: string; + analyze: boolean; +}; + +type NodeDoneStatus = { + nodeId: number; + done: Record | boolean; +}; + +type TilesConverted = { + nodes: NodeDoneStatus[]; +}; + +export class ConversionDump { + /** Conversion options */ + private options?: ConversionDumpOptions; + /** Tiles conversion progress status map */ + tilesConverted: Record; + + constructor() { + this.tilesConverted = {}; + } + + /** + * Create a dump file with convertion options + * @param options - converter options + */ + createDumpFile(options: ConversionDumpOptions): void { + const { + tilesetName, + slpk, + egmFilePath, + inputUrl, + outputPath, + draco = true, + maxDepth, + token, + generateTextures, + generateBoundingVolumes, + mergeMaterials = true, + metadataClass, + analyze = false + } = options; + this.options = { + tilesetName, + slpk, + egmFilePath, + inputUrl, + outputPath, + draco, + maxDepth, + token, + generateTextures, + generateBoundingVolumes, + mergeMaterials, + metadataClass, + analyze + }; + + try { + writeFile( + options.outputPath, + JSON.stringify({options: this.options}), + `${options.tilesetName}${DUMP_FILE_SUFFIX}` + ); + } catch (error) { + console.log("Can't create dump file", error); + } + } + + /** + * Update conversion status in the dump file + */ + updateDumpFile(): void { + if (this.options?.outputPath && this.options.tilesetName) { + try { + writeFile( + this.options.outputPath, + JSON.stringify({ + options: this.options, + tilesConverted: this.tilesConverted + }), + `${this.options.tilesetName}${DUMP_FILE_SUFFIX}` + ); + } catch (error) { + console.log("Can't update dump file", error); + } + } + } + + /** + * Delete a dump file + */ + deleteDumpFile(): void { + if (this.options?.outputPath && this.options.tilesetName) { + removeFile(join(this.options.outputPath, `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`)); + } + } + + /** + * Get record from the tilesConverted Map + * @param fileName - source filename + * @returns existing object from the tilesConverted Map + */ + getRecord(fileName: string) { + return this.tilesConverted[fileName]; + } + + /** + * Set a record for the dump file + * @param fileName - key - source filename + * @param object - value + */ + setRecord(fileName: string, object: any) { + this.tilesConverted[fileName] = object; + } + + /** + * Update done status object for the writing resources + * @param fileName - key - source filename + * @param nodeId - nodeId for the source filename + * @param resourceType - resource type to update status + * @param value - value + */ + updateDoneStatus(filename: string, nodeId: number, resourceType: string, value: boolean) { + const nodeDump = this.tilesConverted[filename]?.nodes.find( + (element) => element.nodeId === nodeId + ); + if (nodeDump) { + nodeDump.done[resourceType] = value; + } + } + + /** + * Update dump file according to writing results + * @param changedRecords - array of parameters ids for the written resources + * @param writeResults - array of writing resource files results + */ + updateConvertedTilesDump( + changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[], + writeResults: PromiseSettledResult[] + ) { + for (let i = 0; i < changedRecords.length; i++) { + if (changedRecords[i] && 'value' in writeResults[i]) { + const {sourceId, resourceType, outputId} = changedRecords[i]; + if (!sourceId || !resourceType || !outputId) continue; + for (const node of this.tilesConverted[sourceId].nodes) { + if (typeof node.done !== 'boolean' && node.nodeId === outputId) { + node.done[resourceType] = true; + } + if (typeof node.done !== 'boolean') { + let done = false; + for (const key in node.done) { + done = node.done[key]; + if (!done) break; + } + if (done) { + node.done = true; + } + } + } + } + } + this.updateDumpFile(); + } +} diff --git a/modules/tile-converter/src/lib/utils/write-queue.ts b/modules/tile-converter/src/lib/utils/write-queue.ts index 38165093c5..cfbbef0c4b 100644 --- a/modules/tile-converter/src/lib/utils/write-queue.ts +++ b/modules/tile-converter/src/lib/utils/write-queue.ts @@ -1,13 +1,15 @@ -import {isMap} from 'util/types'; import {Queue} from './queue'; import process from 'process'; +import {ConversionDump} from './conversion-dump'; /** Memory limit size is based on testing */ const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB export type WriteQueueItem = { archiveKey?: string; - convertedTileDumpMap?: any; + sourceId?: string; + outputId?: number; + resourceType?: string; /** * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises); * Arguments for this call are specified in writeQueue.enqueue call like this: @@ -29,14 +31,19 @@ export type WriteQueueItem = { export default class WriteQueue extends Queue { private intervalId?: NodeJS.Timeout; + private conversionDump: ConversionDump; public writePromise: Promise | null = null; - public writeDumpFile: (() => void) | undefined; public fileMap: {[key: string]: string} = {}; public listeningInterval: number; public writeConcurrency: number; - constructor(listeningInterval: number = 2000, writeConcurrency: number = 400) { + constructor( + conversionDump: ConversionDump, + listeningInterval: number = 2000, + writeConcurrency: number = 400 + ) { super(); + this.conversionDump = conversionDump; this.listeningInterval = listeningInterval; this.writeConcurrency = writeConcurrency; } @@ -84,21 +91,21 @@ export default class WriteQueue extends Queue { while (this.length) { const promises: Promise[] = []; const archiveKeys: (string | undefined)[] = []; - const convertedTileDumpMaps: any[] = []; + const changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[] = []; for (let i = 0; i < this.writeConcurrency; i++) { const item = this.dequeue(); if (!item) { break; } - const {archiveKey, convertedTileDumpMap, writePromise} = item as WriteQueueItem; + const {archiveKey, sourceId, outputId, resourceType, writePromise} = item as WriteQueueItem; archiveKeys.push(archiveKey); - convertedTileDumpMaps.push(convertedTileDumpMap); + changedRecords.push({sourceId, outputId, resourceType}); const promise = writePromise(); promises.push(promise); } const writeResults = await Promise.allSettled(promises); this.updateFileMap(archiveKeys, writeResults); - this.updateConvertedTilesMap(convertedTileDumpMaps, writeResults); + this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults); } } @@ -113,34 +120,4 @@ export default class WriteQueue extends Queue { } } } - - private updateConvertedTilesMap( - convertedTileDumpMaps: any[], - writeResults: PromiseSettledResult[] - ) { - for (let i = 0; i < convertedTileDumpMaps.length; i++) { - if (convertedTileDumpMaps[i] && 'value' in writeResults[i]) { - const {dumpTileRecord} = convertedTileDumpMaps[i]; - for (const node of dumpTileRecord.nodes) { - if (node.nodeId === convertedTileDumpMaps[i].nodeId) { - node.done.set(convertedTileDumpMaps[i].resourceType, true); - } - if (isMap(node.done)) { - let done = false; - for (const [_, value] of node.done) { - done = value; - if (!done) break; - } - if (done) { - delete node.done; - node.done = true; - } - } - } - } - } - if (this.writeDumpFile) { - this.writeDumpFile(); - } - } } diff --git a/modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts b/modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts deleted file mode 100644 index 29d30a9006..0000000000 --- a/modules/tile-converter/test/i3s-converter/helpers/dump-parser.spec.ts +++ /dev/null @@ -1,42 +0,0 @@ -import test from 'tape-promise/tape'; -import {dumpTilesToObject} from '../../../src/i3s-converter/helpers/dump-parser'; - -test('tile-converter(i3s)#dumpTilesToObject - conversion from dump map to object', (t) => { - const inputMap = new Map(); - inputMap.set('file1', { - nodes: [{nodeId: 1, done: true}] - }); - inputMap.set('file2', { - nodes: [ - {nodeId: 2, done: new Map()}, - {nodeId: 3, done: new Map()} - ] - }); - const {nodes} = inputMap.get('file2'); - nodes[0].done.set('geometry', false); - nodes[0].done.set('texture', false); - nodes[0].done.set('shared', true); - nodes[1].done.set('geometry', true); - nodes[1].done.set('texture', true); - nodes[1].done.set('shared', false); - inputMap.set('file3', {nodes: []}); - inputMap.set('file4', {nodes: [{nodeId: 4, done: new Map()}]}); - - const expectedResult = { - file1: { - nodes: [{nodeId: 1, done: true}] - }, - file2: { - nodes: [ - {nodeId: 2, done: {geometry: false, texture: false, shared: true}}, - {nodeId: 3, done: {geometry: true, texture: true, shared: false}} - ] - }, - file3: {nodes: []}, - file4: {nodes: [{nodeId: 4, done: {}}]} - }; - - const result = dumpTilesToObject(inputMap); - t.deepEqual(result, expectedResult); - t.end(); -}); diff --git a/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js b/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js index 35154d431c..3c4d167440 100644 --- a/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js +++ b/modules/tile-converter/test/i3s-converter/helpers/node-index-document.spec.js @@ -3,6 +3,7 @@ import {isBrowser} from '@loaders.gl/core'; import {NodeIndexDocument} from '../../../src/i3s-converter/helpers/node-index-document'; import I3SConverter from '../../../src/i3s-converter/i3s-converter'; import WriteQueue from '../../../src/lib/utils/write-queue'; +import {ConversionDump} from '../../../src/lib/utils/conversion-dump'; const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWriting: false}) => { const converter = new I3SConverter(); @@ -11,7 +12,7 @@ const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWrit instantNodeWriting }; converter.layers0Path = '.data/node-pages-test/layers/0'; - converter.writeQueue = new WriteQueue(); + converter.writeQueue = new WriteQueue(new ConversionDump()); return converter; }; diff --git a/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js b/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js index 8efc9f9a2a..4cd7127c82 100644 --- a/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js +++ b/modules/tile-converter/test/i3s-converter/helpers/node-pages.spec.js @@ -3,6 +3,7 @@ import {default as NodePages} from '../../../src/i3s-converter/helpers/node-page import {isBrowser} from '@loaders.gl/core'; import I3SConverter from '../../../src/i3s-converter/i3s-converter'; import WriteQueue from '../../../src/lib/utils/write-queue'; +import {ConversionDump} from '../../../src/lib/utils/conversion-dump'; const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWriting: false}) => { const converter = new I3SConverter(); @@ -11,7 +12,7 @@ const getConverter = ({slpk, instantNodeWriting} = {slpk: false, instantNodeWrit instantNodeWriting }; converter.layers0Path = '.data/node-pages-test/layers/0'; - converter.writeQueue = new WriteQueue(); + converter.writeQueue = new WriteQueue(new ConversionDump()); return converter; }; diff --git a/modules/tile-converter/test/index.js b/modules/tile-converter/test/index.js index c97cce1a0a..b1b323b1f5 100644 --- a/modules/tile-converter/test/index.js +++ b/modules/tile-converter/test/index.js @@ -13,7 +13,6 @@ import './i3s-converter/helpers/preprocess-3d-tiles.spec'; import './i3s-converter/helpers/geometry-attributes.spec'; import './i3s-converter/helpers/attribute-metadata-info.spec.ts'; import './i3s-converter/helpers/progress.spec.ts'; -import './i3s-converter/helpers/dump-parser.spec.ts'; import './i3s-converter/i3s-converter.spec'; import './slpk-extractor/slpk-extractor.spec'; From 406cfa45480e901be1511866b9eb8f29e08c650e Mon Sep 17 00:00:00 2001 From: Maxim Kuznetsov Date: Tue, 23 Jan 2024 21:16:22 +0300 Subject: [PATCH 4/4] update after review --- .../src/i3s-converter/i3s-converter.ts | 29 ++-- .../src/lib/utils/conversion-dump.ts | 36 +++-- .../src/lib/utils/write-queue.ts | 2 +- modules/tile-converter/test/index.js | 2 + .../test/lib/utils/conversion-dump.spec.ts | 144 ++++++++++++++++++ 5 files changed, 188 insertions(+), 25 deletions(-) create mode 100644 modules/tile-converter/test/lib/utils/conversion-dump.spec.ts diff --git a/modules/tile-converter/src/i3s-converter/i3s-converter.ts b/modules/tile-converter/src/i3s-converter/i3s-converter.ts index 9849f1959a..d8873a6545 100644 --- a/modules/tile-converter/src/i3s-converter/i3s-converter.ts +++ b/modules/tile-converter/src/i3s-converter/i3s-converter.ts @@ -265,7 +265,7 @@ export default class I3SConverter { } //create a dump file with convertion options - this.conversionDump.createDumpFile(options as ConversionDumpOptions); + await this.conversionDump.createDumpFile(options as ConversionDumpOptions); try { const preloadOptions = await this._fetchPreloadOptions(); @@ -563,6 +563,7 @@ export default class I3SConverter { * @param tilesetPath - Path to save file */ private async _createSlpk(tilesetPath: string): Promise { + await this.conversionDump.deleteDumpFile(); if (this.options.slpk) { const slpkTilesetPath = join(tilesetPath, 'SceneServer', 'layers', '0'); const slpkFileName = `${tilesetPath}.slpk`; @@ -767,15 +768,10 @@ export default class I3SConverter { if (nodeInPage.mesh) { //update a record in a dump file - const nodeId = parseInt(node.id); - if (!isNaN(nodeId) && sourceTile.id) { - const {nodes} = this.conversionDump.getRecord(sourceTile.id) || {nodes: []}; - nodes.push({nodeId, done: {}}); - if (nodes.length === 1) { - this.conversionDump.setRecord(sourceTile.id, {nodes}); - } - this.conversionDump.updateDumpFile(); + if (sourceTile.id) { + await this.conversionDump.addNode(sourceTile.id, nodeInPage.index); } + //write resources await this._writeResources(resources, node.id, sourceTile); } @@ -950,7 +946,7 @@ export default class I3SConverter { compressedGeometry!, childPath, slpkChildPath, - sourceTile.id!, + sourceTile.id || '', parseInt(nodePath) ); await this._writeShared( @@ -958,15 +954,21 @@ export default class I3SConverter { childPath, slpkChildPath, nodePath, - sourceTile.id!, + sourceTile.id || '', + parseInt(nodePath) + ); + await this._writeTexture( + texture, + childPath, + slpkChildPath, + sourceTile.id || '', parseInt(nodePath) ); - await this._writeTexture(texture, childPath, slpkChildPath, sourceTile.id!, parseInt(nodePath)); await this._writeAttributes( attributes, childPath, slpkChildPath, - sourceTile.id!, + sourceTile.id || '', parseInt(nodePath) ); } @@ -1407,7 +1409,6 @@ export default class I3SConverter { console.log(`File(s) size: `, filesSize, ' bytes'); // eslint-disable-line no-undef, no-console console.log(`Percentage of tiles with "ADD" refinement type:`, addRefinementPercentage, '%'); // eslint-disable-line no-undef, no-console console.log(`------------------------------------------------`); // eslint-disable-line no-undef, no-console - this.conversionDump.deleteDumpFile(); } /** diff --git a/modules/tile-converter/src/lib/utils/conversion-dump.ts b/modules/tile-converter/src/lib/utils/conversion-dump.ts index 879237b41b..215075ace5 100644 --- a/modules/tile-converter/src/lib/utils/conversion-dump.ts +++ b/modules/tile-converter/src/lib/utils/conversion-dump.ts @@ -41,7 +41,7 @@ export class ConversionDump { * Create a dump file with convertion options * @param options - converter options */ - createDumpFile(options: ConversionDumpOptions): void { + async createDumpFile(options: ConversionDumpOptions): Promise { const { tilesetName, slpk, @@ -74,7 +74,7 @@ export class ConversionDump { }; try { - writeFile( + await writeFile( options.outputPath, JSON.stringify({options: this.options}), `${options.tilesetName}${DUMP_FILE_SUFFIX}` @@ -87,10 +87,10 @@ export class ConversionDump { /** * Update conversion status in the dump file */ - updateDumpFile(): void { + private async updateDumpFile(): Promise { if (this.options?.outputPath && this.options.tilesetName) { try { - writeFile( + await writeFile( this.options.outputPath, JSON.stringify({ options: this.options, @@ -107,9 +107,11 @@ export class ConversionDump { /** * Delete a dump file */ - deleteDumpFile(): void { + async deleteDumpFile(): Promise { if (this.options?.outputPath && this.options.tilesetName) { - removeFile(join(this.options.outputPath, `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`)); + await removeFile( + join(this.options.outputPath, `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`) + ); } } @@ -118,7 +120,7 @@ export class ConversionDump { * @param fileName - source filename * @returns existing object from the tilesConverted Map */ - getRecord(fileName: string) { + private getRecord(fileName: string) { return this.tilesConverted[fileName]; } @@ -127,10 +129,24 @@ export class ConversionDump { * @param fileName - key - source filename * @param object - value */ - setRecord(fileName: string, object: any) { + private setRecord(fileName: string, object: any) { this.tilesConverted[fileName] = object; } + /** + * Add a node into the dump file for the source file record + * @param fileName - source filename + * @param nodeId - nodeId of the node + */ + async addNode(filename: string, nodeId: number) { + const {nodes} = this.getRecord(filename) || {nodes: []}; + nodes.push({nodeId, done: {}}); + if (nodes.length === 1) { + this.setRecord(filename, {nodes}); + } + await this.updateDumpFile(); + } + /** * Update done status object for the writing resources * @param fileName - key - source filename @@ -152,7 +168,7 @@ export class ConversionDump { * @param changedRecords - array of parameters ids for the written resources * @param writeResults - array of writing resource files results */ - updateConvertedTilesDump( + async updateConvertedTilesDump( changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[], writeResults: PromiseSettledResult[] ) { @@ -177,6 +193,6 @@ export class ConversionDump { } } } - this.updateDumpFile(); + await this.updateDumpFile(); } } diff --git a/modules/tile-converter/src/lib/utils/write-queue.ts b/modules/tile-converter/src/lib/utils/write-queue.ts index cfbbef0c4b..f8b8268247 100644 --- a/modules/tile-converter/src/lib/utils/write-queue.ts +++ b/modules/tile-converter/src/lib/utils/write-queue.ts @@ -105,7 +105,7 @@ export default class WriteQueue extends Queue { } const writeResults = await Promise.allSettled(promises); this.updateFileMap(archiveKeys, writeResults); - this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults); + await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults); } } diff --git a/modules/tile-converter/test/index.js b/modules/tile-converter/test/index.js index b1b323b1f5..20b7cd20f3 100644 --- a/modules/tile-converter/test/index.js +++ b/modules/tile-converter/test/index.js @@ -30,3 +30,5 @@ import './i3s-server/controllers/index-controller.spec'; import './i3s-server/controllers/slpk-controller.spec'; import './i3s-server/utils/create-scene-server.spec.ts'; import './i3s-server/utils/server-utils.spec'; + +import './lib/utils/conversion-dump.spec.ts'; diff --git a/modules/tile-converter/test/lib/utils/conversion-dump.spec.ts b/modules/tile-converter/test/lib/utils/conversion-dump.spec.ts new file mode 100644 index 0000000000..056703fbed --- /dev/null +++ b/modules/tile-converter/test/lib/utils/conversion-dump.spec.ts @@ -0,0 +1,144 @@ +import test from 'tape-promise/tape'; +import {ConversionDump, ConversionDumpOptions} from '../../../src/lib/utils/conversion-dump'; +import {join} from 'path'; +import {isFileExists, openJson} from '../../../src/lib/utils/file-utils'; +import {DUMP_FILE_SUFFIX} from '../../../src/constants'; +import {cleanUpPath} from '../../utils/file-utils'; + +test('tile-converter(i3s)#ConversionDump - Should create and delete conversion dump with options', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + inputUrl: 'testInputUrl', + outputPath: 'testPath', + tilesetName: 'testTileset', + maxDepth: 5, + slpk: true, + egmFilePath: 'testEGM', + token: 'testToken', + draco: true, + mergeMaterials: true, + generateTextures: true, + generateBoundingVolumes: true, + metadataClass: 'testMetadataClass', + analyze: true, + something: 'test' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + + let isDumpExists = await isFileExists( + join(testOptions.outputPath, `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}`) + ); + t.equal(isDumpExists, true); + + const {options} = await openJson( + testOptions.outputPath, + `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}` + ); + const {something, ...correctOptions} = testOptions; + t.deepEqual(options, correctOptions); + + await conversionDump.deleteDumpFile(); + isDumpExists = await isFileExists( + join(testOptions.outputPath, `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}`) + ); + t.equal(isDumpExists, false); + await cleanUpPath(testOptions.outputPath); + t.end(); +}); + +test('tile-converter(i3s)#ConversionDump - Add node to the dump', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + outputPath: 'testPath', + tilesetName: 'testTileset' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + + await conversionDump.addNode('1.glb', 1); + await conversionDump.addNode('1.glb', 2); + await conversionDump.addNode('2.glb', 3); + + const {tilesConverted} = await openJson( + testOptions.outputPath, + `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}` + ); + + t.deepEqual(tilesConverted, { + '1.glb': { + nodes: [ + {nodeId: 1, done: {}}, + {nodeId: 2, done: {}} + ] + }, + '2.glb': { + nodes: [{nodeId: 3, done: {}}] + } + }); + + await conversionDump.deleteDumpFile(); + await cleanUpPath(testOptions.outputPath); + t.end(); +}); + +test('tile-converter(i3s)#ConversionDump - update Done Status', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + outputPath: 'testPath', + tilesetName: 'testTileset' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + await conversionDump.addNode('1.glb', 1); + conversionDump.updateDoneStatus('1.glb', 1, 'testResource', false); + + t.deepEqual(conversionDump.tilesConverted, { + '1.glb': { + nodes: [{nodeId: 1, done: {testResource: false}}] + } + }); + + await conversionDump.deleteDumpFile(); + await cleanUpPath(testOptions.outputPath); + t.end(); +}); + +test('tile-converter(i3s)#ConversionDump - updateConvertedTilesDump', async (t) => { + const conversionDump = new ConversionDump(); + const testOptions = { + outputPath: 'testPath', + tilesetName: 'testTileset' + }; + + await conversionDump.createDumpFile(testOptions as ConversionDumpOptions); + await conversionDump.addNode('1.glb', 1); + conversionDump.updateDoneStatus('1.glb', 1, 'testResource', false); + + const promises: Promise[] = []; + promises.push(Promise.resolve('')); + const writeResults = await Promise.allSettled(promises); + const changedRecords = [{sourceId: '1.glb', outputId: 1, resourceType: 'testResource'}]; + await conversionDump.updateConvertedTilesDump(changedRecords, writeResults); + + t.deepEqual(conversionDump.tilesConverted, { + '1.glb': { + nodes: [{nodeId: 1, done: true}] + } + }); + + const {tilesConverted} = await openJson( + testOptions.outputPath, + `${testOptions.tilesetName}${DUMP_FILE_SUFFIX}` + ); + + t.deepEqual(tilesConverted, { + '1.glb': { + nodes: [{nodeId: 1, done: true}] + } + }); + + await conversionDump.deleteDumpFile(); + await cleanUpPath(testOptions.outputPath); + t.end(); +});