Merge pull request #167 from CesiumGS/async-source-and-target

Make tileset sources and targets asynchronous

.eslintignore

@@ -5,3 +5,4 @@ dist
 etc
 generateThirdParty.js
 src/ktx/ktx/external/basis_encoder.cjs
+demos

.eslintrc.json

@@ -6,12 +6,14 @@
   ],
   "parser": "@typescript-eslint/parser",
   "parserOptions": {
-    "ecmaVersion": 2020
+    "ecmaVersion": 2020,
+    "project": "./tsconfig.json"
   },
   "plugins": ["@typescript-eslint"],
   "root": true,
   "rules": {
     "@typescript-eslint/no-explicit-any": "off",
-    "@typescript-eslint/no-empty-interface": "off"
+    "@typescript-eslint/no-empty-interface": "off",
+    "@typescript-eslint/no-floating-promises": "error"
   }
 }

demos/base/ContentDataTypeChecksDemo.ts

@@ -32,4 +32,4 @@ async function testContentDataTypeChecks() {
   }
 }
 
-testContentDataTypeChecks();
+void testContentDataTypeChecks();

demos/base/ContentDataTypeRegistryDemo.ts

@@ -30,4 +30,4 @@ async function testContentDataTypeRegistry() {
   }
 }
 
-testContentDataTypeRegistry();
+void testContentDataTypeRegistry();

demos/gltf-extensions/ExtInstanceFeaturesDemo.ts

@@ -135,5 +135,5 @@ async function runReadingExample() {
   console.log(s);
 }
 
-runCreationExample();
-runReadingExample();
+void runCreationExample();
+void runReadingExample();

demos/gltf-extensions/ExtMeshFeaturesDemo.ts

@@ -107,5 +107,5 @@ async function runReadingExample() {
   console.log(s);
 }
 
-runCreationExample();
-runReadingExample();
+void runCreationExample();
+void runReadingExample();

demos/gltf-extensions/ExtMeshFeaturesGeometryDemo.ts

@@ -272,5 +272,5 @@ async function runReadingExample() {
   console.log(s);
 }
 
-runCreationExample();
-runReadingExample();
+void runCreationExample();
+void runReadingExample();

demos/gltf-extensions/ExtStructuralMetadataPropertyTableDemo.ts

@@ -135,4 +135,4 @@ async function runCreationExample() {
   console.log(JSON.stringify(written.json, null, 2));
 }
 
-runCreationExample();
+void runCreationExample();

demos/ktx/KtxDemo.ts

@@ -80,4 +80,4 @@ async function runDemo() {
   await convertImageData();
 }
 
-runDemo();
+void runDemo();

demos/tilesets/PackageConversion.ts

@@ -77,19 +77,22 @@ async function tilesetPackageConversion(options: any) {
       overwrite
     );
   } else {
-    const tilesetSource = TilesetSources.createAndOpen(input);
-    const tilesetTarget = TilesetTargets.createAndBegin(output, overwrite);
+    const tilesetSource = await TilesetSources.createAndOpen(input);
+    const tilesetTarget = await TilesetTargets.createAndBegin(
+      output,
+      overwrite
+    );
 
-    const keys = tilesetSource.getKeys();
-    for (const key of keys) {
-      const content = tilesetSource.getValue(key);
+    const keys = await tilesetSource.getKeys();
+    for await (const key of keys) {
+      const content = await tilesetSource.getValue(key);
       // TODO Compression or decompression could happen here...
       if (content) {
-        tilesetTarget.addEntry(key, content);
+        await tilesetTarget.addEntry(key, content);
       }
     }
 
-    tilesetSource.close();
+    await tilesetSource.close();
     await tilesetTarget.end();
   }
 }
@@ -103,7 +106,7 @@ async function run() {
   if (options === undefined) {
     return;
   }
-  tilesetPackageConversion(options);
+  await tilesetPackageConversion(options);
 }
 
-run();
+void run();

demos/tilesets/PackageServer.ts

@@ -90,7 +90,7 @@ function determineMimeType(fileName: string, content: Buffer | undefined) {
  * @param req The request
  * @param res The response
  */
-function handleRequest(
+async function handleRequest(
   tilesetSource: TilesetSource,
   req: http.IncomingMessage,
   res: http.ServerResponse<http.IncomingMessage>
@@ -124,7 +124,7 @@ function handleRequest(
   if (nameEnd != -1) {
     path = path.substring(0, nameEnd);
   }
-  const content = tilesetSource.getValue(path);
+  const content = await tilesetSource.getValue(path);
 
   //console.log("Content for " + path + " is " + content?.length);
 
@@ -154,7 +154,7 @@ function handleRequest(
  *
  * @param options The options
  */
-function startServer(options: any) {
+async function startServer(options: any) {
   const hostName = options.hostName;
   const port = options.port;
   const sourceName = options.sourceName;
@@ -165,7 +165,7 @@ function startServer(options: any) {
     console.log("Could not create source for " + sourceName);
     return;
   }
-  tilesetSource.open(sourceName);
+  await tilesetSource.open(sourceName);
   const server = http.createServer((req, res) =>
     handleRequest(tilesetSource, req, res)
   );
@@ -183,7 +183,7 @@ async function run() {
   if (options === undefined) {
     return;
   }
-  startServer(options);
+  await startServer(options);
 }
 
-run();
+void run();

demos/tilesets/PackagesDemo.ts

@@ -12,10 +12,13 @@ async function createPackageExample(fileName: string) {
   console.log("Creating package " + fileName);
 
   const overwrite = true;
-  const tilesetTarget = TilesetTargets.createAndBegin(fileName, overwrite);
+  const tilesetTarget = await TilesetTargets.createAndBegin(
+    fileName,
+    overwrite
+  );
 
-  tilesetTarget.addEntry("example.json", Buffer.alloc(100));
-  tilesetTarget.addEntry("example.glb", Buffer.alloc(1000));
+  await tilesetTarget.addEntry("example.json", Buffer.alloc(100));
+  await tilesetTarget.addEntry("example.glb", Buffer.alloc(1000));
 
   await tilesetTarget.end();
 
@@ -30,20 +33,21 @@ async function createPackageExample(fileName: string) {
 async function readPackageExample(fileName: string) {
   console.log("Reading package " + fileName);
 
-  const tilesetSource = TilesetSources.createAndOpen(fileName);
+  const tilesetSource = await TilesetSources.createAndOpen(fileName);
 
   console.log("Package contents:");
-  for (const key of tilesetSource.getKeys()) {
+  const keys = await tilesetSource.getKeys();
+  for await (const key of keys) {
     console.log("  key: " + key);
   }
 
-  const valueA = tilesetSource.getValue("example.json");
+  const valueA = await tilesetSource.getValue("example.json");
   console.log("Value for example.json ", valueA);
 
-  const valueB = tilesetSource.getValue("example.glb");
+  const valueB = await tilesetSource.getValue("example.glb");
   console.log("Value for example.glb  ", valueB);
 
-  tilesetSource.close();
+  await tilesetSource.close();
 
   console.log("Reading package DONE");
 }
@@ -67,4 +71,4 @@ async function run() {
   console.log("Running test DONE");
 }
 
-run();
+void run();

demos/tilesets/TileFormatsDemo.ts

@@ -42,7 +42,7 @@ async function b3dmOrI3dmToGlb(inputFileName: string, outputFileName: string) {
   fs.writeFileSync(outputFileName, upgradedOutputBuffer);
 }
 
-function testTileFormatsDemotConversions() {
+async function testTileFormatsDemotConversions() {
   Paths.ensureDirectoryExists(SPECS_DATA_BASE_DIRECTORY + "/output");
 
   glbToB3dm(
@@ -55,15 +55,15 @@ function testTileFormatsDemotConversions() {
     SPECS_DATA_BASE_DIRECTORY + "/output/CesiumTexturedBox.i3dm"
   );
 
-  b3dmOrI3dmToGlb(
+  await b3dmOrI3dmToGlb(
     SPECS_DATA_BASE_DIRECTORY + "/batchedWithBatchTableBinary.b3dm",
     SPECS_DATA_BASE_DIRECTORY + "/output/batchedWithBatchTableBinary.glb"
   );
 
-  b3dmOrI3dmToGlb(
+  await b3dmOrI3dmToGlb(
     SPECS_DATA_BASE_DIRECTORY + "/instancedWithBatchTableBinary.i3dm",
     SPECS_DATA_BASE_DIRECTORY + "/output/instancedWithBatchTableBinary.glb"
   );
 }
 
-testTileFormatsDemotConversions();
+void testTileFormatsDemotConversions();

demos/tilesets/TraversalDemo.ts

@@ -58,4 +58,4 @@ async function runDemo() {
   await runExternalDemo();
 }
 
-runDemo();
+void runDemo();

demos/tilesets/TraversalStatsDemo.ts

@@ -257,4 +257,4 @@ async function runDemo() {
   await tilesetTraversalDemo(tilesetFileName);
 }
 
-runDemo();
+void runDemo();

demos/tools/BasicTilesetProcessorExamples.ts

@@ -59,4 +59,4 @@ async function example() {
   await tilesetProcessor.end();
 }
 
-example();
+void example();

demos/tools/ImplicitToExplicitDemo.ts

@@ -116,4 +116,4 @@ async function runDemo() {
   await runConversionDemo(tilesetSourceName, tilesetTargetName);
 }
 
-runDemo();
+void runDemo();

demos/tools/PipelineExperimentsContentStages.ts

@@ -90,4 +90,4 @@ async function example() {
   );
 }
 
-example();
+void example();

demos/tools/PipelineExperimentsTilesetStages.ts

@@ -44,4 +44,4 @@ async function example() {
   await PipelineExecutor.executePipeline(pipeline, overwrite);
 }
 
-example();
+void example();

demos/tools/SubtreeInfoDemos.ts

@@ -69,4 +69,4 @@ async function runDemos() {
   await testSubtreeInfo();
 }
 
-runDemos();
+void runDemos();

demos/tools/TileContentProcessingDemo.ts

@@ -11,7 +11,7 @@ import { TileContentProcessorsGltfpack } from "3d-tiles-tools";
 import { GltfPackOptions } from "3d-tiles-tools";
 
 // The intention of this demo is to show the effects of compression that
-// may be applied with glTF-Transform,  gltf-pipeline, or gltfpack. To
+// may be applied with glTF-Transform, gltf-pipeline, or gltfpack. To
 // show these effects, the input tilesets should be large and complex.
 // Therefore, the input here is not part of the repository.
 const tilesetSourceName = "./input/TestTileset";
@@ -92,6 +92,8 @@ async function runDemoGltfPack() {
   );
 }
 
-runDemo();
-
-runDemoGltfPack();
+async function run() {
+  await runDemo();
+  await runDemoGltfPack();
+}
+void run();

demos/tools/TilesetDataProcessorExamples.ts

@@ -41,4 +41,4 @@ async function example() {
   await tilesetProcessor.end();
 }
 
-example();
+void example();

demos/tools/TilesetProcessingDemos.ts

@@ -75,4 +75,4 @@ async function tilesetProcessingDemos() {
   );
 }
 
-tilesetProcessingDemos();
+void tilesetProcessingDemos();

demos/tools/TilesetUpgraderDemos.ts

@@ -82,4 +82,4 @@ async function tilesetUpgraderDemos() {
   );
 }
 
-tilesetUpgraderDemos();
+void tilesetUpgraderDemos();

specs/SpecHelpers.ts

@@ -238,8 +238,8 @@ export class SpecHelpers {
    * @returns The tileset
    * @throws DeveloperError if the tileset could not be read
    */
-  static parseTileset(tilesetSource: TilesetSource) {
-    const tilesetJsonBuffer = tilesetSource.getValue("tileset.json");
+  static async parseTileset(tilesetSource: TilesetSource): Promise<Tileset> {
+    const tilesetJsonBuffer = await tilesetSource.getValue("tileset.json");
     if (!tilesetJsonBuffer) {
       throw new DeveloperError("No tileset.json found in input");
     }
@@ -262,20 +262,20 @@ export class SpecHelpers {
    * @returns A string describing the difference, or `undefined`
    * if there is no difference.
    */
-  static computePackageDifference(
+  static async computePackageDifference(
     nameA: string,
     nameB: string
-  ): string | undefined {
-    const tilesetSourceA = TilesetSources.createAndOpen(nameA);
-    const tilesetSourceB = TilesetSources.createAndOpen(nameB);
-    const result = SpecHelpers.computePackageDifferenceInternal(
+  ): Promise<string | undefined> {
+    const tilesetSourceA = await TilesetSources.createAndOpen(nameA);
+    const tilesetSourceB = await TilesetSources.createAndOpen(nameB);
+    const result = await SpecHelpers.computePackageDifferenceInternal(
       nameA,
       tilesetSourceA,
       nameB,
       tilesetSourceB
     );
-    tilesetSourceA.close();
-    tilesetSourceB.close();
+    await tilesetSourceA.close();
+    await tilesetSourceB.close();
     return result;
   }
 
@@ -297,14 +297,16 @@ export class SpecHelpers {
    * @returns A string describing the difference, or `undefined`
    * if there is no difference.
    */
-  static computePackageDifferenceInternal(
+  static async computePackageDifferenceInternal(
     nameA: string,
     tilesetSourceA: TilesetSource,
     nameB: string,
     tilesetSourceB: TilesetSource
-  ): string | undefined {
-    const keysA = [...tilesetSourceA.getKeys()].sort();
-    const keysB = [...tilesetSourceB.getKeys()].sort();
+  ): Promise<string | undefined> {
+    const keysA = await Iterables.asyncToArray(await tilesetSourceA.getKeys());
+    const keysB = await Iterables.asyncToArray(await tilesetSourceB.getKeys());
+    keysA.sort();
+    keysB.sort();
 
     if (keysA.length != keysB.length) {
       return `There are ${keysA.length} keys in ${nameA} and ${keysB.length} keys in ${nameB}`;
@@ -315,8 +317,8 @@ export class SpecHelpers {
       }
     }
     for (let i = 0; i < keysA.length; i++) {
-      const valueA = tilesetSourceA.getValue(keysA[i]);
-      const valueB = tilesetSourceB.getValue(keysB[i]);
+      const valueA = await tilesetSourceA.getValue(keysA[i]);
+      const valueB = await tilesetSourceB.getValue(keysB[i]);
       const entryDifference = SpecHelpers.computeEntryDifference(
         keysA[i],
         nameA,

specs/base/contentTypes/LazyContentDataSpec.ts

@@ -5,6 +5,9 @@ import { ResourceResolver } from "../../../src/base";
 
 function createTestResourceResolver(): ResourceResolver {
   return {
+    resolveUri(uri: string) {
+      return uri;
+    },
     async resolveData(uri: string): Promise<Buffer | null> {
       return null;
     },