refactor: Specify common types TDE-1030

src/commands/basemaps-mapsheet/create-mapsheet.ts

@@ -9,6 +9,7 @@ import { gunzip } from 'zlib';
 
 import { CliInfo } from '../../cli.info.js';
 import { logger } from '../../log.js';
+import { UrlParser } from '../../utils/parsers.js';
 import { registerCli, verbose } from '../common.js';
 
 const gunzipProm = promisify(gunzip);
@@ -22,9 +23,9 @@ export function isGzip(b: Buffer): boolean {
  *
  * If the file ends with .gz or is a GZIP like {@link isGzip} file it will automatically be decompressed.
  */
-async function readConfig(config: string): Promise<ConfigBundled> {
-  const obj = await fsa.read(config);
-  if (config.endsWith('.gz') || isGzip(obj)) {
+async function readConfig(config: URL): Promise<ConfigBundled> {
+  const obj = await fsa.read(config.href);
+  if (config.href.endsWith('.gz') || isGzip(obj)) {
     const data = await gunzipProm(obj);
     return JSON.parse(data.toString());
   }
@@ -39,17 +40,17 @@ interface Output {
 export const CommandCreateMapSheetArgs = {
   verbose,
   path: option({
-    type: string,
+    type: UrlParser,
     long: 'path',
     description: 'Path of flatgeobuf, this can be both a local path or s3 location',
   }),
   bmConfig: option({
-    type: string,
+    type: UrlParser,
     long: 'bm-config',
     description: 'Path of basemaps config json, this can be both a local path or s3 location',
   }),
   output: option({
-    type: string,
+    type: UrlParser,
     long: 'output',
     description: 'Output of the mapsheet file',
   }),
@@ -80,7 +81,7 @@ export const basemapsCreateMapSheet = command({
     const exclude = args.exclude ? new RegExp(args.exclude.toLowerCase(), 'i') : undefined;
 
     logger.info({ path }, 'MapSheet:LoadFgb');
-    const buf = await fsa.read(path);
+    const buf = await fsa.read(path.href);
     logger.info({ config }, 'MapSheet:LoadConfig');
     const configJson = await readConfig(config);
     const mem = ConfigProviderMemory.fromJson(configJson);
@@ -95,7 +96,7 @@ export const basemapsCreateMapSheet = command({
     const outputs = await createMapSheet(aerial, mem, rest, include, exclude);
 
     logger.info({ outputPath }, 'MapSheet:WriteOutput');
-    const outputWritePromise = fsa.write(outputPath, JSON.stringify(outputs, null, 2));
+    const outputWritePromise = fsa.write(outputPath.href, JSON.stringify(outputs, null, 2));
 
     await Promise.all([featuresWritePromise, outputWritePromise]);
   },

src/commands/copy/__test__/copy.test.ts

@@ -26,12 +26,12 @@ describe('copyFiles', () => {
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: new URL('memory://source/topographic.json'),
+          target: new URL('memory://target/topographic.json'),
         },
         {
-          source: 'memory://source/foo/bar/topographic.png',
-          target: 'memory://target/topographic.png',
+          source: new URL('memory://source/foo/bar/topographic.png'),
+          target: new URL('memory://target/topographic.png'),
         },
       ],
       start: 0,
@@ -77,12 +77,12 @@ describe('copyFiles', () => {
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: new URL('memory://source/topographic.json'),
+          target: new URL('memory://target/topographic.json'),
         },
         {
-          source: 'memory://source/foo/bar/topographic.tiff',
-          target: 'memory://target/topographic.tiff',
+          source: new URL('memory://source/foo/bar/topographic.tiff'),
+          target: new URL('memory://target/topographic.tiff'),
         },
       ],
       start: 0,
@@ -121,12 +121,12 @@ describe('copyFiles', () => {
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: new URL('memory://source/topographic.json'),
+          target: new URL('memory://target/topographic.json'),
         },
         {
-          source: 'memory://source/foo/bar/topographic.tiff',
-          target: 'memory://target/topographic.tiff',
+          source: new URL('memory://source/foo/bar/topographic.tiff'),
+          target: new URL('memory://target/topographic.tiff'),
         },
       ],
       start: 0,

src/commands/copy/copy-rpc.ts

@@ -6,7 +6,7 @@ export interface CopyContractArgs {
   /** Copy ID for tracing */
   id: string;
   /** List of files that need to be copied */
-  manifest: { source: string; target: string }[];
+  manifest: { source: URL; target: URL }[];
   /** Offset into the manifest to start at */
   start: number;
   /** Number of records to copy */

src/commands/copy/copy-worker.ts

@@ -19,20 +19,20 @@ export const FixableContentType = new Set(['binary/octet-stream', 'application/o
  * If the file has been written with a unknown binary contentType attempt to fix it with common content types
  *
  *
- * @param path File path to fix the metadata of
+ * @param url File path to fix the metadata of
  * @param meta File metadata
  * @returns New fixed file metadata if fixed other wise source file metadata
  */
-export function fixFileMetadata(path: string, meta: FileInfo): FileInfo {
+export function fixFileMetadata(url: URL, meta: FileInfo): FileInfo {
   // If the content is encoded we do not know what the content-type should be
   if (meta.contentEncoding != null) return meta;
   if (!FixableContentType.has(meta.contentType ?? 'binary/octet-stream')) return meta;
 
   // Assume our tiffs are cloud optimized
-  if (isTiff(path)) return { ...meta, contentType: 'image/tiff; application=geotiff; profile=cloud-optimized' };
+  if (isTiff(url)) return { ...meta, contentType: 'image/tiff; application=geotiff; profile=cloud-optimized' };
 
   // overwrite with application/json
-  if (path.endsWith('.json')) return { ...meta, contentType: 'application/json' };
+  if (url.href.endsWith('.json')) return { ...meta, contentType: 'application/json' };
 
   return meta;
 }
@@ -42,13 +42,13 @@ export function fixFileMetadata(path: string, meta: FileInfo): FileInfo {
  *
  * try reading the path {retryCount} times before aborting, with a delay of 250ms between requests
  *
- * @param filePath File to head
+ * @param url File to head
  * @param retryCount number of times to retry
  * @returns file size if it exists or null
  */
-async function tryHead(filePath: string, retryCount = 3): Promise<number | null> {
+async function tryHead(url: URL, retryCount = 3): Promise<number | null> {
   for (let i = 0; i < retryCount; i++) {
-    const ret = await fsa.head(filePath);
+    const ret = await fsa.head(url.href);
     if (ret?.size) return ret.size;
     await new Promise((r) => setTimeout(r, 250));
   }
@@ -66,7 +66,7 @@ export const worker = new WorkerRpc<CopyContract>({
       if (todo == null) continue;
 
       Q.push(async () => {
-        const [source, target] = await Promise.all([fsa.head(todo.source), fsa.head(todo.target)]);
+        const [source, target] = await Promise.all([fsa.head(todo.source.href), fsa.head(todo.target.href)]);
         if (source == null) return;
         if (source.size == null) return;
         if (target != null) {
@@ -86,8 +86,8 @@ export const worker = new WorkerRpc<CopyContract>({
         log.trace(todo, 'File:Copy:start');
         const startTime = performance.now();
         await fsa.write(
-          todo.target,
-          fsa.stream(todo.source),
+          todo.target.href,
+          fsa.stream(todo.source.href),
           args.fixContentType ? fixFileMetadata(todo.source, source) : source,
         );
 
@@ -96,7 +96,7 @@ export const worker = new WorkerRpc<CopyContract>({
         if (targetSize !== source.size) {
           log.fatal({ ...todo }, 'Copy:Failed');
           // Cleanup the failed copy so it can be retried
-          if (targetSize != null) await fsa.delete(todo.target);
+          if (targetSize != null) await fsa.delete(todo.target.href);
           throw new Error(`Failed to copy source:${todo.source} target:${todo.target}`);
         }
         log.debug({ ...todo, size: targetSize, duration: performance.now() - startTime }, 'File:Copy');

src/commands/copy/copy.ts

@@ -1,35 +1,19 @@
 import { fsa } from '@chunkd/fs';
 import { WorkerRpcPool } from '@wtrpc/core';
-import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts';
+import { boolean, command, flag, number, option, restPositionals } from 'cmd-ts';
 import { performance } from 'perf_hooks';
-import { gunzipSync } from 'zlib';
 import * as z from 'zod';
 
 import { CliInfo } from '../../cli.info.js';
 import { logger, logId } from '../../log.js';
 import { ActionCopy } from '../../utils/actions.js';
+import { UrlParser } from '../../utils/parsers.js';
 import { config, registerCli, verbose } from '../common.js';
 import { CopyContract } from './copy-rpc.js';
 
 const CopyValidator = z.object({ source: z.string(), target: z.string() });
 const CopyManifest = z.array(CopyValidator);
 
-/**
- * Attempt to figure out how the configuration is pass to us
- * - Could be a path to a S3 location s3://foo/bar.json
- * - Could be a JSON document "[{}]"
- * - Could be a Base64'd Gzipped document
- */
-async function tryParse(x: string): Promise<unknown> {
-  if (x.startsWith('s3://') || x.startsWith('./') || x.startsWith('/')) {
-    const json = await fsa.readJson<ActionCopy>(x);
-    if (json.action !== 'copy') throw new Error('Invalid action: ' + json.action + ' from:' + x);
-    return json.parameters.manifest;
-  }
-  if (x.startsWith('[') || x.startsWith('{')) return JSON.parse(x);
-  return JSON.parse(gunzipSync(Buffer.from(x, 'base64url')).toString());
-}
-
 export const commandCopy = command({
   name: 'copy',
   description: 'Copy a manifest of files',
@@ -66,7 +50,11 @@ export const commandCopy = command({
       defaultValueIsSerializable: true,
     }),
     concurrency: option({ type: number, long: 'concurrency', defaultValue: () => 4 }),
-    manifest: restPositionals({ type: string, displayName: 'location', description: 'Manifest of file to copy' }),
+    manifest: restPositionals({
+      type: UrlParser,
+      displayName: 'location',
+      description: 'Manifest of file to copy',
+    }),
   },
   async handler(args) {
     registerCli(this, args);
@@ -87,8 +75,12 @@ export const commandCopy = command({
     const chunks = [];
     const startTime = performance.now();
     for (const m of args.manifest) {
-      const data = await tryParse(m);
-      const manifest = CopyManifest.parse(data);
+      const json = await fsa.readJson<ActionCopy>(m.href);
+      if (json.action !== 'copy') throw new Error('Invalid action: ' + json.action + ' from:' + m);
+      const data = CopyManifest.parse(json.parameters.manifest);
+      const manifest: ActionCopy['parameters']['manifest'] = data.map(({ source, target }) => {
+        return { source: new URL(source), target: new URL(target) };
+      });
 
       const chunkSize = Math.ceil(manifest.length / args.concurrency);
       for (let i = 0; i < manifest.length; i += chunkSize) {

src/commands/create-manifest/__test__/create-manifest.test.ts

@@ -18,15 +18,17 @@ describe('createManifest', () => {
       fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
     ]);
 
-    const outputFiles = await createManifest('memory://source/', 'memory://target/', { flatten: true });
+    const outputFiles = await createManifest(new URL('memory://source/'), new URL('memory://target/'), {
+      flatten: true,
+    });
     assert.deepEqual(outputFiles[0], [
       {
-        source: 'memory://source/topographic.json',
-        target: 'memory://target/topographic.json',
+        source: new URL('memory://source/topographic.json'),
+        target: new URL('memory://target/topographic.json'),
       },
       {
-        source: 'memory://source/foo/bar/topographic.png',
-        target: 'memory://target/topographic.png',
+        source: new URL('memory://source/foo/bar/topographic.png'),
+        target: new URL('memory://target/topographic.png'),
       },
     ]);
   });
@@ -36,18 +38,18 @@ describe('createManifest', () => {
       fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true }))),
       fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
     ]);
-    const outputFiles = await createManifest('memory://source/', 'memory://target/sub/', {
+    const outputFiles = await createManifest(new URL('memory://source/'), new URL('memory://target/sub/'), {
       flatten: false,
       transform: 'f.replace("topographic", "test")',
     });
     assert.deepEqual(outputFiles[0], [
       {
-        source: 'memory://source/topographic.json',
-        target: 'memory://target/sub/test.json',
+        source: new URL('memory://source/topographic.json'),
+        target: new URL('memory://target/sub/test.json'),
       },
       {
-        source: 'memory://source/foo/bar/topographic.png',
-        target: 'memory://target/sub/foo/bar/test.png',
+        source: new URL('memory://source/foo/bar/topographic.png'),
+        target: new URL('memory://target/sub/foo/bar/test.png'),
       },
     ]);
   });
@@ -58,15 +60,17 @@ describe('createManifest', () => {
       fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
     ]);
 
-    const outputFiles = await createManifest('memory://source/', 'memory://target/sub/', { flatten: false });
+    const outputFiles = await createManifest(new URL('memory://source/'), new URL('memory://target/sub/'), {
+      flatten: false,
+    });
     assert.deepEqual(outputFiles[0], [
       {
-        source: 'memory://source/topographic.json',
-        target: 'memory://target/sub/topographic.json',
+        source: new URL('memory://source/topographic.json'),
+        target: new URL('memory://target/sub/topographic.json'),
       },
       {
-        source: 'memory://source/foo/bar/topographic.png',
-        target: 'memory://target/sub/foo/bar/topographic.png',
+        source: new URL('memory://source/foo/bar/topographic.png'),
+        target: new URL('memory://target/sub/foo/bar/topographic.png'),
       },
     ]);
   });
@@ -75,26 +79,26 @@ describe('createManifest', () => {
     await Promise.all([fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })))]);
 
     const outputFiles = await createManifest(
-      'memory://source/topographic.json',
-      'memory://target/sub/topographic.json',
+      new URL('memory://source/topographic.json'),
+      new URL('memory://target/sub/topographic.json'),
       { flatten: false },
     );
     assert.deepEqual(outputFiles[0], [
       {
-        source: 'memory://source/topographic.json',
-        target: 'memory://target/sub/topographic.json',
+        source: new URL('memory://source/topographic.json'),
+        target: new URL('memory://target/sub/topographic.json'),
       },
     ]);
   });
   describe('validatePaths', () => {
     it('Should throw error for Missmatch Paths', () => {
       assert.throws(() => {
-        validatePaths('memory://source/', 'memory://target/sub/test.tiff');
+        validatePaths(new URL('memory://source/'), new URL('memory://target/sub/test.tiff'));
       }, Error);
     });
     it('Should also throw error for Missmatch Paths', () => {
       assert.throws(() => {
-        validatePaths('memory://source/test.tiff', 'memory://target/sub/');
+        validatePaths(new URL('memory://source/test.tiff'), new URL('memory://target/sub/'));
       }, Error);
     });
   });