refactor: Specify common types TDE-1030

package.json

@@ -12,11 +12,10 @@
     "organization": true
   },
   "devDependencies": {
-    "@chunkd/source-memory": "^10.1.0",
+    "@chunkd/source-memory": "^11.0.2",
     "@linzjs/style": "^5.1.0",
     "@types/node": "^20.8.9",
     "@types/prettier": "^3.0.0",
-    "esno": "^0.17.0",
     "stac-ts": "^1.0.0"
   },
   "scripts": {
@@ -40,9 +39,10 @@
     "@aws-sdk/lib-storage": "^3.440.0",
     "@basemaps/config": "^6.46.0",
     "@basemaps/geo": "^6.44.0",
-    "@chunkd/fs": "^10.0.9",
-    "@chunkd/source-aws-v3": "^10.1.3",
-    "@cogeotiff/core": "^8.1.1",
+    "@chunkd/fs": "^11.2.0",
+    "@chunkd/fs-aws": "^11.2.1",
+    "@chunkd/source-aws": "^11.0.2",
+    "@cogeotiff/core": "^9.0.3",
     "@linzjs/geojson": "^6.43.0",
     "@octokit/core": "^5.0.0",
     "@octokit/plugin-rest-endpoint-methods": "^10.1.1",

src/commands/basemaps-github/create-pr.ts

@@ -31,7 +31,7 @@ async function parseTargetInfo(
   }
 
   if (epsg == null || name == null) throw new Error(`Invalid target ${target} to parse the epsg and imagery name.`);
-  const collectionPath = fsa.join(target, 'collection.json');
+  const collectionPath = new URL('collection.json', target);
   const collection = await fsa.readJson<StacCollection>(collectionPath);
   if (collection == null) throw new Error(`Failed to get target collection json from ${collectionPath}.`);
   const title = collection.title;

src/commands/basemaps-github/make.cog.github.ts

@@ -5,7 +5,6 @@ import {
   TileSetType,
 } from '@basemaps/config/build/config/tile.set.js';
 import { TileSetConfigSchema } from '@basemaps/config/build/json/parse.tile.set.js';
-import { fsa } from '@chunkd/fs';
 
 import { logger } from '../../log.js';
 import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js';
@@ -83,13 +82,13 @@ export class MakeCogGithub {
         layers: [layer],
       };
       const content = await prettyPrint(JSON.stringify(tileSet, null, 2), ConfigPrettierFormat);
-      const tileSetPath = fsa.joinAll('config', 'tileset', region, `${layer.name}.json`);
+      const tileSetPath = `config/tileset/${region}/${layer.name}.json`;
       const file = { path: tileSetPath, content };
       // Github create pull request
       await createPR(gh, branch, title, botEmail, [file]);
     } else {
       // Prepare new aerial tileset config
-      const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
+      const tileSetPath = `config/tileset/${filename}.json`;
       const tileSetContent = await gh.getContent(tileSetPath);
       const tileSet = JSON.parse(tileSetContent) as ConfigTileSetRaster;
       const newTileSet = await this.prepareRasterTileSetConfig(layer, tileSet, category);
@@ -176,7 +175,7 @@ export class MakeCogGithub {
 
     // Prepare new aerial tileset config
     logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file');
-    const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
+    const tileSetPath = `config/tileset/${filename}.json`;
     const tileSetContent = await gh.getContent(tileSetPath);
     const tileSet = JSON.parse(tileSetContent) as ConfigTileSetVector;
     const newTileSet = await this.prepareVectorTileSetConfig(layer, tileSet);

src/commands/basemaps-mapsheet/create-mapsheet.ts

@@ -9,6 +9,7 @@ import { gunzip } from 'zlib';
 
 import { CliInfo } from '../../cli.info.js';
 import { logger } from '../../log.js';
+import { UrlParser } from '../../utils/parsers.js';
 import { registerCli, verbose } from '../common.js';
 
 const gunzipProm = promisify(gunzip);
@@ -22,9 +23,9 @@ export function isGzip(b: Buffer): boolean {
  *
  * If the file ends with .gz or is a GZIP like {@link isGzip} file it will automatically be decompressed.
  */
-async function readConfig(config: string): Promise<ConfigBundled> {
+async function readConfig(config: URL): Promise<ConfigBundled> {
   const obj = await fsa.read(config);
-  if (config.endsWith('.gz') || isGzip(obj)) {
+  if (config.href.endsWith('.gz') || isGzip(obj)) {
     const data = await gunzipProm(obj);
     return JSON.parse(data.toString());
   }
@@ -39,17 +40,17 @@ interface Output {
 export const CommandCreateMapSheetArgs = {
   verbose,
   path: option({
-    type: string,
+    type: UrlParser,
     long: 'path',
     description: 'Path of flatgeobuf, this can be both a local path or s3 location',
   }),
   bmConfig: option({
-    type: string,
+    type: UrlParser,
     long: 'bm-config',
     description: 'Path of basemaps config json, this can be both a local path or s3 location',
   }),
   output: option({
-    type: string,
+    type: UrlParser,
     long: 'output',
     description: 'Output of the map sheet file',
   }),
@@ -72,30 +73,30 @@ export const basemapsCreateMapSheet = command({
   args: CommandCreateMapSheetArgs,
   async handler(args) {
     registerCli(this, args);
-    const path = args.path;
+    const url = args.path;
     const config = args.bmConfig;
-    const outputPath = args.output;
+    const outputUrl = args.output;
 
     const include = args.include ? new RegExp(args.include.toLowerCase(), 'i') : undefined;
     const exclude = args.exclude ? new RegExp(args.exclude.toLowerCase(), 'i') : undefined;
 
-    logger.info({ path }, 'MapSheet:LoadFgb');
-    const buf = await fsa.read(path);
+    logger.info({ path: url }, 'MapSheet:LoadFgb');
+    const buf = await fsa.read(url);
     logger.info({ config }, 'MapSheet:LoadConfig');
     const configJson = await readConfig(config);
     const mem = ConfigProviderMemory.fromJson(configJson);
 
     const rest = fgb.deserialize(buf) as FeatureCollection;
-    const featuresWritePromise = fsa.write('features.json', JSON.stringify(rest));
+    const featuresWritePromise = fsa.write(new URL(fsa.toUrl('./features.json').href), JSON.stringify(rest));
 
     const aerial = await mem.TileSet.get('ts_aerial');
     if (aerial == null) throw new Error('Invalid config file.');
 
-    logger.info({ path, config }, 'MapSheet:CreateMapSheet');
+    logger.info({ path: url, config }, 'MapSheet:CreateMapSheet');
     const outputs = await createMapSheet(aerial, mem, rest, include, exclude);
 
-    logger.info({ outputPath }, 'MapSheet:WriteOutput');
-    const outputWritePromise = fsa.write(outputPath, JSON.stringify(outputs, null, 2));
+    logger.info({ outputPath: outputUrl }, 'MapSheet:WriteOutput');
+    const outputWritePromise = fsa.write(outputUrl, JSON.stringify(outputs, null, 2));
 
     await Promise.all([featuresWritePromise, outputWritePromise]);
   },

src/commands/common.ts

@@ -1,8 +1,5 @@
-import { fsa } from '@chunkd/fs';
-import { CogTiff } from '@cogeotiff/core';
 import { boolean, flag, option, optional, string } from 'cmd-ts';
-import pLimit from 'p-limit';
-import { fileURLToPath, pathToFileURL } from 'url';
+import { fileURLToPath } from 'url';
 
 import { CliInfo } from '../cli.info.js';
 import { registerFileSystem } from '../fs.register.js';
@@ -80,44 +77,6 @@ export function parseSize(size: string): number {
   return Math.round(fileSize);
 }
 
-/** Limit fetches to 25 concurrently **/
-const TiffQueue = pLimit(25);
-
-/**
- * There is a minor difference between @chunkd/core and @cogeotiff/core
- * because @chunkd/core is a major version behind, when it upgrades this can be removed
- *
- * Because the major version upgrade for chunkd is a lot of work skip it for now (2023-11)
- *
- * @param loc location to load the tiff from
- * @returns Initialized tiff
- */
-export function createTiff(loc: string): Promise<CogTiff> {
-  const source = fsa.source(loc);
-
-  const tiff = new CogTiff({
-    url: tryParseUrl(loc),
-    fetch: (offset, length): Promise<ArrayBuffer> => {
-      /** Limit fetches concurrency see {@link TiffQueue} **/
-      return TiffQueue(() => source.fetchBytes(offset, length));
-    },
-  });
-  return tiff.init();
-}
-
-/**
- * Attempt to parse a location as a string as a URL,
- *
- * Relative paths will be converted into file urls.
- */
-function tryParseUrl(loc: string): URL {
-  try {
-    return new URL(loc);
-  } catch (e) {
-    return pathToFileURL(loc);
-  }
-}
-
 /**
  * When chunkd moves to URLs this can be removed
  *

src/commands/copy/__test__/copy.test.ts

@@ -1,8 +1,7 @@
 import assert from 'node:assert';
 import { beforeEach, describe, it } from 'node:test';
 
-import { fsa } from '@chunkd/fs';
-import { FsMemory } from '@chunkd/source-memory';
+import { fsa, FsMemory } from '@chunkd/fs';
 
 import { worker } from '../copy-worker.js';
 
@@ -16,10 +15,10 @@ describe('copyFiles', () => {
 
   it('should copy to the target location', async () => {
     await Promise.all([
-      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+      fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
         contentType: 'application/json',
       }),
-      fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test'), { contentType: 'image/png' }),
+      fsa.write(new URL('memory://source/foo/bar/topographic.png'), Buffer.from('test'), { contentType: 'image/png' }),
     ]);
 
     await worker.routes.copy({
@@ -42,16 +41,16 @@ describe('copyFiles', () => {
     });
 
     const [jsonSource, jsonTarget] = await Promise.all([
-      fsa.head('memory://source/topographic.json'),
-      fsa.head('memory://target/topographic.json'),
+      fsa.head(new URL('memory://source/topographic.json')),
+      fsa.head(new URL('memory://target/topographic.json')),
     ]);
 
     assert.equal(jsonTarget?.contentType, 'application/json');
     assert.equal(jsonSource?.contentType, 'application/json');
 
     const [pngSource, pngTarget] = await Promise.all([
-      fsa.head('memory://source/foo/bar/topographic.png'),
-      fsa.head('memory://target/topographic.png'),
+      fsa.head(new URL('memory://source/foo/bar/topographic.png')),
+      fsa.head(new URL('memory://target/topographic.png')),
     ]);
 
     assert.equal(pngTarget?.contentType, 'image/png');
@@ -66,10 +65,10 @@ describe('copyFiles', () => {
 
   it('should default to COG/json', async () => {
     await Promise.all([
-      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+      fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
         contentType: 'application/octet-stream',
       }),
-      fsa.write('memory://source/foo/bar/topographic.tiff', Buffer.from('test'), {
+      fsa.write(new URL('memory://source/foo/bar/topographic.tiff'), Buffer.from('test'), {
         contentType: 'binary/octet-stream',
       }),
     ]);
@@ -92,16 +91,16 @@ describe('copyFiles', () => {
       fixContentType: true,
     });
     const [jsonSource, jsonTarget] = await Promise.all([
-      fsa.head('memory://source/topographic.json'),
-      fsa.head('memory://target/topographic.json'),
+      fsa.head(new URL('memory://source/topographic.json')),
+      fsa.head(new URL('memory://target/topographic.json')),
     ]);
 
     assert.equal(jsonSource?.contentType, 'application/octet-stream');
     assert.equal(jsonTarget?.contentType, 'application/json');
 
     const [tiffSource, tiffTarget] = await Promise.all([
-      fsa.head('memory://source/foo/bar/topographic.tiff'),
-      fsa.head('memory://target/topographic.tiff'),
+      fsa.head(new URL('memory://source/foo/bar/topographic.tiff')),
+      fsa.head(new URL('memory://target/topographic.tiff')),
     ]);
 
     assert.equal(tiffSource?.contentType, 'binary/octet-stream');
@@ -110,10 +109,10 @@ describe('copyFiles', () => {
 
   it('should not default COG/json when fixContentType=false', async () => {
     await Promise.all([
-      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+      fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
         contentType: 'application/octet-stream',
       }),
-      fsa.write('memory://source/foo/bar/topographic.tiff', Buffer.from('test'), {
+      fsa.write(new URL('memory://source/foo/bar/topographic.tiff'), Buffer.from('test'), {
         contentType: 'binary/octet-stream',
       }),
     ]);
@@ -136,16 +135,16 @@ describe('copyFiles', () => {
       fixContentType: false,
     });
     const [jsonSource, jsonTarget] = await Promise.all([
-      fsa.head('memory://source/topographic.json'),
-      fsa.head('memory://target/topographic.json'),
+      fsa.head(new URL('memory://source/topographic.json')),
+      fsa.head(new URL('memory://target/topographic.json')),
     ]);
 
     assert.equal(jsonSource?.contentType, 'application/octet-stream');
     assert.equal(jsonTarget?.contentType, 'application/octet-stream');
 
     const [tiffSource, tiffTarget] = await Promise.all([
-      fsa.head('memory://source/foo/bar/topographic.tiff'),
-      fsa.head('memory://target/topographic.tiff'),
+      fsa.head(new URL('memory://source/foo/bar/topographic.tiff')),
+      fsa.head(new URL('memory://target/topographic.tiff')),
     ]);
 
     assert.equal(tiffSource?.contentType, 'binary/octet-stream');

src/commands/copy/copy-worker.ts

@@ -1,8 +1,7 @@
 import { performance } from 'node:perf_hooks';
 import { parentPort, threadId } from 'node:worker_threads';
 
-import { FileInfo } from '@chunkd/core';
-import { fsa } from '@chunkd/fs';
+import { FileInfo, fsa } from '@chunkd/fs';
 import { WorkerRpc } from '@wtrpc/core';
 
 import { baseLogger } from '../../log.js';
@@ -19,20 +18,20 @@ export const FixableContentType = new Set(['binary/octet-stream', 'application/o
  * If the file has been written with a unknown binary contentType attempt to fix it with common content types
  *
  *
- * @param path File path to fix the metadata of
+ * @param url File path to fix the metadata of
  * @param meta File metadata
  * @returns New fixed file metadata if fixed other wise source file metadata
  */
-export function fixFileMetadata(path: string, meta: FileInfo): FileInfo {
+export function fixFileMetadata(url: URL, meta: FileInfo): FileInfo {
   // If the content is encoded we do not know what the content-type should be
   if (meta.contentEncoding != null) return meta;
   if (!FixableContentType.has(meta.contentType ?? 'binary/octet-stream')) return meta;
 
   // Assume our tiffs are cloud optimized
-  if (isTiff(path)) return { ...meta, contentType: 'image/tiff; application=geotiff; profile=cloud-optimized' };
+  if (isTiff(url)) return { ...meta, contentType: 'image/tiff; application=geotiff; profile=cloud-optimized' };
 
   // overwrite with application/json
-  if (path.endsWith('.json')) return { ...meta, contentType: 'application/json' };
+  if (url.href.endsWith('.json')) return { ...meta, contentType: 'application/json' };
 
   return meta;
 }
@@ -42,13 +41,13 @@ export function fixFileMetadata(path: string, meta: FileInfo): FileInfo {
  *
  * try reading the path {retryCount} times before aborting, with a delay of 250ms between requests
  *
- * @param filePath File to head
+ * @param url File to head
  * @param retryCount number of times to retry
  * @returns file size if it exists or null
  */
-async function tryHead(filePath: string, retryCount = 3): Promise<number | null> {
+async function tryHead(url: URL, retryCount = 3): Promise<number | null> {
   for (let i = 0; i < retryCount; i++) {
-    const ret = await fsa.head(filePath);
+    const ret = await fsa.head(url);
     if (ret?.size) return ret.size;
     await new Promise((r) => setTimeout(r, 250));
   }
@@ -62,8 +61,9 @@ export const worker = new WorkerRpc<CopyContract>({
     const log = baseLogger.child({ id: args.id, threadId });
 
     for (let i = args.start; i < end; i++) {
-      const todo = args.manifest[i];
-      if (todo == null) continue;
+      const manifest = args.manifest[i];
+      if (manifest == null) continue;
+      const todo = { source: new URL(manifest.source), target: new URL(manifest.target) };
 
       Q.push(async () => {
         const [source, target] = await Promise.all([fsa.head(todo.source), fsa.head(todo.target)]);
@@ -78,7 +78,7 @@ export const worker = new WorkerRpc<CopyContract>({
           }
 
           if (!args.force) {
-            log.error({ target: target.path, source: source.path }, 'File:Overwrite');
+            log.error({ target: target, source: source }, 'File:Overwrite');
             throw new Error('Cannot overwrite file: ' + todo.target + ' source:' + todo.source);
           }
         }
@@ -87,7 +87,7 @@ export const worker = new WorkerRpc<CopyContract>({
         const startTime = performance.now();
         await fsa.write(
           todo.target,
-          fsa.stream(todo.source),
+          fsa.readStream(todo.source),
           args.fixContentType ? fixFileMetadata(todo.source, source) : source,
         );