refactor: Specify common types TDE-1030

package.json

@@ -12,11 +12,10 @@
     "organization": true
   },
   "devDependencies": {
-    "@chunkd/source-memory": "^10.1.0",
+    "@chunkd/source-memory": "^11.0.2",
     "@linzjs/style": "^5.1.0",
     "@types/node": "^20.8.9",
     "@types/prettier": "^3.0.0",
-    "esno": "^0.17.0",
     "stac-ts": "^1.0.0"
   },
   "scripts": {
@@ -40,9 +39,10 @@
     "@aws-sdk/lib-storage": "^3.440.0",
     "@basemaps/config": "^6.46.0",
     "@basemaps/geo": "^6.44.0",
-    "@chunkd/fs": "^10.0.9",
-    "@chunkd/source-aws-v3": "^10.1.3",
-    "@cogeotiff/core": "^8.1.1",
+    "@chunkd/fs": "^11.2.0",
+    "@chunkd/fs-aws": "^11.2.1",
+    "@chunkd/source-aws": "^11.0.2",
+    "@cogeotiff/core": "^9.0.3",
     "@linzjs/geojson": "^6.43.0",
     "@octokit/core": "^5.0.0",
     "@octokit/plugin-rest-endpoint-methods": "^10.1.1",

src/commands/basemaps-github/create-pr.ts

@@ -31,7 +31,7 @@ async function parseTargetInfo(
   }
 
   if (epsg == null || name == null) throw new Error(`Invalid target ${target} to parse the epsg and imagery name.`);
-  const collectionPath = fsa.join(target, 'collection.json');
+  const collectionPath = new URL('collection.json', target);
   const collection = await fsa.readJson<StacCollection>(collectionPath);
   if (collection == null) throw new Error(`Failed to get target collection json from ${collectionPath}.`);
   const title = collection.title;

src/commands/basemaps-github/make.cog.github.ts

@@ -5,7 +5,6 @@ import {
   TileSetType,
 } from '@basemaps/config/build/config/tile.set.js';
 import { TileSetConfigSchema } from '@basemaps/config/build/json/parse.tile.set.js';
-import { fsa } from '@chunkd/fs';
 
 import { logger } from '../../log.js';
 import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js';
@@ -83,13 +82,13 @@ export class MakeCogGithub {
         layers: [layer],
       };
       const content = await prettyPrint(JSON.stringify(tileSet, null, 2), ConfigPrettierFormat);
-      const tileSetPath = fsa.joinAll('config', 'tileset', region, `${layer.name}.json`);
+      const tileSetPath = `config/tileset/${region}/${layer.name}.json`;
       const file = { path: tileSetPath, content };
       // Github create pull request
       await createPR(gh, branch, title, botEmail, [file]);
     } else {
       // Prepare new aerial tileset config
-      const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
+      const tileSetPath = `config/tileset/${filename}.json`;
       const tileSetContent = await gh.getContent(tileSetPath);
       const tileSet = JSON.parse(tileSetContent) as ConfigTileSetRaster;
       const newTileSet = await this.prepareRasterTileSetConfig(layer, tileSet, category);
@@ -176,7 +175,7 @@ export class MakeCogGithub {
 
     // Prepare new aerial tileset config
     logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file');
-    const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
+    const tileSetPath = `config/tileset/${filename}.json`;
     const tileSetContent = await gh.getContent(tileSetPath);
     const tileSet = JSON.parse(tileSetContent) as ConfigTileSetVector;
     const newTileSet = await this.prepareVectorTileSetConfig(layer, tileSet);

src/commands/basemaps-mapsheet/create-mapsheet.ts

@@ -9,6 +9,7 @@ import { gunzip } from 'zlib';
 
 import { CliInfo } from '../../cli.info.js';
 import { logger } from '../../log.js';
+import { UrlParser } from '../../utils/parsers.js';
 import { registerCli, verbose } from '../common.js';
 
 const gunzipProm = promisify(gunzip);
@@ -22,9 +23,9 @@ export function isGzip(b: Buffer): boolean {
  *
  * If the file ends with .gz or is a GZIP like {@link isGzip} file it will automatically be decompressed.
  */
-async function readConfig(config: string): Promise<ConfigBundled> {
+async function readConfig(config: URL): Promise<ConfigBundled> {
   const obj = await fsa.read(config);
-  if (config.endsWith('.gz') || isGzip(obj)) {
+  if (config.href.endsWith('.gz') || isGzip(obj)) {
     const data = await gunzipProm(obj);
     return JSON.parse(data.toString());
   }
@@ -39,17 +40,17 @@ interface Output {
 export const CommandCreateMapSheetArgs = {
   verbose,
   path: option({
-    type: string,
+    type: UrlParser,
     long: 'path',
     description: 'Path of flatgeobuf, this can be both a local path or s3 location',
   }),
   bmConfig: option({
-    type: string,
+    type: UrlParser,
     long: 'bm-config',
     description: 'Path of basemaps config json, this can be both a local path or s3 location',
   }),
   output: option({
-    type: string,
+    type: UrlParser,
     long: 'output',
     description: 'Output of the map sheet file',
   }),
@@ -72,30 +73,30 @@ export const basemapsCreateMapSheet = command({
   args: CommandCreateMapSheetArgs,
   async handler(args) {
     registerCli(this, args);
-    const path = args.path;
+    const url = args.path;
     const config = args.bmConfig;
-    const outputPath = args.output;
+    const outputUrl = args.output;
 
     const include = args.include ? new RegExp(args.include.toLowerCase(), 'i') : undefined;
     const exclude = args.exclude ? new RegExp(args.exclude.toLowerCase(), 'i') : undefined;
 
-    logger.info({ path }, 'MapSheet:LoadFgb');
-    const buf = await fsa.read(path);
+    logger.info({ path: url }, 'MapSheet:LoadFgb');
+    const buf = await fsa.read(url);
     logger.info({ config }, 'MapSheet:LoadConfig');
     const configJson = await readConfig(config);
     const mem = ConfigProviderMemory.fromJson(configJson);
 
     const rest = fgb.deserialize(buf) as FeatureCollection;
-    const featuresWritePromise = fsa.write('features.json', JSON.stringify(rest));
+    const featuresWritePromise = fsa.write(new URL(`file://${process.cwd()}/features.json`), JSON.stringify(rest));
 
     const aerial = await mem.TileSet.get('ts_aerial');
     if (aerial == null) throw new Error('Invalid config file.');
 
-    logger.info({ path, config }, 'MapSheet:CreateMapSheet');
+    logger.info({ path: url, config }, 'MapSheet:CreateMapSheet');
     const outputs = await createMapSheet(aerial, mem, rest, include, exclude);
 
-    logger.info({ outputPath }, 'MapSheet:WriteOutput');
-    const outputWritePromise = fsa.write(outputPath, JSON.stringify(outputs, null, 2));
+    logger.info({ outputPath: outputUrl }, 'MapSheet:WriteOutput');
+    const outputWritePromise = fsa.write(outputUrl, JSON.stringify(outputs, null, 2));
 
     await Promise.all([featuresWritePromise, outputWritePromise]);
   },

src/commands/common.ts

@@ -1,8 +1,5 @@
-import { fsa } from '@chunkd/fs';
-import { CogTiff } from '@cogeotiff/core';
 import { boolean, flag, option, optional, string } from 'cmd-ts';
-import pLimit from 'p-limit';
-import { fileURLToPath, pathToFileURL } from 'url';
+import { fileURLToPath } from 'url';
 
 import { CliInfo } from '../cli.info.js';
 import { registerFileSystem } from '../fs.register.js';
@@ -80,44 +77,6 @@ export function parseSize(size: string): number {
   return Math.round(fileSize);
 }
 
-/** Limit fetches to 25 concurrently **/
-const TiffQueue = pLimit(25);
-
-/**
- * There is a minor difference between @chunkd/core and @cogeotiff/core
- * because @chunkd/core is a major version behind, when it upgrades this can be removed
- *
- * Because the major version upgrade for chunkd is a lot of work skip it for now (2023-11)
- *
- * @param loc location to load the tiff from
- * @returns Initialized tiff
- */
-export function createTiff(loc: string): Promise<CogTiff> {
-  const source = fsa.source(loc);
-
-  const tiff = new CogTiff({
-    url: tryParseUrl(loc),
-    fetch: (offset, length): Promise<ArrayBuffer> => {
-      /** Limit fetches concurrency see {@link TiffQueue} **/
-      return TiffQueue(() => source.fetchBytes(offset, length));
-    },
-  });
-  return tiff.init();
-}
-
-/**
- * Attempt to parse a location as a string as a URL,
- *
- * Relative paths will be converted into file urls.
- */
-function tryParseUrl(loc: string): URL {
-  try {
-    return new URL(loc);
-  } catch (e) {
-    return pathToFileURL(loc);
-  }
-}
-
 /**
  * When chunkd moves to URLs this can be removed
  *

src/commands/copy/__test__/copy.test.ts

@@ -1,8 +1,7 @@
 import assert from 'node:assert';
 import { beforeEach, describe, it } from 'node:test';
 
-import { fsa } from '@chunkd/fs';
-import { FsMemory } from '@chunkd/source-memory';
+import { fsa, FsMemory } from '@chunkd/fs';
 
 import { worker } from '../copy-worker.js';
 
@@ -16,22 +15,22 @@ describe('copyFiles', () => {
 
   it('should copy to the target location', async () => {
     await Promise.all([
-      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+      fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
         contentType: 'application/json',
       }),
-      fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test'), { contentType: 'image/png' }),
+      fsa.write(new URL('memory://source/foo/bar/topographic.png'), Buffer.from('test'), { contentType: 'image/png' }),
     ]);
 
     await worker.routes.copy({
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: new URL('memory://source/topographic.json'),
+          target: new URL('memory://target/topographic.json'),
         },
         {
-          source: 'memory://source/foo/bar/topographic.png',
-          target: 'memory://target/topographic.png',
+          source: new URL('memory://source/foo/bar/topographic.png'),
+          target: new URL('memory://target/topographic.png'),
         },
       ],
       start: 0,
@@ -42,16 +41,16 @@ describe('copyFiles', () => {
     });
 
     const [jsonSource, jsonTarget] = await Promise.all([
-      fsa.head('memory://source/topographic.json'),
-      fsa.head('memory://target/topographic.json'),
+      fsa.head(new URL('memory://source/topographic.json')),
+      fsa.head(new URL('memory://target/topographic.json')),
     ]);
 
     assert.equal(jsonTarget?.contentType, 'application/json');
     assert.equal(jsonSource?.contentType, 'application/json');
 
     const [pngSource, pngTarget] = await Promise.all([
-      fsa.head('memory://source/foo/bar/topographic.png'),
-      fsa.head('memory://target/topographic.png'),
+      fsa.head(new URL('memory://source/foo/bar/topographic.png')),
+      fsa.head(new URL('memory://target/topographic.png')),
     ]);
 
     assert.equal(pngTarget?.contentType, 'image/png');
@@ -66,23 +65,23 @@ describe('copyFiles', () => {
 
   it('should default to COG/json', async () => {
     await Promise.all([
-      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+      fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
         contentType: 'application/octet-stream',
       }),
-      fsa.write('memory://source/foo/bar/topographic.tiff', Buffer.from('test'), {
+      fsa.write(new URL('memory://source/foo/bar/topographic.tiff'), Buffer.from('test'), {
         contentType: 'binary/octet-stream',
       }),
     ]);
     await worker.routes.copy({
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: new URL('memory://source/topographic.json'),
+          target: new URL('memory://target/topographic.json'),
         },
         {
-          source: 'memory://source/foo/bar/topographic.tiff',
-          target: 'memory://target/topographic.tiff',
+          source: new URL('memory://source/foo/bar/topographic.tiff'),
+          target: new URL('memory://target/topographic.tiff'),
         },
       ],
       start: 0,
@@ -92,16 +91,16 @@ describe('copyFiles', () => {
       fixContentType: true,
     });
     const [jsonSource, jsonTarget] = await Promise.all([
-      fsa.head('memory://source/topographic.json'),
-      fsa.head('memory://target/topographic.json'),
+      fsa.head(new URL('memory://source/topographic.json')),
+      fsa.head(new URL('memory://target/topographic.json')),
     ]);
 
     assert.equal(jsonSource?.contentType, 'application/octet-stream');
     assert.equal(jsonTarget?.contentType, 'application/json');
 
     const [tiffSource, tiffTarget] = await Promise.all([
-      fsa.head('memory://source/foo/bar/topographic.tiff'),
-      fsa.head('memory://target/topographic.tiff'),
+      fsa.head(new URL('memory://source/foo/bar/topographic.tiff')),
+      fsa.head(new URL('memory://target/topographic.tiff')),
     ]);
 
     assert.equal(tiffSource?.contentType, 'binary/octet-stream');
@@ -110,23 +109,23 @@ describe('copyFiles', () => {
 
   it('should not default COG/json when fixContentType=false', async () => {
     await Promise.all([
-      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+      fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
         contentType: 'application/octet-stream',
       }),
-      fsa.write('memory://source/foo/bar/topographic.tiff', Buffer.from('test'), {
+      fsa.write(new URL('memory://source/foo/bar/topographic.tiff'), Buffer.from('test'), {
         contentType: 'binary/octet-stream',
       }),
     ]);
     await worker.routes.copy({
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: new URL('memory://source/topographic.json'),
+          target: new URL('memory://target/topographic.json'),
         },
         {
-          source: 'memory://source/foo/bar/topographic.tiff',
-          target: 'memory://target/topographic.tiff',
+          source: new URL('memory://source/foo/bar/topographic.tiff'),
+          target: new URL('memory://target/topographic.tiff'),
         },
       ],
       start: 0,
@@ -136,16 +135,16 @@ describe('copyFiles', () => {
       fixContentType: false,
     });
     const [jsonSource, jsonTarget] = await Promise.all([
-      fsa.head('memory://source/topographic.json'),
-      fsa.head('memory://target/topographic.json'),
+      fsa.head(new URL('memory://source/topographic.json')),
+      fsa.head(new URL('memory://target/topographic.json')),
     ]);
 
     assert.equal(jsonSource?.contentType, 'application/octet-stream');
     assert.equal(jsonTarget?.contentType, 'application/octet-stream');
 
     const [tiffSource, tiffTarget] = await Promise.all([
-      fsa.head('memory://source/foo/bar/topographic.tiff'),
-      fsa.head('memory://target/topographic.tiff'),
+      fsa.head(new URL('memory://source/foo/bar/topographic.tiff')),
+      fsa.head(new URL('memory://target/topographic.tiff')),
     ]);
 
     assert.equal(tiffSource?.contentType, 'binary/octet-stream');

src/commands/copy/copy-rpc.ts

@@ -6,7 +6,7 @@ export interface CopyContractArgs {
   /** Copy ID for tracing */
   id: string;
   /** List of files that need to be copied */
-  manifest: { source: string; target: string }[];
+  manifest: { source: URL; target: URL }[];
   /** Offset into the manifest to start at */
   start: number;
   /** Number of records to copy */