refactor: Specify common types TDE-1030

package.json

@@ -12,11 +12,10 @@
  "organization": true
  },
  "devDependencies": {
- "@chunkd/source-memory": "^10.1.0",
+ "@chunkd/source-memory": "^11.0.2",
  "@linzjs/style": "^5.1.0",
  "@types/node": "^20.8.9",
  "@types/prettier": "^3.0.0",
- "esno": "^0.17.0",
  "stac-ts": "^1.0.0"
  },
  "scripts": {
@@ -40,9 +39,10 @@
  "@aws-sdk/lib-storage": "^3.440.0",
  "@basemaps/config": "^6.46.0",
  "@basemaps/geo": "^6.44.0",
- "@chunkd/fs": "^10.0.9",
- "@chunkd/source-aws-v3": "^10.1.3",
- "@cogeotiff/core": "^8.1.1",
+ "@chunkd/fs": "^11.2.0",
+ "@chunkd/fs-aws": "^11.2.1",
+ "@chunkd/source-aws": "^11.0.2",
+ "@cogeotiff/core": "^9.0.3",
  "@linzjs/geojson": "^6.43.0",
  "@octokit/core": "^5.0.0",
  "@octokit/plugin-rest-endpoint-methods": "^10.1.1",

src/commands/basemaps-github/create-pr.ts

@@ -31,7 +31,7 @@ async function parseTargetInfo(
  }
 
  if (epsg == null || name == null) throw new Error(`Invalid target ${target} to parse the epsg and imagery name.`);
- const collectionPath = fsa.join(target, 'collection.json');
+ const collectionPath = new URL('collection.json', target);
  const collection = await fsa.readJson<StacCollection>(collectionPath);
  if (collection == null) throw new Error(`Failed to get target collection json from ${collectionPath}.`);
  const title = collection.title;

src/commands/basemaps-github/make.cog.github.ts

@@ -5,7 +5,6 @@ import {
  TileSetType,
 } from '@basemaps/config/build/config/tile.set.js';
 import { TileSetConfigSchema } from '@basemaps/config/build/json/parse.tile.set.js';
-import { fsa } from '@chunkd/fs';
 
 import { logger } from '../../log.js';
 import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js';
@@ -83,13 +82,13 @@ export class MakeCogGithub {
  layers: [layer],
  };
  const content = await prettyPrint(JSON.stringify(tileSet, null, 2), ConfigPrettierFormat);
- const tileSetPath = fsa.joinAll('config', 'tileset', region, `${layer.name}.json`);
+ const tileSetPath = `config/tileset/${region}/${layer.name}.json`;
  const file = { path: tileSetPath, content };
  // Github create pull request
  await createPR(gh, branch, title, botEmail, [file]);
  } else {
  // Prepare new aerial tileset config
- const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
+ const tileSetPath = `config/tileset/${filename}.json`;
  const tileSetContent = await gh.getContent(tileSetPath);
  const tileSet = JSON.parse(tileSetContent) as ConfigTileSetRaster;
  const newTileSet = await this.prepareRasterTileSetConfig(layer, tileSet, category);
@@ -176,7 +175,7 @@ export class MakeCogGithub {
 
  // Prepare new aerial tileset config
  logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file');
- const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
+ const tileSetPath = `config/tileset/${filename}.json`;
  const tileSetContent = await gh.getContent(tileSetPath);
  const tileSet = JSON.parse(tileSetContent) as ConfigTileSetVector;
  const newTileSet = await this.prepareVectorTileSetConfig(layer, tileSet);

src/commands/basemaps-mapsheet/create-mapsheet.ts

@@ -9,6 +9,7 @@ import { gunzip } from 'zlib';
 
 import { CliInfo } from '../../cli.info.js';
 import { logger } from '../../log.js';
+import { UrlParser } from '../../utils/parsers.js';
 import { registerCli, verbose } from '../common.js';
 
 const gunzipProm = promisify(gunzip);
@@ -22,9 +23,9 @@ export function isGzip(b: Buffer): boolean {
  *
  * If the file ends with .gz or is a GZIP like {@link isGzip} file it will automatically be decompressed.
  */
-async function readConfig(config: string): Promise<ConfigBundled> {
+async function readConfig(config: URL): Promise<ConfigBundled> {
  const obj = await fsa.read(config);
- if (config.endsWith('.gz') || isGzip(obj)) {
+ if (config.href.endsWith('.gz') || isGzip(obj)) {
  const data = await gunzipProm(obj);
  return JSON.parse(data.toString());
  }
@@ -39,17 +40,17 @@ interface Output {
 export const CommandCreateMapSheetArgs = {
  verbose,
  path: option({
- type: string,
+ type: UrlParser,
  long: 'path',
  description: 'Path of flatgeobuf, this can be both a local path or s3 location',
  }),
  bmConfig: option({
- type: string,
+ type: UrlParser,
  long: 'bm-config',
  description: 'Path of basemaps config json, this can be both a local path or s3 location',
  }),
  output: option({
- type: string,
+ type: UrlParser,
  long: 'output',
  description: 'Output of the map sheet file',
  }),
@@ -72,30 +73,30 @@ export const basemapsCreateMapSheet = command({
  args: CommandCreateMapSheetArgs,
  async handler(args) {
  registerCli(this, args);
- const path = args.path;
+ const url = args.path;
  const config = args.bmConfig;
- const outputPath = args.output;
+ const outputUrl = args.output;
 
  const include = args.include ? new RegExp(args.include.toLowerCase(), 'i') : undefined;
  const exclude = args.exclude ? new RegExp(args.exclude.toLowerCase(), 'i') : undefined;
 
- logger.info({ path }, 'MapSheet:LoadFgb');
- const buf = await fsa.read(path);
+ logger.info({ path: url }, 'MapSheet:LoadFgb');
+ const buf = await fsa.read(url);
  logger.info({ config }, 'MapSheet:LoadConfig');
  const configJson = await readConfig(config);
  const mem = ConfigProviderMemory.fromJson(configJson);
 
  const rest = fgb.deserialize(buf) as FeatureCollection;
- const featuresWritePromise = fsa.write('features.json', JSON.stringify(rest));
+ const featuresWritePromise = fsa.write(new URL(`file://${process.cwd()}/features.json`), JSON.stringify(rest));
 
  const aerial = await mem.TileSet.get('ts_aerial');
  if (aerial == null) throw new Error('Invalid config file.');
 
- logger.info({ path, config }, 'MapSheet:CreateMapSheet');
+ logger.info({ path: url, config }, 'MapSheet:CreateMapSheet');
  const outputs = await createMapSheet(aerial, mem, rest, include, exclude);
 
- logger.info({ outputPath }, 'MapSheet:WriteOutput');
- const outputWritePromise = fsa.write(outputPath, JSON.stringify(outputs, null, 2));
+ logger.info({ outputPath: outputUrl }, 'MapSheet:WriteOutput');
+ const outputWritePromise = fsa.write(outputUrl, JSON.stringify(outputs, null, 2));
 
  await Promise.all([featuresWritePromise, outputWritePromise]);
  },

src/commands/common.ts

@@ -1,8 +1,5 @@
-import { fsa } from '@chunkd/fs';
-import { CogTiff } from '@cogeotiff/core';
 import { boolean, flag, option, optional, string } from 'cmd-ts';
-import pLimit from 'p-limit';
-import { fileURLToPath, pathToFileURL } from 'url';
+import { fileURLToPath } from 'url';
 
 import { CliInfo } from '../cli.info.js';
 import { registerFileSystem } from '../fs.register.js';
@@ -80,44 +77,6 @@ export function parseSize(size: string): number {
  return Math.round(fileSize);
 }
 
-/** Limit fetches to 25 concurrently **/
-const TiffQueue = pLimit(25);
-
-/**
- * There is a minor difference between @chunkd/core and @cogeotiff/core
- * because @chunkd/core is a major version behind, when it upgrades this can be removed
- *
- * Because the major version upgrade for chunkd is a lot of work skip it for now (2023-11)
- *
- * @param loc location to load the tiff from
- * @returns Initialized tiff
- */
-export function createTiff(loc: string): Promise<CogTiff> {
- const source = fsa.source(loc);
-
- const tiff = new CogTiff({
- url: tryParseUrl(loc),
- fetch: (offset, length): Promise<ArrayBuffer> => {
- /** Limit fetches concurrency see {@link TiffQueue} **/
- return TiffQueue(() => source.fetchBytes(offset, length));
- },
- });
- return tiff.init();
-}
-
-/**
- * Attempt to parse a location as a string as a URL,
- *
- * Relative paths will be converted into file urls.
- */
-function tryParseUrl(loc: string): URL {
- try {
- return new URL(loc);
- } catch (e) {
- return pathToFileURL(loc);
- }
-}
-
 /**
  * When chunkd moves to URLs this can be removed
  *

src/commands/copy/__test__/copy.test.ts

@@ -1,8 +1,7 @@
 import assert from 'node:assert';
 import { beforeEach, describe, it } from 'node:test';
 
-import { fsa } from '@chunkd/fs';
-import { FsMemory } from '@chunkd/source-memory';
+import { fsa, FsMemory } from '@chunkd/fs';
 
 import { worker } from '../copy-worker.js';
 
@@ -16,22 +15,22 @@ describe('copyFiles', () => {
 
  it('should copy to the target location', async () => {
  await Promise.all([
- fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+ fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
  contentType: 'application/json',
  }),
- fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test'), { contentType: 'image/png' }),
+ fsa.write(new URL('memory://source/foo/bar/topographic.png'), Buffer.from('test'), { contentType: 'image/png' }),
  ]);
 
  await worker.routes.copy({
  id: '1',
  manifest: [
  {
- source: 'memory://source/topographic.json',
- target: 'memory://target/topographic.json',
+ source: new URL('memory://source/topographic.json'),
+ target: new URL('memory://target/topographic.json'),
  },
  {
- source: 'memory://source/foo/bar/topographic.png',
- target: 'memory://target/topographic.png',
+ source: new URL('memory://source/foo/bar/topographic.png'),
+ target: new URL('memory://target/topographic.png'),
  },
  ],
  start: 0,
@@ -42,16 +41,16 @@ describe('copyFiles', () => {
  });
 
  const [jsonSource, jsonTarget] = await Promise.all([
- fsa.head('memory://source/topographic.json'),
- fsa.head('memory://target/topographic.json'),
+ fsa.head(new URL('memory://source/topographic.json')),
+ fsa.head(new URL('memory://target/topographic.json')),
  ]);
 
  assert.equal(jsonTarget?.contentType, 'application/json');
  assert.equal(jsonSource?.contentType, 'application/json');
 
  const [pngSource, pngTarget] = await Promise.all([
- fsa.head('memory://source/foo/bar/topographic.png'),
- fsa.head('memory://target/topographic.png'),
+ fsa.head(new URL('memory://source/foo/bar/topographic.png')),
+ fsa.head(new URL('memory://target/topographic.png')),
  ]);
 
  assert.equal(pngTarget?.contentType, 'image/png');
@@ -66,23 +65,23 @@ describe('copyFiles', () => {
 
  it('should default to COG/json', async () => {
  await Promise.all([
- fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+ fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
  contentType: 'application/octet-stream',
  }),
- fsa.write('memory://source/foo/bar/topographic.tiff', Buffer.from('test'), {
+ fsa.write(new URL('memory://source/foo/bar/topographic.tiff'), Buffer.from('test'), {
  contentType: 'binary/octet-stream',
  }),
  ]);
  await worker.routes.copy({
  id: '1',
  manifest: [
  {
- source: 'memory://source/topographic.json',
- target: 'memory://target/topographic.json',
+ source: new URL('memory://source/topographic.json'),
+ target: new URL('memory://target/topographic.json'),
  },
  {
- source: 'memory://source/foo/bar/topographic.tiff',
- target: 'memory://target/topographic.tiff',
+ source: new URL('memory://source/foo/bar/topographic.tiff'),
+ target: new URL('memory://target/topographic.tiff'),
  },
  ],
  start: 0,
@@ -92,16 +91,16 @@ describe('copyFiles', () => {
  fixContentType: true,
  });
  const [jsonSource, jsonTarget] = await Promise.all([
- fsa.head('memory://source/topographic.json'),
- fsa.head('memory://target/topographic.json'),
+ fsa.head(new URL('memory://source/topographic.json')),
+ fsa.head(new URL('memory://target/topographic.json')),
  ]);
 
  assert.equal(jsonSource?.contentType, 'application/octet-stream');
  assert.equal(jsonTarget?.contentType, 'application/json');
 
  const [tiffSource, tiffTarget] = await Promise.all([
- fsa.head('memory://source/foo/bar/topographic.tiff'),
- fsa.head('memory://target/topographic.tiff'),
+ fsa.head(new URL('memory://source/foo/bar/topographic.tiff')),
+ fsa.head(new URL('memory://target/topographic.tiff')),
  ]);
 
  assert.equal(tiffSource?.contentType, 'binary/octet-stream');
@@ -110,23 +109,23 @@ describe('copyFiles', () => {
 
  it('should not default COG/json when fixContentType=false', async () => {
  await Promise.all([
- fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })), {
+ fsa.write(new URL('memory://source/topographic.json'), Buffer.from(JSON.stringify({ test: true })), {
  contentType: 'application/octet-stream',
  }),
- fsa.write('memory://source/foo/bar/topographic.tiff', Buffer.from('test'), {
+ fsa.write(new URL('memory://source/foo/bar/topographic.tiff'), Buffer.from('test'), {
  contentType: 'binary/octet-stream',
  }),
  ]);
  await worker.routes.copy({
  id: '1',
  manifest: [
  {
- source: 'memory://source/topographic.json',
- target: 'memory://target/topographic.json',
+ source: new URL('memory://source/topographic.json'),
+ target: new URL('memory://target/topographic.json'),
  },
  {
- source: 'memory://source/foo/bar/topographic.tiff',
- target: 'memory://target/topographic.tiff',
+ source: new URL('memory://source/foo/bar/topographic.tiff'),
+ target: new URL('memory://target/topographic.tiff'),
  },
  ],
  start: 0,
@@ -136,16 +135,16 @@ describe('copyFiles', () => {
  fixContentType: false,
  });
  const [jsonSource, jsonTarget] = await Promise.all([
- fsa.head('memory://source/topographic.json'),
- fsa.head('memory://target/topographic.json'),
+ fsa.head(new URL('memory://source/topographic.json')),
+ fsa.head(new URL('memory://target/topographic.json')),
  ]);
 
  assert.equal(jsonSource?.contentType, 'application/octet-stream');
  assert.equal(jsonTarget?.contentType, 'application/octet-stream');
 
  const [tiffSource, tiffTarget] = await Promise.all([
- fsa.head('memory://source/foo/bar/topographic.tiff'),
- fsa.head('memory://target/topographic.tiff'),
+ fsa.head(new URL('memory://source/foo/bar/topographic.tiff')),
+ fsa.head(new URL('memory://target/topographic.tiff')),
  ]);
 
  assert.equal(tiffSource?.contentType, 'binary/octet-stream');

src/commands/copy/copy-rpc.ts

@@ -6,7 +6,7 @@ export interface CopyContractArgs {
  /** Copy ID for tracing */
  id: string;
  /** List of files that need to be copied */
- manifest: { source: string; target: string }[];
+ manifest: { source: URL; target: URL }[];
  /** Offset into the manifest to start at */
  start: number;
  /** Number of records to copy */