feat(cli): adding info/validate command (#6)

* build: add a sample mbtiles

* wip: create info/serve cli

* feat(cli): adding info command

package.json

@@ -28,5 +28,6 @@
       "nohoist": [
           "**/@types/**"
       ]
-  }
+  },
+  "files": []
 }

packages/cli/README.md

@@ -0,0 +1,12 @@
+#  Cloud Optimized Vector Tiles (COVT) - CLI (@covt/cli)
+
+## Create cloud optimized 
+
+```bash
+covt create --output sample.covt sample.mbtiles --verbose
+```
+
+
+```bash
+covt serve sample.covt
+```

packages/cli/bin/covt

@@ -1,4 +1,4 @@
 #!/usr/bin/env node
 
-
-require('../build/src/index.js').CreateCovt.run().catch(require('@oclif/errors/handle'))
+require('@oclif/command').run()
+.catch(require('@oclif/errors/handle'))

packages/cli/package.json

@@ -7,6 +7,7 @@
     "covt": "./bin/covt"
   },
   "dependencies": {
+    "@cogeotiff/source-file": "^4.2.0",
     "@covt/core": "^1.1.0",
     "@oclif/command": "^1.8.0",
     "better-sqlite3": "^7.1.2",
@@ -24,6 +25,10 @@
   "publishConfig": {
     "access": "public"
   },
+  "oclif": {
+    "commands": "./build/src/commands",
+    "bin": "covt"
+  },
   "files": [
     "build/src",
     "bin",

packages/cli/src/commands/create.ts

@@ -0,0 +1,44 @@
+import { Command, flags } from '@oclif/command';
+import { existsSync } from 'fs';
+import { logger } from '../log';
+import { toTarTiles } from '../create/mbtiles.to.ttiles';
+import { toTarTilesIndex } from '../create/tar.to.ttiles';
+
+export class CreateCovt extends Command {
+  static flags = {
+    force: flags.boolean({ description: 'force overwriting existing files' }),
+    decompress: flags.boolean({
+      description: 'decompress gzip encoded tiles before storing in tar',
+      default: false,
+    }),
+
+    output: flags.string({ description: 'output file', required: true }),
+    verbose: flags.boolean({ description: 'verbose logging' }),
+    index: flags.boolean({ description: 'Only create the index', default: false }),
+  };
+
+  static args = [{ name: 'inputFile', required: true }];
+
+  async run(): Promise<void> {
+    const { args, flags } = this.parse(CreateCovt);
+    if (flags.verbose) logger.level = 'debug';
+
+    if (existsSync(flags.output) && !flags.force) {
+      logger.error({ output: flags.output }, 'Output file exists, aborting..');
+      return;
+    }
+
+    if (!args.inputFile.endsWith('.mbtiles')) {
+      logger.error({ input: args.inputFile }, 'Input file must be a mbtiles file');
+      return;
+    }
+    logger.info({ output: flags.output }, 'Covt:Create');
+
+    const startTime = Date.now();
+    if (flags.index === false) await toTarTiles(args.inputFile, flags.output, flags.decompress, logger);
+    await toTarTilesIndex(flags.output, flags.output + '.index', logger);
+
+    const duration = Date.now() - startTime;
+    logger.info({ output: flags.output, duration }, 'Covt:Created');
+  }
+}

packages/cli/src/commands/info.ts

@@ -0,0 +1,50 @@
+import { SourceFile } from '@cogeotiff/source-file';
+import { Covt, xyzToPath } from '@covt/core';
+import Command, { flags } from '@oclif/command';
+import { promises as fs } from 'fs';
+import { readMbTiles } from '../create/mbtiles.to.ttiles';
+import { logger } from '../log';
+
+export class CreateCovt extends Command {
+  static flags = {
+    verbose: flags.boolean({ description: 'verbose logging' }),
+    mbtiles: flags.string({ description: 'Source MBTiles for validation' }),
+  };
+
+  static args = [{ name: 'inputFile', required: true }];
+
+  async run(): Promise<void> {
+    const { args, flags } = this.parse(CreateCovt);
+    if (flags.verbose) logger.level = 'debug';
+
+    logger.info({ fileName: args.inputFile }, 'Covt.Load');
+
+    const source = new SourceFile(args.inputFile);
+    logger.debug({ indexPath: args.inputFile + '.index' }, 'Index:Load');
+    const sourceIndex = await fs.readFile(args.inputFile + '.index');
+
+    const index = JSON.parse(sourceIndex.toString());
+
+    const covt = await Covt.create(source, index);
+    logger.info({ fileName: args.inputFile, files: covt.index.size }, 'Covt.Loaded');
+
+    if (flags.mbtiles) {
+      const failures = [];
+
+      for await (const { tile, index, total } of readMbTiles(flags.mbtiles)) {
+        if (index === 0) logger.info({ total }, 'Covt.Validation:Start');
+        else if (index % 25_000 === 0) logger.debug({ total, index }, 'Covt.Validation:Progress');
+
+        const tileName = xyzToPath(tile.tile_column, tile.tile_row, tile.zoom_level);
+
+        const covtTile = covt.index.get(tileName);
+        if (covtTile == null) failures.push(tileName);
+      }
+
+      if (failures.length > 0) {
+        logger.error({ total: failures.length }, 'Covt.Validation:Failure');
+        logger.error({ failures: failures.slice(0, 25) }, 'Covt.Validation:Failures');
+      }
+    }
+  }
+}

packages/cli/src/mbtiles.to.ttiles.ts → packages/cli/src/create/mbtiles.to.ttiles.ts

@@ -15,41 +15,50 @@ export interface TileTable {
 const LimitCount = 0;
 const Limit = LimitCount > 0 ? `LIMIT ${LimitCount}` : '';
 
+export async function* readMbTiles(
+  fileName: string,
+): AsyncGenerator<{ tile: TileTable; index: number; total: number }, null> {
+  const db = bs3(fileName);
+
+  const total = await db.prepare('SELECT count(*) from tiles;').pluck().get();
+  const query = db.prepare(`SELECT * from tiles order by zoom_level ${Limit}`);
+
+  let index = 0;
+  for (const tile of query.iterate()) yield { tile, index: index++, total };
+  return null;
+}
+
 export async function toTarTiles(
-  filename: string,
+  fileName: string,
   tarFileName: string,
   decompress: boolean,
   logger: Logger,
 ): Promise<void> {
   const packer = tar.pack();
-  const db = bs3(filename);
-
-  const tileCount = await db.prepare('SELECT count(*) from tiles;').pluck().get();
-
-  const query = db.prepare(`SELECT * from tiles ${Limit}`);
-
   const startTime = Date.now();
   let writeCount = 0;
   const writeProm = new Promise((resolve) => packer.on('end', resolve));
 
   packer.pipe(createWriteStream(tarFileName));
 
   let startTileTime = Date.now();
-  for (const tile of query.iterate()) {
+  for await (const { tile, index, total } of readMbTiles(fileName)) {
+    if (index === 0) logger.info({ path: tarFileName, count: total }, 'Covt.Tar:Start');
+
     const tileName = xyzToPath(tile.tile_column, tile.tile_row, tile.zoom_level);
     const tileData = decompress ? zlib.gunzipSync(tile.tile_data) : tile.tile_data;
     packer.entry({ name: tileName }, tileData);
     if (writeCount % 25_000 === 0) {
-      const percent = ((writeCount / tileCount) * 100).toFixed(2);
+      const percent = ((writeCount / index) * 100).toFixed(2);
       const duration = Date.now() - startTileTime;
       startTileTime = Date.now();
-      logger.debug({ count: writeCount, total: tileCount, percent, duration }, 'Tar:WriteTile');
+      logger.debug({ current: writeCount, total: total, percent, duration }, 'Covt.Tar:WriteTile');
     }
     writeCount++;
   }
 
-  logger.debug('Tar:Finalize');
+  logger.debug('Covt.Tar:Finalize');
   packer.finalize();
   await writeProm;
-  logger.info({ path: tarFileName, count: writeCount, duration: Date.now() - startTime }, 'Tar:Done');
+  logger.info({ path: tarFileName, count: writeCount, duration: Date.now() - startTime }, 'Covt.Tar:Done');
 }

packages/cli/src/tar.to.ttiles.ts → packages/cli/src/create/tar.to.ttiles.ts

@@ -1,5 +1,5 @@
 import { bp } from 'binparse';
-import { promises as fs } from 'fs';
+import { createWriteStream, promises as fs } from 'fs';
 import type { Logger } from 'pino';
 
 export enum TarFileType {
@@ -44,8 +44,12 @@ export async function toTarTilesIndex(filename: string, indexFileName: string, l
   const Files: Record<string, { o: number; s: number }> = {};
   let fileCount = 0;
   const headBuffer = Buffer.alloc(512);
+  logger.info({ index: indexFileName }, 'Covt.Index:Start');
+  const outputBuffer = createWriteStream(indexFileName);
+  outputBuffer.write(`[\n`);
 
-  let startTime = Date.now();
+  const startTime = Date.now();
+  let currentTime = startTime;
   while (ctx.offset < stat.size) {
     alignOffsetToBlock(ctx);
 
@@ -58,19 +62,24 @@ export async function toTarTilesIndex(filename: string, indexFileName: string, l
     if (TarFileType[head.type] == null) throw new Error('Unknown header');
 
     if (head.type === TarFileType.File) {
+      if (fileCount > 0) outputBuffer.write(',\n');
+      outputBuffer.write(JSON.stringify([head.path, ctx.offset, head.size]));
       Files[head.path] = { o: ctx.offset, s: head.size };
       fileCount++;
       if (fileCount % 25_000 === 0) {
-        const duration = Date.now() - startTime;
-        startTime = Date.now();
+        const duration = Date.now() - currentTime;
+        currentTime = Date.now();
         const percent = ((ctx.offset / stat.size) * 100).toFixed(2);
-        logger.debug({ count: fileCount, percent, duration }, 'TarIndex:Write');
+        logger.debug({ current: fileCount, percent, duration }, 'Covt.Index:Write');
       }
     }
 
     ctx.offset += head.size;
   }
 
-  logger.info({ index: indexFileName, count: Object.keys(Files).length }, 'IndexCreated');
-  await fs.writeFile(indexFileName, JSON.stringify(Files, null, 2));
+  await new Promise<void>((r) => outputBuffer.write('\n]', () => r()));
+  logger.info(
+    { index: indexFileName, count: Object.keys(Files).length, duration: Date.now() - startTime },
+    'Covt.Index:Created',
+  );
 }

packages/cli/src/index.ts

@@ -1,47 +1 @@
-import { Command, flags } from '@oclif/command';
-import { existsSync } from 'fs';
-import pino from 'pino';
-import { PrettyTransform } from 'pretty-json-log';
-import { toTarTiles } from './mbtiles.to.ttiles';
-import { toTarTilesIndex } from './tar.to.ttiles';
-
-const logger = process.stdout.isTTY ? pino(PrettyTransform.stream()) : pino();
-
-export class CreateCovt extends Command {
-  static flags = {
-    force: flags.boolean({ description: 'force overwriting existing files' }),
-    decompress: flags.boolean({
-      description: 'decompress gzip encoded tiles before storing in tar',
-      default: false,
-    }),
-    verbose: flags.boolean({ description: 'verbose logging' }),
-  };
-
-  static args = [
-    { name: 'outputFile', required: true },
-    { name: 'inputFile', required: true },
-  ];
-
-  async run(): Promise<void> {
-    const { args, flags } = this.parse(CreateCovt);
-    if (flags.verbose) logger.level = 'debug';
-
-    if (existsSync(args.outputFile) && !flags.force) {
-      logger.error({ output: args.outputFile }, 'Output file exists, aborting..');
-      return;
-    }
-
-    if (!args.inputFile.endsWith('.mbtiles')) {
-      logger.error({ input: args.inputFile }, 'Input file must be a mbtiles file');
-      return;
-    }
-    logger.info({ output: args.outputFile }, 'Covt:Create');
-
-    const startTime = Date.now();
-    await toTarTiles(args.inputFile, args.outputFile, flags.decompress, logger);
-    await toTarTilesIndex(args.outputFile, args.outputFile + '.index', logger);
-
-    const duration = Date.now() - startTime;
-    logger.info({ output: args.outputFile, duration }, 'Covt:Created');
-  }
-}
+export { run } from '@oclif/command';

packages/cli/src/log.ts

@@ -0,0 +1,4 @@
+import pino from 'pino';
+import { PrettyTransform } from 'pretty-json-log';
+
+export const logger = process.stdout.isTTY ? pino(PrettyTransform.stream()) : pino();

packages/core/src/__test__/covt.test.ts

@@ -28,15 +28,20 @@ export class MemorySource extends ChunkSource {
 }
 
 o.spec('Covt', () => {
-  const tarIndex: TarIndex = { 'tiles/0/0/0.pbf': { o: 0, s: 1 }, 'tiles/1/1/1.pbf': { o: 4, s: 4 } };
-  const sourceIndex = new MemorySource('TarIndex', JSON.stringify(tarIndex));
+  const tarIndex: TarIndex = [
+    ['tiles/0/0/0.pbf', 0, 1],
+    ['tiles/1/1/1.pbf', 4, 4],
+  ];
   o('should load index', async () => {
-    const covt = await Covt.create(new MemorySource('Tar', '0123456789'), sourceIndex);
-    o(covt.index).deepEquals(tarIndex);
+    const covt = await Covt.create(new MemorySource('Tar', '0123456789'), tarIndex);
+
+    o(covt.index.get(tarIndex[0][0])).deepEquals(tarIndex[0]);
+    o(covt.index.get(tarIndex[1][0])).deepEquals(tarIndex[1]);
   });
   o('should load a tile', async () => {
-    const covt = await Covt.create(new MemorySource('Tar', '0123456789'), sourceIndex);
-    o(covt.index).deepEquals(tarIndex);
+    const covt = await Covt.create(new MemorySource('Tar', '0123456789'), tarIndex);
+    o(covt.index.get(tarIndex[0][0])).deepEquals(tarIndex[0]);
+    o(covt.index.get(tarIndex[1][0])).deepEquals(tarIndex[1]);
 
     const tile0 = await covt.getTile(0, 0, 0);
     o(tile0).notEquals(null);