feat(core): simplify index building with CotarIndexBuilder (#29)

README.md

@@ -27,7 +27,7 @@ This makes it very easy to add new files to a archive as more files can just be
 ![TarFileBackground](./static/TarFileBackground.png)
 
 ### Tar Index
-TAR Index (.tar.index) is a NDJSON document containing the file location and size inside of a tar file. with this index a tar file can be randomly read.
+TAR Index (.tar.index) is a NDJSON document or binary file containing the file location and size inside of a tar file. with this index a tar file can be randomly read.
 
 ![TarFileIndex](./static/TarFileIndex.png)
 

packages/cli/README.md

@@ -6,7 +6,7 @@
 ## Create cloud optimized 
 
 ```bash
-cotar create sample.tar --verbose
+cotar create sample.tar --verbose --binary
 ```
 
 

packages/cli/src/create/create.index.ts

@@ -1,4 +1,4 @@
-import { CotarIndexBinaryBuilder, CotarIndexNdjsonBuilder } from '@cotar/core';
+import { CotarIndexBuilder } from '@cotar/core';
 import { promises as fs } from 'fs';
 import type { LogType } from '@cogeotiff/chunk';
 
@@ -12,8 +12,8 @@ export async function toTarIndex(
   logger.info({ index: indexFileName }, 'Cotar.Index:Start');
   const startTime = Date.now();
 
-  const writer = binary ? CotarIndexBinaryBuilder : CotarIndexNdjsonBuilder;
-  const { buffer, count } = await writer.create(fd, logger);
+  const indexType = binary ? CotarIndexBuilder.Binary : CotarIndexBuilder.NdJson;
+  const { buffer, count } = await CotarIndexBuilder.create(fd, indexType, logger);
 
   await fs.writeFile(indexFileName, buffer);
 

packages/core/README.md

@@ -11,27 +11,28 @@ For example `@cotar/core` can fetch a 1KB file from a 100GB tar file with only 1
 
 To fetch a single tile, the index has to be loaded into memory then the cotar object provides a `get(fileName)` interface to access any file inside the tar
 ```typescript
-import { Cotar } from '@cotar/core';
+import { Cotar, CotarIndexBinary } from '@cotar/core';
 import { SourceAwsS3 } from '@cogeotiff/source-aws'
 
 const source = SourceAwsS3.fromUri('s3://linz-basemaps/topographic.tar');
-const tarIndex = fs.readFileSync('./file.tar.index').toString().split('\n');
+const index = SourceAwsS3.fromUri('s3://linz-basemaps/topographic.tar.index');;
 
-const cotar = new Cotar(source, index);
+const cotar = new Cotar(source, new CotarIndexBinary(index));
 
 // Fetch a gzipped PBF file from  a tar
 const bytes  = await cotar.get(`tiles/z10/5/5.pbf.gz`);
 ```
 
-Creating indexes, indexes can be created using the `@cotar/cli` package or programmatically using the `TarReader.index`
+### Creating indexes
+
+Indexes can be created using the `@cotar/cli` package or programmatically using the `CotarIndexBuilder`
 
 ```typescript
-import { TarReader } from '@cotar/core'
+import { CotarIndexBuilder } from '@cotar/core'
 import * as fs from 'fs/promises';
 
 const fd = await fs.open('tarFile.tar', 'r');
-const tarIndex = await TarReader.index(readBytes, outputStream);
-await fs.write('tarFile.tar.index', tarIndex.join('\n'))
-
+const res = await CotarIndexBuilder.create(fd, CotarIndex.Binary);
+await fs.write('tarFile.tar.index', res.buffer)
 await fd.close();
 ```

packages/core/src/__test__/tar.test.ts

@@ -6,8 +6,8 @@ import o from 'ospec';
 import * as path from 'path';
 import { toArrayBuffer } from '../binary/build.binary';
 import { Cotar } from '../cotar';
+import { CotarIndexBuilder } from '../cotar.index';
 import { CotarIndexNdjson } from '../ndjson';
-import { CotarIndexNdjsonBuilder } from '../ndjson/build.ndjson';
 import { TarFileHeader, TarReader } from '../tar';
 
 o.spec('TarReader', () => {
@@ -56,7 +56,7 @@ o.spec('TarReader', () => {
   o('should create a index', async () => {
     const source = await fs.open(tarFilePath, 'r');
 
-    const res = await CotarIndexNdjsonBuilder.create(source);
+    const res = await CotarIndexBuilder.create(source, CotarIndexBuilder.NdJson);
     fs.writeFile(tarFileIndexPath, res.buffer);
 
     await source.close();

packages/core/src/cotar.index.ts

@@ -0,0 +1,28 @@
+import { LogType } from '@cogeotiff/chunk';
+import { CotarIndexBinaryBuilder } from './binary/build.binary';
+import { CotarIndexNdjsonBuilder } from './ndjson/build.ndjson';
+import { AsyncFileDescriptor, AsyncFileRead, TarIndexResult } from './tar.index';
+
+export enum CotarIndexType {
+  Binary,
+  NdJson,
+}
+
+export const CotarIndexBuilder = {
+  Binary: CotarIndexType.Binary,
+  NdJson: CotarIndexType.NdJson,
+  create(
+    fd: AsyncFileRead | AsyncFileDescriptor,
+    type: CotarIndexType = CotarIndexType.Binary,
+    logger?: LogType,
+  ): Promise<TarIndexResult> {
+    switch (type) {
+      case CotarIndexType.Binary:
+        return CotarIndexBinaryBuilder.create(fd, logger);
+      case CotarIndexType.NdJson:
+        return CotarIndexNdjsonBuilder.create(fd, logger);
+      default:
+        throw new Error('Unknown index type:' + type);
+    }
+  },
+};

packages/core/src/index.ts

@@ -1,6 +1,5 @@
 export { Cotar } from './cotar';
 export { TarHeader, TarReader } from './tar';
-export { CotarIndexBinaryBuilder } from './binary/build.binary';
 export { CotarIndexBinary } from './binary';
-export { CotarIndexNdjsonBuilder } from './ndjson/build.ndjson';
 export { CotarIndexNdjson } from './ndjson';
+export { CotarIndexBuilder } from './cotar.index';