From d53c29d6c9c202c877dbcbfc380dcf498366a65d Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Thu, 24 Jun 2021 12:51:11 +1200 Subject: [PATCH] feat(core): simplify index building with CotarIndexBuilder (#29) --- README.md | 2 +- packages/cli/README.md | 2 +- packages/cli/src/create/create.index.ts | 6 +++--- packages/core/README.md | 17 ++++++++------- packages/core/src/__test__/tar.test.ts | 4 ++-- packages/core/src/cotar.index.ts | 28 +++++++++++++++++++++++++ packages/core/src/index.ts | 3 +-- 7 files changed, 45 insertions(+), 17 deletions(-) create mode 100644 packages/core/src/cotar.index.ts diff --git a/README.md b/README.md index 39250f5f..8afef891 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This makes it very easy to add new files to a archive as more files can just be ![TarFileBackground](./static/TarFileBackground.png) ### Tar Index -TAR Index (.tar.index) is a NDJSON document containing the file location and size inside of a tar file. with this index a tar file can be randomly read. +TAR Index (.tar.index) is a NDJSON document or binary file containing the file location and size inside of a tar file. with this index a tar file can be randomly read. ![TarFileIndex](./static/TarFileIndex.png) diff --git a/packages/cli/README.md b/packages/cli/README.md index c26ddb5f..add7e042 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -6,7 +6,7 @@ ## Create cloud optimized ```bash -cotar create sample.tar --verbose +cotar create sample.tar --verbose --binary ``` diff --git a/packages/cli/src/create/create.index.ts b/packages/cli/src/create/create.index.ts index 725722fd..f25bc8e7 100644 --- a/packages/cli/src/create/create.index.ts +++ b/packages/cli/src/create/create.index.ts @@ -1,4 +1,4 @@ -import { CotarIndexBinaryBuilder, CotarIndexNdjsonBuilder } from '@cotar/core'; +import { CotarIndexBuilder } from '@cotar/core'; import { promises as fs } from 'fs'; import type { LogType } from '@cogeotiff/chunk'; @@ -12,8 +12,8 @@ export async function toTarIndex( logger.info({ index: indexFileName }, 'Cotar.Index:Start'); const startTime = Date.now(); - const writer = binary ? CotarIndexBinaryBuilder : CotarIndexNdjsonBuilder; - const { buffer, count } = await writer.create(fd, logger); + const indexType = binary ? CotarIndexBuilder.Binary : CotarIndexBuilder.NdJson; + const { buffer, count } = await CotarIndexBuilder.create(fd, indexType, logger); await fs.writeFile(indexFileName, buffer); diff --git a/packages/core/README.md b/packages/core/README.md index 77e1c5e1..45267e77 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -11,27 +11,28 @@ For example `@cotar/core` can fetch a 1KB file from a 100GB tar file with only 1 To fetch a single tile, the index has to be loaded into memory then the cotar object provides a `get(fileName)` interface to access any file inside the tar ```typescript -import { Cotar } from '@cotar/core'; +import { Cotar, CotarIndexBinary } from '@cotar/core'; import { SourceAwsS3 } from '@cogeotiff/source-aws' const source = SourceAwsS3.fromUri('s3://linz-basemaps/topographic.tar'); -const tarIndex = fs.readFileSync('./file.tar.index').toString().split('\n'); +const index = SourceAwsS3.fromUri('s3://linz-basemaps/topographic.tar.index');; -const cotar = new Cotar(source, index); +const cotar = new Cotar(source, new CotarIndexBinary(index)); // Fetch a gzipped PBF file from a tar const bytes = await cotar.get(`tiles/z10/5/5.pbf.gz`); ``` -Creating indexes, indexes can be created using the `@cotar/cli` package or programmatically using the `TarReader.index` +### Creating indexes + +Indexes can be created using the `@cotar/cli` package or programmatically using the `CotarIndexBuilder` ```typescript -import { TarReader } from '@cotar/core' +import { CotarIndexBuilder } from '@cotar/core' import * as fs from 'fs/promises'; const fd = await fs.open('tarFile.tar', 'r'); -const tarIndex = await TarReader.index(readBytes, outputStream); -await fs.write('tarFile.tar.index', tarIndex.join('\n')) - +const res = await CotarIndexBuilder.create(fd, CotarIndex.Binary); +await fs.write('tarFile.tar.index', res.buffer) await fd.close(); ``` \ No newline at end of file diff --git a/packages/core/src/__test__/tar.test.ts b/packages/core/src/__test__/tar.test.ts index 48f6e706..00c9ad45 100644 --- a/packages/core/src/__test__/tar.test.ts +++ b/packages/core/src/__test__/tar.test.ts @@ -6,8 +6,8 @@ import o from 'ospec'; import * as path from 'path'; import { toArrayBuffer } from '../binary/build.binary'; import { Cotar } from '../cotar'; +import { CotarIndexBuilder } from '../cotar.index'; import { CotarIndexNdjson } from '../ndjson'; -import { CotarIndexNdjsonBuilder } from '../ndjson/build.ndjson'; import { TarFileHeader, TarReader } from '../tar'; o.spec('TarReader', () => { @@ -56,7 +56,7 @@ o.spec('TarReader', () => { o('should create a index', async () => { const source = await fs.open(tarFilePath, 'r'); - const res = await CotarIndexNdjsonBuilder.create(source); + const res = await CotarIndexBuilder.create(source, CotarIndexBuilder.NdJson); fs.writeFile(tarFileIndexPath, res.buffer); await source.close(); diff --git a/packages/core/src/cotar.index.ts b/packages/core/src/cotar.index.ts new file mode 100644 index 00000000..22d01adc --- /dev/null +++ b/packages/core/src/cotar.index.ts @@ -0,0 +1,28 @@ +import { LogType } from '@cogeotiff/chunk'; +import { CotarIndexBinaryBuilder } from './binary/build.binary'; +import { CotarIndexNdjsonBuilder } from './ndjson/build.ndjson'; +import { AsyncFileDescriptor, AsyncFileRead, TarIndexResult } from './tar.index'; + +export enum CotarIndexType { + Binary, + NdJson, +} + +export const CotarIndexBuilder = { + Binary: CotarIndexType.Binary, + NdJson: CotarIndexType.NdJson, + create( + fd: AsyncFileRead | AsyncFileDescriptor, + type: CotarIndexType = CotarIndexType.Binary, + logger?: LogType, + ): Promise { + switch (type) { + case CotarIndexType.Binary: + return CotarIndexBinaryBuilder.create(fd, logger); + case CotarIndexType.NdJson: + return CotarIndexNdjsonBuilder.create(fd, logger); + default: + throw new Error('Unknown index type:' + type); + } + }, +}; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index afe0279e..2d565170 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,6 +1,5 @@ export { Cotar } from './cotar'; export { TarHeader, TarReader } from './tar'; -export { CotarIndexBinaryBuilder } from './binary/build.binary'; export { CotarIndexBinary } from './binary'; -export { CotarIndexNdjsonBuilder } from './ndjson/build.ndjson'; export { CotarIndexNdjson } from './ndjson'; +export { CotarIndexBuilder } from './cotar.index';