feat: add file system abstraction layer (#34)

BREAKING CHANGE: this renames `@chunkd/source-url` to `@chunkd/source-http`

Author: blacha

README.md

@@ -3,14 +3,15 @@
 [![Build Status](https://github.com/blacha/chunkd/workflows/Main/badge.svg)](https://github.com/blacha/chunkd/actions)
 
 
-File abstraction to read chunks of files from various sources
+File system abstraction to read files from various sources
 
 ## Usage
 
-Load a chunks from a URL using `fetch`
+Load a chunks of data from a URL using `fetch`
 
 ```typescript
-const source = new SourceUrl('https://example.com/foo')
+import {SourceHttp} from '@chunkd/source-http'
+const source = new SourceHttp('https://example.com/foo')
 // Read 1KB chunks
 source.chunkSize = 1024;
 
@@ -28,6 +29,13 @@ bytes.getBigUint64(1024);
 ```
 
 
+```typescript
+import {fsa} from '@chunkd/fs'
+
+const source = fsa.source('https://example.com/foo.zip');
+source.chunkSize = 1024;
+```
+
 # Building
 
 This requires [NodeJs](https://nodejs.org/en/) > 12 & [Yarn](https://yarnpkg.com/en/)

package.json

@@ -18,10 +18,12 @@
     "devDependencies": {
         "@linzjs/style": "^3.1.1",
         "@types/ospec": "^4.0.2",
+        "@types/sinon": "^10.0.2",
         "conventional-github-releaser": "^3.1.5",
         "lerna": "^4.0.0",
         "ospec": "^4.1.1",
         "rimraf": "^3.0.0",
+        "sinon": "^11.1.2",
         "source-map-support": "^0.5.19"
     },
     "workspaces": {

packages/core/src/composite.ts

@@ -0,0 +1,24 @@
+export const enum ErrorCodes {
+  PermissionDenied = 403,
+  NotFound = 404,
+  InternalError = 500,
+}
+/**
+ * Utility error to wrap other errors to make them more understandable
+ */
+export class CompositeError extends Error {
+  name = 'CompositeError';
+  code: ErrorCodes;
+  reason: unknown;
+
+  constructor(msg: string, code: ErrorCodes, reason: unknown) {
+    super(msg);
+    this.code = code;
+    this.reason = reason;
+  }
+
+  static isCompositeError(e: unknown): e is CompositeError {
+    if (typeof e !== 'object' || e == null) return false;
+    return (e as CompositeError).name === 'CompositeError';
+  }
+}

packages/core/src/fs.ts

@@ -0,0 +1,39 @@
+import type { Readable } from 'node:stream';
+import { ChunkSource } from '.';
+
+export interface FileInfo {
+  /** file path */
+  path: string;
+  /**
+   * Size of file in bytes
+   * undefined if no size found
+   */
+  size?: number;
+}
+
+export interface FileSystem<T extends ChunkSource = ChunkSource> {
+  /**
+   * Protocol used for communication
+   * @example
+   * file
+   * s3
+   * http
+   */
+  protocol: string;
+  /** Read a file into a buffer */
+  read(filePath: string): Promise<Buffer>;
+  /** Create a read stream */
+  stream(filePath: string): Readable;
+  /** Write a file from either a buffer or stream */
+  write(filePath: string, buffer: Buffer | Readable | string): Promise<void>;
+  /** Recursively list all files in path */
+  list(filePath: string): AsyncGenerator<string>;
+  /** Recursively list all files in path with additional details */
+  details(filePath: string): AsyncGenerator<FileInfo>;
+  /** Does the path exists */
+  exists(filePath: string): Promise<boolean>;
+  /** Get information about the path  */
+  head(filePath: string): Promise<FileInfo | null>;
+  /** Create a file source to read chunks out of */
+  source(filePath: string): T | null;
+}

packages/core/src/index.ts

@@ -3,3 +3,9 @@ export { ChunkSourceBase } from './chunk.source.js';
 export { LogType } from './log.js';
 export { SourceMemory } from './chunk.source.memory.js';
 export { ChunkSource } from './source.js';
+export { ErrorCodes, CompositeError } from './composite.js';
+export { FileSystem, FileInfo } from './fs.js';
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}

packages/fs/CHANGELOG.md

@@ -0,0 +1,50 @@
+# @chunkd/fs
+
+Utility functions for working with files that could either reside on the local file system or other sources like AWS S3
+
+## Usage
+
+```typescript
+import { fsa } from '@chunkd/fs';
+
+for await (const file of fsa.list('s3://foo/bar')) {
+    // ['s3://foo/bar/baz.html', 's3://foo/bar/index.html']
+}
+
+for await (const file of fsa.list('/home/blacha')) {
+    // '/home/blacha/index.html'
+}
+
+// Convert the generator to an array
+const files = await fsa.toArray(fsa.list('s3://foo/bar'));
+```
+
+This is designed for use with multiple s3 credentials 
+
+```typescript
+import {fsa, FsAwsS3} from '@chunkd/fs'
+
+const bucketA = new S3({ credentials: bucketACredentials })
+const bucketB = new S3({ credentials: bucketBCredentials  })
+
+fsa.register('s3://bucket-a', new FsAwsS3(bucketA))
+fsa.register('s3://bucket-b', new FsAwsS3(bucketB))
+
+// Stream a file from bucketA to bucketB
+await fsa.write('s3://bucket-b/foo', fsa.stream('s3://bucket-a/foo'))
+```
+
+Or even any s3 compatible api
+
+```typescript
+import {fsa, FsAwsS3} from '@chunkd/fs'
+
+const bucketA = new S3({ endpoint: 'http://10.0.0.1:8080' })
+const bucketB = new S3({ endpoint: 'http://10.0.0.99:8080' })
+
+fsa.register('s3://bucket-a', new FsAwsS3(bucketA))
+fsa.register('s3://bucket-b', new FsAwsS3(bucketB))
+
+// Stream a file from bucketA (10.0.0.1) to bucketB (10.0.0.99)
+await fsa.write('s3://bucket-b/foo', fsa.stream('s3://bucket-a/foo'))
+```

packages/fs/README.md

@@ -0,0 +1,30 @@
+{
+    "name": "@chunkd/fs",
+    "version": "6.0.0",
+    "type": "module",
+    "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+    },
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/blacha/chunkd.git",
+        "directory": "packages/fs"
+    },
+    "author": "Blayne Chard",
+    "main": "./build/index.node.js",
+    "browser": "./build/index.js",
+    "types": "./build/index.d.ts",
+    "license": "MIT",
+    "scripts": {},
+    "dependencies": {
+        "@chunkd/core": "^6.0.0",
+        "@chunkd/source-file": "^6.0.0",
+        "@chunkd/source-aws": "^6.0.0"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "devDependencies": {
+        "@types/node": "^14.14.31"
+    }
+}

packages/fs/package.json

@@ -0,0 +1,61 @@
+import { FsFile } from '@chunkd/source-file';
+import o from 'ospec';
+import { FileSystemAbstraction } from '../fs.abstraction.js';
+
+export class FakeSystem extends FsFile {
+  constructor(protocol = 'fake') {
+    super();
+    this.protocol = protocol;
+  }
+}
+
+o.spec('FileSystemAbstraction', () => {
+  o('should find file systems', () => {
+    const fsa = new FileSystemAbstraction();
+
+    o(fsa.get('/foo').protocol).equals('file');
+    o(fsa.get('/').protocol).equals('file');
+    o(fsa.get('./').protocol).equals('file');
+  });
+
+  o('should register new file systems', () => {
+    const fsa = new FileSystemAbstraction();
+
+    const fakeLocal = new FakeSystem('fake');
+    fsa.register('fake://', fakeLocal);
+
+    o(fsa.get('/').protocol).equals('file');
+    o(fsa.get('//').protocol).equals('file');
+
+    o(fsa.get('fake://foo').protocol).equals('fake');
+    o(fsa.get('fake:/foo').protocol).equals('file');
+    o(fsa.get('fake//foo').protocol).equals('file');
+    o(fsa.get('fake').protocol).equals('file');
+  });
+
+  o('should find file systems in order they were registered', () => {
+    const fakeA = new FakeSystem('fake');
+    const fakeB = new FakeSystem('fakeSpecific');
+    const fsa = new FileSystemAbstraction();
+
+    fsa.register('fake://', fakeA);
+    fsa.register('fake://some-prefix-string/', fakeB);
+
+    o(fsa.get('fake://foo').protocol).equals('fake');
+    o(fsa.get('fake://some-prefix-string/').protocol).equals('fakeSpecific');
+    o(fsa.get('fake://some-prefix-string/some-key').protocol).equals('fakeSpecific');
+  });
+
+  o('should order file systems by length', () => {
+    const fakeA = new FakeSystem('fake');
+    const fakeB = new FakeSystem('fakeSpecific');
+    const fsa = new FileSystemAbstraction();
+
+    fsa.register('fake://some-prefix-string/', fakeB);
+    fsa.register('fake://', fakeA);
+
+    o(fsa.get('fake://foo').protocol).equals('fake');
+    o(fsa.get('fake://some-prefix-string/').protocol).equals('fakeSpecific');
+    o(fsa.get('fake://some-prefix-string/some-key').protocol).equals('fakeSpecific');
+  });
+});