feat: initial google cloud support

packages/core/src/index.ts

@@ -8,3 +8,28 @@ export { FileSystem, FileInfo, WriteOptions } from './fs.js';
 export function isRecord<T = unknown>(value: unknown): value is Record<string, T> {
   return typeof value === 'object' && value !== null;
 }
+
+/**
+ * Parse a s3/google storage URI into a bucket and key if the key exists
+ *
+ * @example
+ * ```typescript
+ * parseUri('s3://bucket-name') // { bucket: 'bucket-name', protocol: 's3' }
+ * parseUri('gs://bucket-name') // { bucket: 'bucket-name', protocol: 'gs' }
+ * ```
+ */
+export function parseUri(uri: string): { protocol: string; bucket: string; key?: string } | null {
+  const parts = uri.split('/');
+
+  let protocol = parts[0];
+  if (protocol == null || protocol === '') return null;
+  if (protocol.endsWith(':')) protocol = protocol.slice(0, protocol.length - 1);
+
+  const bucket = parts[2];
+  if (bucket == null || bucket.trim() === '') return null;
+  if (parts.length === 3) return { protocol, bucket };
+
+  const key = parts.slice(3).join('/');
+  if (key == null || key.trim() === '') return { protocol, bucket };
+  return { key, bucket, protocol };
+}

packages/fs/package.json

@@ -18,10 +18,13 @@
   "scripts": {},
   "dependencies": {
     "@chunkd/core": "^8.2.0",
-    "@chunkd/source-aws": "^8.2.0",
     "@chunkd/source-file": "^8.2.0",
     "@chunkd/source-http": "^8.2.0"
   },
+  "optionalDependencies": {
+    "@chunkd/source-aws": "*",
+    "@chunkd/source-google-cloud": "*"
+  },
   "publishConfig": {
     "access": "public"
   },

packages/fs/src/index.node.ts

@@ -1,11 +1,7 @@
-import { FsAwsS3 } from '@chunkd/source-aws';
 import { FsFile } from '@chunkd/source-file';
 import { FsHttp } from '@chunkd/source-http';
-import S3 from 'aws-sdk/clients/s3.js';
 import { fsa } from './fs.abstraction.js';
 
-export { FsAwsS3 } from '@chunkd/source-aws';
-export { FsHttp } from '@chunkd/source-http';
 export { FileSystemAbstraction, fsa } from './fs.abstraction.js';
 
 // Include local files by default in nodejs
@@ -16,6 +12,3 @@ fsa.register('file://', fsFile);
 const fsHttp = new FsHttp();
 fsa.register('http://', fsHttp);
 fsa.register('https://', fsHttp);
-
-const fsAwsS3 = new FsAwsS3(new S3());
-fsa.register('s3://', fsAwsS3);

packages/fs/src/index.ts

@@ -1,3 +1,2 @@
 export { FileSystemAbstraction, fsa } from './fs.abstraction.js';
-export { FsAwsS3 } from '@chunkd/source-aws';
 export { FsHttp } from '@chunkd/source-http';

packages/fs/tsconfig.json

@@ -6,5 +6,10 @@
     "lib": ["es2020"]
   },
   "include": ["src/**/*"],
-  "references": [{ "path": "../core" }, { "path": "../source-aws" }, { "path": "../source-file" }]
+  "references": [
+    { "path": "../core" },
+    { "path": "../source-aws" },
+    { "path": "../source-file" },
+    { "path": "../source-google-cloud" }
+  ]
 }

packages/source-aws/src/__test__/s3.fs.test.ts

@@ -2,6 +2,7 @@ import { FsAwsS3 } from '../s3.fs.js';
 import o from 'ospec';
 import S3 from 'aws-sdk/clients/s3.js';
 import sinon from 'sinon';
+import { parseUri } from '@chunkd/core';
 
 /** Utility to convert async generators into arrays */
 async function toArray<T>(generator: AsyncGenerator<T>): Promise<T[]> {
@@ -17,21 +18,18 @@ o.spec('file.s3', () => {
   o.afterEach(() => sandbox.restore());
 
   o.spec('parse', () => {
-    o('should only parse s3://', () => {
-      o(() => fs.parse('https://')).throws(Error);
-      o(() => fs.parse('https://google.com')).throws(Error);
-      o(() => fs.parse('/home/foo/bar')).throws(Error);
-      o(() => fs.parse('c:\\Program Files\\')).throws(Error);
-    });
-
     o('should parse s3 uris', () => {
-      o(fs.parse('s3://bucket/key')).deepEquals({ bucket: 'bucket', key: 'key' });
-      o(fs.parse('s3://bucket/key/')).deepEquals({ bucket: 'bucket', key: 'key/' });
-      o(fs.parse('s3://bucket/key/is/deep.txt')).deepEquals({ bucket: 'bucket', key: 'key/is/deep.txt' });
+      o(parseUri('s3://bucket/key')).deepEquals({ bucket: 'bucket', key: 'key', protocol: 's3' });
+      o(parseUri('s3://bucket/key/')).deepEquals({ bucket: 'bucket', key: 'key/', protocol: 's3' });
+      o(parseUri('s3://bucket/key/is/deep.txt')).deepEquals({
+        bucket: 'bucket',
+        key: 'key/is/deep.txt',
+        protocol: 's3',
+      });
     });
     o('should parse bucket only uris', () => {
-      o(fs.parse('s3://bucket')).deepEquals({ bucket: 'bucket' });
-      o(fs.parse('s3://bucket/')).deepEquals({ bucket: 'bucket' });
+      o(parseUri('s3://bucket')).deepEquals({ bucket: 'bucket', protocol: 's3' });
+      o(parseUri('s3://bucket/')).deepEquals({ bucket: 'bucket', protocol: 's3' });
     });
   });
 

packages/source-aws/src/s3.fs.ts

@@ -1,4 +1,4 @@
-import { FileInfo, FileSystem, isRecord, WriteOptions } from '@chunkd/core';
+import { FileInfo, FileSystem, isRecord, parseUri, WriteOptions } from '@chunkd/core';
 import type { Readable } from 'stream';
 import { getCompositeError, SourceAwsS3 } from './s3.source.js';
 import { ListRes, S3Like, toPromise } from './type.js';
@@ -34,30 +34,14 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
   }
 
   /** Parse a s3:// URI into the bucket and key components */
-  static parse(uri: string): { bucket: string; key?: string } {
-    if (!uri.startsWith('s3://')) throw new Error(`Unable to parse s3 uri: "${uri}"`);
-    const parts = uri.split('/');
-    const bucket = parts[2];
-    if (bucket == null || bucket.trim() === '') {
-      throw new Error(`Unable to parse s3 uri: "${uri}"`);
-    }
-
-    if (parts.length === 3) return { bucket };
-
-    const key = parts.slice(3).join('/');
-    if (key == null || key.trim() === '') {
-      return { bucket };
-    }
-    return { key, bucket };
-  }
-  parse = FsAwsS3.parse;
 
   async *list(filePath: string): AsyncGenerator<string> {
     for await (const obj of this.details(filePath)) yield obj.path;
   }
 
   async *details(filePath: string): AsyncGenerator<FileInfo> {
-    const opts = this.parse(filePath);
+    const opts = parseUri(filePath);
+    if (opts == null) return;
     let ContinuationToken: string | undefined = undefined;
     const Bucket = opts.bucket;
     const Prefix = opts.key;
@@ -91,8 +75,8 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
   }
 
   async read(filePath: string): Promise<Buffer> {
-    const opts = this.parse(filePath);
-    if (opts.key == null) throw new Error(`Failed to read:  "${filePath}"`);
+    const opts = parseUri(filePath);
+    if (opts == null || opts.key == null) throw new Error(`Failed to read:  "${filePath}"`);
 
     try {
       const res = await this.s3.getObject({ Bucket: opts.bucket, Key: opts.key }).promise();
@@ -103,8 +87,8 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
   }
 
   async write(filePath: string, buf: Buffer | Readable | string, ctx?: WriteOptions): Promise<void> {
-    const opts = this.parse(filePath);
-    if (opts.key == null) throw new Error(`Failed to write: "${filePath}"`);
+    const opts = parseUri(filePath);
+    if (opts == null || opts.key == null) throw new Error(`Failed to write: "${filePath}"`);
 
     try {
       await toPromise(
@@ -126,15 +110,15 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
   }
 
   stream(filePath: string): Readable {
-    const opts = this.parse(filePath);
-    if (opts.key == null) throw new Error(`S3: Unable to read "${filePath}"`);
+    const opts = parseUri(filePath);
+    if (opts == null || opts.key == null) throw new Error(`S3: Unable to read "${filePath}"`);
 
     return this.s3.getObject({ Bucket: opts.bucket, Key: opts.key }).createReadStream();
   }
 
   async head(filePath: string): Promise<FileInfo | null> {
-    const opts = this.parse(filePath);
-    if (opts.key == null) throw new Error(`Failed to exists: "${filePath}"`);
+    const opts = parseUri(filePath);
+    if (opts == null || opts.key == null) throw new Error(`Failed to exists: "${filePath}"`);
     try {
       const res = await toPromise(this.s3.headObject({ Bucket: opts.bucket, Key: opts.key }));
       return { size: res.ContentLength, path: filePath };

packages/source-aws/src/s3.source.ts

@@ -1,4 +1,4 @@
-import { ChunkSource, ChunkSourceBase, CompositeError, isRecord } from '@chunkd/core';
+import { ChunkSource, ChunkSourceBase, CompositeError, isRecord, parseUri } from '@chunkd/core';
 import { S3Like, toPromise } from './type.js';
 
 export function getCompositeError(e: unknown, msg: string): CompositeError {
@@ -51,36 +51,21 @@ export class SourceAwsS3 extends ChunkSourceBase {
     });
     return this._size;
   }
-  /**
-   * Parse a s3 URI and return the components
-   *
-   * @example
-   * `s3://foo/bar/baz.tiff`
-   *
-   * @param uri URI to parse
-   */
-  static parse(uri: string): { key: string; bucket: string } | null {
-    if (!uri.startsWith('s3://')) return null;
-
-    const parts = uri.split('/');
-    const bucket = parts[2];
-    if (bucket == null || bucket.trim() === '') return null;
-    const key = parts.slice(3).join('/');
-    if (key == null || key.trim() === '') return null;
-    return { key, bucket };
-  }
 
   /**
    * Parse a URI and create a source
    *
    * @example
-   * `s3://foo/bar/baz.tiff`
+   * ```
+   * fromUri('s3://foo/bar/baz.tiff')
+   * ```
    *
    * @param uri URI to parse
    */
   static fromUri(uri: string, remote: S3Like): SourceAwsS3 | null {
-    const res = SourceAwsS3.parse(uri);
-    if (res == null) return null;
+    const res = parseUri(uri);
+    if (res == null || res.key == null) return null;
+    if (res.protocol !== 's3') return null;
     return new SourceAwsS3(res.bucket, res.key, remote);
   }
 

packages/source-google-cloud/.npmignore

@@ -0,0 +1 @@
+tsconfig.tsbuildinfo

packages/source-google-cloud/README.md

@@ -0,0 +1,15 @@
+# @chunkd/source-gcp
+
+Load a chunks of a file from a AWS using `aws-sdk`
+
+## Usage
+
+```typescript
+import { SourceAwsS3 } from '@chunkd/source-google-cloud';
+import { Storage } from '@google-cloud/storage';
+
+const source = SourceGoogleStorage.fromUri('gs://bucket/path/to/cog.tif', new Storage());
+
+// Load the first 1KB
+await source.fetchBytes(0, 1024);
+```

packages/source-google-cloud/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@chunkd/source-google-cloud",
+  "version": "8.2.0",
+  "type": "module",
+  "engines": {
+    "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/blacha/chunkd.git",
+    "directory": "packages/source-google-cloud"
+  },
+  "author": "Blayne Chard",
+  "main": "./build/index.js",
+  "types": "./build/index.d.ts",
+  "license": "MIT",
+  "scripts": {},
+  "dependencies": {
+    "@chunkd/core": "^8.2.0"
+  },
+  "peerDependencies": {
+    "@google-cloud/storage": "*"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@google-cloud/storage": "^5.19.4",
+    "@types/node": "^16.11.7"
+  }
+}

packages/source-google-cloud/src/__test__/gcp.source.test.ts

@@ -0,0 +1,22 @@
+import o from 'ospec';
+import 'source-map-support/register.js';
+import { SourceGoogleStorage } from '../gcp.source.js';
+import { Storage } from '@google-cloud/storage';
+
+o.spec('SourceGoogleStorage', () => {
+  const fakeRemote = new Storage();
+
+  o('should round trip uri', () => {
+    o(SourceGoogleStorage.fromUri('gs://foo/bar.tiff', fakeRemote)?.uri).equals('gs://foo/bar.tiff');
+    o(SourceGoogleStorage.fromUri('gs://foo/bar/baz.tiff', fakeRemote)?.uri).equals('gs://foo/bar/baz.tiff');
+
+    // No Key
+    o(SourceGoogleStorage.fromUri('gs://foo', fakeRemote)).equals(null);
+
+    // No Bucket
+    o(SourceGoogleStorage.fromUri('gs:///foo', fakeRemote)).equals(null);
+
+    // Not s3
+    o(SourceGoogleStorage.fromUri('http://example.com/foo.tiff', fakeRemote)).equals(null);
+  });
+});