feat(source-aws): create a credential provider

packages/source-aws-v2/src/index.ts

@@ -1 +1,2 @@
 export { AwsCredentials, RoleConfig } from './s3.credentials.js';
+export { FsAwsS3ProviderV2, CredentialSourceJson, CredentialSource } from './s3.provider.js';

packages/source-aws-v2/src/s3.credentials.ts

@@ -1,7 +1,7 @@
-import { Credentials } from 'aws-sdk/lib/credentials.js';
-import aws from 'aws-sdk/lib/core.js';
-import S3 from 'aws-sdk/clients/s3.js';
 import { FsAwsS3 } from '@chunkd/source-aws';
+import S3 from 'aws-sdk/clients/s3.js';
+import aws from 'aws-sdk/lib/core.js';
+import { Credentials } from 'aws-sdk/lib/credentials.js';
 
 export interface RoleConfig {
  roleArn: string;

packages/source-aws-v2/src/s3.provider.ts

@@ -0,0 +1,65 @@
+import { FsAwsS3, FsAwsS3Provider } from '@chunkd/source-aws';
+import { FileSystem } from '@chunkd/core';
+import { AwsCredentials } from './s3.credentials.js';
+
+export interface CredentialSource {
+ /** Prefix type generally s3 */
+ type: 's3';
+ /** Prefix should always start with `s3://${bucket}` */
+ prefix: string;
+ /** Role to use to access */
+ roleArn: string;
+ /** Aws account this bucket belongs to */
+ accountId: string;
+ /** Bucket name */
+ bucket: string;
+ /** ExternalId if required */
+ externalId?: string;
+ /** Max role session duration */
+ roleSessionDuration?: number;
+ /** Can this be used for "read" or "read-write" access */
+ flags?: 'r' | 'rw';
+}
+
+export interface CredentialSourceJson {
+ prefixes: CredentialSource[];
+ v: 2;
+}
+
+export class FsAwsS3ProviderV2 implements FsAwsS3Provider {
+ /** Should the resulting file system be registered onto the top level file system */
+ isRegisterable = true;
+ path: string;
+ fs: FileSystem;
+
+ constructor(path: string, fs: FileSystem) {
+ this.path = path;
+ this.fs = fs;
+ }
+
+ onFileSystemCreated?: (ro: CredentialSource, fs: FileSystem) => void;
+
+ _config: Promise<CredentialSourceJson> | null = null;
+ get config(): Promise<CredentialSourceJson> {
+ if (this._config == null) this._config = this.fs.read(this.path).then((buf) => JSON.parse(buf.toString()));
+ return this._config;
+ }
+
+ async find(path: string): Promise<FsAwsS3 | null> {
+ if (this.path === path) return null;
+
+ const cfg = await this.config;
+ if (cfg == null) return null;
+
+ if (cfg.v !== 2) throw new Error('Invalid bucket config version: ' + cfg.v + ' from ' + this.path);
+ if (cfg.prefixes == null || !Array.isArray(cfg.prefixes)) {
+ throw new Error('Invalid bucket config missing "prefixes" from ' + this.path);
+ }
+ const ro = cfg.prefixes.find((f) => path.startsWith(f.prefix));
+
+ if (ro == null) return null;
+ const fs = AwsCredentials.fsFromRole(ro.roleArn, ro.externalId, ro.roleSessionDuration);
+ if (this.onFileSystemCreated) this.onFileSystemCreated(ro, fs);
+ return fs;
+ }
+}

packages/source-aws/src/credentials.ts

@@ -0,0 +1,5 @@
+import { FsAwsS3 } from './s3.fs.js';
+
+export interface FsAwsS3Provider {
+ find(path: string): Promise<FsAwsS3 | null>;
+}

packages/source-aws/src/index.ts

@@ -1,2 +1,3 @@
 export { SourceAwsS3 } from './s3.source.js';
 export { FsAwsS3 } from './s3.fs.js';
+export { FsAwsS3Provider } from './credentials.js';

packages/source-aws/src/s3.fs.ts

@@ -1,5 +1,6 @@
 import { FileInfo, FileSystem, isRecord, ListOptions, parseUri, WriteOptions } from '@chunkd/core';
 import type { Readable } from 'stream';
+import { FsAwsS3Provider } from './credentials.js';
 import { getCompositeError, SourceAwsS3 } from './s3.source.js';
 import { ListRes, S3Like, toPromise } from './type.js';
 
@@ -9,11 +10,14 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
  /** Max list requests to run before erroring */
  static MaxListCount = 100;
 
+ credentials: FsAwsS3Provider | undefined;
+
  /** AWS-SDK s3 to use */
  s3: S3Like;
 
- constructor(s3: S3Like) {
+ constructor(s3: S3Like, credentials?: FsAwsS3Provider) {
  this.s3 = s3;
+ this.credentials = credentials;
  }
 
  source(filePath: string): SourceAwsS3 {
@@ -77,7 +81,12 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
  ContinuationToken = res.NextContinuationToken;
  }
  } catch (e) {
- throw getCompositeError(e, `Failed to list: "${filePath}"`);
+ const ce = getCompositeError(e, `Failed to list: "${filePath}"`);
+ if (this.credentials != null && ce.code === 403) {
+ const newFs = await this.credentials.find(filePath);
+ if (newFs) return newFs.details(filePath, opts);
+ }
+ throw ce;
  }
  }
 
@@ -89,7 +98,12 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
  const res = await this.s3.getObject({ Bucket: opts.bucket, Key: opts.key }).promise();
  return res.Body as Buffer;
  } catch (e) {
- throw getCompositeError(e, `Failed to read: "${filePath}"`);
+ const ce = getCompositeError(e, `Failed to read: "${filePath}"`);
+ if (this.credentials != null && ce.code === 403) {
+ const newFs = await this.credentials.find(filePath);
+ if (newFs) return newFs.read(filePath);
+ }
+ throw ce;
  }
  }
 
@@ -108,7 +122,12 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
  }),
  );
  } catch (e) {
- throw getCompositeError(e, `Failed to write: "${filePath}"`);
+ const ce = getCompositeError(e, `Failed to write: "${filePath}"`);
+ if (this.credentials != null && ce.code === 403) {
+ const newFs = await this.credentials.find(filePath);
+ if (newFs) return newFs.write(filePath, buf, ctx);
+ }
+ throw ce;
  }
  }
 
@@ -125,13 +144,19 @@ export class FsAwsS3 implements FileSystem<SourceAwsS3> {
 
  async head(filePath: string): Promise<FileInfo | null> {
  const opts = parseUri(filePath);
- if (opts == null || opts.key == null) throw new Error(`Failed to exists: "${filePath}"`);
+ if (opts == null || opts.key == null) throw new Error(`Failed to head: "${filePath}"`);
  try {
  const res = await toPromise(this.s3.headObject({ Bucket: opts.bucket, Key: opts.key }));
  return { size: res.ContentLength, path: filePath };
  } catch (e) {
  if (isRecord(e) && e.code === 'NotFound') return null;
- throw getCompositeError(e, `Failed to exists: "${filePath}"`);
+
+ const ce = getCompositeError(e, `Failed to head: "${filePath}"`);
+ if (this.credentials != null && ce.code === 403) {
+ const newFs = await this.credentials.find(filePath);
+ if (newFs) return newFs.head(filePath);
+ }
+ throw ce;
  }
  }
 }