feat(fs-aws): add credential config for public and requester pays (#1409

)

#### Motivation

Requester pays and public buckets are pretty common ways of accessing s3
locations, access needs to be configured so that a bucket can be
accessed

with either:

- "assume this role then use requester pays"
- "use the current role but as requester pays"

or as a public bucket with no credentials.

#### Modification

allow credentials to be defined as requester pays or public

packages/fs-aws/src/__tests__/credentials.test.ts

@@ -0,0 +1,37 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { AwsS3CredentialProvider, getPublicS3 } from '../credentials.js';
+import { AwsCredentialConfig } from '../types.js';
+
+describe('AwsS3CredentialProvider', () => {
+ const baseConfig: AwsCredentialConfig = { type: 's3', prefix: 's3://' };
+ it('should set requester pays', () => {
+ const creds = new AwsS3CredentialProvider();
+
+ const fs = creds.createFileSystem({ ...baseConfig, access: 'public' });
+ assert.ok(fs.s3 === getPublicS3());
+ assert.ok(fs.requestPayer === 'public');
+ });
+
+ it('should support requester pays', () => {
+ const creds = new AwsS3CredentialProvider();
+
+ const fs = creds.createFileSystem({ ...baseConfig, roleArn: 'arn:...', access: 'requesterPays' });
+ assert.ok(fs.s3 !== getPublicS3());
+ assert.ok(fs.requestPayer === 'requester');
+ });
+
+ it('should support requester pays from the current role', () => {
+ const creds = new AwsS3CredentialProvider();
+
+ const fs = creds.createFileSystem({ ...baseConfig, roleArn: undefined, access: 'requesterPays' });
+ assert.ok(fs.s3 !== getPublicS3());
+ assert.ok(fs.requestPayer === 'requester');
+ });
+
+ it('should require a roleArn', () => {
+ const creds = new AwsS3CredentialProvider();
+ assert.throws(() => creds.createFileSystem({ ...baseConfig }));
+ });
+});

packages/fs-aws/src/credentials.ts

@@ -1,8 +1,9 @@
 import { S3Client } from '@aws-sdk/client-s3';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
 import { FileSystem, FileSystemProvider } from '@chunkd/fs';
-import { AwsCredentialConfig, AwsCredentialProvider } from './types.js';
+
 import { FsAwsS3 } from './fs.s3.js';
+import { AwsCredentialConfig, AwsCredentialProvider } from './types.js';
 
 export function isPromise<T>(t: AwsCredentialProvider | Promise<T>): t is Promise<T> {
  return 'then' in t && typeof t['then'] === 'function';
@@ -48,6 +49,14 @@ export class FsConfigFetcher {
  }
 }
 
+let PublicClient: S3Client | undefined;
+/** Creating a public s3 client is somewhat hard, where the signing method needs to be overriden */
+export function getPublicS3(): S3Client {
+ if (PublicClient) return PublicClient;
+ PublicClient = new S3Client({ signer: { sign: async (req) => req } });
+ return PublicClient;
+}
+
 export type AwsCredentialProviderLoader = () => Promise<AwsCredentialProvider>;
 export class AwsS3CredentialProvider implements FileSystemProvider<FsAwsS3> {
  /**
@@ -63,6 +72,22 @@ export class AwsS3CredentialProvider implements FileSystemProvider<FsAwsS3> {
 
  /** Given a config create a file system */
  createFileSystem(cs: AwsCredentialConfig): FsAwsS3 {
+ // Public access
+ if (cs.access === 'public') {
+ const fs = new FsAwsS3(getPublicS3());
+ fs.requestPayer = 'public';
+ return fs;
+ }
+
+ // Requester pays off the current credentials
+ if (cs.access === 'requesterPays' && cs.roleArn == null) {
+ const fs = new FsAwsS3(new S3Client({}));
+ if (cs.access === 'requesterPays') fs.requestPayer = 'requester';
+ return fs;
+ }
+
+ // All other credentials need a role assumed
+ if (cs.roleArn == null) throw new Error('No roleArn is defined for prefix: ' + cs.prefix);
  const client = new S3Client({
  credentials: fromTemporaryCredentials({
  params: {
@@ -74,7 +99,9 @@ export class AwsS3CredentialProvider implements FileSystemProvider<FsAwsS3> {
  }),
  });
 
- return new FsAwsS3(client);
+ const fs = new FsAwsS3(client);
+ if (cs.access === 'requesterPays') fs.requestPayer = 'requester';
+ return fs;
  }
  /** Version for session name generally v2 or v3 for aws-sdk versions */
  version = 'v3';

packages/fs-aws/src/fs.s3.ts

@@ -1,3 +1,6 @@
+import type { Readable } from 'node:stream';
+import { PassThrough } from 'node:stream';
+
 import {
  DeleteObjectCommand,
  GetObjectCommand,
@@ -7,10 +10,9 @@ import {
  S3Client,
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
-import { FileInfo, FileSystem, FileSystemAction, FsError, ListOptions, WriteOptions, isRecord } from '@chunkd/fs';
+import { FileInfo, FileSystem, FileSystemAction, FsError, isRecord, ListOptions, WriteOptions } from '@chunkd/fs';
 import { SourceAwsS3 } from '@chunkd/source-aws';
-import type { Readable } from 'node:stream';
-import { PassThrough } from 'node:stream';
+
 import { AwsS3CredentialProvider } from './credentials.js';
 
 function isReadable(r: any): r is Readable {
@@ -49,7 +51,7 @@ export class FsAwsS3 implements FileSystem {
  writeTests = new Map<string, Promise<void | FsAwsS3>>();
 
  /** Request Payment option */
- requestPayer?: 'requester';
+ requestPayer?: 'requester' | 'public';
 
  /**
  * When testing write permissions add a suffix to the file name, this file will be deleted up after writing completes

packages/fs-aws/src/types.ts

@@ -1,20 +1,53 @@
 export interface AwsCredentialConfig {
- /** Prefix type generally s3 */
+ /**
+ * Prefix type generally s3
+ */
  type: 's3';
- /** Prefix should always start with `s3://${bucket}` */
+
+ /**
+ * Location that these credentials are valid for,
+ *
+ * prefixes should always start with `s3://${bucket}/`
+ *
+ * @example
+ * ```typescript
+ * "s3://example/" // Matches all files in "s3://example/**"
+ * "s3://example" // Matches all files in all buckets starting with "s3://example*"
+ * ```
+ */
  prefix: string;
- /** Role to use to access */
- roleArn: string;
- /** Aws account this bucket belongs to */
+
+ /**
+ * Role to use to access
+ *
+ * roleArn is not required if access is "public"
+ */
+ roleArn?: string;
+
+ /**
+ * Aws account this bucket belongs to
+ */
  accountId?: string;
- /** Bucket name */
- bucket?: string;
- /** ExternalId if required */
+
+ /**
+ * ExternalId if required
+ */
  externalId?: string;
- /** Max role session duration */
+
+ /**
+ * Max role session duration
+ */
  roleSessionDuration?: number;
- /** Can these credentials be used for "read" or "read-write" access */
+
+ /**
+ * Can these credentials be used for "read" or "read-write" access
+ */
  flags?: 'r' | 'rw';
+
+ /**
+ * Can this prefix be accessed without credentials or as requesterPays
+ */
+ access?: 'public' | 'requesterPays';
 }
 
 export interface AwsCredentialProvider {