From 5cfe274fc402e3df4fd4bc8520cefb178f0eba65 Mon Sep 17 00:00:00 2001 From: Benjamin Goering <171782+gobengo@users.noreply.github.com> Date: Mon, 6 Feb 2023 15:40:04 -0800 Subject: [PATCH] feat: rm upload-api-proxy ability to route to separate environment audiences (#407) Motivation: * https://github.com/web3-storage/w3protocol/issues/406 * tl;dr remove some code/behavior that's not strictly needed and @hugomrdias asked me to remove --- packages/access-api/src/bindings.d.ts | 8 +-- packages/access-api/src/config.js | 3 +- .../src/service/upload-api-proxy.js | 70 ++++--------------- packages/access-api/src/utils/context.js | 15 ++-- packages/access-api/test/helpers/context.js | 1 - packages/access-api/test/store-list.js | 4 +- packages/access-api/wrangler.toml | 13 ++-- 7 files changed, 32 insertions(+), 82 deletions(-) diff --git a/packages/access-api/src/bindings.d.ts b/packages/access-api/src/bindings.d.ts index 0363c45ab..66567164c 100644 --- a/packages/access-api/src/bindings.d.ts +++ b/packages/access-api/src/bindings.d.ts @@ -9,7 +9,7 @@ import { Email } from './utils/email.js' import { Spaces } from './models/spaces.js' import { Validations } from './models/validations.js' import { loadConfig } from './config.js' -import { Signer as EdSigner } from '@ucanto/principal/ed25519' +import { ConnectionView, Signer as EdSigner } from '@ucanto/principal/ed25519' import { Accounts } from './models/accounts.js' export {} @@ -35,7 +35,6 @@ export interface Env { DID: `did:web:${string}` // URLs to upload-api so we proxy invocations to it UPLOAD_API_URL: string - UPLOAD_API_URL_STAGING: string // secrets PRIVATE_KEY: string SENTRY_DSN: string @@ -60,10 +59,7 @@ export interface RouteContext { validations: Validations accounts: Accounts } - uploadApi: { - production?: URL - staging?: URL - } + uploadApi: ConnectionView } export type Handler = _Handler diff --git a/packages/access-api/src/config.js b/packages/access-api/src/config.js index 0561c9f25..87bf17ce3 100644 --- a/packages/access-api/src/config.js +++ b/packages/access-api/src/config.js @@ -57,8 +57,7 @@ export function loadConfig(env) { PRIVATE_KEY: vars.PRIVATE_KEY, DID: DID.parse(vars.DID).did(), - UPLOAD_API_URL: env.UPLOAD_API_URL, - UPLOAD_API_URL_STAGING: env.UPLOAD_API_URL_STAGING, + UPLOAD_API_URL: env.UPLOAD_API_URL || 'https://up.web3.storage/', // bindings METRICS: /** @type {import("./bindings").AnalyticsEngine} */ ( diff --git a/packages/access-api/src/service/upload-api-proxy.js b/packages/access-api/src/service/upload-api-proxy.js index 606645bab..484457ff8 100644 --- a/packages/access-api/src/service/upload-api-proxy.js +++ b/packages/access-api/src/service/upload-api-proxy.js @@ -37,10 +37,13 @@ function createProxyService(options) { */ /** + * Create a ucanto connection to an upload api url. + * Assumes upload-api at that URL decodes requests as CAR and encodes responses as CBOR. + * * @param {UcantoHttpConnectionOptions} options * @returns {Ucanto.ConnectionView} */ -function createUcantoHttpConnection(options) { +export function createUploadApiConnection(options) { return Client.connect({ id: DID.parse(options.audience), encoder: CAR, @@ -52,79 +55,30 @@ function createUcantoHttpConnection(options) { }) } -const uploadApiEnvironments = { - production: { - audience: /** @type {const} */ ('did:web:web3.storage'), - // dont use up.web3.storage because it won't resolve from inside cloudflare workers - // until resolution of https://github.com/web3-storage/w3protocol/issues/363 - url: new URL('https://3bd9h7xn3j.execute-api.us-west-2.amazonaws.com/'), - }, - staging: { - audience: /** @type {const} */ ('did:web:staging.web3.storage'), - url: new URL('https://staging.up.web3.storage'), - }, -} - -/** - * @typedef {keyof typeof uploadApiEnvironments} UploadApiEnvironmentName - * @typedef {typeof uploadApiEnvironments[UploadApiEnvironmentName]['audience']} UploadApiAudience - */ - -/** - * @param {object} options - * @param {typeof globalThis.fetch} [options.fetch] - * @param {object} options.uploadApi - * @param {URL} [options.uploadApi.production] - * @param {URL} [options.uploadApi.staging] - */ -function getDefaultConnections(options) { - const { fetch = globalThis.fetch.bind(globalThis), uploadApi } = options - return { - default: createUcantoHttpConnection({ - ...uploadApiEnvironments.production, - ...(uploadApi.production && { url: uploadApi.production }), - fetch, - }), - [uploadApiEnvironments.staging.audience]: createUcantoHttpConnection({ - ...uploadApiEnvironments.staging, - url: uploadApi.staging ?? uploadApiEnvironments.staging.url, - fetch, - }), - } -} - /** - * @template {Ucanto.ConnectionView} [Connection=Ucanto.ConnectionView] * @param {object} options - * @param {typeof globalThis.fetch} [options.fetch] - * @param {{ default: Connection, [K: Ucanto.UCAN.DID]: Connection }} [options.connections] - * @param {Record} [options.audienceToUrl] - * @param {object} options.uploadApi - * @param {URL} [options.uploadApi.production] - * @param {URL} [options.uploadApi.staging] + * @param {import('../bindings.js').RouteContext['uploadApi']} options.uploadApi */ export function createUploadProxy(options) { return createProxyService({ ...options, - connections: options.connections || getDefaultConnections(options), + connections: { + default: options.uploadApi, + }, methods: ['list', 'add', 'remove', 'upload'], }) } /** - * @template {Ucanto.ConnectionView} [Connection=Ucanto.ConnectionView] * @param {object} options - * @param {typeof globalThis.fetch} [options.fetch] - * @param {{ default: Connection, [K: Ucanto.UCAN.DID]: Connection }} [options.connections] - * @param {Record} [options.audienceToUrl] - * @param {object} options.uploadApi - * @param {URL} [options.uploadApi.production] - * @param {URL} [options.uploadApi.staging] + * @param {import('../bindings.js').RouteContext['uploadApi']} options.uploadApi */ export function createStoreProxy(options) { return createProxyService({ ...options, - connections: options.connections || getDefaultConnections(options), + connections: { + default: options.uploadApi, + }, methods: ['list', 'add', 'remove', 'store'], }) } diff --git a/packages/access-api/src/utils/context.js b/packages/access-api/src/utils/context.js index 653d3343b..731ed65d2 100644 --- a/packages/access-api/src/utils/context.js +++ b/packages/access-api/src/utils/context.js @@ -7,6 +7,8 @@ import { Accounts } from '../models/accounts.js' import { Spaces } from '../models/spaces.js' import { Validations } from '../models/validations.js' import { Email } from './email.js' +import { createUploadApiConnection } from '../service/upload-api-proxy.js' +import { DID } from '@ucanto/core' /** * Obtains a route context object. @@ -57,13 +59,10 @@ export function getContext(request, env, ctx) { accounts: new Accounts(config.DB), }, email: new Email({ token: config.POSTMARK_TOKEN }), - uploadApi: { - production: config.UPLOAD_API_URL - ? new URL(config.UPLOAD_API_URL) - : undefined, - staging: config.UPLOAD_API_URL_STAGING - ? new URL(config.UPLOAD_API_URL_STAGING) - : undefined, - }, + uploadApi: createUploadApiConnection({ + audience: DID.parse(config.DID).did(), + url: new URL(config.UPLOAD_API_URL), + fetch: globalThis.fetch.bind(globalThis), + }), } } diff --git a/packages/access-api/test/helpers/context.js b/packages/access-api/test/helpers/context.js index ca9f84d7c..366a90daa 100644 --- a/packages/access-api/test/helpers/context.js +++ b/packages/access-api/test/helpers/context.js @@ -34,7 +34,6 @@ function createBindings(env) { LOGTAIL_TOKEN: env.LOGTAIL_TOKEN || '', W3ACCESS_METRICS: createAnalyticsEngine(), UPLOAD_API_URL: env.UPLOAD_API_URL || '', - UPLOAD_API_URL_STAGING: env.UPLOAD_API_URL_STAGING || '', } } diff --git a/packages/access-api/test/store-list.js b/packages/access-api/test/store-list.js index 7fa80c157..cc27326ce 100644 --- a/packages/access-api/test/store-list.js +++ b/packages/access-api/test/store-list.js @@ -13,8 +13,7 @@ import { describe('proxy store/list invocations to upload-api', function () { for (const web3storageDid of /** @type {const} */ ([ - 'did:web:web3.storage', - 'did:web:staging.web3.storage', + 'did:web:test.web3.storage', ])) { it(`forwards invocations with aud=${web3storageDid}`, async function () { const mockUpstream = createMockUploadApiServer({ @@ -47,7 +46,6 @@ describe('proxy store/list invocations to upload-api', function () { // @ts-ignore PRIVATE_KEY: privateKeyFromEnv ?? process.env.PRIVATE_KEY, UPLOAD_API_URL: mockUpstreamUrl.toString(), - UPLOAD_API_URL_STAGING: mockUpstreamUrl.toString(), }) const spaceCreation = await createSpace( issuer, diff --git a/packages/access-api/wrangler.toml b/packages/access-api/wrangler.toml index 8ce8d2dfe..e9579d28d 100644 --- a/packages/access-api/wrangler.toml +++ b/packages/access-api/wrangler.toml @@ -29,6 +29,7 @@ database_id = "7c676e0c-b9e7-4711-97c8-7b1c8eb229ae" ENV = "dev" DEBUG = "true" DID = "did:web:local.web3.storage" +UPLOAD_API_URL = "https://up.web3.storage" [build] command = "scripts/cli.js build" @@ -37,12 +38,11 @@ watch_dir = "src" [miniflare] d1_persist = ".wrangler/miniflare" - # Dev [env.dev] name = "w3access-dev" workers_dev = true -vars = { ENV = "dev", DEBUG = "false", DID = "did:web:dev.web3.storage" } +vars = { ENV = "dev", DEBUG = "false", DID = "did:web:dev.web3.storage", UPLOAD_API_URL = "https://staging.up.web3.storage" } build = { command = "scripts/cli.js build --env dev", watch_dir = "src" } kv_namespaces = [ { binding = "SPACES", id = "5697e95e1aaa436788e6d697fd3350be" }, @@ -60,7 +60,7 @@ unsafe = { bindings = [ [env.staging] name = "w3access-staging" workers_dev = true -vars = { ENV = "staging", DEBUG = "false", DID = "did:web:staging.web3.storage" } +vars = { ENV = "staging", DEBUG = "false", DID = "did:web:staging.web3.storage", UPLOAD_API_URL = "https://staging.up.web3.storage" } build = { command = "scripts/cli.js build --env staging", watch_dir = "src" } kv_namespaces = [ { binding = "SPACES", id = "b0e5ca990dda4e3784a1741dfa28a52e" }, @@ -78,7 +78,6 @@ unsafe = { bindings = [ [env.production] name = "w3access" routes = [{ pattern = "access.web3.storage", custom_domain = true }] -vars = { ENV = "production", DEBUG = "false", DID = "did:web:web3.storage" } build = { command = "scripts/cli.js build --env production", watch_dir = "src" } kv_namespaces = [ { binding = "SPACES", id = "5437954e8cfd4f7d98557132b0a2e93f" }, @@ -90,3 +89,9 @@ d1_databases = [ unsafe = { bindings = [ { type = "analytics_engine", dataset = "W3ACCESS_METRICS", name = "W3ACCESS_METRICS" }, ] } +[env.production.vars] +DEBUG = "false" +DID = "did:web:web3.storage" +ENV = "production" +# production upload-api - bypass up.web3.storage due to cloudflare dns issue https://github.com/web3-storage/w3protocol/issues/363#issuecomment-1410887488 +UPLOAD_API_URL = "https://3bd9h7xn3j.execute-api.us-west-2.amazonaws.com/"