From eb8201a27bb8be99e14f50b2e0300a91e3017010 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 9 Nov 2022 16:31:39 +0000 Subject: [PATCH 01/47] feat: add static uploads api --- packages/upload-client/README.md | 287 ++++++++++++++++++ packages/upload-client/package.json | 56 +++- packages/upload-client/src/car.js | 31 ++ packages/upload-client/src/index.js | 76 ++++- packages/upload-client/src/sharding.js | 104 +++++++ packages/upload-client/src/storage.js | 134 ++++++++ packages/upload-client/src/types.ts | 93 ++++++ packages/upload-client/src/unixfs.js | 126 ++++++++ packages/upload-client/src/utils.js | 39 +++ packages/upload-client/test/fixtures.js | 16 + .../upload-client/test/helpers/ok-server.js | 12 + packages/upload-client/test/helpers/random.js | 36 +++ packages/upload-client/test/helpers/shims.js | 10 + packages/upload-client/test/index.test.js | 133 +++++++- packages/upload-client/test/sharding.test.js | 106 +++++++ packages/upload-client/test/storage.test.js | 165 ++++++++++ packages/upload-client/test/unixfs.test.js | 74 +++++ 17 files changed, 1482 insertions(+), 16 deletions(-) create mode 100644 packages/upload-client/README.md create mode 100644 packages/upload-client/src/car.js create mode 100644 packages/upload-client/src/sharding.js create mode 100644 packages/upload-client/src/storage.js create mode 100644 packages/upload-client/src/types.ts create mode 100644 packages/upload-client/src/unixfs.js create mode 100644 packages/upload-client/src/utils.js create mode 100644 packages/upload-client/test/fixtures.js create mode 100644 packages/upload-client/test/helpers/ok-server.js create mode 100644 packages/upload-client/test/helpers/random.js create mode 100644 packages/upload-client/test/helpers/shims.js create mode 100644 packages/upload-client/test/sharding.test.js create mode 100644 packages/upload-client/test/storage.test.js create mode 100644 packages/upload-client/test/unixfs.test.js diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md new file mode 100644 index 000000000..274f25c4b --- /dev/null +++ b/packages/upload-client/README.md @@ -0,0 +1,287 @@ +

⁂
web3.storage

+

The upload client for https://web3.storage

+ +## Install + +Install the package using npm: + +```console +npm install @web3-storage/upload-client +``` + +## Usage + +[API Reference](#api) + +TODO: how to obtain account/signer + +### Uploading files + +```js +import { uploadFile } from '@web3-storage/upload-client' + +const cid = await uploadFile(account, signer, new Blob(['Hello World!'])) +``` + +```js +import { uploadDirectory } from '@web3-storage/upload-client' + +const cid = await uploadDirectory(account, signer, [ + new File(['doc0'], 'doc0.txt'), + new File(['doc1'], 'dir/doc1.txt'), +]) + +// Note: you can use https://npm.im/files-from-path to read files from the +// filesystem in Nodejs. +``` + +### Advanced usage + +#### Buffering API + +The buffering API loads all data into memory so is suitable only for small files. The root data CID is obtained before any transfer to the service takes place. + +```js +import { UnixFS, CAR, Storage } from '@web3-storage/upload-client' + +// Encode a file as a DAG, get back a root data CID and a set of blocks +const { cid, blocks } = await UnixFS.encodeFile(file) +// Encode the DAG as a CAR file +const car = await CAR.encode(blocks, cid) +// Store the CAR file to the service +const carCID = await Storage.store(account, signer, car) +// Register an "upload" - a root CID contained within the passed CAR file(s) +await Storage.registerUpload(account, signer, cid, [carCID]) +``` + +#### Streaming API + +This API offers streaming DAG generation, allowing CAR "shards" to be sent to the service as the DAG is built. It allows files and directories of arbitrary size to be sent to the service while keeping within memory limits of the device. The _last_ CAR file sent contains the root data CID. + +```js +import { + UnixFS, + ShardingStream, + ShardStoringStream, + Storage, +} from '@web3-storage/upload-client' + +const cars = [] +// Encode a file as a DAG, get back a readable stream of blocks. +await UnixFS.createFileEncoderStream(file) + // Pipe blocks to a stream that yields CARs files - shards of the DAG. + .pipeThrough(new ShardingStream()) + // Pipe CARs to a stream that stores them to the service and yields metadata + // about the CARs that were stored. + .pipeThrough(new ShardStoringStream(account, issuer)) + // Collect the metadata, we're mostly interested in the CID of each CAR file + // and the root data CID (which can be found in the _last_ CAR file). + .pipeTo( + new WritableStream({ + write: (car) => { + cars.push(car) + }, + }) + ) + +// The last CAR stored contains the root data CID +const rootCID = cars[cars.length - 1].roots[0] +const carCIDs = cars.map((car) => car.cid) + +// Register an "upload" - a root CID contained within the passed CAR file(s) +await Storage.registerUpload(account, signer, rootCID, carCIDs) +``` + +## API + +- `CAR` + - [`encode`](#carencode) +- [`ShardingStream`](#shardingstream) +- [`ShardStoringStream`](#shardstoringstream) +- `Storage` + - [`registerUpload`](#storageregisterupload) + - [`store`](#storagestoredag) +- `UnixFS` + - [`createDirectoryEncoderStream`](#unixfscreatedirectoryencoderstream) + - [`createFileEncoderStream`](#unixfscreatefileencoderstream) + - [`encodeDirectory`](#unixfsencodedirectory) + - [`encodeFile`](#unixfsencodefile) +- [`uploadDirectory`](#uploaddirectory) +- [`uploadFile`](#uploadfile) + +--- + +### `CAR.encode` + +```ts +function encode(blocks: Iterable, root?: CID): Promise +``` + +Encode a DAG as a CAR file. + +Note: `CARFile` is just a `Blob` with two extra properties: + +```ts +type CARFile = Blob & { version: 1; roots: CID[] } +``` + +Example: + +```js +const { cid, blocks } = await UnixFS.encodeFile(new Blob(['data'])) +const car = await CAR.encode(blocks, cid) +``` + +### `ShardingStream` + +```ts +class ShardingStream extends TransformStream +``` + +Shard a set of blocks into a set of CAR files. The last block written to the stream is assumed to be the DAG root and becomes the CAR root CID for the last CAR output. + +Note: `CARFile` is just a `Blob` with two extra properties: + +```ts +type CARFile = Blob & { version: 1; roots: CID[] } +``` + +### `ShardStoringStream` + +```ts +class ShardStoringStream extends TransformStream +``` + +Stores multiple DAG shards (encoded as CAR files) to the service. + +Note: an "upload" must be registered in order to link multiple shards together as a complete upload. + +The writeable side of this transform stream accepts `CARFile`s and the readable side yields `CARMetadata`, which contains the CAR CID, it's size (in bytes) and it's roots (if it has any). + +### `Storage.registerUpload` + +```ts +function registerUpload( + account: DID, + signer: Signer, + root: CID, + shards: CID[], + options: { retries?: number; signal?: AbortSignal } = {} +): Promise +``` + +Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload. + +### `Storage.store` + +```ts +function store( + account: DID, + signer: Signer, + car: Blob, + options: { retries?: number; signal?: AbortSignal } = {} +): Promise +``` + +Store a CAR file to the service. + +### `UnxiFS.createDirectoryEncoderStream` + +```ts +function createDirectoryEncoderStream( + files: Iterable +): ReadableStream +``` + +Creates a `ReadableStream` that yields UnixFS DAG blocks. All files are added to a container directory, with paths in file names preserved. + +Note: you can use https://npm.im/files-from-path to read files from the filesystem in Nodejs. + +### `UnxiFS.createFileEncoderStream` + +```ts +function createFileEncoderStream(file: Blob): ReadableStream +``` + +Creates a `ReadableStream` that yields UnixFS DAG blocks. + +### `UnxiFS.encodeDirectory` + +```ts +function encodeDirectory( + files: Iterable +): Promise<{ cid: CID; blocks: Block[] }> +``` + +Create a UnixFS DAG from the passed file data. All files are added to a container directory, with paths in file names preserved. + +Note: you can use https://npm.im/files-from-path to read files from the filesystem in Nodejs. + +Example: + +```js +const { cid, blocks } = encodeDirectory([ + new File(['doc0'], 'doc0.txt'), + new File(['doc1'], 'dir/doc1.txt'), +]) +// DAG structure will be: +// bafybei.../doc0.txt +// bafybei.../dir/doc1.txt +``` + +### `UnxiFS.encodeFile` + +```ts +function encodeFile(file: Blob): Promise<{ cid: CID; blocks: Block[] }> +``` + +Create a UnixFS DAG from the passed file data. + +Example: + +```js +const { cid, blocks } = await encodeFile(new File(['data'], 'doc.txt')) +// Note: file name is not preserved - use encodeDirectory if required. +``` + +### `uploadDirectory` + +```ts +function uploadDirectory( + account: DID, + signer: Signer, + files: File[], + options: { + retries?: number + signal?: AbortSignal + onShardStored: ShardStoredCallback + } = {} +): Promise +``` + +Uploads a directory of files to the service and returns the root data CID for the generated DAG. All files are added to a container directory, with paths in file names preserved. + +### `uploadFile` + +```ts +function uploadFile( + account: DID, + signer: Signer, + file: Blob, + options: { + retries?: number + signal?: AbortSignal + onShardStored: ShardStoredCallback + } = {} +): Promise +``` + +Uploads a file to the service and returns the root data CID for the generated DAG. + +## Contributing + +Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! + +## License + +Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/w3protocol/blob/main/license.md) diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index de4b4af80..0964079cb 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -1,12 +1,12 @@ { "name": "@web3-storage/upload-client", "version": "0.0.0", - "description": "The web3.storage client", - "homepage": "https://github.com/web3-storage/w3protocol/tree/main/packages/upload-client", + "description": "The web3.storage upload client", + "homepage": "https://github.com/web3-storage/w3-protocol/tree/main/packages/client", "repository": { "type": "git", - "url": "https://github.com/web3-storage/w3protocol.git", - "directory": "packages/upload-client" + "url": "https://github.com/web3-storage/w3-protocol.git", + "directory": "packages/client" }, "author": "Alan Shaw", "license": "Apache-2.0 OR MIT", @@ -16,13 +16,19 @@ "scripts": { "lint": "tsc && eslint '**/*.{js,ts}' && prettier --check '**/*.{js,ts,yml,json}' --ignore-path ../../.gitignore", "build": "tsc --build", - "test": "npm run test:node && npm run test:browser", - "test:node": "mocha 'test/**/!(*.browser).test.js' -n experimental-vm-modules -n no-warnings", + "test": "npm-run-all -p -r mock:bucket test:all", + "test:all": "run-s test:browser test:node", + "test:node": "c8 -r html -r text mocha 'test/**/!(*.browser).test.js' -n experimental-vm-modules -n no-warnings", "test:browser": "playwright-test 'test/**/!(*.node).test.js'", + "mock:bucket": "node test/helpers/ok-server.js", "rc": "npm version prerelease --preid rc" }, "exports": { - ".": "./src/index.js" + ".": "./src/index.js", + "./car": "./src/car.js", + "./sharding": "./src/sharding.js", + "./storage": "./src/storage.js", + "./unixfs": "./src/unixfs.js" }, "typesVersions": { "*": { @@ -36,12 +42,31 @@ "dist/src/**/*.d.ts", "dist/src/**/*.d.ts.map" ], + "dependencies": { + "@ipld/car": "^5.0.0", + "@ipld/dag-ucan": "^2.0.1", + "@ipld/unixfs": "^2.0.0", + "@ucanto/client": "^3.0.1", + "@ucanto/interface": "^3.0.0", + "@ucanto/transport": "^3.0.1", + "@web3-storage/access": "workspace:^", + "multiformats": "^10.0.2", + "p-queue": "^7.3.0", + "p-retry": "^5.1.1" + }, "devDependencies": { "@types/assert": "^1.5.6", "@types/mocha": "^10.0.0", + "@ucanto/principal": "^3.0.0", + "@ucanto/server": "^3.0.1", "assert": "^2.0.0", + "blockstore-core": "^2.0.2", + "c8": "^7.12.0", "hd-scripts": "^3.0.2", + "ipfs-unixfs-exporter": "^9.0.1", "mocha": "^10.1.0", + "npm-run-all": "^4.1.5", + "path": "^0.12.7", "playwright-test": "^8.1.1", "typescript": "^4.8.4" }, @@ -53,13 +78,26 @@ "project": "./tsconfig.json" }, "rules": { - "unicorn/prefer-number-properties": "off" + "unicorn/prefer-number-properties": "off", + "unicorn/no-null": "off", + "unicorn/prefer-set-has": "off", + "unicorn/no-array-for-each": "off", + "unicorn/prefer-export-from": "off", + "unicorn/catch-error-name": "off", + "unicorn/explicit-length-check": "off", + "unicorn/prefer-type-error": "off", + "eqeqeq": "off", + "no-void": "off", + "no-console": "off", + "no-continue": "off", + "jsdoc/require-hyphen-before-param-description": "off" }, "env": { "mocha": true }, "ignorePatterns": [ - "dist" + "dist", + "coverage" ] } } diff --git a/packages/upload-client/src/car.js b/packages/upload-client/src/car.js new file mode 100644 index 000000000..b60d5cd1d --- /dev/null +++ b/packages/upload-client/src/car.js @@ -0,0 +1,31 @@ +import { CarWriter } from '@ipld/car' +import { collect } from './utils.js' + +/** + * @param {Iterable|AsyncIterable} blocks + * @param {import('multiformats').Link} [root] + * @returns {Promise} + */ +export async function encode(blocks, root) { + // @ts-expect-error + const { writer, out } = CarWriter.create(root) + /** @type {Error?} */ + let error + void (async () => { + try { + for await (const block of blocks) { + // @ts-expect-error + await writer.put(block) + } + } catch (/** @type {any} */ err) { + error = err + } finally { + await writer.close() + } + })() + const chunks = await collect(out) + // @ts-expect-error + if (error != null) throw error + const roots = root != null ? [root] : [] + return Object.assign(new Blob(chunks), { version: 1, roots }) +} diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 8901d0373..46fa4f245 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -1,4 +1,74 @@ -class Client {} +import * as Storage from './storage.js' +import * as UnixFS from './unixfs.js' +import * as CAR from './car.js' +import { ShardingStream, ShardStoringStream } from './sharding.js' -export default Client -export { Client } +export { Storage, UnixFS, CAR } +export * from './sharding.js' + +/** + * @typedef {(meta: import('./types').CARMetadata) => void} StoredShardCallback + * @typedef {import('./types').RequestOptions & { onStoredShard?: StoredShardCallback }} UploadOptions + */ + +/** + * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. + * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {Blob} file File data. + * @param {UploadOptions} [options] + */ +export async function uploadFile(account, signer, file, options = {}) { + return await uploadBlockStream( + account, + signer, + UnixFS.createFileEncoderStream(file), + options + ) +} + +/** + * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. + * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('./types').FileLike[]} files File data. + * @param {UploadOptions} [options] + */ +export async function uploadDirectory(account, signer, files, options = {}) { + return await uploadBlockStream( + account, + signer, + UnixFS.createDirectoryEncoderStream(files), + options + ) +} + +/** + * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. + * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {ReadableStream} blocks UnixFS blocks. + * @param {UploadOptions} [options] + */ +async function uploadBlockStream(account, signer, blocks, options = {}) { + const onStoredShard = options.onStoredShard ?? (() => {}) + + /** @type {import('./types').CARLink[]} */ + const shards = [] + /** @type {import('multiformats').Link?} */ + let root = null + await blocks + .pipeThrough(new ShardingStream()) + .pipeThrough(new ShardStoringStream(account, signer, options)) + .pipeTo( + new WritableStream({ + write(meta) { + root = root || meta.roots[0] + shards.push(meta.cid) + onStoredShard(meta) + }, + }) + ) + + if (root == null) throw new Error('missing root CID') + + await Storage.registerUpload(account, signer, root, shards, options) + return root +} diff --git a/packages/upload-client/src/sharding.js b/packages/upload-client/src/sharding.js new file mode 100644 index 000000000..4c1ecc956 --- /dev/null +++ b/packages/upload-client/src/sharding.js @@ -0,0 +1,104 @@ +import Queue from 'p-queue' +import { encode } from './car.js' +import { store } from './storage.js' + +// most thing are < 30MB +const SHARD_SIZE = 1024 * 1024 * 30 +const CONCURRENT_UPLOADS = 3 + +/** + * Shard a set of blocks into a set of CAR files. The last block is assumed to + * be the DAG root and becomes the CAR root CID for the last CAR output. + * + * @extends {TransformStream} + */ +export class ShardingStream extends TransformStream { + /** + * @param {object} [options] + * @param {number} [options.shardSize] The target shard size. Actual size of + * CAR output may be bigger due to CAR header and block encoding data. + */ + constructor(options = {}) { + const shardSize = options.shardSize ?? SHARD_SIZE + /** @type {import('@ipld/unixfs').Block[]} */ + let shard = [] + /** @type {import('@ipld/unixfs').Block[] | null} */ + let readyShard = null + let size = 0 + + super({ + async transform(block, controller) { + if (readyShard != null) { + controller.enqueue(await encode(readyShard)) + readyShard = null + } + if (shard.length && size + block.bytes.length > shardSize) { + readyShard = shard + shard = [] + size = 0 + } + shard.push(block) + size += block.bytes.length + }, + + async flush(controller) { + if (readyShard != null) { + controller.enqueue(await encode(readyShard)) + } + + const rootBlock = shard.at(-1) + if (rootBlock != null) { + controller.enqueue(await encode(shard, rootBlock.cid)) + } + }, + }) + } +} + +/** + * Upload multiple DAG shards (encoded as CAR files) to the service. + * + * Note: an "upload" must be registered in order to link multiple shards + * together as a complete upload. + * + * The writeable side of this transform stream accepts CAR files and the + * readable side yields `CARMetadata`. + * + * @extends {TransformStream} + */ +export class ShardStoringStream extends TransformStream { + /** + * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. + * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('./types').RequestOptions} [options] + */ + constructor(account, signer, options = {}) { + const queue = new Queue({ concurrency: CONCURRENT_UPLOADS }) + const abortController = new AbortController() + super({ + async transform(car, controller) { + void queue.add( + async () => { + try { + const opts = { ...options, signal: abortController.signal } + const cid = await store(account, signer, car, opts) + const { version, roots, size } = car + controller.enqueue({ version, roots, cid, size }) + } catch (err) { + controller.error(err) + abortController.abort(err) + } + }, + { signal: abortController.signal } + ) + + // retain backpressure by not returning until no items queued to be run + await queue.onSizeLessThan(1) + }, + async flush() { + // wait for queue empty AND pending items complete + await queue.onIdle() + }, + }) + } +} diff --git a/packages/upload-client/src/storage.js b/packages/upload-client/src/storage.js new file mode 100644 index 000000000..a90fb3503 --- /dev/null +++ b/packages/upload-client/src/storage.js @@ -0,0 +1,134 @@ +import { connect } from '@ucanto/client' +import { CAR, CBOR, HTTP } from '@ucanto/transport' +import { parse } from '@ipld/dag-ucan/did' +import { add as storeAdd } from '@web3-storage/access/capabilities/store' +import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' +import retry, { AbortError } from 'p-retry' + +// Production +const serviceURL = new URL( + 'https://8609r1772a.execute-api.us-east-1.amazonaws.com' +) +const serviceDID = parse( + 'did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z' +) + +const RETRIES = 3 + +const connection = connect({ + id: serviceDID, + encoder: CAR, + decoder: CBOR, + channel: HTTP.open({ + url: serviceURL, + method: 'POST', + }), +}) + +/** + * Register an "upload" with the service. + * + * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. + * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. + * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. + * @param {import('./types').RequestOptions} [options] + */ +export async function registerUpload( + account, + signer, + root, + shards, + options = {} +) { + /** @type {import('@ucanto/interface').ConnectionView} */ + const conn = options.connection ?? connection + await retry( + async () => { + const result = await uploadAdd + .invoke({ + issuer: signer, + audience: serviceDID, + with: account, + nb: { + // @ts-expect-error should allow v0 CIDs! + root, + shards, + }, + }) + .execute(conn) + if (result?.error === true) throw result + }, + { onFailedAttempt: console.warn, retries: options.retries ?? RETRIES } + ) +} + +/** + * Store a DAG encoded as a CAR file. + * + * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. + * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {Blob} car CAR file data. + * @param {import('./types').RequestOptions} [options] + * @returns {Promise} + */ +export async function store(account, signer, car, options = {}) { + // TODO: validate blob contains CAR data + const bytes = new Uint8Array(await car.arrayBuffer()) + const link = await CAR.codec.link(bytes) + /** @type {import('@ucanto/interface').ConnectionView} */ + const conn = options.connection ?? connection + const result = await retry( + async () => { + const res = await storeAdd + .invoke({ + issuer: signer, + audience: serviceDID, + with: account, + nb: { link }, + }) + .execute(conn) + return res + }, + { onFailedAttempt: console.warn, retries: options.retries ?? RETRIES } + ) + + if (result.error != null) { + throw new Error('failed store/add invocation', { cause: result }) + } + + // Return early if it was already uploaded. + if (result.status === 'done') { + return link + } + + const res = await retry( + async () => { + try { + const res = await fetch(result.url, { + method: 'PUT', + mode: 'cors', + body: car, + headers: result.headers, + signal: options.signal, + }) + if (res.status >= 400 && res.status < 500) { + throw new AbortError(`upload failed: ${res.status}`) + } + return res + } catch (err) { + if (options.signal?.aborted === true) { + throw new AbortError('upload aborted') + } + throw err + } + }, + { onFailedAttempt: console.warn, retries: options.retries ?? RETRIES } + ) + + if (!res.ok) { + throw new Error('upload failed') + } + + return link +} diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts new file mode 100644 index 000000000..129c5f061 --- /dev/null +++ b/packages/upload-client/src/types.ts @@ -0,0 +1,93 @@ +import { Link, UnknownLink, Version } from 'multiformats/link' +import { Block } from '@ipld/unixfs' +import { CAR } from '@ucanto/transport' +import { ServiceMethod, ConnectionView } from '@ucanto/interface' +import { StoreAdd, UploadAdd } from '@web3-storage/access/capabilities/types' + +export type { StoreAdd, UploadAdd } + +export interface Service { + store: { add: ServiceMethod } + upload: { add: ServiceMethod } +} + +export interface StoreAddResponse { + status: string + headers: Record + url: string +} + +export interface UnixFSEncodeResult { + /** + * Root CID for the DAG. + */ + cid: UnknownLink + /** + * Blocks for the generated DAG. + */ + blocks: Block[] +} + +/** + * Information present in the CAR file header. + */ +export interface CARHeaderInfo { + /** + * CAR version number. + */ + version: number + /** + * Root CIDs present in the CAR header. + */ + roots: Array> +} + +/** + * A DAG encoded as a CAR. + */ +export interface CARFile extends CARHeaderInfo, Blob {} + +/** + * An IPLD Link that has the CAR codec code. + */ +export type CARLink = Link + +/** + * Metadata pertaining to a CAR file. + */ +export interface CARMetadata extends CARHeaderInfo { + /** + * CID of the CAR file (not the data it contains). + */ + cid: CARLink + /** + * Size of the CAR file in bytes. + */ + size: number +} + +export interface Retryable { + retries?: number +} + +export interface Abortable { + signal?: AbortSignal +} + +export interface Connectable { + connection?: ConnectionView +} + +export type RequestOptions = Retryable & Abortable & Connectable + +export interface FileLike { + /** + * Name of the file. May include path information. + */ + name: string + /** + * Returns a ReadableStream which upon reading returns the data contained + * within the File. + */ + stream: () => ReadableStream +} diff --git a/packages/upload-client/src/unixfs.js b/packages/upload-client/src/unixfs.js new file mode 100644 index 000000000..e581672f9 --- /dev/null +++ b/packages/upload-client/src/unixfs.js @@ -0,0 +1,126 @@ +import * as UnixFS from '@ipld/unixfs' +import * as raw from 'multiformats/codecs/raw' +import { toIterable, collect } from './utils.js' + +const queuingStrategy = UnixFS.withCapacity(1_048_576 * 175) + +// TODO: configure chunk size and max children https://github.com/ipld/js-unixfs/issues/36 +const settings = UnixFS.configure({ + fileChunkEncoder: raw, + smallFileEncoder: raw, +}) + +/** + * @param {Blob} blob + * @returns {Promise} + */ +export async function encodeFile(blob) { + const readable = createFileEncoderStream(blob) + const blocks = await collect(toIterable(readable)) + const rootBlock = blocks.at(-1) + if (rootBlock == null) throw new Error('missing root block') + return { cid: rootBlock.cid, blocks } +} + +/** + * @param {Blob} blob + * @returns {ReadableStream} + */ +export function createFileEncoderStream(blob) { + /** @type {TransformStream} */ + const { readable, writable } = new TransformStream({}, queuingStrategy) + const unixfsWriter = UnixFS.createWriter({ writable, settings }) + const fileBuilder = new UnixFsFileBuilder(blob) + void (async () => { + await fileBuilder.finalize(unixfsWriter) + await unixfsWriter.close() + })() + return readable +} + +class UnixFsFileBuilder { + #file + + /** @param {{ stream: () => ReadableStream }} file */ + constructor(file) { + this.#file = file + } + + /** @param {import('@ipld/unixfs').View} writer */ + async finalize(writer) { + const unixfsFileWriter = UnixFS.createFileWriter(writer) + const stream = toIterable(this.#file.stream()) + for await (const chunk of stream) { + await unixfsFileWriter.write(chunk) + } + return await unixfsFileWriter.close() + } +} + +class UnixFSDirectoryBuilder { + /** @type {Map} */ + entries = new Map() + + /** @param {import('@ipld/unixfs').View} writer */ + async finalize(writer) { + const dirWriter = UnixFS.createDirectoryWriter(writer) + for (const [name, entry] of this.entries) { + const link = await entry.finalize(writer) + dirWriter.set(name, link) + } + return await dirWriter.close() + } +} + +/** + * @param {Iterable} files + * @returns {Promise} + */ +export async function encodeDirectory(files) { + const readable = createDirectoryEncoderStream(files) + const blocks = await collect(toIterable(readable)) + const rootBlock = blocks.at(-1) + if (rootBlock == null) throw new Error('missing root block') + return { cid: rootBlock.cid, blocks } +} + +/** + * @param {Iterable} files + * @returns {ReadableStream} + */ +export function createDirectoryEncoderStream(files) { + const rootDir = new UnixFSDirectoryBuilder() + + for (const file of files) { + const path = file.name.split('/') + if (path[0] === '' || path[0] === '.') { + path.shift() + } + let dir = rootDir + for (const [i, name] of path.entries()) { + if (i === path.length - 1) { + dir.entries.set(name, new UnixFsFileBuilder(file)) + break + } + let dirBuilder = dir.entries.get(name) + if (dirBuilder == null) { + dirBuilder = new UnixFSDirectoryBuilder() + dir.entries.set(name, dirBuilder) + } + if (!(dirBuilder instanceof UnixFSDirectoryBuilder)) { + throw new Error(`"${name}" cannot be a file and a directory`) + } + dir = dirBuilder + } + } + + /** @type {TransformStream} */ + const { readable, writable } = new TransformStream({}, queuingStrategy) + const unixfsWriter = UnixFS.createWriter({ writable, settings }) + void (async () => { + await rootDir.finalize(unixfsWriter) + await unixfsWriter.close() + })() + + return readable +} diff --git a/packages/upload-client/src/utils.js b/packages/upload-client/src/utils.js new file mode 100644 index 000000000..b40045799 --- /dev/null +++ b/packages/upload-client/src/utils.js @@ -0,0 +1,39 @@ +/** + * @template T + * @param {ReadableStream | NodeJS.ReadableStream} readable + * @returns {AsyncIterable} + */ +export function toIterable(readable) { + // @ts-expect-error + if (readable[Symbol.asyncIterator] != null) return readable + + // Browser ReadableStream + if ('getReader' in readable) { + return (async function* () { + const reader = readable.getReader() + + try { + while (true) { + const { done, value } = await reader.read() + if (done) return + yield value + } + } finally { + reader.releaseLock() + } + })() + } + + throw new Error('unknown stream') +} + +/** + * @template T + * @param {AsyncIterable|Iterable} collectable + * @returns {Promise} + */ +export async function collect(collectable) { + const chunks = [] + for await (const chunk of collectable) chunks.push(chunk) + return chunks +} diff --git a/packages/upload-client/test/fixtures.js b/packages/upload-client/test/fixtures.js new file mode 100644 index 000000000..5b6d70e72 --- /dev/null +++ b/packages/upload-client/test/fixtures.js @@ -0,0 +1,16 @@ +import * as ed25519 from '@ucanto/principal/ed25519' + +/** did:key:z6Mkqa4oY9Z5Pf5tUcjLHLUsDjKwMC95HGXdE1j22jkbhz6r */ +export const alice = ed25519.parse( + 'MgCZT5vOnYZoVAeyjnzuJIVY9J4LNtJ+f8Js0cTPuKUpFne0BVEDJjEu6quFIU8yp91/TY/+MYK8GvlKoTDnqOCovCVM=' +) + +/** did:key:z6MkffDZCkCTWreg8868fG1FGFogcJj5X6PY93pPcWDn9bob */ +export const bob = ed25519.parse( + 'MgCYbj5AJfVvdrjkjNCxB3iAUwx7RQHVQ7H1sKyHy46Iose0BEevXgL1V73PD9snOCIoONgb+yQ9sycYchQC8kygR4qY=' +) + +/** did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z */ +export const service = ed25519.parse( + 'MgCYKXoHVy7Vk4/QjcEGi+MCqjntUiasxXJ8uJKY0qh11e+0Bs8WsdqGK7xothgrDzzWD0ME7ynPjz2okXDh8537lId8=' +) diff --git a/packages/upload-client/test/helpers/ok-server.js b/packages/upload-client/test/helpers/ok-server.js new file mode 100644 index 000000000..ba07494b5 --- /dev/null +++ b/packages/upload-client/test/helpers/ok-server.js @@ -0,0 +1,12 @@ +import { createServer } from 'http' + +const port = process.env.PORT ?? 9000 + +const server = createServer((_, res) => { + res.setHeader('Access-Control-Allow-Origin', '*') + res.setHeader('Access-Control-Allow-Methods', '*') + res.setHeader('Access-Control-Allow-Headers', '*') + res.end() +}) + +server.listen(port, () => console.log(`Listening on :${port}`)) diff --git a/packages/upload-client/test/helpers/random.js b/packages/upload-client/test/helpers/random.js new file mode 100644 index 000000000..489ba6ca1 --- /dev/null +++ b/packages/upload-client/test/helpers/random.js @@ -0,0 +1,36 @@ +import { CarWriter } from '@ipld/car' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' +import * as CAR from '@ucanto/transport/car' + +/** @param {number} size */ +export function randomBytes(size) { + const bytes = new Uint8Array(size) + while (size) { + const chunk = crypto.getRandomValues(new Uint8Array(Math.min(size, 65_536))) + size -= bytes.length + bytes.set(chunk, size) + } + return bytes +} + +/** @param {number} size */ +export async function randomCAR(size) { + const bytes = randomBytes(128) + const hash = await sha256.digest(bytes) + const root = CID.create(1, raw.code, hash) + + const { writer, out } = CarWriter.create(root) + writer.put({ cid: root, bytes }) + writer.close() + + const chunks = [] + for await (const chunk of out) { + chunks.push(chunk) + } + const blob = new Blob(chunks) + const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) + + return Object.assign(blob, { cid, roots: [root] }) +} diff --git a/packages/upload-client/test/helpers/shims.js b/packages/upload-client/test/helpers/shims.js new file mode 100644 index 000000000..26b0c5c1f --- /dev/null +++ b/packages/upload-client/test/helpers/shims.js @@ -0,0 +1,10 @@ +export class File extends Blob { + /** + * @param {BlobPart[]} blobParts + * @param {string} name + */ + constructor(blobParts, name) { + super(blobParts) + this.name = name + } +} diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index bfcd8cc1e..8a41e332d 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -1,8 +1,133 @@ import assert from 'assert' -import Client from '../src/index.js' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as CBOR from '@ucanto/transport/cbor' +import * as Signer from '@ucanto/principal/ed25519' +import { uploadFile, uploadDirectory } from '../src/index.js' +import { service as id, alice } from './fixtures.js' +import { randomBytes } from './helpers/random.js' +import { File } from './helpers/shims.js' -describe('index', function () { - it('should export a client object', () => { - assert(Client) +describe('uploadFile', () => { + it('uploads a file to the service', async () => { + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9000', + } + + const account = alice.did() + const signer = await Signer.generate() + const file = new Blob([randomBytes(128)]) + /** @type {import('../src/types').CARLink|undefined} */ + let carCID + + const service = { + store: { + /** @param {Server.Invocation} invocation */ + add(invocation) { + assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'store/add') + assert.equal(invCap.with, account) + return res + }, + }, + upload: { + /** @param {Server.Invocation} invocation */ + add: (invocation) => { + assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'upload/add') + assert.equal(invCap.with, account) + assert.equal(invCap.nb.shards?.length, 1) + assert.equal(String(invCap.nb.shards?.[0]), carCID?.toString()) + return null + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + const dataCID = await uploadFile(account, signer, file, { + connection, + onStoredShard: (meta) => { + carCID = meta.cid + }, + }) + + assert(carCID) + assert(dataCID) + }) +}) + +describe('uploadDirectory', () => { + it('uploads a directory to the service', async () => { + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9000', + } + + const account = alice.did() + const signer = await Signer.generate() + const files = [ + new File([randomBytes(128)], '1.txt'), + new File([randomBytes(32)], '2.txt'), + ] + /** @type {import('../src/types').CARLink?} */ + let carCID = null + + const service = { + store: { + /** @param {Server.Invocation} invocation */ + add(invocation) { + assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'store/add') + assert.equal(invCap.with, account) + return res + }, + }, + upload: { + /** @param {Server.Invocation} invocation */ + add: (invocation) => { + assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'upload/add') + assert.equal(invCap.with, account) + assert.equal(invCap.nb.shards?.length, 1) + assert.equal(String(invCap.nb.shards?.[0]), carCID?.toString()) + return null + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + const dataCID = await uploadDirectory(account, signer, files, { + connection, + onStoredShard: (meta) => { + carCID = meta.cid + }, + }) + + assert(carCID) + assert(dataCID) }) }) diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js new file mode 100644 index 000000000..5a7f72e8e --- /dev/null +++ b/packages/upload-client/test/sharding.test.js @@ -0,0 +1,106 @@ +import assert from 'assert' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as CBOR from '@ucanto/transport/cbor' +import * as Signer from '@ucanto/principal/ed25519' +import { createFileEncoderStream } from '../src/unixfs.js' +import { ShardingStream, ShardStoringStream } from '../src/sharding.js' +import { service as id, alice } from './fixtures.js' +import { randomBytes, randomCAR } from './helpers/random.js' + +describe('ShardingStream', () => { + it('creates shards from blocks', async () => { + const file = new Blob([randomBytes(1024 * 1024)]) + const shardSize = 512 * 1024 + + /** @type {import('../src/types').CARFile[]} */ + const shards = [] + + await createFileEncoderStream(file) + .pipeThrough(new ShardingStream({ shardSize })) + .pipeTo( + new WritableStream({ + write: (s) => { + shards.push(s) + }, + }) + ) + + assert(shards.length > 1) + + for (const car of shards) { + // add 100 bytes leeway to the chunk size for encoded CAR data + assert(car.size <= shardSize + 100) + } + }) +}) + +describe('ShardStoringStream', () => { + it('stores multiple DAGs with the service', async () => { + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9000', + } + + const account = alice.did() + const signer = await Signer.generate() + const cars = await Promise.all([randomCAR(128), randomCAR(128)]) + let invokes = 0 + + const service = { + store: { + /** @param {Server.Invocation} invocation */ + add(invocation) { + assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'store/add') + assert.equal(invCap.with, account) + assert.equal(String(invCap.nb.link), cars[invokes].cid.toString()) + invokes++ + return res + }, + }, + upload: { + add: () => { + throw new Server.Failure('not expected to be called') + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + let pulls = 0 + const carStream = new ReadableStream({ + pull(controller) { + if (pulls >= cars.length) return controller.close() + controller.enqueue(cars[pulls]) + pulls++ + }, + }) + + /** @type {import('../src/types').CARLink[]} */ + const carCIDs = [] + await carStream + .pipeThrough(new ShardStoringStream(account, signer, { connection })) + .pipeTo( + new WritableStream({ + write: ({ cid }) => { + carCIDs.push(cid) + }, + }) + ) + + cars.forEach(({ cid }, i) => + assert.equal(cid.toString(), carCIDs[i].toString()) + ) + }) +}) diff --git a/packages/upload-client/test/storage.test.js b/packages/upload-client/test/storage.test.js new file mode 100644 index 000000000..83a6d231e --- /dev/null +++ b/packages/upload-client/test/storage.test.js @@ -0,0 +1,165 @@ +import assert from 'assert' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as CBOR from '@ucanto/transport/cbor' +import * as Signer from '@ucanto/principal/ed25519' +import { registerUpload, store } from '../src/storage.js' +import { service as id, alice } from './fixtures.js' +import { randomCAR } from './helpers/random.js' + +describe('Storage', () => { + it('stores a DAG with the service', async () => { + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9000', + } + + const account = alice.did() + const signer = await Signer.generate() + const car = await randomCAR(128) + + const service = { + store: { + /** @param {Server.Invocation} invocation */ + add(invocation) { + assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'store/add') + assert.equal(invCap.with, account) + assert.equal(String(invCap.nb.link), car.cid.toString()) + return res + }, + }, + upload: { + add: () => { + throw new Server.Failure('not expected to be called') + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + const carCID = await store(account, signer, car, { connection }) + assert(carCID) + assert.equal(carCID.toString(), car.cid.toString()) + }) + + it('skips sending CAR if status = done', async () => { + const res = { + status: 'done', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9001', // will fail the test if called + } + + const account = alice.did() + const signer = await Signer.generate() + const car = await randomCAR(128) + + const service = { + store: { add: () => res }, + upload: { + add: () => { + throw new Server.Failure('not expected to be called') + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + const carCID = await store(account, signer, car, { connection }) + assert(carCID) + assert.equal(carCID.toString(), car.cid.toString()) + }) + + it('aborts', async () => { + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9001', // will fail the test if called + } + + const service = { + store: { add: () => res }, + upload: { + add: () => { + throw new Server.Failure('not expected to be called') + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + const account = alice.did() + const signer = await Signer.generate() + const car = await randomCAR(128) + + const controller = new AbortController() + controller.abort() // already aborted + + await assert.rejects( + store(account, signer, car, { connection, signal: controller.signal }), + { name: 'Error', message: 'upload aborted' } + ) + }) + + it('registers an upload with the service', async () => { + const account = alice.did() + const signer = await Signer.generate() + const car = await randomCAR(128) + + const service = { + store: { + add: () => { + throw new Server.Failure('not expected to be called') + }, + }, + upload: { + /** @param {Server.Invocation} invocation */ + add: (invocation) => { + assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'upload/add') + assert.equal(invCap.with, account) + assert.equal(String(invCap.nb.root), car.roots[0].toString()) + assert.equal(invCap.nb.shards?.length, 1) + assert.equal(String(invCap.nb.shards?.[0]), car.cid.toString()) + return null + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await registerUpload(account, signer, car.roots[0], [car.cid], { + connection, + }) + }) +}) diff --git a/packages/upload-client/test/unixfs.test.js b/packages/upload-client/test/unixfs.test.js new file mode 100644 index 000000000..a3a5b5b33 --- /dev/null +++ b/packages/upload-client/test/unixfs.test.js @@ -0,0 +1,74 @@ +import assert from 'assert' +import { exporter } from 'ipfs-unixfs-exporter' +import { MemoryBlockstore } from 'blockstore-core/memory' +import * as raw from 'multiformats/codecs/raw' +import path from 'path' +import { encodeFile, encodeDirectory } from '../src/unixfs.js' +import { collect } from '../src/utils.js' +import { File } from './helpers/shims.js' + +/** @param {import('ipfs-unixfs-exporter').UnixFSDirectory} dir */ +async function collectDir(dir) { + /** @type {import('ipfs-unixfs-exporter').UnixFSEntry[]} */ + const entries = [] + for await (const entry of dir.content()) { + if (entry.type === 'directory') { + entries.push(...(await collectDir(entry))) + } else { + entries.push(entry) + } + } + return entries +} + +/** @param {Iterable} blocks */ +async function blocksToBlockstore(blocks) { + const blockstore = new MemoryBlockstore() + for (const block of blocks) { + // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 + await blockstore.put(block.cid, block.bytes) + } + return blockstore +} + +describe('UnixFS', () => { + it('encodes a file', async () => { + const file = new Blob(['test']) + const { cid, blocks } = await encodeFile(file) + const blockstore = await blocksToBlockstore(blocks) + const entry = await exporter(cid.toString(), blockstore) + const out = new Blob(await collect(entry.content())) + assert.equal(await out.text(), await file.text()) + }) + + it('encodes a directory', async () => { + const files = [ + new File(['top level'], 'aaaaa.txt'), + new File(['top level dot prefix'], './bbb.txt'), + new File(['top level slash prefix'], '/c.txt'), + new File(['in a dir'], 'dir/two.txt'), + new File(['another in a dir'], 'dir/three.txt'), + new File(['in deeper in dir'], 'dir/deeper/four.png'), + new File(['back in the parent'], 'dir/five.pdf'), + new File(['another in the child'], 'dir/deeper/six.mp4'), + ] + + const { cid, blocks } = await encodeDirectory(files) + const blockstore = await blocksToBlockstore(blocks) + const dirEntry = await exporter(cid.toString(), blockstore) + assert.equal(dirEntry.type, 'directory') + + const expectedPaths = files.map((f) => path.join(cid.toString(), f.name)) + // @ts-expect-error + const entries = await collectDir(dirEntry) + const actualPaths = entries.map((e) => e.path) + + expectedPaths.forEach((p) => assert(actualPaths.includes(p))) + }) + + it('configured to use raw leaves', async () => { + const file = new Blob(['test']) + const { cid } = await encodeFile(file) + assert.equal(cid.code, raw.code) + }) +}) From 1fdae9d8d9684cfd595a12fd6cd0788057c58b1c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 9 Nov 2022 16:42:33 +0000 Subject: [PATCH 02/47] docs: rename var --- packages/upload-client/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 274f25c4b..a7a45a4bc 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -66,7 +66,7 @@ import { Storage, } from '@web3-storage/upload-client' -const cars = [] +const metadatas = [] // Encode a file as a DAG, get back a readable stream of blocks. await UnixFS.createFileEncoderStream(file) // Pipe blocks to a stream that yields CARs files - shards of the DAG. @@ -78,15 +78,15 @@ await UnixFS.createFileEncoderStream(file) // and the root data CID (which can be found in the _last_ CAR file). .pipeTo( new WritableStream({ - write: (car) => { - cars.push(car) + write: (meta) => { + metadatas.push(meta) }, }) ) // The last CAR stored contains the root data CID -const rootCID = cars[cars.length - 1].roots[0] -const carCIDs = cars.map((car) => car.cid) +const rootCID = metadatas[metadatas.length - 1].roots[0] +const carCIDs = metadatas.map((meta) => meta.cid) // Register an "upload" - a root CID contained within the passed CAR file(s) await Storage.registerUpload(account, signer, rootCID, carCIDs) From 2f010719449157cfe0100f142f52c0103dac8ea0 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 9 Nov 2022 16:44:10 +0000 Subject: [PATCH 03/47] fix: lockfile --- pnpm-lock.yaml | 525 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 492 insertions(+), 33 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index defdcdc0c..7a938feec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -259,24 +259,6 @@ importers: typescript: 4.8.4 wrangler: 2.1.15 - packages/upload-client: - specifiers: - '@types/assert': ^1.5.6 - '@types/mocha': ^10.0.0 - assert: ^2.0.0 - hd-scripts: ^3.0.2 - mocha: ^10.1.0 - playwright-test: ^8.1.1 - typescript: ^4.8.4 - devDependencies: - '@types/assert': 1.5.6 - '@types/mocha': 10.0.0 - assert: 2.0.0 - hd-scripts: 3.0.2 - mocha: 10.1.0 - playwright-test: 8.1.1 - typescript: 4.8.4 - packages/store: specifiers: '@types/chai': ^4.3.0 @@ -320,6 +302,59 @@ importers: playwright-test: 8.1.1 typescript: 4.8.4 + packages/upload-client: + specifiers: + '@ipld/car': ^5.0.0 + '@ipld/dag-ucan': ^2.0.1 + '@ipld/unixfs': ^2.0.0 + '@types/assert': ^1.5.6 + '@types/mocha': ^10.0.0 + '@ucanto/client': ^3.0.1 + '@ucanto/interface': ^3.0.0 + '@ucanto/principal': ^3.0.0 + '@ucanto/server': ^3.0.1 + '@ucanto/transport': ^3.0.1 + '@web3-storage/access': workspace:^ + assert: ^2.0.0 + blockstore-core: ^2.0.2 + c8: ^7.12.0 + hd-scripts: ^3.0.2 + ipfs-unixfs-exporter: ^9.0.1 + mocha: ^10.1.0 + multiformats: ^10.0.2 + npm-run-all: ^4.1.5 + p-queue: ^7.3.0 + p-retry: ^5.1.1 + path: ^0.12.7 + playwright-test: ^8.1.1 + typescript: ^4.8.4 + dependencies: + '@ipld/car': 5.0.0 + '@ipld/dag-ucan': 2.0.1 + '@ipld/unixfs': 2.0.0 + '@ucanto/client': 3.0.1 + '@ucanto/interface': 3.0.0 + '@ucanto/transport': 3.0.1 + '@web3-storage/access': link:../access + multiformats: 10.0.2 + p-queue: 7.3.0 + p-retry: 5.1.1 + devDependencies: + '@types/assert': 1.5.6 + '@types/mocha': 10.0.0 + '@ucanto/principal': 3.0.0 + '@ucanto/server': 3.0.1 + assert: 2.0.0 + blockstore-core: 2.0.2 + c8: 7.12.0 + hd-scripts: 3.0.2 + ipfs-unixfs-exporter: 9.0.1 + mocha: 10.1.0 + npm-run-all: 4.1.5 + path: 0.12.7 + playwright-test: 8.1.1 + typescript: 4.8.4 + packages/wallet: specifiers: '@types/node': ^18.11.7 @@ -388,6 +423,10 @@ packages: regenerator-runtime: 0.13.10 dev: true + /@bcoe/v8-coverage/0.2.3: + resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} + dev: true + /@cloudflare/kv-asset-handler/0.2.0: resolution: {integrity: sha512-MVbXLbTcAotOPUj0pAMhVtJ+3/kFkwJqc5qNOleOZTv6QkZZABDMS21dSrSlVswEHwrpWC03e4fWytjqKvuE2A==} dependencies: @@ -506,7 +545,6 @@ packages: cborg: 1.9.5 multiformats: 10.0.2 varint: 6.0.0 - dev: false /@ipld/dag-cbor/8.0.0: resolution: {integrity: sha512-VfedC21yAD/ZIahcrHTeMcc17kEVRlCmHQl0JY9/Rwbd102v0QcuXtBN8KGH8alNO82S89+H6MM/hxP85P4Veg==} @@ -514,7 +552,6 @@ packages: dependencies: cborg: 1.9.5 multiformats: 10.0.2 - dev: false /@ipld/dag-json/9.0.1: resolution: {integrity: sha512-dL5Xhrk0XXoq3lSsY2LNNraH2Nxx4nlgQwSarl2J3oir2jBDQEiBDW8bjgr30ni8/epdWDhXm5mdxat8dFWwGQ==} @@ -522,14 +559,36 @@ packages: dependencies: cborg: 1.9.5 multiformats: 10.0.2 + + /@ipld/dag-pb/2.1.18: + resolution: {integrity: sha512-ZBnf2fuX9y3KccADURG5vb9FaOeMjFkCrNysB0PtftME/4iCTjxfaLoNq/IAh5fTqUOMXvryN6Jyka4ZGuMLIg==} + dependencies: + multiformats: 9.9.0 dev: false + /@ipld/dag-pb/3.0.0: + resolution: {integrity: sha512-d9TYsiS/8ixtUnXRkpHU+4kkI1ZUN57n/HjMvK75uhezL1p9+heWryb/rv+Ztlbux9OW9Zus75lMjFoPbG36bw==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + multiformats: 10.0.2 + dev: true + /@ipld/dag-ucan/2.0.1: resolution: {integrity: sha512-0cqnXPmjmFhz9JVtgU/wCaNvbnFr/HYzl4LaVm7Q7c8FsF9u671rOvNgCbSpJL6f+YTM4Q4fihxLYWPHaUSEww==} dependencies: '@ipld/dag-cbor': 8.0.0 '@ipld/dag-json': 9.0.1 multiformats: 10.0.2 + + /@ipld/unixfs/2.0.0: + resolution: {integrity: sha512-Li6ObZWlnQPM8R1O6mjUWQWlxjf+4yjZDERZIvNILOXeTvF0G36WFIdr3c2s9M6Aiez8gCMzodNnJLRXzXnJ0Q==} + dependencies: + '@ipld/dag-pb': 2.1.18 + '@web-std/stream': 1.0.1 + actor: 2.3.1 + multiformats: 10.0.2 + protobufjs: 7.1.2 + rabin-rs: 2.1.0 dev: false /@istanbuljs/schema/0.1.3: @@ -895,6 +954,14 @@ packages: - utf-8-validate dev: true + /@multiformats/murmur3/2.0.0: + resolution: {integrity: sha512-rnmRpmHMMlgnDQEL5IJ8GiNECMrNv0PR5tlmBTDlb9cnx4oSgFQRdqw7a0uqo3ftt/lUbc9cM1pOE/NxZGX7NQ==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + multiformats: 10.0.2 + murmurhash3js-revisited: 3.0.0 + dev: true + /@next/env/12.3.1: resolution: {integrity: sha512-9P9THmRFVKGKt9DYqeC2aKIxm8rlvkK38V1P1sRE7qyoPBIs8l9oo79QoSdPtOWfzkbDAVUqvbQGgTMsb8BtJg==} dev: false @@ -1024,7 +1091,6 @@ packages: /@noble/ed25519/1.7.1: resolution: {integrity: sha512-Rk4SkJFaXZiznFyC/t77Q0NKS4FL7TLJJsVG2V2oiEq3kJVeTdxysEe/yRWSpnWMe808XRDJ+VFh5pt/FN5plw==} - dev: false /@noble/hashes/1.1.3: resolution: {integrity: sha512-CE0FCR57H2acVI5UOzIGSSIYxZ6v/HOhDR0Ro9VLyhnzLwx0o8W1mmgaqlEUx4049qJDlIBRztv5k+MM8vbO3A==} @@ -1068,6 +1134,39 @@ packages: resolution: {integrity: sha512-a5Sab1C4/icpTZVzZc5Ghpz88yQtGOyNqYXcZgOssB2uuAr+wF/MvN6bgtW32q7HHrvBki+BsZ0OuNv6EV3K9g==} dev: true + /@protobufjs/aspromise/1.1.2: + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + /@protobufjs/base64/1.1.2: + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + /@protobufjs/codegen/2.0.4: + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + /@protobufjs/eventemitter/1.1.0: + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + /@protobufjs/fetch/1.1.0: + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + /@protobufjs/float/1.0.2: + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + /@protobufjs/inquire/1.1.0: + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + /@protobufjs/path/1.1.2: + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + /@protobufjs/pool/1.1.0: + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + /@protobufjs/utf8/1.1.0: + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + /@rushstack/eslint-patch/1.2.0: resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==} dev: true @@ -1437,14 +1536,12 @@ packages: '@ipld/dag-ucan': 2.0.1 '@ucanto/interface': 3.0.0 multiformats: 10.0.2 - dev: false /@ucanto/interface/3.0.0: resolution: {integrity: sha512-CZ+wPD9Zxv3UL/0s3+d1Zf0LPktbkMLGtpdt08+/gKLVU95qxvHzEcpBp9toDYvbKNYBIKQtEc2VbOvwUlMLZA==} dependencies: '@ipld/dag-ucan': 2.0.1 multiformats: 10.0.2 - dev: false /@ucanto/principal/3.0.0: resolution: {integrity: sha512-7CwVbYQc+CdWta+lNuI6vRjVT+GkPwf2VQVXFBm8i4fQomFOTmOq8I/Be3TBS9N2OaIbL7b/eKy1WBdMMzfz5g==} @@ -1454,7 +1551,6 @@ packages: '@ucanto/interface': 3.0.0 multiformats: 10.0.2 one-webcrypto: 1.0.3 - dev: false /@ucanto/server/3.0.1: resolution: {integrity: sha512-PMBb9gPk+5+qKl7bANusPq9xCm2S0dVNIT33EmmtNhXWQSj8nLvWLInnxXYGc+ziyQonPZtDPgKi02uREFEdXw==} @@ -1462,7 +1558,6 @@ packages: '@ucanto/core': 3.0.1 '@ucanto/interface': 3.0.0 '@ucanto/validator': 3.0.1 - dev: false /@ucanto/transport/3.0.1: resolution: {integrity: sha512-stUryA8e6Npkt8RVcxko61gE11+c2Gta+eGzv1XOrWGCk5WqdzequWs/zLxgHxq52EqCWZhLYKQpY5nkrz5UAA==} @@ -1482,7 +1577,6 @@ packages: '@ucanto/core': 3.0.1 '@ucanto/interface': 3.0.0 multiformats: 10.0.2 - dev: false /@web-std/blob/3.0.4: resolution: {integrity: sha512-+dibyiw+uHYK4dX5cJ7HA+gtDAaUUe6JsOryp2ZpAC7h4ICsh49E34JwHoEKPlPvP0llCrNzz45vvD+xX5QDBg==} @@ -1571,6 +1665,10 @@ packages: hasBin: true dev: true + /actor/2.3.1: + resolution: {integrity: sha512-ST/3wnvcP2tKDXnum7nLCLXm+/rsf8vPocXH2Fre6D8FQwNkGDd4JEitBlXj007VQJfiGYRQvXqwOBZVi+JtRg==} + dev: false + /agent-base/6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} @@ -1931,6 +2029,20 @@ packages: resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} dev: true + /blockstore-core/2.0.2: + resolution: {integrity: sha512-ALry3rBp2pTEi4F/usjCJGRluAKYFWI9Np7uE0pZHfDeScMJSj/fDkHEWvY80tPYu4kj03sLKRDGJlZH+V7VzQ==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + err-code: 3.0.1 + interface-blockstore: 3.0.1 + interface-store: 3.0.1 + it-all: 1.0.6 + it-drain: 1.0.5 + it-filter: 1.0.3 + it-take: 1.0.2 + multiformats: 10.0.2 + dev: true + /blueimp-md5/2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} dev: true @@ -1993,6 +2105,25 @@ packages: dependencies: streamsearch: 1.1.0 + /c8/7.12.0: + resolution: {integrity: sha512-CtgQrHOkyxr5koX1wEUmN/5cfDa2ckbHRA4Gy5LAL0zaCFtVWJS5++n+w4/sr2GWGerBxgTjpKeDclk/Qk6W/A==} + engines: {node: '>=10.12.0'} + hasBin: true + dependencies: + '@bcoe/v8-coverage': 0.2.3 + '@istanbuljs/schema': 0.1.3 + find-up: 5.0.0 + foreground-child: 2.0.0 + istanbul-lib-coverage: 3.2.0 + istanbul-lib-report: 3.0.0 + istanbul-reports: 3.1.5 + rimraf: 3.0.2 + test-exclude: 6.0.0 + v8-to-istanbul: 9.0.1 + yargs: 16.2.0 + yargs-parser: 20.2.9 + dev: true + /call-bind/1.0.2: resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==} dependencies: @@ -2033,7 +2164,6 @@ packages: /cborg/1.9.5: resolution: {integrity: sha512-fLBv8wmqtlXqy1Yu+pHzevAIkW6k2K0ZtMujNzWphLsA34vzzg9BHn+5GmZqOJkSA9V7EMKsWrf6K976c1QMjQ==} hasBin: true - dev: false /chai-subset/1.6.0: resolution: {integrity: sha512-K3d+KmqdS5XKW5DWPd5sgNffL3uxdDe+6GdnJh3AYPhwnBGRY5urfvfcbRtWIvvpz+KxkL9FeBB6MZewLUNwug==} @@ -2352,6 +2482,17 @@ packages: resolution: {integrity: sha512-izfGgKyzzIyLaeb1EtZ3KbglkS6AKp9cv7LxmiyoOu+fXfol1tQDC0Cof0enVZGNtudTHW+3lfuW9ZkLQss4Wg==} dev: true + /cross-spawn/6.0.5: + resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} + engines: {node: '>=4.8'} + dependencies: + nice-try: 1.0.5 + path-key: 2.0.1 + semver: 5.7.1 + shebang-command: 1.2.0 + which: 1.3.1 + dev: true + /cross-spawn/7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} @@ -2630,6 +2771,10 @@ packages: engines: {node: '>=6'} dev: false + /err-code/3.0.1: + resolution: {integrity: sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA==} + dev: true + /error-ex/1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: @@ -3893,7 +4038,6 @@ packages: /eventemitter3/4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} - dev: false /events/3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} @@ -3942,6 +4086,10 @@ packages: resolution: {integrity: sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==} dev: true + /fast-fifo/1.1.0: + resolution: {integrity: sha512-Kl29QoNbNvn4nhDsLYjyIAaIqaJB6rBx5p3sL9VjaefJ+eMFBWVZiaoguaoZfzEKr5RhAti0UgM8703akGPJ6g==} + dev: true + /fast-glob/3.2.12: resolution: {integrity: sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==} engines: {node: '>=8.6.0'} @@ -4044,6 +4192,14 @@ packages: dependencies: is-callable: 1.2.7 + /foreground-child/2.0.0: + resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} + engines: {node: '>=8.0.0'} + dependencies: + cross-spawn: 7.0.3 + signal-exit: 3.0.7 + dev: true + /fs-constants/1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} dev: true @@ -4243,6 +4399,14 @@ packages: resolution: {integrity: sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==} dev: true + /hamt-sharding/3.0.2: + resolution: {integrity: sha512-f0DzBD2tSmLFdFsLAvOflIBqFPjerbA7BfmwO8mVho/5hXwgyyYhv+ijIzidQf/DpDX3bRjAQvhGoBFj+DBvPw==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + sparse-array: 1.3.2 + uint8arrays: 4.0.2 + dev: true + /has-bigints/1.0.2: resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==} dev: true @@ -4329,6 +4493,10 @@ packages: resolution: {integrity: sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==} dev: true + /html-escaper/2.0.2: + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + dev: true + /html-rewriter-wasm/0.4.1: resolution: {integrity: sha512-lNovG8CMCCmcVB1Q7xggMSf7tqPCijZXaH4gL6iE8BFghdQCbaY5Met9i1x2Ex8m/cZHDUtXK9H6/znKamRP8Q==} dev: true @@ -4402,6 +4570,10 @@ packages: wrappy: 1.0.2 dev: true + /inherits/2.0.3: + resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} + dev: true + /inherits/2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -4430,6 +4602,19 @@ packages: wrap-ansi: 8.0.1 dev: false + /interface-blockstore/3.0.1: + resolution: {integrity: sha512-yZcLm+ewUbWhvAhvqd+Xbt+w5Sm5SeG0s1HTb0gkGESZVM7MEc1cC5uDRUe6i+X4hEzWO10HCqENbpTgHuWerQ==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + interface-store: 3.0.1 + multiformats: 10.0.2 + dev: true + + /interface-store/3.0.1: + resolution: {integrity: sha512-S5JcwBV+cJorsD0zGKHcBa8A2e578gw9vhZX0QhkV4Xyl4lAMAg5N2GJceUnjCfj/FOKzxTdABzJKPOF2Id8Ig==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dev: true + /internal-slot/1.0.3: resolution: {integrity: sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==} engines: {node: '>= 0.4'} @@ -4444,6 +4629,35 @@ packages: engines: {node: '>= 0.10'} dev: true + /ipfs-unixfs-exporter/9.0.1: + resolution: {integrity: sha512-n/nHhnW9ec4UHI0eQq9VTGgm0+k3FP0OmAFmbICCqwRrmTkgguXOgHb/Z51wWJ/TXvbI5CPz9xqAzG1/lGRyBA==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + '@ipld/dag-cbor': 8.0.0 + '@ipld/dag-pb': 3.0.0 + '@multiformats/murmur3': 2.0.0 + err-code: 3.0.1 + hamt-sharding: 3.0.2 + interface-blockstore: 3.0.1 + ipfs-unixfs: 8.0.0 + it-last: 2.0.0 + it-map: 2.0.0 + it-parallel: 3.0.0 + it-pipe: 2.0.4 + it-pushable: 3.1.0 + multiformats: 10.0.2 + p-queue: 7.3.0 + uint8arrays: 4.0.2 + dev: true + + /ipfs-unixfs/8.0.0: + resolution: {integrity: sha512-PAHtfyjiFs2PZBbeft5QRyXpVOvZ2zsGqID+zVRla7fjC1zRTqJkrGY9h6dF03ldGv/mSmFlNZh479qPC6aZKg==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + err-code: 3.0.1 + protobufjs: 7.1.2 + dev: true + /irregular-plurals/3.3.0: resolution: {integrity: sha512-MVBLKUTangM3EfRPFROhmWQQKRDsrgI83J8GS3jXy+OwYqiR2/aoWndYQ5416jLE3uaGgLH7ncme3X9y09gZ3g==} engines: {node: '>=8'} @@ -4703,6 +4917,90 @@ packages: ws: 8.10.0 dev: false + /istanbul-lib-coverage/3.2.0: + resolution: {integrity: sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==} + engines: {node: '>=8'} + dev: true + + /istanbul-lib-report/3.0.0: + resolution: {integrity: sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==} + engines: {node: '>=8'} + dependencies: + istanbul-lib-coverage: 3.2.0 + make-dir: 3.1.0 + supports-color: 7.2.0 + dev: true + + /istanbul-reports/3.1.5: + resolution: {integrity: sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w==} + engines: {node: '>=8'} + dependencies: + html-escaper: 2.0.2 + istanbul-lib-report: 3.0.0 + dev: true + + /it-all/1.0.6: + resolution: {integrity: sha512-3cmCc6Heqe3uWi3CVM/k51fa/XbMFpQVzFoDsV0IZNHSQDyAXl3c4MjHkFX5kF3922OGj7Myv1nSEUgRtcuM1A==} + dev: true + + /it-drain/1.0.5: + resolution: {integrity: sha512-r/GjkiW1bZswC04TNmUnLxa6uovme7KKwPhc+cb1hHU65E3AByypHH6Pm91WHuvqfFsm+9ws0kPtDBV3/8vmIg==} + dev: true + + /it-filter/1.0.3: + resolution: {integrity: sha512-EI3HpzUrKjTH01miLHWmhNWy3Xpbx4OXMXltgrNprL5lDpF3giVpHIouFpr5l+evXw6aOfxhnt01BIB+4VQA+w==} + dev: true + + /it-last/2.0.0: + resolution: {integrity: sha512-u0GHZ01tWYtPvDkOaqZSLLWjFv3IJw9cPL9mbEV7wnE8DOsbVoXIuKpnz3U6pySl5RzPVjTzSHOc961ZYttBxg==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dev: true + + /it-map/2.0.0: + resolution: {integrity: sha512-mLgtk/NZaN7NZ06iLrMXCA6jjhtZO0vZT5Ocsp31H+nsGI18RSPVmUbFyA1sWx7q+g92J22Sixya7T2QSSAwfA==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dev: true + + /it-merge/1.0.4: + resolution: {integrity: sha512-DcL6GksTD2HQ7+5/q3JznXaLNfwjyG3/bObaF98da+oHfUiPmdo64oJlT9J8R8G5sJRU7thwaY5zxoAKCn7FJw==} + dependencies: + it-pushable: 1.4.2 + dev: true + + /it-parallel/3.0.0: + resolution: {integrity: sha512-/y70cY7VoZ7natLbWrPxoRaKWMD67RvtWx21cyLJr6kkuHrUWOrHNr8CPMBqzDRh73aig/uUT82hzTTmTTkDUg==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + p-defer: 4.0.0 + dev: true + + /it-pipe/2.0.4: + resolution: {integrity: sha512-lK0BV0egwfc64DFJva+0Jh1z8UxwmYBpAHDwq21s0OenRCaEDIntx/iOyWH/jg5efBU6Xa8igzmOqm2CPPNDgg==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + it-merge: 1.0.4 + it-pushable: 3.1.0 + it-stream-types: 1.0.4 + dev: true + + /it-pushable/1.4.2: + resolution: {integrity: sha512-vVPu0CGRsTI8eCfhMknA7KIBqqGFolbRx+1mbQ6XuZ7YCz995Qj7L4XUviwClFunisDq96FdxzF5FnAbw15afg==} + dependencies: + fast-fifo: 1.1.0 + dev: true + + /it-pushable/3.1.0: + resolution: {integrity: sha512-sEAdT86u6aIWvLkH4hlOmgvHpRyUOUG22HD365H+Dh67zYpaPdILmT4Om7Wjdb+m/SjEB81z3nYCoIrgVYpOFA==} + dev: true + + /it-stream-types/1.0.4: + resolution: {integrity: sha512-0F3CqTIcIHwtnmIgqd03a7sw8BegAmE32N2w7anIGdALea4oAN4ltqPgDMZ7zn4XPLZifXEZlBXSzgg64L1Ebw==} + dev: true + + /it-take/1.0.2: + resolution: {integrity: sha512-u7I6qhhxH7pSevcYNaMECtkvZW365ARqAIt9K+xjdK1B2WUDEjQSfETkOCT8bxFq/59LqrN3cMLUtTgmDBaygw==} + dev: true + /js-sdsl/4.1.5: resolution: {integrity: sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==} dev: true @@ -4735,6 +5033,10 @@ packages: engines: {node: '>=12.0.0'} dev: true + /json-parse-better-errors/1.0.2: + resolution: {integrity: sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==} + dev: true + /json-parse-even-better-errors/2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true @@ -4854,6 +5156,16 @@ packages: wrap-ansi: 7.0.0 dev: true + /load-json-file/4.0.0: + resolution: {integrity: sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==} + engines: {node: '>=4'} + dependencies: + graceful-fs: 4.2.10 + parse-json: 4.0.0 + pify: 3.0.0 + strip-bom: 3.0.0 + dev: true + /load-json-file/7.0.1: resolution: {integrity: sha512-Gnxj3ev3mB5TkVBGad0JM6dmLiQL+o0t23JPBZ9sd+yvSLk05mFoqKBw5N8gbbkU4TNXyqCgIrl/VM17OgUIgQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -4919,6 +5231,9 @@ packages: wrap-ansi: 6.2.0 dev: true + /long/5.2.1: + resolution: {integrity: sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A==} + /loose-envify/1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -4986,6 +5301,11 @@ packages: mimic-fn: 4.0.0 dev: true + /memorystream/0.3.1: + resolution: {integrity: sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==} + engines: {node: '>= 0.10.0'} + dev: true + /merge-options/3.0.4: resolution: {integrity: sha512-2Sug1+knBjkaMsMgf1ctR1Ujx+Ayku4EdJN4Z+C2+JzoeF7A3OZ9KM2GY0CpQS51NR61LTurMJrRKPhSs3ZRTQ==} engines: {node: '>=10'} @@ -5208,8 +5528,16 @@ packages: /multiformats/10.0.2: resolution: {integrity: sha512-nJEHLFOYhO4L+aNApHhCnWqa31FyqAHv9Q77AhmwU3KsM2f1j7tuJpCk5ByZ33smzycNCpSG5klNIejIyfFx2A==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} + + /multiformats/9.9.0: + resolution: {integrity: sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==} dev: false + /murmurhash3js-revisited/3.0.0: + resolution: {integrity: sha512-/sF3ee6zvScXMb1XFJ8gDsSnY+X8PbOyjIuBhtgis10W2Jx4ZjIhikUCIF9c4gpJxVnQIsPAFrSwTCuAjicP6g==} + engines: {node: '>=8.0.0'} + dev: true + /mustache/4.2.0: resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} hasBin: true @@ -5297,6 +5625,10 @@ packages: - babel-plugin-macros dev: false + /nice-try/1.0.5: + resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} + dev: true + /node-abi/3.28.0: resolution: {integrity: sha512-fRlDb4I0eLcQeUvGq7IY3xHrSb0c9ummdvDSYWfT9+LKP+3jCKw/tKoqaM7r1BAoiAC6GtwyjaGnOz6B3OtF+A==} engines: {node: '>=10'} @@ -5340,6 +5672,22 @@ packages: engines: {node: '>=0.10.0'} dev: true + /npm-run-all/4.1.5: + resolution: {integrity: sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==} + engines: {node: '>= 4'} + hasBin: true + dependencies: + ansi-styles: 3.2.1 + chalk: 2.4.2 + cross-spawn: 6.0.5 + memorystream: 0.3.1 + minimatch: 3.1.2 + pidtree: 0.3.1 + read-pkg: 3.0.0 + shell-quote: 1.7.4 + string.prototype.padend: 3.1.4 + dev: true + /npm-run-path/5.1.0: resolution: {integrity: sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -5454,7 +5802,6 @@ packages: /one-webcrypto/1.0.3: resolution: {integrity: sha512-fu9ywBVBPx0gS9K0etIROTiCkvI5S1TDjFsYFb3rC1ewFxeOqsbzq7aIMBHsYfrTHBcGXJaONXXjTl8B01cW1Q==} - dev: false /onetime/5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} @@ -5508,7 +5855,6 @@ packages: /p-defer/4.0.0: resolution: {integrity: sha512-Vb3QRvQ0Y5XnF40ZUWW7JfLogicVh/EnA5gBIvKDJoYpeI82+1E3AlB9yOcKFS0AhHrWVnAQO39fbR0G99IVEQ==} engines: {node: '>=12'} - dev: false /p-event/4.2.0: resolution: {integrity: sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ==} @@ -5603,7 +5949,6 @@ packages: dependencies: eventemitter3: 4.0.7 p-timeout: 5.1.0 - dev: false /p-retry/5.1.1: resolution: {integrity: sha512-i69WkEU5ZAL8mrmdmVviWwU+DN+IUF8f4sSJThoJ3z5A7Nn5iuO5ROX3Boye0u+uYQLOSfgFl7SuFZCjlAVbQA==} @@ -5652,6 +5997,14 @@ packages: callsites: 3.1.0 dev: true + /parse-json/4.0.0: + resolution: {integrity: sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==} + engines: {node: '>=4'} + dependencies: + error-ex: 1.3.2 + json-parse-better-errors: 1.0.2 + dev: true + /parse-json/5.2.0: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} @@ -5694,6 +6047,11 @@ packages: engines: {node: '>=0.10.0'} dev: true + /path-key/2.0.1: + resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} + engines: {node: '>=4'} + dev: true + /path-key/3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -5712,11 +6070,25 @@ packages: resolution: {integrity: sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==} dev: true + /path-type/3.0.0: + resolution: {integrity: sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==} + engines: {node: '>=4'} + dependencies: + pify: 3.0.0 + dev: true + /path-type/4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} dev: true + /path/0.12.7: + resolution: {integrity: sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==} + dependencies: + process: 0.11.10 + util: 0.10.4 + dev: true + /pathval/1.1.1: resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} dev: true @@ -5730,12 +6102,23 @@ packages: engines: {node: '>=8.6'} dev: true + /pidtree/0.3.1: + resolution: {integrity: sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA==} + engines: {node: '>=0.10'} + hasBin: true + dev: true + /pidtree/0.6.0: resolution: {integrity: sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==} engines: {node: '>=0.10'} hasBin: true dev: true + /pify/3.0.0: + resolution: {integrity: sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==} + engines: {node: '>=4'} + dev: true + /pkg-conf/4.0.0: resolution: {integrity: sha512-7dmgi4UY4qk+4mj5Cd8v/GExPo0K+SlY+hulOSdfZ/T6jVH6//y7NtzZo5WrfhDBxuQ0jCa7fLZmNaNh7EWL/w==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -5907,6 +6290,24 @@ packages: react-is: 16.13.1 dev: true + /protobufjs/7.1.2: + resolution: {integrity: sha512-4ZPTPkXCdel3+L81yw3dG6+Kq3umdWKh7Dc7GW/CpNk4SX3hK58iPCWeCyhVTDrbkNeKrYNZ7EojM5WDaEWTLQ==} + engines: {node: '>=12.0.0'} + requiresBuild: true + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 18.11.9 + long: 5.2.1 + /proxy-from-env/1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} dev: true @@ -5937,6 +6338,10 @@ packages: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} dev: true + /rabin-rs/2.1.0: + resolution: {integrity: sha512-5y72gAXPzIBsAMHcpxZP8eMDuDT98qMP1BqSDHRbHkJJXEgWIN1lA47LxUqzsK6jknOJtgfkQr9v+7qMlFDm6g==} + dev: false + /randombytes/2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} dependencies: @@ -5983,6 +6388,15 @@ packages: type-fest: 0.8.1 dev: true + /read-pkg/3.0.0: + resolution: {integrity: sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA==} + engines: {node: '>=4'} + dependencies: + load-json-file: 4.0.0 + normalize-package-data: 2.5.0 + path-type: 3.0.0 + dev: true + /read-pkg/5.2.0: resolution: {integrity: sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==} engines: {node: '>=8'} @@ -6279,6 +6693,13 @@ packages: resolution: {integrity: sha512-1jeBGaKNGdEq4FgIrORu/N570dwoPYio8lSoYLWmX7sQ//0JY08Xh9o5pBcgmHQ/MbsYp/aZnOe1s1lIsbLprQ==} dev: true + /shebang-command/1.2.0: + resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} + engines: {node: '>=0.10.0'} + dependencies: + shebang-regex: 1.0.0 + dev: true + /shebang-command/2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -6286,11 +6707,20 @@ packages: shebang-regex: 3.0.0 dev: true + /shebang-regex/1.0.0: + resolution: {integrity: sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==} + engines: {node: '>=0.10.0'} + dev: true + /shebang-regex/3.0.0: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} dev: true + /shell-quote/1.7.4: + resolution: {integrity: sha512-8o/QEhSSRb1a5i7TFR0iM4G16Z0vYB2OQVs4G3aAFXjn3T6yEx8AZxy1PgDF7I00LZHYA3WxaSYIf5e5sAX8Rw==} + dev: true + /shelljs/0.8.5: resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} engines: {node: '>=4'} @@ -6406,6 +6836,10 @@ packages: resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==} dev: true + /sparse-array/1.3.2: + resolution: {integrity: sha512-ZT711fePGn3+kQyLuv1fpd3rNSkNF8vd5Kv2D+qnOANeyKs3fx6bUMGWRPvgTTcYV64QMqZKZwcuaQSP3AZ0tg==} + dev: true + /spdx-correct/3.1.1: resolution: {integrity: sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==} dependencies: @@ -6522,6 +6956,15 @@ packages: side-channel: 1.0.4 dev: true + /string.prototype.padend/3.1.4: + resolution: {integrity: sha512-67otBXoksdjsnXXRUq+KMVTdlVRZ2af422Y0aTyTjVaoQkGr3mxl2Bc5emi7dOQ3OGVVQQskmLEWwFXwommpNw==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.4 + es-abstract: 1.20.4 + dev: true + /string.prototype.trim/1.2.6: resolution: {integrity: sha512-8lMR2m+U0VJTPp6JjvJTtGyc4FIGq9CdRt7O9p6T0e6K4vjU+OP+SQJpbe/SBmRcCUIvNUnjsbmY6lnMp8MhsQ==} engines: {node: '>= 0.4'} @@ -6888,7 +7331,6 @@ packages: engines: {node: '>=16.0.0', npm: '>=7.0.0'} dependencies: multiformats: 10.0.2 - dev: false /unbox-primitive/1.0.2: resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} @@ -6938,6 +7380,12 @@ packages: /util-deprecate/1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + /util/0.10.4: + resolution: {integrity: sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==} + dependencies: + inherits: 2.0.3 + dev: true + /util/0.12.5: resolution: {integrity: sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==} dependencies: @@ -6972,7 +7420,6 @@ packages: /varint/6.0.0: resolution: {integrity: sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==} - dev: false /watch/1.0.2: resolution: {integrity: sha512-1u+Z5n9Jc1E2c7qDO8SinPoZuHj7FgbgU1olSFoyaklduDvvtX7GMMtlE6OC9FTXq4KvNAOfj6Zu4vI1e9bAKA==} @@ -7056,6 +7503,13 @@ packages: has-tostringtag: 1.0.0 is-typed-array: 1.1.10 + /which/1.3.1: + resolution: {integrity: sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==} + hasBin: true + dependencies: + isexe: 2.0.0 + dev: true + /which/2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -7197,6 +7651,11 @@ packages: engines: {node: '>=10'} dev: true + /yargs-parser/20.2.9: + resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} + engines: {node: '>=10'} + dev: true + /yargs-parser/21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} From a81a446465b808a70bacc0daf26c3aed712efdc8 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 9 Nov 2022 16:47:06 +0000 Subject: [PATCH 04/47] fix: repo name --- packages/upload-client/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 0964079cb..513b6cb9d 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -2,11 +2,11 @@ "name": "@web3-storage/upload-client", "version": "0.0.0", "description": "The web3.storage upload client", - "homepage": "https://github.com/web3-storage/w3-protocol/tree/main/packages/client", + "homepage": "https://github.com/web3-storage/w3protocol/tree/main/packages/client", "repository": { "type": "git", - "url": "https://github.com/web3-storage/w3-protocol.git", - "directory": "packages/client" + "url": "https://github.com/web3-storage/w3protocol.git", + "directory": "packages/upload-client" }, "author": "Alan Shaw", "license": "Apache-2.0 OR MIT", From 591681ac5bdd765d3b7fb9f3c5907fbd2cf31ea6 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 9 Nov 2022 16:47:46 +0000 Subject: [PATCH 05/47] fixL homepage --- packages/upload-client/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 513b6cb9d..f03bd65e6 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -2,7 +2,7 @@ "name": "@web3-storage/upload-client", "version": "0.0.0", "description": "The web3.storage upload client", - "homepage": "https://github.com/web3-storage/w3protocol/tree/main/packages/client", + "homepage": "https://github.com/web3-storage/w3protocol/tree/main/packages/upload-client", "repository": { "type": "git", "url": "https://github.com/web3-storage/w3protocol.git", From 97b59aeaff8b43e8ce3b32a3a6cfdcc3e11dc316 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 14:38:23 +0000 Subject: [PATCH 06/47] refactor: accept signer and proof --- packages/upload-client/package.json | 1 + packages/upload-client/src/index.js | 32 +++++------ packages/upload-client/src/sharding.js | 8 +-- packages/upload-client/src/storage.js | 59 ++++++++++++++++---- packages/upload-client/test/fixtures.js | 10 ---- packages/upload-client/test/sharding.test.js | 14 ++++- packages/upload-client/test/storage.test.js | 52 +++++++++++++---- pnpm-lock.yaml | 2 + 8 files changed, 122 insertions(+), 56 deletions(-) diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index f03bd65e6..7cdf145a8 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -47,6 +47,7 @@ "@ipld/dag-ucan": "^2.0.1", "@ipld/unixfs": "^2.0.0", "@ucanto/client": "^3.0.1", + "@ucanto/core": "^3.0.1", "@ucanto/interface": "^3.0.0", "@ucanto/transport": "^3.0.1", "@web3-storage/access": "workspace:^", diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 46fa4f245..bce5c220a 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -12,42 +12,42 @@ export * from './sharding.js' */ /** - * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. - * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Delegation} proof Proof the signer has the capability to perform the action. * @param {Blob} file File data. * @param {UploadOptions} [options] */ -export async function uploadFile(account, signer, file, options = {}) { +export async function uploadFile(issuer, proof, file, options = {}) { return await uploadBlockStream( - account, - signer, + issuer, + proof, UnixFS.createFileEncoderStream(file), options ) } /** - * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. - * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Delegation} proof Proof the signer has the capability to perform the action. * @param {import('./types').FileLike[]} files File data. * @param {UploadOptions} [options] */ -export async function uploadDirectory(account, signer, files, options = {}) { +export async function uploadDirectory(issuer, proof, files, options = {}) { return await uploadBlockStream( - account, - signer, + issuer, + proof, UnixFS.createDirectoryEncoderStream(files), options ) } /** - * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. - * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. - * @param {ReadableStream} blocks UnixFS blocks. + * @param {import('@ucanto/interface').Signer} issuer + * @param {import('@ucanto/interface').Delegation} proof + * @param {ReadableStream} blocks * @param {UploadOptions} [options] */ -async function uploadBlockStream(account, signer, blocks, options = {}) { +async function uploadBlockStream(issuer, proof, blocks, options = {}) { const onStoredShard = options.onStoredShard ?? (() => {}) /** @type {import('./types').CARLink[]} */ @@ -56,7 +56,7 @@ async function uploadBlockStream(account, signer, blocks, options = {}) { let root = null await blocks .pipeThrough(new ShardingStream()) - .pipeThrough(new ShardStoringStream(account, signer, options)) + .pipeThrough(new ShardStoringStream(issuer, proof, options)) .pipeTo( new WritableStream({ write(meta) { @@ -69,6 +69,6 @@ async function uploadBlockStream(account, signer, blocks, options = {}) { if (root == null) throw new Error('missing root CID') - await Storage.registerUpload(account, signer, root, shards, options) + await Storage.registerUpload(issuer, proof, root, shards, options) return root } diff --git a/packages/upload-client/src/sharding.js b/packages/upload-client/src/sharding.js index 4c1ecc956..7b41d4f94 100644 --- a/packages/upload-client/src/sharding.js +++ b/packages/upload-client/src/sharding.js @@ -68,11 +68,11 @@ export class ShardingStream extends TransformStream { */ export class ShardStoringStream extends TransformStream { /** - * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. - * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Delegation} proof Proof the signer has the capability to perform the action. * @param {import('./types').RequestOptions} [options] */ - constructor(account, signer, options = {}) { + constructor(issuer, proof, options = {}) { const queue = new Queue({ concurrency: CONCURRENT_UPLOADS }) const abortController = new AbortController() super({ @@ -81,7 +81,7 @@ export class ShardStoringStream extends TransformStream { async () => { try { const opts = { ...options, signal: abortController.signal } - const cid = await store(account, signer, car, opts) + const cid = await store(issuer, proof, car, opts) const { version, roots, size } = car controller.enqueue({ version, roots, cid, size }) } catch (err) { diff --git a/packages/upload-client/src/storage.js b/packages/upload-client/src/storage.js index a90fb3503..31f463493 100644 --- a/packages/upload-client/src/storage.js +++ b/packages/upload-client/src/storage.js @@ -1,3 +1,4 @@ +import { isDelegation } from '@ucanto/core' import { connect } from '@ucanto/client' import { CAR, CBOR, HTTP } from '@ucanto/transport' import { parse } from '@ipld/dag-ucan/did' @@ -28,28 +29,31 @@ const connection = connect({ /** * Register an "upload" with the service. * - * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. - * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Proof} proof Proof the signer has the capability to perform the action. * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. * @param {import('./types').RequestOptions} [options] */ export async function registerUpload( - account, - signer, + issuer, + proof, root, shards, options = {} ) { + validateProof(proof, serviceDID.did(), uploadAdd.can) + /** @type {import('@ucanto/interface').ConnectionView} */ const conn = options.connection ?? connection await retry( async () => { const result = await uploadAdd .invoke({ - issuer: signer, + issuer, audience: serviceDID, - with: account, + // @ts-ignore expects did:${string} but cap with is ${string}:${string} + with: proof.capabilities[0].with, nb: { // @ts-expect-error should allow v0 CIDs! root, @@ -66,13 +70,15 @@ export async function registerUpload( /** * Store a DAG encoded as a CAR file. * - * @param {import('@ucanto/interface').DID} account DID of the account that is receiving the upload. - * @param {import('@ucanto/interface').Signer} signer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. + * @param {import('@ucanto/interface').Proof} proof Proof the signer has the capability to perform the action. * @param {Blob} car CAR file data. * @param {import('./types').RequestOptions} [options] * @returns {Promise} */ -export async function store(account, signer, car, options = {}) { +export async function store(issuer, proof, car, options = {}) { + validateProof(proof, serviceDID.did(), storeAdd.can) + // TODO: validate blob contains CAR data const bytes = new Uint8Array(await car.arrayBuffer()) const link = await CAR.codec.link(bytes) @@ -82,10 +88,12 @@ export async function store(account, signer, car, options = {}) { async () => { const res = await storeAdd .invoke({ - issuer: signer, + issuer, audience: serviceDID, - with: account, + // @ts-ignore expects did:${string} but cap with is ${string}:${string} + with: proof.capabilities[0].with, nb: { link }, + proofs: [proof], }) .execute(conn) return res @@ -94,7 +102,7 @@ export async function store(account, signer, car, options = {}) { ) if (result.error != null) { - throw new Error('failed store/add invocation', { cause: result }) + throw new Error(`failed ${storeAdd.can} invocation`, { cause: result }) } // Return early if it was already uploaded. @@ -132,3 +140,30 @@ export async function store(account, signer, car, options = {}) { return link } + +/** + * @param {import('@ucanto/interface').Proof} proof + * @param {import('@ucanto/interface').DID} audience + * @param {import('@ucanto/interface').Ability} ability + */ +function validateProof(proof, audience, ability) { + if (!isDelegation(proof)) { + throw new Error('Linked proofs not supported') + } + if (proof.audience.did() !== audience) { + throw new Error(`Unexpected audience: ${proof.audience}`) + } + if (!proof.capabilities.some((c) => capabilityMatches(c.can, ability))) { + throw new Error(`Missing proof of delegated capability: ${ability}`) + } +} + +/** + * @param {string} can + * @param {import('@ucanto/interface').Ability} ability + */ +function capabilityMatches(can, ability) { + return can === ability + ? true + : can.endsWith('*') && ability.startsWith(can.split('*')[0]) +} diff --git a/packages/upload-client/test/fixtures.js b/packages/upload-client/test/fixtures.js index 5b6d70e72..03ee05d2f 100644 --- a/packages/upload-client/test/fixtures.js +++ b/packages/upload-client/test/fixtures.js @@ -1,15 +1,5 @@ import * as ed25519 from '@ucanto/principal/ed25519' -/** did:key:z6Mkqa4oY9Z5Pf5tUcjLHLUsDjKwMC95HGXdE1j22jkbhz6r */ -export const alice = ed25519.parse( - 'MgCZT5vOnYZoVAeyjnzuJIVY9J4LNtJ+f8Js0cTPuKUpFne0BVEDJjEu6quFIU8yp91/TY/+MYK8GvlKoTDnqOCovCVM=' -) - -/** did:key:z6MkffDZCkCTWreg8868fG1FGFogcJj5X6PY93pPcWDn9bob */ -export const bob = ed25519.parse( - 'MgCYbj5AJfVvdrjkjNCxB3iAUwx7RQHVQ7H1sKyHy46Iose0BEevXgL1V73PD9snOCIoONgb+yQ9sycYchQC8kygR4qY=' -) - /** did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z */ export const service = ed25519.parse( 'MgCYKXoHVy7Vk4/QjcEGi+MCqjntUiasxXJ8uJKY0qh11e+0Bs8WsdqGK7xothgrDzzWD0ME7ynPjz2okXDh8537lId8=' diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index 5a7f72e8e..06a134ebc 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -4,9 +4,10 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' +import { add as storeAdd } from '@web3-storage/access/capabilities/store' import { createFileEncoderStream } from '../src/unixfs.js' import { ShardingStream, ShardStoringStream } from '../src/sharding.js' -import { service as id, alice } from './fixtures.js' +import { service as id } from './fixtures.js' import { randomBytes, randomCAR } from './helpers/random.js' describe('ShardingStream', () => { @@ -44,11 +45,18 @@ describe('ShardStoringStream', () => { url: 'http://localhost:9000', } - const account = alice.did() + const account = await Signer.generate() const signer = await Signer.generate() const cars = await Promise.all([randomCAR(128), randomCAR(128)]) let invokes = 0 + const proof = await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }) + const service = { store: { /** @param {Server.Invocation} invocation */ @@ -90,7 +98,7 @@ describe('ShardStoringStream', () => { /** @type {import('../src/types').CARLink[]} */ const carCIDs = [] await carStream - .pipeThrough(new ShardStoringStream(account, signer, { connection })) + .pipeThrough(new ShardStoringStream(signer, proof, { connection })) .pipeTo( new WritableStream({ write: ({ cid }) => { diff --git a/packages/upload-client/test/storage.test.js b/packages/upload-client/test/storage.test.js index 83a6d231e..57a8a0c50 100644 --- a/packages/upload-client/test/storage.test.js +++ b/packages/upload-client/test/storage.test.js @@ -4,8 +4,10 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' +import { add as storeAdd } from '@web3-storage/access/capabilities/store' +import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' import { registerUpload, store } from '../src/storage.js' -import { service as id, alice } from './fixtures.js' +import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' describe('Storage', () => { @@ -16,10 +18,17 @@ describe('Storage', () => { url: 'http://localhost:9000', } - const account = alice.did() + const account = await Signer.generate() const signer = await Signer.generate() const car = await randomCAR(128) + const proof = await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }) + const service = { store: { /** @param {Server.Invocation} invocation */ @@ -28,7 +37,7 @@ describe('Storage', () => { assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') - assert.equal(invCap.with, account) + assert.equal(invCap.with, account.did()) assert.equal(String(invCap.nb.link), car.cid.toString()) return res }, @@ -48,7 +57,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store(account, signer, car, { connection }) + const carCID = await store(signer, proof, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -60,10 +69,17 @@ describe('Storage', () => { url: 'http://localhost:9001', // will fail the test if called } - const account = alice.did() + const account = await Signer.generate() const signer = await Signer.generate() const car = await randomCAR(128) + const proof = await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }) + const service = { store: { add: () => res }, upload: { @@ -81,7 +97,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store(account, signer, car, { connection }) + const carCID = await store(signer, proof, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -110,24 +126,38 @@ describe('Storage', () => { channel: server, }) - const account = alice.did() + const account = await Signer.generate() const signer = await Signer.generate() const car = await randomCAR(128) + const proof = await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }) + const controller = new AbortController() controller.abort() // already aborted await assert.rejects( - store(account, signer, car, { connection, signal: controller.signal }), + store(signer, proof, car, { connection, signal: controller.signal }), { name: 'Error', message: 'upload aborted' } ) }) it('registers an upload with the service', async () => { - const account = alice.did() + const account = await Signer.generate() const signer = await Signer.generate() const car = await randomCAR(128) + const proof = await uploadAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }) + const service = { store: { add: () => { @@ -141,7 +171,7 @@ describe('Storage', () => { assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'upload/add') - assert.equal(invCap.with, account) + assert.equal(invCap.with, account.did()) assert.equal(String(invCap.nb.root), car.roots[0].toString()) assert.equal(invCap.nb.shards?.length, 1) assert.equal(String(invCap.nb.shards?.[0]), car.cid.toString()) @@ -158,7 +188,7 @@ describe('Storage', () => { channel: server, }) - await registerUpload(account, signer, car.roots[0], [car.cid], { + await registerUpload(signer, proof, car.roots[0], [car.cid], { connection, }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a938feec..cab3a4955 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -310,6 +310,7 @@ importers: '@types/assert': ^1.5.6 '@types/mocha': ^10.0.0 '@ucanto/client': ^3.0.1 + '@ucanto/core': ^3.0.1 '@ucanto/interface': ^3.0.0 '@ucanto/principal': ^3.0.0 '@ucanto/server': ^3.0.1 @@ -333,6 +334,7 @@ importers: '@ipld/dag-ucan': 2.0.1 '@ipld/unixfs': 2.0.0 '@ucanto/client': 3.0.1 + '@ucanto/core': 3.0.1 '@ucanto/interface': 3.0.0 '@ucanto/transport': 3.0.1 '@web3-storage/access': link:../access From 21edef8141868dbf370094ed7a729fc683fc644b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 21:11:57 +0000 Subject: [PATCH 07/47] fix: pass multiple proofs --- packages/upload-client/README.md | 38 +++++++---- packages/upload-client/src/index.js | 30 +++++---- packages/upload-client/src/sharding.js | 11 ++-- packages/upload-client/src/storage.js | 67 ++++++++++++-------- packages/upload-client/test/index.test.js | 50 ++++++++++++--- packages/upload-client/test/sharding.test.js | 18 +++--- packages/upload-client/test/storage.test.js | 64 +++++++++++-------- 7 files changed, 176 insertions(+), 102 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index a7a45a4bc..72cdafc7b 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -13,20 +13,32 @@ npm install @web3-storage/upload-client [API Reference](#api) -TODO: how to obtain account/signer +### Step 0 + +Obtain the issuer (the signing authority) and proofs that the issuer has been delegated the capabilities to store data and register uploads: + +```js +import { Agent } from '@web3-storage/access-client' +import { add as storeAdd } from '@web3-storage/access-client/capabilities/store' +import { add as uploadAdd } from '@web3-storage/access-client/capabilities/upload' + +const agent = await Agent.create({ store }) +const issuer = agent.issuer +const proofs = agent.getProofs([storeAdd, uploadAdd]) +``` ### Uploading files ```js import { uploadFile } from '@web3-storage/upload-client' -const cid = await uploadFile(account, signer, new Blob(['Hello World!'])) +const cid = await uploadFile(issuer, proofs, new Blob(['Hello World!'])) ``` ```js import { uploadDirectory } from '@web3-storage/upload-client' -const cid = await uploadDirectory(account, signer, [ +const cid = await uploadDirectory(issuer, proofs, [ new File(['doc0'], 'doc0.txt'), new File(['doc1'], 'dir/doc1.txt'), ]) @@ -49,9 +61,9 @@ const { cid, blocks } = await UnixFS.encodeFile(file) // Encode the DAG as a CAR file const car = await CAR.encode(blocks, cid) // Store the CAR file to the service -const carCID = await Storage.store(account, signer, car) +const carCID = await Storage.store(issuer, proofs, car) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Storage.registerUpload(account, signer, cid, [carCID]) +await Storage.registerUpload(issuer, proofs, cid, [carCID]) ``` #### Streaming API @@ -73,7 +85,7 @@ await UnixFS.createFileEncoderStream(file) .pipeThrough(new ShardingStream()) // Pipe CARs to a stream that stores them to the service and yields metadata // about the CARs that were stored. - .pipeThrough(new ShardStoringStream(account, issuer)) + .pipeThrough(new ShardStoringStream(issuer, proofs)) // Collect the metadata, we're mostly interested in the CID of each CAR file // and the root data CID (which can be found in the _last_ CAR file). .pipeTo( @@ -89,7 +101,7 @@ const rootCID = metadatas[metadatas.length - 1].roots[0] const carCIDs = metadatas.map((meta) => meta.cid) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Storage.registerUpload(account, signer, rootCID, carCIDs) +await Storage.registerUpload(issuer, proofs, rootCID, carCIDs) ``` ## API @@ -162,8 +174,8 @@ The writeable side of this transform stream accepts `CARFile`s and the readable ```ts function registerUpload( - account: DID, - signer: Signer, + issuer: Signer, + proofs: Proof[], root: CID, shards: CID[], options: { retries?: number; signal?: AbortSignal } = {} @@ -248,8 +260,8 @@ const { cid, blocks } = await encodeFile(new File(['data'], 'doc.txt')) ```ts function uploadDirectory( - account: DID, - signer: Signer, + issuer: Signer, + proofs: Proof[], files: File[], options: { retries?: number @@ -265,8 +277,8 @@ Uploads a directory of files to the service and returns the root data CID for th ```ts function uploadFile( - account: DID, - signer: Signer, + issuer: Signer, + proofs: DID, file: Blob, options: { retries?: number diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index bce5c220a..59cffa9c0 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -12,30 +12,36 @@ export * from './sharding.js' */ /** - * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. - * @param {import('@ucanto/interface').Delegation} proof Proof the signer has the capability to perform the action. + * @param {import('@ucanto/interface').Signer} issuer Signing authority that is + * issuing the UCAN invocations. Typically the user _agent_. + * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the issuer + * has the capability to perform the action. At minimum the issuer needs the + * `store/add` and `upload/add` delegated capability. * @param {Blob} file File data. * @param {UploadOptions} [options] */ -export async function uploadFile(issuer, proof, file, options = {}) { +export async function uploadFile(issuer, proofs, file, options = {}) { return await uploadBlockStream( issuer, - proof, + proofs, UnixFS.createFileEncoderStream(file), options ) } /** - * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. - * @param {import('@ucanto/interface').Delegation} proof Proof the signer has the capability to perform the action. + * @param {import('@ucanto/interface').Signer} issuer Signing authority that is + * issuing the UCAN invocations. Typically the user _agent_. + * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the issuer + * has the capability to perform the action. At minimum the issuer needs the + * `store/add` and `upload/add` delegated capability. * @param {import('./types').FileLike[]} files File data. * @param {UploadOptions} [options] */ -export async function uploadDirectory(issuer, proof, files, options = {}) { +export async function uploadDirectory(issuer, proofs, files, options = {}) { return await uploadBlockStream( issuer, - proof, + proofs, UnixFS.createDirectoryEncoderStream(files), options ) @@ -43,11 +49,11 @@ export async function uploadDirectory(issuer, proof, files, options = {}) { /** * @param {import('@ucanto/interface').Signer} issuer - * @param {import('@ucanto/interface').Delegation} proof + * @param {import('@ucanto/interface').Proof[]} proofs * @param {ReadableStream} blocks * @param {UploadOptions} [options] */ -async function uploadBlockStream(issuer, proof, blocks, options = {}) { +async function uploadBlockStream(issuer, proofs, blocks, options = {}) { const onStoredShard = options.onStoredShard ?? (() => {}) /** @type {import('./types').CARLink[]} */ @@ -56,7 +62,7 @@ async function uploadBlockStream(issuer, proof, blocks, options = {}) { let root = null await blocks .pipeThrough(new ShardingStream()) - .pipeThrough(new ShardStoringStream(issuer, proof, options)) + .pipeThrough(new ShardStoringStream(issuer, proofs, options)) .pipeTo( new WritableStream({ write(meta) { @@ -69,6 +75,6 @@ async function uploadBlockStream(issuer, proof, blocks, options = {}) { if (root == null) throw new Error('missing root CID') - await Storage.registerUpload(issuer, proof, root, shards, options) + await Storage.registerUpload(issuer, proofs, root, shards, options) return root } diff --git a/packages/upload-client/src/sharding.js b/packages/upload-client/src/sharding.js index 7b41d4f94..668572ff7 100644 --- a/packages/upload-client/src/sharding.js +++ b/packages/upload-client/src/sharding.js @@ -68,11 +68,14 @@ export class ShardingStream extends TransformStream { */ export class ShardStoringStream extends TransformStream { /** - * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. - * @param {import('@ucanto/interface').Delegation} proof Proof the signer has the capability to perform the action. + * @param {import('@ucanto/interface').Signer} issuer Signing authority that + * is issuing the UCAN invocations. Typically the user _agent_. + * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the + * issuer has the capability to perform the action. At minimum the issuer + * needs the `store/add` delegated capability. * @param {import('./types').RequestOptions} [options] */ - constructor(issuer, proof, options = {}) { + constructor(issuer, proofs, options = {}) { const queue = new Queue({ concurrency: CONCURRENT_UPLOADS }) const abortController = new AbortController() super({ @@ -81,7 +84,7 @@ export class ShardStoringStream extends TransformStream { async () => { try { const opts = { ...options, signal: abortController.signal } - const cid = await store(issuer, proof, car, opts) + const cid = await store(issuer, proofs, car, opts) const { version, roots, size } = car controller.enqueue({ version, roots, cid, size }) } catch (err) { diff --git a/packages/upload-client/src/storage.js b/packages/upload-client/src/storage.js index 31f463493..cf4aaaae2 100644 --- a/packages/upload-client/src/storage.js +++ b/packages/upload-client/src/storage.js @@ -1,7 +1,7 @@ import { isDelegation } from '@ucanto/core' import { connect } from '@ucanto/client' import { CAR, CBOR, HTTP } from '@ucanto/transport' -import { parse } from '@ipld/dag-ucan/did' +import * as DID from '@ipld/dag-ucan/did' import { add as storeAdd } from '@web3-storage/access/capabilities/store' import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' import retry, { AbortError } from 'p-retry' @@ -10,7 +10,7 @@ import retry, { AbortError } from 'p-retry' const serviceURL = new URL( 'https://8609r1772a.execute-api.us-east-1.amazonaws.com' ) -const serviceDID = parse( +const serviceDID = DID.parse( 'did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z' ) @@ -29,21 +29,23 @@ const connection = connect({ /** * Register an "upload" with the service. * - * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. - * @param {import('@ucanto/interface').Proof} proof Proof the signer has the capability to perform the action. + * @param {import('@ucanto/interface').Signer} issuer Signing authority that is + * issuing the UCAN invocations. Typically the user _agent_. + * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the issuer + * has the capability to perform the action. At minimum the issuer needs the + * `upload/add` delegated capability. * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. * @param {import('./types').RequestOptions} [options] */ export async function registerUpload( issuer, - proof, + proofs, root, shards, options = {} ) { - validateProof(proof, serviceDID.did(), uploadAdd.can) - + const capability = findCapability(proofs, serviceDID.did(), uploadAdd.can) /** @type {import('@ucanto/interface').ConnectionView} */ const conn = options.connection ?? connection await retry( @@ -52,10 +54,9 @@ export async function registerUpload( .invoke({ issuer, audience: serviceDID, - // @ts-ignore expects did:${string} but cap with is ${string}:${string} - with: proof.capabilities[0].with, + // @ts-expect-error expects did:${string} but cap with is ${string}:${string} + with: capability.with, nb: { - // @ts-expect-error should allow v0 CIDs! root, shards, }, @@ -70,15 +71,17 @@ export async function registerUpload( /** * Store a DAG encoded as a CAR file. * - * @param {import('@ucanto/interface').Signer} issuer Signing authority. Usually the user agent. - * @param {import('@ucanto/interface').Proof} proof Proof the signer has the capability to perform the action. + * @param {import('@ucanto/interface').Signer} issuer Signing authority that + * is issuing the UCAN invocations. Typically the user _agent_. + * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the + * issuer has the capability to perform the action. At minimum the issuer + * needs the `store/add` delegated capability. * @param {Blob} car CAR file data. * @param {import('./types').RequestOptions} [options] * @returns {Promise} */ -export async function store(issuer, proof, car, options = {}) { - validateProof(proof, serviceDID.did(), storeAdd.can) - +export async function store(issuer, proofs, car, options = {}) { + const capability = findCapability(proofs, serviceDID.did(), storeAdd.can) // TODO: validate blob contains CAR data const bytes = new Uint8Array(await car.arrayBuffer()) const link = await CAR.codec.link(bytes) @@ -90,10 +93,10 @@ export async function store(issuer, proof, car, options = {}) { .invoke({ issuer, audience: serviceDID, - // @ts-ignore expects did:${string} but cap with is ${string}:${string} - with: proof.capabilities[0].with, + // @ts-expect-error expects did:${string} but cap with is ${string}:${string} + with: capability.with, nb: { link }, - proofs: [proof], + proofs, }) .execute(conn) return res @@ -101,7 +104,7 @@ export async function store(issuer, proof, car, options = {}) { { onFailedAttempt: console.warn, retries: options.retries ?? RETRIES } ) - if (result.error != null) { + if (result.error) { throw new Error(`failed ${storeAdd.can} invocation`, { cause: result }) } @@ -142,20 +145,28 @@ export async function store(issuer, proof, car, options = {}) { } /** - * @param {import('@ucanto/interface').Proof} proof + * @param {import('@ucanto/interface').Proof[]} proofs * @param {import('@ucanto/interface').DID} audience * @param {import('@ucanto/interface').Ability} ability */ -function validateProof(proof, audience, ability) { - if (!isDelegation(proof)) { - throw new Error('Linked proofs not supported') - } - if (proof.audience.did() !== audience) { - throw new Error(`Unexpected audience: ${proof.audience}`) +function findCapability(proofs, audience, ability) { + let capability + for (const proof of proofs) { + if (!isDelegation(proof)) continue + if (proof.audience.did() !== audience) continue + capability = proof.capabilities.find((c) => + capabilityMatches(c.can, ability) + ) + if (capability) break } - if (!proof.capabilities.some((c) => capabilityMatches(c.can, ability))) { - throw new Error(`Missing proof of delegated capability: ${ability}`) + if (!capability) { + throw new Error( + `Missing proof of delegated capability "${ + uploadAdd.can + }" for audience "${serviceDID.did()}"` + ) } + return capability } /** diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 8a41e332d..d285ca14d 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -4,8 +4,10 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' +import { add as storeAdd } from '@web3-storage/access/capabilities/store' +import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' import { uploadFile, uploadDirectory } from '../src/index.js' -import { service as id, alice } from './fixtures.js' +import { service as id } from './fixtures.js' import { randomBytes } from './helpers/random.js' import { File } from './helpers/shims.js' @@ -17,12 +19,27 @@ describe('uploadFile', () => { url: 'http://localhost:9000', } - const account = alice.did() + const account = await Signer.generate() const signer = await Signer.generate() const file = new Blob([randomBytes(128)]) /** @type {import('../src/types').CARLink|undefined} */ let carCID + const proofs = await Promise.all([ + storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + uploadAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ]) + const service = { store: { /** @param {Server.Invocation} invocation */ @@ -31,7 +48,7 @@ describe('uploadFile', () => { assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') - assert.equal(invCap.with, account) + assert.equal(invCap.with, account.did()) return res }, }, @@ -42,7 +59,7 @@ describe('uploadFile', () => { assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'upload/add') - assert.equal(invCap.with, account) + assert.equal(invCap.with, account.did()) assert.equal(invCap.nb.shards?.length, 1) assert.equal(String(invCap.nb.shards?.[0]), carCID?.toString()) return null @@ -57,7 +74,7 @@ describe('uploadFile', () => { decoder: CBOR, channel: server, }) - const dataCID = await uploadFile(account, signer, file, { + const dataCID = await uploadFile(signer, proofs, file, { connection, onStoredShard: (meta) => { carCID = meta.cid @@ -77,7 +94,7 @@ describe('uploadDirectory', () => { url: 'http://localhost:9000', } - const account = alice.did() + const account = await Signer.generate() const signer = await Signer.generate() const files = [ new File([randomBytes(128)], '1.txt'), @@ -86,6 +103,21 @@ describe('uploadDirectory', () => { /** @type {import('../src/types').CARLink?} */ let carCID = null + const proofs = await Promise.all([ + storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + uploadAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ]) + const service = { store: { /** @param {Server.Invocation} invocation */ @@ -94,7 +126,7 @@ describe('uploadDirectory', () => { assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') - assert.equal(invCap.with, account) + assert.equal(invCap.with, account.did()) return res }, }, @@ -105,7 +137,7 @@ describe('uploadDirectory', () => { assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'upload/add') - assert.equal(invCap.with, account) + assert.equal(invCap.with, account.did()) assert.equal(invCap.nb.shards?.length, 1) assert.equal(String(invCap.nb.shards?.[0]), carCID?.toString()) return null @@ -120,7 +152,7 @@ describe('uploadDirectory', () => { decoder: CBOR, channel: server, }) - const dataCID = await uploadDirectory(account, signer, files, { + const dataCID = await uploadDirectory(signer, proofs, files, { connection, onStoredShard: (meta) => { carCID = meta.cid diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index 06a134ebc..226347fde 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -50,12 +50,14 @@ describe('ShardStoringStream', () => { const cars = await Promise.all([randomCAR(128), randomCAR(128)]) let invokes = 0 - const proof = await storeAdd.delegate({ - issuer: account, - audience: id, - with: account.did(), - expiration: Infinity, - }) + const proofs = [ + await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] const service = { store: { @@ -65,7 +67,7 @@ describe('ShardStoringStream', () => { assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') - assert.equal(invCap.with, account) + assert.equal(invCap.with, account.did()) assert.equal(String(invCap.nb.link), cars[invokes].cid.toString()) invokes++ return res @@ -98,7 +100,7 @@ describe('ShardStoringStream', () => { /** @type {import('../src/types').CARLink[]} */ const carCIDs = [] await carStream - .pipeThrough(new ShardStoringStream(signer, proof, { connection })) + .pipeThrough(new ShardStoringStream(signer, proofs, { connection })) .pipeTo( new WritableStream({ write: ({ cid }) => { diff --git a/packages/upload-client/test/storage.test.js b/packages/upload-client/test/storage.test.js index 57a8a0c50..7d7cf53eb 100644 --- a/packages/upload-client/test/storage.test.js +++ b/packages/upload-client/test/storage.test.js @@ -22,12 +22,14 @@ describe('Storage', () => { const signer = await Signer.generate() const car = await randomCAR(128) - const proof = await storeAdd.delegate({ - issuer: account, - audience: id, - with: account.did(), - expiration: Infinity, - }) + const proofs = [ + await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] const service = { store: { @@ -57,7 +59,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store(signer, proof, car, { connection }) + const carCID = await store(signer, proofs, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -73,12 +75,14 @@ describe('Storage', () => { const signer = await Signer.generate() const car = await randomCAR(128) - const proof = await storeAdd.delegate({ - issuer: account, - audience: id, - with: account.did(), - expiration: Infinity, - }) + const proofs = [ + await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] const service = { store: { add: () => res }, @@ -97,7 +101,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store(signer, proof, car, { connection }) + const carCID = await store(signer, proofs, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -130,18 +134,20 @@ describe('Storage', () => { const signer = await Signer.generate() const car = await randomCAR(128) - const proof = await storeAdd.delegate({ - issuer: account, - audience: id, - with: account.did(), - expiration: Infinity, - }) + const proofs = [ + await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] const controller = new AbortController() controller.abort() // already aborted await assert.rejects( - store(signer, proof, car, { connection, signal: controller.signal }), + store(signer, proofs, car, { connection, signal: controller.signal }), { name: 'Error', message: 'upload aborted' } ) }) @@ -151,12 +157,14 @@ describe('Storage', () => { const signer = await Signer.generate() const car = await randomCAR(128) - const proof = await uploadAdd.delegate({ - issuer: account, - audience: id, - with: account.did(), - expiration: Infinity, - }) + const proofs = [ + await uploadAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] const service = { store: { @@ -188,7 +196,7 @@ describe('Storage', () => { channel: server, }) - await registerUpload(signer, proof, car.roots[0], [car.cid], { + await registerUpload(signer, proofs, car.roots[0], [car.cid], { connection, }) }) From 41d8efa59ebf5e50df94d147fe82cd43d50b5470 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 21:17:53 +0000 Subject: [PATCH 08/47] docs: add required capability proofs --- packages/upload-client/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 72cdafc7b..b083014f0 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -184,6 +184,8 @@ function registerUpload( Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload. +Required delegated capability proofs: `upload/add` + ### `Storage.store` ```ts @@ -197,6 +199,8 @@ function store( Store a CAR file to the service. +Required delegated capability proofs: `store/add` + ### `UnxiFS.createDirectoryEncoderStream` ```ts @@ -273,6 +277,8 @@ function uploadDirectory( Uploads a directory of files to the service and returns the root data CID for the generated DAG. All files are added to a container directory, with paths in file names preserved. +Required delegated capability proofs: `store/add`, `uplaod/add` + ### `uploadFile` ```ts @@ -290,6 +296,8 @@ function uploadFile( Uploads a file to the service and returns the root data CID for the generated DAG. +Required delegated capability proofs: `store/add`, `uplaod/add` + ## Contributing Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! From f51e26dcc721ebcaa691fbd841914d91162de049 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 21:36:56 +0000 Subject: [PATCH 09/47] chore: IDK make it work in CI --- .github/workflows/{client.yml => upload-client.yml} | 3 ++- packages/upload-client/package.json | 6 ++++-- packages/upload-client/tsconfig.json | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) rename .github/workflows/{client.yml => upload-client.yml} (89%) diff --git a/.github/workflows/client.yml b/.github/workflows/upload-client.yml similarity index 89% rename from .github/workflows/client.yml rename to .github/workflows/upload-client.yml index 953d5c8f8..82913e0ca 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/upload-client.yml @@ -1,4 +1,4 @@ -name: Client +name: Upload Client env: CI: true FORCE_COLOR: 1 @@ -28,5 +28,6 @@ jobs: node-version: 18 cache: 'pnpm' - run: pnpm install + - run: pnpm -r --filter @web3-storage/upload-client run build - run: pnpm -r --filter @web3-storage/upload-client run lint - run: pnpm -r --filter @web3-storage/upload-client run test diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 7cdf145a8..c9c3fdc69 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -14,8 +14,10 @@ "types": "dist/src/index.d.ts", "main": "src/index.js", "scripts": { - "lint": "tsc && eslint '**/*.{js,ts}' && prettier --check '**/*.{js,ts,yml,json}' --ignore-path ../../.gitignore", - "build": "tsc --build", + "lint": "eslint '**/*.{js,ts}' && prettier --check '**/*.{js,ts,yml,json}' --ignore-path ../../.gitignore", + "build": "run-s build:*", + "build:deps": "pnpm -r --filter @web3-storage/access run build", + "build:tsc": "tsc --build", "test": "npm-run-all -p -r mock:bucket test:all", "test:all": "run-s test:browser test:node", "test:node": "c8 -r html -r text mocha 'test/**/!(*.browser).test.js' -n experimental-vm-modules -n no-warnings", diff --git a/packages/upload-client/tsconfig.json b/packages/upload-client/tsconfig.json index 180297270..c3fbadc5a 100644 --- a/packages/upload-client/tsconfig.json +++ b/packages/upload-client/tsconfig.json @@ -6,5 +6,6 @@ "emitDeclarationOnly": true }, "include": ["src", "scripts", "test", "package.json"], - "exclude": ["**/node_modules/**"] + "exclude": ["**/node_modules/**"], + "references": [{ "path": "../access" }] } From c3df0a9bb591dba417edbaf76aa3300c7648a554 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 21:58:10 +0000 Subject: [PATCH 10/47] fix: tests in Node.js 18 --- packages/upload-client/test/helpers/random.js | 19 ++++++++++++++++--- packages/upload-client/test/index.test.js | 6 +++--- packages/upload-client/test/sharding.test.js | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/upload-client/test/helpers/random.js b/packages/upload-client/test/helpers/random.js index 489ba6ca1..1b0360864 100644 --- a/packages/upload-client/test/helpers/random.js +++ b/packages/upload-client/test/helpers/random.js @@ -5,10 +5,23 @@ import { sha256 } from 'multiformats/hashes/sha2' import * as CAR from '@ucanto/transport/car' /** @param {number} size */ -export function randomBytes(size) { +export async function randomBytes(size) { const bytes = new Uint8Array(size) while (size) { - const chunk = crypto.getRandomValues(new Uint8Array(Math.min(size, 65_536))) + const chunk = new Uint8Array(Math.min(size, 65_536)) + if (!globalThis.crypto) { + try { + const { webcrypto } = await import('node:crypto') + webcrypto.getRandomValues(chunk) + } catch (err) { + throw new Error( + 'unknown environment - no global crypto and not Node.js', + { cause: err } + ) + } + } else { + crypto.getRandomValues(chunk) + } size -= bytes.length bytes.set(chunk, size) } @@ -17,7 +30,7 @@ export function randomBytes(size) { /** @param {number} size */ export async function randomCAR(size) { - const bytes = randomBytes(128) + const bytes = await randomBytes(size) const hash = await sha256.digest(bytes) const root = CID.create(1, raw.code, hash) diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index d285ca14d..c24219285 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -21,7 +21,7 @@ describe('uploadFile', () => { const account = await Signer.generate() const signer = await Signer.generate() - const file = new Blob([randomBytes(128)]) + const file = new Blob([await randomBytes(128)]) /** @type {import('../src/types').CARLink|undefined} */ let carCID @@ -97,8 +97,8 @@ describe('uploadDirectory', () => { const account = await Signer.generate() const signer = await Signer.generate() const files = [ - new File([randomBytes(128)], '1.txt'), - new File([randomBytes(32)], '2.txt'), + new File([await randomBytes(128)], '1.txt'), + new File([await randomBytes(32)], '2.txt'), ] /** @type {import('../src/types').CARLink?} */ let carCID = null diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index 226347fde..f7bf113fb 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -12,7 +12,7 @@ import { randomBytes, randomCAR } from './helpers/random.js' describe('ShardingStream', () => { it('creates shards from blocks', async () => { - const file = new Blob([randomBytes(1024 * 1024)]) + const file = new Blob([await randomBytes(1024 * 1024)]) const shardSize = 512 * 1024 /** @type {import('../src/types').CARFile[]} */ From aea43048a4112cefb9409f61a36090cdfdc324f7 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 22:59:45 +0000 Subject: [PATCH 11/47] refactor: use object for invocation configuration --- packages/upload-client/README.md | 54 ++++++++++++------ packages/upload-client/src/index.js | 60 +++++++++++++------- packages/upload-client/src/sharding.js | 19 ++++--- packages/upload-client/src/storage.js | 45 ++++++++++----- packages/upload-client/src/types.ts | 13 ++++- packages/upload-client/test/index.test.js | 16 +++--- packages/upload-client/test/sharding.test.js | 6 +- packages/upload-client/test/storage.test.js | 20 +++---- 8 files changed, 150 insertions(+), 83 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index b083014f0..9a759a465 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -15,7 +15,7 @@ npm install @web3-storage/upload-client ### Step 0 -Obtain the issuer (the signing authority) and proofs that the issuer has been delegated the capabilities to store data and register uploads: +Obtain the invocation configuration. i.e. the issuer (the signing authority) and proofs that the issuer has been delegated the capabilities to store data and register uploads: ```js import { Agent } from '@web3-storage/access-client' @@ -23,8 +23,10 @@ import { add as storeAdd } from '@web3-storage/access-client/capabilities/store' import { add as uploadAdd } from '@web3-storage/access-client/capabilities/upload' const agent = await Agent.create({ store }) -const issuer = agent.issuer -const proofs = agent.getProofs([storeAdd, uploadAdd]) +const conf = { + issuer: agent.issuer, + proofs: agent.getProofs([storeAdd, uploadAdd]), +} ``` ### Uploading files @@ -32,13 +34,13 @@ const proofs = agent.getProofs([storeAdd, uploadAdd]) ```js import { uploadFile } from '@web3-storage/upload-client' -const cid = await uploadFile(issuer, proofs, new Blob(['Hello World!'])) +const cid = await uploadFile(conf, new Blob(['Hello World!'])) ``` ```js import { uploadDirectory } from '@web3-storage/upload-client' -const cid = await uploadDirectory(issuer, proofs, [ +const cid = await uploadDirectory(conf, [ new File(['doc0'], 'doc0.txt'), new File(['doc1'], 'dir/doc1.txt'), ]) @@ -61,9 +63,9 @@ const { cid, blocks } = await UnixFS.encodeFile(file) // Encode the DAG as a CAR file const car = await CAR.encode(blocks, cid) // Store the CAR file to the service -const carCID = await Storage.store(issuer, proofs, car) +const carCID = await Storage.store(conf, car) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Storage.registerUpload(issuer, proofs, cid, [carCID]) +await Storage.registerUpload(conf, cid, [carCID]) ``` #### Streaming API @@ -85,7 +87,7 @@ await UnixFS.createFileEncoderStream(file) .pipeThrough(new ShardingStream()) // Pipe CARs to a stream that stores them to the service and yields metadata // about the CARs that were stored. - .pipeThrough(new ShardStoringStream(issuer, proofs)) + .pipeThrough(new ShardStoringStream(conf)) // Collect the metadata, we're mostly interested in the CID of each CAR file // and the root data CID (which can be found in the _last_ CAR file). .pipeTo( @@ -174,8 +176,7 @@ The writeable side of this transform stream accepts `CARFile`s and the readable ```ts function registerUpload( - issuer: Signer, - proofs: Proof[], + conf: InvocationConfig, root: CID, shards: CID[], options: { retries?: number; signal?: AbortSignal } = {} @@ -184,14 +185,18 @@ function registerUpload( Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload. +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + Required delegated capability proofs: `upload/add` ### `Storage.store` ```ts function store( - account: DID, - signer: Signer, + conf: InvocationConfig, car: Blob, options: { retries?: number; signal?: AbortSignal } = {} ): Promise @@ -199,6 +204,11 @@ function store( Store a CAR file to the service. +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + Required delegated capability proofs: `store/add` ### `UnxiFS.createDirectoryEncoderStream` @@ -264,8 +274,7 @@ const { cid, blocks } = await encodeFile(new File(['data'], 'doc.txt')) ```ts function uploadDirectory( - issuer: Signer, - proofs: Proof[], + conf: InvocationConfig, files: File[], options: { retries?: number @@ -277,14 +286,18 @@ function uploadDirectory( Uploads a directory of files to the service and returns the root data CID for the generated DAG. All files are added to a container directory, with paths in file names preserved. -Required delegated capability proofs: `store/add`, `uplaod/add` +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `store/add`, `upload/add` ### `uploadFile` ```ts function uploadFile( - issuer: Signer, - proofs: DID, + conf: InvocationConfig, file: Blob, options: { retries?: number @@ -296,7 +309,12 @@ function uploadFile( Uploads a file to the service and returns the root data CID for the generated DAG. -Required delegated capability proofs: `store/add`, `uplaod/add` +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `store/add`, `upload/add` ## Contributing diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 59cffa9c0..4e3456b14 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -12,48 +12,66 @@ export * from './sharding.js' */ /** - * @param {import('@ucanto/interface').Signer} issuer Signing authority that is - * issuing the UCAN invocations. Typically the user _agent_. - * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the issuer - * has the capability to perform the action. At minimum the issuer needs the - * `store/add` and `upload/add` delegated capability. + * Uploads a file to the service and returns the root data CID for the + * generated DAG. + * + * Required delegated capability proofs: `store/add`, `upload/add` + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `store/add` and `upload/add` delegated capability. * @param {Blob} file File data. * @param {UploadOptions} [options] */ -export async function uploadFile(issuer, proofs, file, options = {}) { +export async function uploadFile({ issuer, proofs }, file, options = {}) { return await uploadBlockStream( - issuer, - proofs, + { issuer, proofs }, UnixFS.createFileEncoderStream(file), options ) } /** - * @param {import('@ucanto/interface').Signer} issuer Signing authority that is - * issuing the UCAN invocations. Typically the user _agent_. - * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the issuer - * has the capability to perform the action. At minimum the issuer needs the - * `store/add` and `upload/add` delegated capability. + * Uploads a directory of files to the service and returns the root data CID + * for the generated DAG. All files are added to a container directory, with + * paths in file names preserved. + * + * Required delegated capability proofs: `store/add`, `upload/add` + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `store/add` and `upload/add` delegated capability. * @param {import('./types').FileLike[]} files File data. * @param {UploadOptions} [options] */ -export async function uploadDirectory(issuer, proofs, files, options = {}) { +export async function uploadDirectory({ issuer, proofs }, files, options = {}) { return await uploadBlockStream( - issuer, - proofs, + { issuer, proofs }, UnixFS.createDirectoryEncoderStream(files), options ) } /** - * @param {import('@ucanto/interface').Signer} issuer - * @param {import('@ucanto/interface').Proof[]} proofs + * @param {import('./types').InvocationConfig} invocationConfig * @param {ReadableStream} blocks * @param {UploadOptions} [options] */ -async function uploadBlockStream(issuer, proofs, blocks, options = {}) { +async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { const onStoredShard = options.onStoredShard ?? (() => {}) /** @type {import('./types').CARLink[]} */ @@ -62,7 +80,7 @@ async function uploadBlockStream(issuer, proofs, blocks, options = {}) { let root = null await blocks .pipeThrough(new ShardingStream()) - .pipeThrough(new ShardStoringStream(issuer, proofs, options)) + .pipeThrough(new ShardStoringStream({ issuer, proofs }, options)) .pipeTo( new WritableStream({ write(meta) { @@ -75,6 +93,6 @@ async function uploadBlockStream(issuer, proofs, blocks, options = {}) { if (root == null) throw new Error('missing root CID') - await Storage.registerUpload(issuer, proofs, root, shards, options) + await Storage.registerUpload({ issuer, proofs }, root, shards, options) return root } diff --git a/packages/upload-client/src/sharding.js b/packages/upload-client/src/sharding.js index 668572ff7..2502efd7c 100644 --- a/packages/upload-client/src/sharding.js +++ b/packages/upload-client/src/sharding.js @@ -68,14 +68,19 @@ export class ShardingStream extends TransformStream { */ export class ShardStoringStream extends TransformStream { /** - * @param {import('@ucanto/interface').Signer} issuer Signing authority that - * is issuing the UCAN invocations. Typically the user _agent_. - * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the - * issuer has the capability to perform the action. At minimum the issuer - * needs the `store/add` delegated capability. + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `store/add` delegated capability. * @param {import('./types').RequestOptions} [options] */ - constructor(issuer, proofs, options = {}) { + constructor({ issuer, proofs }, options = {}) { const queue = new Queue({ concurrency: CONCURRENT_UPLOADS }) const abortController = new AbortController() super({ @@ -84,7 +89,7 @@ export class ShardStoringStream extends TransformStream { async () => { try { const opts = { ...options, signal: abortController.signal } - const cid = await store(issuer, proofs, car, opts) + const cid = await store({ issuer, proofs }, car, opts) const { version, roots, size } = car controller.enqueue({ version, roots, cid, size }) } catch (err) { diff --git a/packages/upload-client/src/storage.js b/packages/upload-client/src/storage.js index cf4aaaae2..8d3778584 100644 --- a/packages/upload-client/src/storage.js +++ b/packages/upload-client/src/storage.js @@ -27,20 +27,27 @@ const connection = connect({ }) /** - * Register an "upload" with the service. + * Register an "upload" with the service. The issuer needs the `upload/add` + * delegated capability. * - * @param {import('@ucanto/interface').Signer} issuer Signing authority that is - * issuing the UCAN invocations. Typically the user _agent_. - * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the issuer - * has the capability to perform the action. At minimum the issuer needs the - * `upload/add` delegated capability. + * Required delegated capability proofs: `upload/add` + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/add` delegated capability. * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. * @param {import('./types').RequestOptions} [options] */ export async function registerUpload( - issuer, - proofs, + { issuer, proofs }, root, shards, options = {} @@ -69,18 +76,26 @@ export async function registerUpload( } /** - * Store a DAG encoded as a CAR file. + * Store a DAG encoded as a CAR file. The issuer needs the `store/add` + * delegated capability. + * + * Required delegated capability proofs: `store/add` + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. * - * @param {import('@ucanto/interface').Signer} issuer Signing authority that - * is issuing the UCAN invocations. Typically the user _agent_. - * @param {import('@ucanto/interface').Proof[]} proofs Proof(s) the - * issuer has the capability to perform the action. At minimum the issuer - * needs the `store/add` delegated capability. + * The issuer needs the `store/add` delegated capability. * @param {Blob} car CAR file data. * @param {import('./types').RequestOptions} [options] * @returns {Promise} */ -export async function store(issuer, proofs, car, options = {}) { +export async function store({ issuer, proofs }, car, options = {}) { const capability = findCapability(proofs, serviceDID.did(), storeAdd.can) // TODO: validate blob contains CAR data const bytes = new Uint8Array(await car.arrayBuffer()) diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 129c5f061..7ec1764a4 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -1,7 +1,7 @@ import { Link, UnknownLink, Version } from 'multiformats/link' import { Block } from '@ipld/unixfs' import { CAR } from '@ucanto/transport' -import { ServiceMethod, ConnectionView } from '@ucanto/interface' +import { ServiceMethod, ConnectionView, Signer, Proof } from '@ucanto/interface' import { StoreAdd, UploadAdd } from '@web3-storage/access/capabilities/types' export type { StoreAdd, UploadAdd } @@ -17,6 +17,17 @@ export interface StoreAddResponse { url: string } +export interface InvocationConfig { + /** + * Signing authority that is issuing the UCAN invocations. + */ + issuer: Signer + /** + * Proof(s) the issuer has the capability to perform the action. + */ + proofs: Proof[] +} + export interface UnixFSEncodeResult { /** * Root CID for the DAG. diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index c24219285..6acfb726a 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -20,7 +20,7 @@ describe('uploadFile', () => { } const account = await Signer.generate() - const signer = await Signer.generate() + const issuer = await Signer.generate() const file = new Blob([await randomBytes(128)]) /** @type {import('../src/types').CARLink|undefined} */ let carCID @@ -44,7 +44,7 @@ describe('uploadFile', () => { store: { /** @param {Server.Invocation} invocation */ add(invocation) { - assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') @@ -55,7 +55,7 @@ describe('uploadFile', () => { upload: { /** @param {Server.Invocation} invocation */ add: (invocation) => { - assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'upload/add') @@ -74,7 +74,7 @@ describe('uploadFile', () => { decoder: CBOR, channel: server, }) - const dataCID = await uploadFile(signer, proofs, file, { + const dataCID = await uploadFile({ issuer, proofs }, file, { connection, onStoredShard: (meta) => { carCID = meta.cid @@ -95,7 +95,7 @@ describe('uploadDirectory', () => { } const account = await Signer.generate() - const signer = await Signer.generate() + const issuer = await Signer.generate() const files = [ new File([await randomBytes(128)], '1.txt'), new File([await randomBytes(32)], '2.txt'), @@ -122,7 +122,7 @@ describe('uploadDirectory', () => { store: { /** @param {Server.Invocation} invocation */ add(invocation) { - assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') @@ -133,7 +133,7 @@ describe('uploadDirectory', () => { upload: { /** @param {Server.Invocation} invocation */ add: (invocation) => { - assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'upload/add') @@ -152,7 +152,7 @@ describe('uploadDirectory', () => { decoder: CBOR, channel: server, }) - const dataCID = await uploadDirectory(signer, proofs, files, { + const dataCID = await uploadDirectory({ issuer, proofs }, files, { connection, onStoredShard: (meta) => { carCID = meta.cid diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index f7bf113fb..948d83a3a 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -46,7 +46,7 @@ describe('ShardStoringStream', () => { } const account = await Signer.generate() - const signer = await Signer.generate() + const issuer = await Signer.generate() const cars = await Promise.all([randomCAR(128), randomCAR(128)]) let invokes = 0 @@ -63,7 +63,7 @@ describe('ShardStoringStream', () => { store: { /** @param {Server.Invocation} invocation */ add(invocation) { - assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') @@ -100,7 +100,7 @@ describe('ShardStoringStream', () => { /** @type {import('../src/types').CARLink[]} */ const carCIDs = [] await carStream - .pipeThrough(new ShardStoringStream(signer, proofs, { connection })) + .pipeThrough(new ShardStoringStream({ issuer, proofs }, { connection })) .pipeTo( new WritableStream({ write: ({ cid }) => { diff --git a/packages/upload-client/test/storage.test.js b/packages/upload-client/test/storage.test.js index 7d7cf53eb..079e773d7 100644 --- a/packages/upload-client/test/storage.test.js +++ b/packages/upload-client/test/storage.test.js @@ -19,7 +19,7 @@ describe('Storage', () => { } const account = await Signer.generate() - const signer = await Signer.generate() + const issuer = await Signer.generate() const car = await randomCAR(128) const proofs = [ @@ -35,7 +35,7 @@ describe('Storage', () => { store: { /** @param {Server.Invocation} invocation */ add(invocation) { - assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'store/add') @@ -59,7 +59,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store(signer, proofs, car, { connection }) + const carCID = await store({ issuer, proofs }, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -72,7 +72,7 @@ describe('Storage', () => { } const account = await Signer.generate() - const signer = await Signer.generate() + const issuer = await Signer.generate() const car = await randomCAR(128) const proofs = [ @@ -101,7 +101,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store(signer, proofs, car, { connection }) + const carCID = await store({ issuer, proofs }, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -131,7 +131,7 @@ describe('Storage', () => { }) const account = await Signer.generate() - const signer = await Signer.generate() + const issuer = await Signer.generate() const car = await randomCAR(128) const proofs = [ @@ -147,14 +147,14 @@ describe('Storage', () => { controller.abort() // already aborted await assert.rejects( - store(signer, proofs, car, { connection, signal: controller.signal }), + store({ issuer, proofs }, car, { connection, signal: controller.signal }), { name: 'Error', message: 'upload aborted' } ) }) it('registers an upload with the service', async () => { const account = await Signer.generate() - const signer = await Signer.generate() + const issuer = await Signer.generate() const car = await randomCAR(128) const proofs = [ @@ -175,7 +175,7 @@ describe('Storage', () => { upload: { /** @param {Server.Invocation} invocation */ add: (invocation) => { - assert.equal(invocation.issuer.did(), signer.did()) + assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] assert.equal(invCap.can, 'upload/add') @@ -196,7 +196,7 @@ describe('Storage', () => { channel: server, }) - await registerUpload(signer, proofs, car.roots[0], [car.cid], { + await registerUpload({ issuer, proofs }, car.roots[0], [car.cid], { connection, }) }) From d704175d4295a2f42e07a9c2b5868288539d709a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 23:02:06 +0000 Subject: [PATCH 12/47] fix: typo --- packages/upload-client/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 9a759a465..ee2d2ad88 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -211,7 +211,7 @@ Note: `InvocationConfig` is configuration for the UCAN invocation. It's values c Required delegated capability proofs: `store/add` -### `UnxiFS.createDirectoryEncoderStream` +### `UnixFS.createDirectoryEncoderStream` ```ts function createDirectoryEncoderStream( @@ -223,7 +223,7 @@ Creates a `ReadableStream` that yields UnixFS DAG blocks. All files are added to Note: you can use https://npm.im/files-from-path to read files from the filesystem in Nodejs. -### `UnxiFS.createFileEncoderStream` +### `UnixFS.createFileEncoderStream` ```ts function createFileEncoderStream(file: Blob): ReadableStream @@ -231,7 +231,7 @@ function createFileEncoderStream(file: Blob): ReadableStream Creates a `ReadableStream` that yields UnixFS DAG blocks. -### `UnxiFS.encodeDirectory` +### `UnixFS.encodeDirectory` ```ts function encodeDirectory( @@ -255,7 +255,7 @@ const { cid, blocks } = encodeDirectory([ // bafybei.../dir/doc1.txt ``` -### `UnxiFS.encodeFile` +### `UnixFS.encodeFile` ```ts function encodeFile(file: Blob): Promise<{ cid: CID; blocks: Block[] }> From 900f1ee12fd93b331d81da1869ea9f7bbdd985fa Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Nov 2022 23:36:20 +0000 Subject: [PATCH 13/47] fix: link --- packages/upload-client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index ee2d2ad88..78a7aa4c0 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -114,7 +114,7 @@ await Storage.registerUpload(issuer, proofs, rootCID, carCIDs) - [`ShardStoringStream`](#shardstoringstream) - `Storage` - [`registerUpload`](#storageregisterupload) - - [`store`](#storagestoredag) + - [`store`](#storagestore) - `UnixFS` - [`createDirectoryEncoderStream`](#unixfscreatedirectoryencoderstream) - [`createFileEncoderStream`](#unixfscreatefileencoderstream) From a5387d627b87d4c8761432883a9007ec3cc4ba1a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 11 Nov 2022 09:34:59 +0000 Subject: [PATCH 14/47] docs: add note about creating an account --- packages/upload-client/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 78a7aa4c0..d9af351a3 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -23,6 +23,8 @@ import { add as storeAdd } from '@web3-storage/access-client/capabilities/store' import { add as uploadAdd } from '@web3-storage/access-client/capabilities/upload' const agent = await Agent.create({ store }) +// Note: you need to create and register an account 1st time: +// await agent.createAccount('you@youremail.com') const conf = { issuer: agent.issuer, proofs: agent.getProofs([storeAdd, uploadAdd]), From ea62bb6e95fd61bc4e02c17e15d08f75524d9f1a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 11 Nov 2022 14:17:14 +0000 Subject: [PATCH 15/47] refactor: use Upload namespace --- packages/upload-client/README.md | 51 ++++---- packages/upload-client/package.json | 1 + packages/upload-client/src/constants.js | 1 + packages/upload-client/src/index.js | 5 +- packages/upload-client/src/service.js | 20 ++++ packages/upload-client/src/storage.js | 123 ++------------------ packages/upload-client/src/upload.js | 52 +++++++++ packages/upload-client/src/utils.js | 35 ++++++ packages/upload-client/test/storage.test.js | 52 +-------- packages/upload-client/test/upload.test.js | 61 ++++++++++ 10 files changed, 212 insertions(+), 189 deletions(-) create mode 100644 packages/upload-client/src/constants.js create mode 100644 packages/upload-client/src/service.js create mode 100644 packages/upload-client/src/upload.js create mode 100644 packages/upload-client/test/upload.test.js diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index d9af351a3..429127da8 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -58,7 +58,7 @@ const cid = await uploadDirectory(conf, [ The buffering API loads all data into memory so is suitable only for small files. The root data CID is obtained before any transfer to the service takes place. ```js -import { UnixFS, CAR, Storage } from '@web3-storage/upload-client' +import { UnixFS, CAR, Storage, Upload } from '@web3-storage/upload-client' // Encode a file as a DAG, get back a root data CID and a set of blocks const { cid, blocks } = await UnixFS.encodeFile(file) @@ -67,7 +67,7 @@ const car = await CAR.encode(blocks, cid) // Store the CAR file to the service const carCID = await Storage.store(conf, car) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Storage.registerUpload(conf, cid, [carCID]) +await Upload.register(conf, cid, [carCID]) ``` #### Streaming API @@ -79,7 +79,7 @@ import { UnixFS, ShardingStream, ShardStoringStream, - Storage, + Upload, } from '@web3-storage/upload-client' const metadatas = [] @@ -105,7 +105,7 @@ const rootCID = metadatas[metadatas.length - 1].roots[0] const carCIDs = metadatas.map((meta) => meta.cid) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Storage.registerUpload(issuer, proofs, rootCID, carCIDs) +await Upload.register(issuer, proofs, rootCID, carCIDs) ``` ## API @@ -115,13 +115,14 @@ await Storage.registerUpload(issuer, proofs, rootCID, carCIDs) - [`ShardingStream`](#shardingstream) - [`ShardStoringStream`](#shardstoringstream) - `Storage` - - [`registerUpload`](#storageregisterupload) - [`store`](#storagestore) - `UnixFS` - [`createDirectoryEncoderStream`](#unixfscreatedirectoryencoderstream) - [`createFileEncoderStream`](#unixfscreatefileencoderstream) - [`encodeDirectory`](#unixfsencodedirectory) - [`encodeFile`](#unixfsencodefile) +- `Upload` + - - [`register`](#uploadregister) - [`uploadDirectory`](#uploaddirectory) - [`uploadFile`](#uploadfile) @@ -174,26 +175,6 @@ Note: an "upload" must be registered in order to link multiple shards together a The writeable side of this transform stream accepts `CARFile`s and the readable side yields `CARMetadata`, which contains the CAR CID, it's size (in bytes) and it's roots (if it has any). -### `Storage.registerUpload` - -```ts -function registerUpload( - conf: InvocationConfig, - root: CID, - shards: CID[], - options: { retries?: number; signal?: AbortSignal } = {} -): Promise -``` - -Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload. - -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - -Required delegated capability proofs: `upload/add` - ### `Storage.store` ```ts @@ -272,6 +253,26 @@ const { cid, blocks } = await encodeFile(new File(['data'], 'doc.txt')) // Note: file name is not preserved - use encodeDirectory if required. ``` +### `Upload.register` + +```ts +function register( + conf: InvocationConfig, + root: CID, + shards: CID[], + options: { retries?: number; signal?: AbortSignal } = {} +): Promise +``` + +Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload. + +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `upload/add` + ### `uploadDirectory` ```ts diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index c9c3fdc69..fdd4d6f01 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -29,6 +29,7 @@ ".": "./src/index.js", "./car": "./src/car.js", "./sharding": "./src/sharding.js", + "./upload": "./src/upload.js", "./storage": "./src/storage.js", "./unixfs": "./src/unixfs.js" }, diff --git a/packages/upload-client/src/constants.js b/packages/upload-client/src/constants.js new file mode 100644 index 000000000..a98ccf42a --- /dev/null +++ b/packages/upload-client/src/constants.js @@ -0,0 +1 @@ +export const REQUEST_RETRIES = 3 diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 4e3456b14..429dd5907 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -1,9 +1,10 @@ import * as Storage from './storage.js' +import * as Upload from './upload.js' import * as UnixFS from './unixfs.js' import * as CAR from './car.js' import { ShardingStream, ShardStoringStream } from './sharding.js' -export { Storage, UnixFS, CAR } +export { Storage, Upload, UnixFS, CAR } export * from './sharding.js' /** @@ -93,6 +94,6 @@ async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { if (root == null) throw new Error('missing root CID') - await Storage.registerUpload({ issuer, proofs }, root, shards, options) + await Upload.register({ issuer, proofs }, root, shards, options) return root } diff --git a/packages/upload-client/src/service.js b/packages/upload-client/src/service.js new file mode 100644 index 000000000..b59e78eaa --- /dev/null +++ b/packages/upload-client/src/service.js @@ -0,0 +1,20 @@ +import { connect } from '@ucanto/client' +import { CAR, CBOR, HTTP } from '@ucanto/transport' +import * as DID from '@ipld/dag-ucan/did' + +export const serviceURL = new URL( + 'https://8609r1772a.execute-api.us-east-1.amazonaws.com' +) +export const serviceDID = DID.parse( + 'did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z' +) + +export const connection = connect({ + id: serviceDID, + encoder: CAR, + decoder: CBOR, + channel: HTTP.open({ + url: serviceURL, + method: 'POST', + }), +}) diff --git a/packages/upload-client/src/storage.js b/packages/upload-client/src/storage.js index 8d3778584..4e9ed1613 100644 --- a/packages/upload-client/src/storage.js +++ b/packages/upload-client/src/storage.js @@ -1,79 +1,9 @@ -import { isDelegation } from '@ucanto/core' -import { connect } from '@ucanto/client' -import { CAR, CBOR, HTTP } from '@ucanto/transport' -import * as DID from '@ipld/dag-ucan/did' +import { CAR } from '@ucanto/transport' import { add as storeAdd } from '@web3-storage/access/capabilities/store' -import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' import retry, { AbortError } from 'p-retry' - -// Production -const serviceURL = new URL( - 'https://8609r1772a.execute-api.us-east-1.amazonaws.com' -) -const serviceDID = DID.parse( - 'did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z' -) - -const RETRIES = 3 - -const connection = connect({ - id: serviceDID, - encoder: CAR, - decoder: CBOR, - channel: HTTP.open({ - url: serviceURL, - method: 'POST', - }), -}) - -/** - * Register an "upload" with the service. The issuer needs the `upload/add` - * delegated capability. - * - * Required delegated capability proofs: `upload/add` - * - * @param {import('./types').InvocationConfig} invocationConfig Configuration - * for the UCAN invocation. An object with `issuer` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/add` delegated capability. - * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. - * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. - * @param {import('./types').RequestOptions} [options] - */ -export async function registerUpload( - { issuer, proofs }, - root, - shards, - options = {} -) { - const capability = findCapability(proofs, serviceDID.did(), uploadAdd.can) - /** @type {import('@ucanto/interface').ConnectionView} */ - const conn = options.connection ?? connection - await retry( - async () => { - const result = await uploadAdd - .invoke({ - issuer, - audience: serviceDID, - // @ts-expect-error expects did:${string} but cap with is ${string}:${string} - with: capability.with, - nb: { - root, - shards, - }, - }) - .execute(conn) - if (result?.error === true) throw result - }, - { onFailedAttempt: console.warn, retries: options.retries ?? RETRIES } - ) -} +import { serviceDID, connection } from './service.js' +import { findCapability } from './utils.js' +import { REQUEST_RETRIES } from './constants.js' /** * Store a DAG encoded as a CAR file. The issuer needs the `store/add` @@ -116,7 +46,10 @@ export async function store({ issuer, proofs }, car, options = {}) { .execute(conn) return res }, - { onFailedAttempt: console.warn, retries: options.retries ?? RETRIES } + { + onFailedAttempt: console.warn, + retries: options.retries ?? REQUEST_RETRIES, + } ) if (result.error) { @@ -149,7 +82,10 @@ export async function store({ issuer, proofs }, car, options = {}) { throw err } }, - { onFailedAttempt: console.warn, retries: options.retries ?? RETRIES } + { + onFailedAttempt: console.warn, + retries: options.retries ?? REQUEST_RETRIES, + } ) if (!res.ok) { @@ -158,38 +94,3 @@ export async function store({ issuer, proofs }, car, options = {}) { return link } - -/** - * @param {import('@ucanto/interface').Proof[]} proofs - * @param {import('@ucanto/interface').DID} audience - * @param {import('@ucanto/interface').Ability} ability - */ -function findCapability(proofs, audience, ability) { - let capability - for (const proof of proofs) { - if (!isDelegation(proof)) continue - if (proof.audience.did() !== audience) continue - capability = proof.capabilities.find((c) => - capabilityMatches(c.can, ability) - ) - if (capability) break - } - if (!capability) { - throw new Error( - `Missing proof of delegated capability "${ - uploadAdd.can - }" for audience "${serviceDID.did()}"` - ) - } - return capability -} - -/** - * @param {string} can - * @param {import('@ucanto/interface').Ability} ability - */ -function capabilityMatches(can, ability) { - return can === ability - ? true - : can.endsWith('*') && ability.startsWith(can.split('*')[0]) -} diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js new file mode 100644 index 000000000..ddae72fa1 --- /dev/null +++ b/packages/upload-client/src/upload.js @@ -0,0 +1,52 @@ +import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' +import retry from 'p-retry' +import { serviceDID, connection } from './service.js' +import { findCapability } from './utils.js' +import { REQUEST_RETRIES } from './constants.js' + +/** + * Register an "upload" with the service. The issuer needs the `upload/add` + * delegated capability. + * + * Required delegated capability proofs: `upload/add` + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/add` delegated capability. + * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. + * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. + * @param {import('./types').RequestOptions} [options] + */ +export async function register({ issuer, proofs }, root, shards, options = {}) { + const capability = findCapability(proofs, serviceDID.did(), uploadAdd.can) + /** @type {import('@ucanto/interface').ConnectionView} */ + const conn = options.connection ?? connection + await retry( + async () => { + const result = await uploadAdd + .invoke({ + issuer, + audience: serviceDID, + // @ts-expect-error expects did:${string} but cap with is ${string}:${string} + with: capability.with, + nb: { + root, + shards, + }, + }) + .execute(conn) + if (result?.error === true) throw result + }, + { + onFailedAttempt: console.warn, + retries: options.retries ?? REQUEST_RETRIES, + } + ) +} diff --git a/packages/upload-client/src/utils.js b/packages/upload-client/src/utils.js index b40045799..ea81fd6e2 100644 --- a/packages/upload-client/src/utils.js +++ b/packages/upload-client/src/utils.js @@ -1,3 +1,5 @@ +import { isDelegation } from '@ucanto/core' + /** * @template T * @param {ReadableStream | NodeJS.ReadableStream} readable @@ -37,3 +39,36 @@ export async function collect(collectable) { for await (const chunk of collectable) chunks.push(chunk) return chunks } + +/** + * @param {import('@ucanto/interface').Proof[]} proofs + * @param {import('@ucanto/interface').DID} audience + * @param {import('@ucanto/interface').Ability} ability + */ +export function findCapability(proofs, audience, ability) { + let capability + for (const proof of proofs) { + if (!isDelegation(proof)) continue + if (proof.audience.did() !== audience) continue + capability = proof.capabilities.find((c) => + capabilityMatches(c.can, ability) + ) + if (capability) break + } + if (!capability) { + throw new Error( + `Missing proof of delegated capability "${ability}" for audience "${audience}"` + ) + } + return capability +} + +/** + * @param {string} can + * @param {import('@ucanto/interface').Ability} ability + */ +function capabilityMatches(can, ability) { + return can === ability + ? true + : can.endsWith('*') && ability.startsWith(can.split('*')[0]) +} diff --git a/packages/upload-client/test/storage.test.js b/packages/upload-client/test/storage.test.js index 079e773d7..935aedda3 100644 --- a/packages/upload-client/test/storage.test.js +++ b/packages/upload-client/test/storage.test.js @@ -5,8 +5,7 @@ import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' import { add as storeAdd } from '@web3-storage/access/capabilities/store' -import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' -import { registerUpload, store } from '../src/storage.js' +import { store } from '../src/storage.js' import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' @@ -151,53 +150,4 @@ describe('Storage', () => { { name: 'Error', message: 'upload aborted' } ) }) - - it('registers an upload with the service', async () => { - const account = await Signer.generate() - const issuer = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await uploadAdd.delegate({ - issuer: account, - audience: id, - with: account.did(), - expiration: Infinity, - }), - ] - - const service = { - store: { - add: () => { - throw new Server.Failure('not expected to be called') - }, - }, - upload: { - /** @param {Server.Invocation} invocation */ - add: (invocation) => { - assert.equal(invocation.issuer.did(), issuer.did()) - assert.equal(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.equal(invCap.can, 'upload/add') - assert.equal(invCap.with, account.did()) - assert.equal(String(invCap.nb.root), car.roots[0].toString()) - assert.equal(invCap.nb.shards?.length, 1) - assert.equal(String(invCap.nb.shards?.[0]), car.cid.toString()) - return null - }, - }, - } - - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) - const connection = Client.connect({ - id, - encoder: CAR, - decoder: CBOR, - channel: server, - }) - - await registerUpload({ issuer, proofs }, car.roots[0], [car.cid], { - connection, - }) - }) }) diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js new file mode 100644 index 000000000..8a46a858a --- /dev/null +++ b/packages/upload-client/test/upload.test.js @@ -0,0 +1,61 @@ +import assert from 'assert' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as CBOR from '@ucanto/transport/cbor' +import * as Signer from '@ucanto/principal/ed25519' +import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' +import { register } from '../src/upload.js' +import { service as id } from './fixtures.js' +import { randomCAR } from './helpers/random.js' + +describe('Upload', () => { + it('registers an upload with the service', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await uploadAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = { + store: { + add: () => { + throw new Server.Failure('not expected to be called') + }, + }, + upload: { + /** @param {Server.Invocation} invocation */ + add: (invocation) => { + assert.equal(invocation.issuer.did(), issuer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, 'upload/add') + assert.equal(invCap.with, account.did()) + assert.equal(String(invCap.nb.root), car.roots[0].toString()) + assert.equal(invCap.nb.shards?.length, 1) + assert.equal(String(invCap.nb.shards?.[0]), car.cid.toString()) + return null + }, + }, + } + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await register({ issuer, proofs }, car.roots[0], [car.cid], { + connection, + }) + }) +}) From ef29b042d36acf33309a56f309cf1bd647a2150a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 11 Nov 2022 14:22:26 +0000 Subject: [PATCH 16/47] fix: exmaple --- packages/upload-client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 429127da8..a167de6a6 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -105,7 +105,7 @@ const rootCID = metadatas[metadatas.length - 1].roots[0] const carCIDs = metadatas.map((meta) => meta.cid) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Upload.register(issuer, proofs, rootCID, carCIDs) +await Upload.register(conf, rootCID, carCIDs) ``` ## API From 10a04965a7ac1fd983c2ef56d6f63228b022e134 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 12 Nov 2022 13:11:13 +0000 Subject: [PATCH 17/47] fix: type declarations paths --- packages/upload-client/package.json | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index fdd4d6f01..8bc2e0b28 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -35,8 +35,20 @@ }, "typesVersions": { "*": { - "*": [ - "dist/*" + "car": [ + "dist/src/car.d.ts" + ], + "sharding": [ + "dist/src/sharding.d.ts" + ], + "upload": [ + "dist/src/upload.d.ts" + ], + "storage": [ + "dist/src/storage.d.ts" + ], + "unixfs": [ + "dist/src/unixfs.d.ts" ] } }, From e85f6e22f8c0e1df72e189613f82362533755b0b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 12 Nov 2022 13:13:29 +0000 Subject: [PATCH 18/47] fix: return type --- packages/upload-client/src/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 429dd5907..7ecadd1f3 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -71,6 +71,7 @@ export async function uploadDirectory({ issuer, proofs }, files, options = {}) { * @param {import('./types').InvocationConfig} invocationConfig * @param {ReadableStream} blocks * @param {UploadOptions} [options] + * @returns {Promise>} */ async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { const onStoredShard = options.onStoredShard ?? (() => {}) From 90da73f41453ed85edc5e777cbc9db5ac8ba0873 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 12 Nov 2022 13:27:15 +0000 Subject: [PATCH 19/47] fix: use BlobLike instead of Blob --- packages/upload-client/README.md | 10 +++++++++- packages/upload-client/src/index.js | 26 ++++++++++++++++++++++++++ packages/upload-client/src/storage.js | 2 +- packages/upload-client/src/types.ts | 26 ++++++++++++++++++++------ packages/upload-client/src/unixfs.js | 4 ++-- packages/upload-client/src/upload.js | 2 +- packages/upload-client/src/utils.js | 6 +++--- 7 files changed, 62 insertions(+), 14 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index a167de6a6..ad1e240dc 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -18,13 +18,21 @@ npm install @web3-storage/upload-client Obtain the invocation configuration. i.e. the issuer (the signing authority) and proofs that the issuer has been delegated the capabilities to store data and register uploads: ```js +import { delegateCapabilities } from '@web3-storage/upload-client' import { Agent } from '@web3-storage/access-client' import { add as storeAdd } from '@web3-storage/access-client/capabilities/store' import { add as uploadAdd } from '@web3-storage/access-client/capabilities/upload' const agent = await Agent.create({ store }) -// Note: you need to create and register an account 1st time: + +// // Note: you need to create and register an account 1st time: // await agent.createAccount('you@youremail.com') +// +// // ...and delegate capabilities from the account, to the agent, allowing it +// // to use the upload API: +// const delegation = await delegateCapabilities(agent.data.accounts[0], agent.issuer) +// await agent.addDelegation(delegation) + const conf = { issuer: agent.issuer, proofs: agent.getProofs([storeAdd, uploadAdd]), diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 7ecadd1f3..9b4ac6e1c 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -1,3 +1,6 @@ +import { delegate } from '@ucanto/core' +import { store as storeStar } from '@web3-storage/access/capabilities/store' +import { upload as uploadStar } from '@web3-storage/access/capabilities/upload' import * as Storage from './storage.js' import * as Upload from './upload.js' import * as UnixFS from './unixfs.js' @@ -98,3 +101,26 @@ async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { await Upload.register({ issuer, proofs }, root, shards, options) return root } + +/** + * Delegate upload API capabilities from the issuer to the audience. + * + * @param {import('@ucanto/interface').Signer} issuer + * @param {import('@ucanto/interface').Principal} audience + * @param {import('@ucanto/interface').Tuple} [abilities] + */ +export async function delegateCapabilities( + issuer, + audience, + abilities = [storeStar.can, uploadStar.can] +) { + const capabilities = abilities.map((can) => ({ can, with: issuer.did() })) + if (!capabilities.length) throw new Error('at least one ability is required') + return await delegate({ + issuer, + audience, + // @ts-expect-error + capabilities, + expiration: Infinity, + }) +} diff --git a/packages/upload-client/src/storage.js b/packages/upload-client/src/storage.js index 4e9ed1613..92d9c3d44 100644 --- a/packages/upload-client/src/storage.js +++ b/packages/upload-client/src/storage.js @@ -26,7 +26,7 @@ import { REQUEST_RETRIES } from './constants.js' * @returns {Promise} */ export async function store({ issuer, proofs }, car, options = {}) { - const capability = findCapability(proofs, serviceDID.did(), storeAdd.can) + const capability = findCapability(proofs, storeAdd.can) // TODO: validate blob contains CAR data const bytes = new Uint8Array(await car.arrayBuffer()) const link = await CAR.codec.link(bytes) diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 7ec1764a4..778d2919e 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -3,6 +3,8 @@ import { Block } from '@ipld/unixfs' import { CAR } from '@ucanto/transport' import { ServiceMethod, ConnectionView, Signer, Proof } from '@ucanto/interface' import { StoreAdd, UploadAdd } from '@web3-storage/access/capabilities/types' +import * as StoreCapabilities from '@web3-storage/access/capabilities/store' +import * as UploadCapabilities from '@web3-storage/access/capabilities/upload' export type { StoreAdd, UploadAdd } @@ -11,6 +13,16 @@ export interface Service { upload: { add: ServiceMethod } } +export type ServiceAbilities = + | typeof StoreCapabilities.store.can + | typeof StoreCapabilities.add.can + | typeof StoreCapabilities.remove.can + | typeof StoreCapabilities.list.can + | typeof UploadCapabilities.upload.can + | typeof UploadCapabilities.add.can + | typeof UploadCapabilities.remove.can + | typeof UploadCapabilities.list.can + export interface StoreAddResponse { status: string headers: Record @@ -91,14 +103,16 @@ export interface Connectable { export type RequestOptions = Retryable & Abortable & Connectable -export interface FileLike { +export interface BlobLike { /** - * Name of the file. May include path information. + * Returns a ReadableStream which yields the Blob data. */ - name: string + stream: () => ReadableStream +} + +export interface FileLike extends BlobLike { /** - * Returns a ReadableStream which upon reading returns the data contained - * within the File. + * Name of the file. May include path information. */ - stream: () => ReadableStream + name: string } diff --git a/packages/upload-client/src/unixfs.js b/packages/upload-client/src/unixfs.js index e581672f9..44a6bd177 100644 --- a/packages/upload-client/src/unixfs.js +++ b/packages/upload-client/src/unixfs.js @@ -11,7 +11,7 @@ const settings = UnixFS.configure({ }) /** - * @param {Blob} blob + * @param {import('./types').BlobLike} blob * @returns {Promise} */ export async function encodeFile(blob) { @@ -23,7 +23,7 @@ export async function encodeFile(blob) { } /** - * @param {Blob} blob + * @param {import('./types').BlobLike} blob * @returns {ReadableStream} */ export function createFileEncoderStream(blob) { diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js index ddae72fa1..847cb28db 100644 --- a/packages/upload-client/src/upload.js +++ b/packages/upload-client/src/upload.js @@ -25,7 +25,7 @@ import { REQUEST_RETRIES } from './constants.js' * @param {import('./types').RequestOptions} [options] */ export async function register({ issuer, proofs }, root, shards, options = {}) { - const capability = findCapability(proofs, serviceDID.did(), uploadAdd.can) + const capability = findCapability(proofs, uploadAdd.can) /** @type {import('@ucanto/interface').ConnectionView} */ const conn = options.connection ?? connection await retry( diff --git a/packages/upload-client/src/utils.js b/packages/upload-client/src/utils.js index ea81fd6e2..387ae7b85 100644 --- a/packages/upload-client/src/utils.js +++ b/packages/upload-client/src/utils.js @@ -42,14 +42,14 @@ export async function collect(collectable) { /** * @param {import('@ucanto/interface').Proof[]} proofs - * @param {import('@ucanto/interface').DID} audience * @param {import('@ucanto/interface').Ability} ability + * @param {import('@ucanto/interface').DID} [audience] */ -export function findCapability(proofs, audience, ability) { +export function findCapability(proofs, ability, audience) { let capability for (const proof of proofs) { if (!isDelegation(proof)) continue - if (proof.audience.did() !== audience) continue + if (audience != null && proof.audience.did() !== audience) continue capability = proof.capabilities.find((c) => capabilityMatches(c.can, ability) ) From dd7e50a90a2d20c966f6ea8eabded099a9aaaa87 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Nov 2022 12:15:49 +0000 Subject: [PATCH 20/47] refactor: follow capability names --- packages/upload-client/README.md | 27 +++++++---------- packages/upload-client/package.json | 6 ++-- packages/upload-client/src/index.js | 30 ++----------------- packages/upload-client/src/sharding.js | 4 +-- .../src/{storage.js => store.js} | 2 +- packages/upload-client/src/upload.js | 2 +- .../test/{storage.test.js => store.test.js} | 11 ++++--- packages/upload-client/test/upload.test.js | 7 ++--- 8 files changed, 30 insertions(+), 59 deletions(-) rename packages/upload-client/src/{storage.js => store.js} (97%) rename packages/upload-client/test/{storage.test.js => store.test.js} (92%) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index ad1e240dc..c7b659d13 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -27,11 +27,6 @@ const agent = await Agent.create({ store }) // // Note: you need to create and register an account 1st time: // await agent.createAccount('you@youremail.com') -// -// // ...and delegate capabilities from the account, to the agent, allowing it -// // to use the upload API: -// const delegation = await delegateCapabilities(agent.data.accounts[0], agent.issuer) -// await agent.addDelegation(delegation) const conf = { issuer: agent.issuer, @@ -66,16 +61,16 @@ const cid = await uploadDirectory(conf, [ The buffering API loads all data into memory so is suitable only for small files. The root data CID is obtained before any transfer to the service takes place. ```js -import { UnixFS, CAR, Storage, Upload } from '@web3-storage/upload-client' +import { UnixFS, CAR, Store, Upload } from '@web3-storage/upload-client' // Encode a file as a DAG, get back a root data CID and a set of blocks const { cid, blocks } = await UnixFS.encodeFile(file) // Encode the DAG as a CAR file const car = await CAR.encode(blocks, cid) // Store the CAR file to the service -const carCID = await Storage.store(conf, car) +const carCID = await Store.add(conf, car) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Upload.register(conf, cid, [carCID]) +await Upload.add(conf, cid, [carCID]) ``` #### Streaming API @@ -113,7 +108,7 @@ const rootCID = metadatas[metadatas.length - 1].roots[0] const carCIDs = metadatas.map((meta) => meta.cid) // Register an "upload" - a root CID contained within the passed CAR file(s) -await Upload.register(conf, rootCID, carCIDs) +await Upload.add(conf, rootCID, carCIDs) ``` ## API @@ -122,15 +117,15 @@ await Upload.register(conf, rootCID, carCIDs) - [`encode`](#carencode) - [`ShardingStream`](#shardingstream) - [`ShardStoringStream`](#shardstoringstream) -- `Storage` - - [`store`](#storagestore) +- `Store` + - [`add`](#storeadd) - `UnixFS` - [`createDirectoryEncoderStream`](#unixfscreatedirectoryencoderstream) - [`createFileEncoderStream`](#unixfscreatefileencoderstream) - [`encodeDirectory`](#unixfsencodedirectory) - [`encodeFile`](#unixfsencodefile) - `Upload` - - - [`register`](#uploadregister) + - - [`add`](#uploadadd) - [`uploadDirectory`](#uploaddirectory) - [`uploadFile`](#uploadfile) @@ -183,10 +178,10 @@ Note: an "upload" must be registered in order to link multiple shards together a The writeable side of this transform stream accepts `CARFile`s and the readable side yields `CARMetadata`, which contains the CAR CID, it's size (in bytes) and it's roots (if it has any). -### `Storage.store` +### `Store.add` ```ts -function store( +function add( conf: InvocationConfig, car: Blob, options: { retries?: number; signal?: AbortSignal } = {} @@ -261,10 +256,10 @@ const { cid, blocks } = await encodeFile(new File(['data'], 'doc.txt')) // Note: file name is not preserved - use encodeDirectory if required. ``` -### `Upload.register` +### `Upload.add` ```ts -function register( +function add( conf: InvocationConfig, root: CID, shards: CID[], diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 8bc2e0b28..7c1a696ee 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -30,7 +30,7 @@ "./car": "./src/car.js", "./sharding": "./src/sharding.js", "./upload": "./src/upload.js", - "./storage": "./src/storage.js", + "./store": "./src/store.js", "./unixfs": "./src/unixfs.js" }, "typesVersions": { @@ -44,8 +44,8 @@ "upload": [ "dist/src/upload.d.ts" ], - "storage": [ - "dist/src/storage.d.ts" + "store": [ + "dist/src/store.d.ts" ], "unixfs": [ "dist/src/unixfs.d.ts" diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 9b4ac6e1c..cc9286301 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -1,7 +1,4 @@ -import { delegate } from '@ucanto/core' -import { store as storeStar } from '@web3-storage/access/capabilities/store' -import { upload as uploadStar } from '@web3-storage/access/capabilities/upload' -import * as Storage from './storage.js' +import * as Storage from './store.js' import * as Upload from './upload.js' import * as UnixFS from './unixfs.js' import * as CAR from './car.js' @@ -98,29 +95,6 @@ async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { if (root == null) throw new Error('missing root CID') - await Upload.register({ issuer, proofs }, root, shards, options) + await Upload.add({ issuer, proofs }, root, shards, options) return root } - -/** - * Delegate upload API capabilities from the issuer to the audience. - * - * @param {import('@ucanto/interface').Signer} issuer - * @param {import('@ucanto/interface').Principal} audience - * @param {import('@ucanto/interface').Tuple} [abilities] - */ -export async function delegateCapabilities( - issuer, - audience, - abilities = [storeStar.can, uploadStar.can] -) { - const capabilities = abilities.map((can) => ({ can, with: issuer.did() })) - if (!capabilities.length) throw new Error('at least one ability is required') - return await delegate({ - issuer, - audience, - // @ts-expect-error - capabilities, - expiration: Infinity, - }) -} diff --git a/packages/upload-client/src/sharding.js b/packages/upload-client/src/sharding.js index 2502efd7c..92b3bef16 100644 --- a/packages/upload-client/src/sharding.js +++ b/packages/upload-client/src/sharding.js @@ -1,6 +1,6 @@ import Queue from 'p-queue' import { encode } from './car.js' -import { store } from './storage.js' +import { add } from './store.js' // most thing are < 30MB const SHARD_SIZE = 1024 * 1024 * 30 @@ -89,7 +89,7 @@ export class ShardStoringStream extends TransformStream { async () => { try { const opts = { ...options, signal: abortController.signal } - const cid = await store({ issuer, proofs }, car, opts) + const cid = await add({ issuer, proofs }, car, opts) const { version, roots, size } = car controller.enqueue({ version, roots, cid, size }) } catch (err) { diff --git a/packages/upload-client/src/storage.js b/packages/upload-client/src/store.js similarity index 97% rename from packages/upload-client/src/storage.js rename to packages/upload-client/src/store.js index 92d9c3d44..661cca6c1 100644 --- a/packages/upload-client/src/storage.js +++ b/packages/upload-client/src/store.js @@ -25,7 +25,7 @@ import { REQUEST_RETRIES } from './constants.js' * @param {import('./types').RequestOptions} [options] * @returns {Promise} */ -export async function store({ issuer, proofs }, car, options = {}) { +export async function add({ issuer, proofs }, car, options = {}) { const capability = findCapability(proofs, storeAdd.can) // TODO: validate blob contains CAR data const bytes = new Uint8Array(await car.arrayBuffer()) diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js index 847cb28db..1f71cc69d 100644 --- a/packages/upload-client/src/upload.js +++ b/packages/upload-client/src/upload.js @@ -24,7 +24,7 @@ import { REQUEST_RETRIES } from './constants.js' * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. * @param {import('./types').RequestOptions} [options] */ -export async function register({ issuer, proofs }, root, shards, options = {}) { +export async function add({ issuer, proofs }, root, shards, options = {}) { const capability = findCapability(proofs, uploadAdd.can) /** @type {import('@ucanto/interface').ConnectionView} */ const conn = options.connection ?? connection diff --git a/packages/upload-client/test/storage.test.js b/packages/upload-client/test/store.test.js similarity index 92% rename from packages/upload-client/test/storage.test.js rename to packages/upload-client/test/store.test.js index 935aedda3..fb581acbc 100644 --- a/packages/upload-client/test/storage.test.js +++ b/packages/upload-client/test/store.test.js @@ -5,7 +5,7 @@ import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' import { add as storeAdd } from '@web3-storage/access/capabilities/store' -import { store } from '../src/storage.js' +import * as Store from '../src/store.js' import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' @@ -58,7 +58,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store({ issuer, proofs }, car, { connection }) + const carCID = await Store.add({ issuer, proofs }, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -100,7 +100,7 @@ describe('Storage', () => { channel: server, }) - const carCID = await store({ issuer, proofs }, car, { connection }) + const carCID = await Store.add({ issuer, proofs }, car, { connection }) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -146,7 +146,10 @@ describe('Storage', () => { controller.abort() // already aborted await assert.rejects( - store({ issuer, proofs }, car, { connection, signal: controller.signal }), + Store.add({ issuer, proofs }, car, { + connection, + signal: controller.signal, + }), { name: 'Error', message: 'upload aborted' } ) }) diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index 8a46a858a..1921846b3 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -5,7 +5,7 @@ import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' -import { register } from '../src/upload.js' +import * as Upload from '../src/upload.js' import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' @@ -54,8 +54,7 @@ describe('Upload', () => { channel: server, }) - await register({ issuer, proofs }, car.roots[0], [car.cid], { - connection, - }) + const root = car.roots[0] + await Upload.add({ issuer, proofs }, root, [car.cid], { connection }) }) }) From d023aea23f9ec57d811d4096824e9acea3b3d8ae Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Nov 2022 12:16:58 +0000 Subject: [PATCH 21/47] fix: remove unneeded import --- packages/upload-client/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index c7b659d13..7766d4c05 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -18,7 +18,6 @@ npm install @web3-storage/upload-client Obtain the invocation configuration. i.e. the issuer (the signing authority) and proofs that the issuer has been delegated the capabilities to store data and register uploads: ```js -import { delegateCapabilities } from '@web3-storage/upload-client' import { Agent } from '@web3-storage/access-client' import { add as storeAdd } from '@web3-storage/access-client/capabilities/store' import { add as uploadAdd } from '@web3-storage/access-client/capabilities/upload' From d96bc1beb77e89ffebfc1bd0c17a3999a04fb3db Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Nov 2022 13:07:35 +0000 Subject: [PATCH 22/47] feat: add list and remove functionality --- packages/upload-client/package.json | 1 + packages/upload-client/src/service.js | 1 + packages/upload-client/src/store.js | 77 ++++++++++++++++++-- packages/upload-client/src/types.ts | 49 ++++++++++++- packages/upload-client/src/upload.js | 71 +++++++++++++++++- packages/upload-client/test/helpers/mocks.js | 27 +++++++ packages/upload-client/test/index.test.js | 9 ++- packages/upload-client/test/store.test.js | 28 ++----- packages/upload-client/test/upload.test.js | 10 +-- 9 files changed, 225 insertions(+), 48 deletions(-) create mode 100644 packages/upload-client/test/helpers/mocks.js diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 7c1a696ee..b5b0c8228 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -106,6 +106,7 @@ "no-void": "off", "no-console": "off", "no-continue": "off", + "jsdoc/check-indentation": "off", "jsdoc/require-hyphen-before-param-description": "off" }, "env": { diff --git a/packages/upload-client/src/service.js b/packages/upload-client/src/service.js index b59e78eaa..282ae6644 100644 --- a/packages/upload-client/src/service.js +++ b/packages/upload-client/src/service.js @@ -9,6 +9,7 @@ export const serviceDID = DID.parse( 'did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z' ) +/** @type {import('@ucanto/interface').ConnectionView} */ export const connection = connect({ id: serviceDID, encoder: CAR, diff --git a/packages/upload-client/src/store.js b/packages/upload-client/src/store.js index 661cca6c1..6c74028ef 100644 --- a/packages/upload-client/src/store.js +++ b/packages/upload-client/src/store.js @@ -1,5 +1,5 @@ import { CAR } from '@ucanto/transport' -import { add as storeAdd } from '@web3-storage/access/capabilities/store' +import * as StoreCapabilities from '@web3-storage/access/capabilities/store' import retry, { AbortError } from 'p-retry' import { serviceDID, connection } from './service.js' import { findCapability } from './utils.js' @@ -26,15 +26,14 @@ import { REQUEST_RETRIES } from './constants.js' * @returns {Promise} */ export async function add({ issuer, proofs }, car, options = {}) { - const capability = findCapability(proofs, storeAdd.can) + const capability = findCapability(proofs, StoreCapabilities.add.can) // TODO: validate blob contains CAR data const bytes = new Uint8Array(await car.arrayBuffer()) const link = await CAR.codec.link(bytes) - /** @type {import('@ucanto/interface').ConnectionView} */ const conn = options.connection ?? connection const result = await retry( async () => { - const res = await storeAdd + const res = await StoreCapabilities.add .invoke({ issuer, audience: serviceDID, @@ -53,7 +52,9 @@ export async function add({ issuer, proofs }, car, options = {}) { ) if (result.error) { - throw new Error(`failed ${storeAdd.can} invocation`, { cause: result }) + throw new Error(`failed ${StoreCapabilities.add.can} invocation`, { + cause: result, + }) } // Return early if it was already uploaded. @@ -89,8 +90,72 @@ export async function add({ issuer, proofs }, car, options = {}) { ) if (!res.ok) { - throw new Error('upload failed') + throw new Error('store failed') } return link } + +/** + * List CAR files stored by the issuer. + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `store/list` delegated capability. + * @param {import('./types').RequestOptions} [options] + */ +export async function list({ issuer, proofs }, options = {}) { + const capability = findCapability(proofs, StoreCapabilities.list.can) + const conn = options.connection ?? connection + + const result = await StoreCapabilities.list + .invoke({ + issuer, + audience: serviceDID, + // @ts-expect-error expects did:${string} but cap with is ${string}:${string} + with: capability.with, + }) + .execute(conn) + if (result.error === true) throw result + + return result +} + +/** + * Remove a stored CAR file by CAR CID. + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `store/remove` delegated capability. + * @param {import('./types').CARLink} link CID of CAR file to remove. + * @param {import('./types').RequestOptions} [options] + */ +export async function remove({ issuer, proofs }, link, options = {}) { + const capability = findCapability(proofs, StoreCapabilities.remove.can) + const conn = options.connection ?? connection + + const result = await StoreCapabilities.remove + .invoke({ + issuer, + audience: serviceDID, + // @ts-expect-error expects did:${string} but cap with is ${string}:${string} + with: capability.with, + nb: { link }, + }) + .execute(conn) + if (result?.error === true) throw result +} diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 778d2919e..ef73851d8 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -2,15 +2,37 @@ import { Link, UnknownLink, Version } from 'multiformats/link' import { Block } from '@ipld/unixfs' import { CAR } from '@ucanto/transport' import { ServiceMethod, ConnectionView, Signer, Proof } from '@ucanto/interface' -import { StoreAdd, UploadAdd } from '@web3-storage/access/capabilities/types' +import { + StoreAdd, + StoreList, + StoreRemove, + UploadAdd, + UploadList, + UploadRemove, +} from '@web3-storage/access/capabilities/types' import * as StoreCapabilities from '@web3-storage/access/capabilities/store' import * as UploadCapabilities from '@web3-storage/access/capabilities/upload' -export type { StoreAdd, UploadAdd } +export type { + StoreAdd, + StoreList, + StoreRemove, + UploadAdd, + UploadList, + UploadRemove, +} export interface Service { - store: { add: ServiceMethod } - upload: { add: ServiceMethod } + store: { + add: ServiceMethod + list: ServiceMethod, never> + remove: ServiceMethod + } + upload: { + add: ServiceMethod + list: ServiceMethod, never> + remove: ServiceMethod + } } export type ServiceAbilities = @@ -29,6 +51,25 @@ export interface StoreAddResponse { url: string } +export interface ListResponse { + count: number + page: number + pageSize: number + results?: R[] +} + +export interface StoreListResult { + payloadCID: CARLink + size: number + uploadedAt: number +} + +export interface UploadListResult { + carCID: CARLink + dataCID: Link + uploadedAt: number +} + export interface InvocationConfig { /** * Signing authority that is issuing the UCAN invocations. diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js index 1f71cc69d..3ae480f16 100644 --- a/packages/upload-client/src/upload.js +++ b/packages/upload-client/src/upload.js @@ -1,4 +1,4 @@ -import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' +import * as UploadCapabilities from '@web3-storage/access/capabilities/upload' import retry from 'p-retry' import { serviceDID, connection } from './service.js' import { findCapability } from './utils.js' @@ -25,12 +25,11 @@ import { REQUEST_RETRIES } from './constants.js' * @param {import('./types').RequestOptions} [options] */ export async function add({ issuer, proofs }, root, shards, options = {}) { - const capability = findCapability(proofs, uploadAdd.can) - /** @type {import('@ucanto/interface').ConnectionView} */ + const capability = findCapability(proofs, UploadCapabilities.add.can) const conn = options.connection ?? connection await retry( async () => { - const result = await uploadAdd + const result = await UploadCapabilities.add .invoke({ issuer, audience: serviceDID, @@ -50,3 +49,67 @@ export async function add({ issuer, proofs }, root, shards, options = {}) { } ) } + +/** + * List uploads created by the issuer. + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/list` delegated capability. + * @param {import('./types').RequestOptions} [options] + */ +export async function list({ issuer, proofs }, options = {}) { + const capability = findCapability(proofs, UploadCapabilities.list.can) + const conn = options.connection ?? connection + + const result = await UploadCapabilities.list + .invoke({ + issuer, + audience: serviceDID, + // @ts-expect-error expects did:${string} but cap with is ${string}:${string} + with: capability.with, + }) + .execute(conn) + if (result.error === true) throw result + + return result +} + +/** + * Remove an upload by root data CID. + * + * @param {import('./types').InvocationConfig} invocationConfig Configuration + * for the UCAN invocation. An object with `issuer` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/remove` delegated capability. + * @param {import('multiformats').UnknownLink} root Root data CID to remove. + * @param {import('./types').RequestOptions} [options] + */ +export async function remove({ issuer, proofs }, root, options = {}) { + const capability = findCapability(proofs, UploadCapabilities.remove.can) + const conn = options.connection ?? connection + + const result = await UploadCapabilities.remove + .invoke({ + issuer, + audience: serviceDID, + // @ts-expect-error expects did:${string} but cap with is ${string}:${string} + with: capability.with, + nb: { root }, + }) + .execute(conn) + if (result?.error === true) throw result +} diff --git a/packages/upload-client/test/helpers/mocks.js b/packages/upload-client/test/helpers/mocks.js new file mode 100644 index 000000000..9a5e53b51 --- /dev/null +++ b/packages/upload-client/test/helpers/mocks.js @@ -0,0 +1,27 @@ +import * as Server from '@ucanto/server' + +const notImplemented = () => { + throw new Server.Failure('not implemented') +} + +/** + * @param {Partial<{ + * store: Partial + * upload: Partial + * }>} impl + * @returns {import('../../src/types').Service} + */ +export function mockService(impl) { + return { + store: { + add: impl.store?.add ?? notImplemented, + list: impl.store?.list ?? notImplemented, + remove: impl.store?.remove ?? notImplemented, + }, + upload: { + add: impl.upload?.add ?? notImplemented, + list: impl.upload?.list ?? notImplemented, + remove: impl.upload?.remove ?? notImplemented, + }, + } +} diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 6acfb726a..9f521aef3 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -10,6 +10,7 @@ import { uploadFile, uploadDirectory } from '../src/index.js' import { service as id } from './fixtures.js' import { randomBytes } from './helpers/random.js' import { File } from './helpers/shims.js' +import { mockService } from './helpers/mocks.js' describe('uploadFile', () => { it('uploads a file to the service', async () => { @@ -40,7 +41,7 @@ describe('uploadFile', () => { }), ]) - const service = { + const service = mockService({ store: { /** @param {Server.Invocation} invocation */ add(invocation) { @@ -65,7 +66,7 @@ describe('uploadFile', () => { return null }, }, - } + }) const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) const connection = Client.connect({ @@ -118,7 +119,7 @@ describe('uploadDirectory', () => { }), ]) - const service = { + const service = mockService({ store: { /** @param {Server.Invocation} invocation */ add(invocation) { @@ -143,7 +144,7 @@ describe('uploadDirectory', () => { return null }, }, - } + }) const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) const connection = Client.connect({ diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index fb581acbc..ef512c2cb 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -8,6 +8,7 @@ import { add as storeAdd } from '@web3-storage/access/capabilities/store' import * as Store from '../src/store.js' import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' describe('Storage', () => { it('stores a DAG with the service', async () => { @@ -30,7 +31,7 @@ describe('Storage', () => { }), ] - const service = { + const service = mockService({ store: { /** @param {Server.Invocation} invocation */ add(invocation) { @@ -43,12 +44,7 @@ describe('Storage', () => { return res }, }, - upload: { - add: () => { - throw new Server.Failure('not expected to be called') - }, - }, - } + }) const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) const connection = Client.connect({ @@ -83,14 +79,7 @@ describe('Storage', () => { }), ] - const service = { - store: { add: () => res }, - upload: { - add: () => { - throw new Server.Failure('not expected to be called') - }, - }, - } + const service = mockService({ store: { add: () => res } }) const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) const connection = Client.connect({ @@ -112,14 +101,7 @@ describe('Storage', () => { url: 'http://localhost:9001', // will fail the test if called } - const service = { - store: { add: () => res }, - upload: { - add: () => { - throw new Server.Failure('not expected to be called') - }, - }, - } + const service = mockService({ store: { add: () => res } }) const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) const connection = Client.connect({ diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index 1921846b3..66e3cb8c7 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -8,6 +8,7 @@ import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' import * as Upload from '../src/upload.js' import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' describe('Upload', () => { it('registers an upload with the service', async () => { @@ -24,12 +25,7 @@ describe('Upload', () => { }), ] - const service = { - store: { - add: () => { - throw new Server.Failure('not expected to be called') - }, - }, + const service = mockService({ upload: { /** @param {Server.Invocation} invocation */ add: (invocation) => { @@ -44,7 +40,7 @@ describe('Upload', () => { return null }, }, - } + }) const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) const connection = Client.connect({ From 5c98ebd052fdfe38da7a4105e200109ffff2fffe Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Nov 2022 14:32:00 +0000 Subject: [PATCH 23/47] feat: add tests and docs for store/upload list/remove --- packages/upload-client/README.md | 80 +++++++++++++- packages/upload-client/test/index.test.js | 4 - packages/upload-client/test/store.test.js | 115 +++++++++++++++++++-- packages/upload-client/test/upload.test.js | 108 ++++++++++++++++++- 4 files changed, 292 insertions(+), 15 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 7766d4c05..efb986189 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -118,13 +118,17 @@ await Upload.add(conf, rootCID, carCIDs) - [`ShardStoringStream`](#shardstoringstream) - `Store` - [`add`](#storeadd) + - [`list`](#storelist) + - [`remove`](#storeremove) - `UnixFS` - [`createDirectoryEncoderStream`](#unixfscreatedirectoryencoderstream) - [`createFileEncoderStream`](#unixfscreatefileencoderstream) - [`encodeDirectory`](#unixfsencodedirectory) - [`encodeFile`](#unixfsencodefile) - `Upload` - - - [`add`](#uploadadd) + - [`add`](#uploadadd) + - [`list`](#uploadlist) + - [`remove`](#uploadremove) - [`uploadDirectory`](#uploaddirectory) - [`uploadFile`](#uploadfile) @@ -196,6 +200,43 @@ Note: `InvocationConfig` is configuration for the UCAN invocation. It's values c Required delegated capability proofs: `store/add` +### `Store.list` + +```ts +function list( + conf: InvocationConfig, + options: { retries?: number; signal?: AbortSignal } = {} +): Promise> +``` + +List CAR files stored by the issuer. + +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `store/list` + +### `Store.remove` + +```ts +function remove( + conf: InvocationConfig, + link: CID, + options: { retries?: number; signal?: AbortSignal } = {} +): Promise +``` + +Remove a stored CAR file by CAR CID. + +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `store/remove` + ### `UnixFS.createDirectoryEncoderStream` ```ts @@ -275,6 +316,43 @@ Note: `InvocationConfig` is configuration for the UCAN invocation. It's values c Required delegated capability proofs: `upload/add` +### `Upload.list` + +```ts +function list( + conf: InvocationConfig, + options: { retries?: number; signal?: AbortSignal } = {} +): Promise> +``` + +List uploads created by the issuer. + +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `upload/list` + +### `Upload.remove` + +```ts +function remove( + conf: InvocationConfig, + link: CID, + options: { retries?: number; signal?: AbortSignal } = {} +): Promise +``` + +Remove a upload by root data CID. + +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `upload/remove` + ### `uploadDirectory` ```ts diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 9f521aef3..f54422025 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -43,7 +43,6 @@ describe('uploadFile', () => { const service = mockService({ store: { - /** @param {Server.Invocation} invocation */ add(invocation) { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) @@ -54,7 +53,6 @@ describe('uploadFile', () => { }, }, upload: { - /** @param {Server.Invocation} invocation */ add: (invocation) => { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) @@ -121,7 +119,6 @@ describe('uploadDirectory', () => { const service = mockService({ store: { - /** @param {Server.Invocation} invocation */ add(invocation) { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) @@ -132,7 +129,6 @@ describe('uploadDirectory', () => { }, }, upload: { - /** @param {Server.Invocation} invocation */ add: (invocation) => { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index ef512c2cb..25fa9a7d2 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -4,7 +4,7 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' -import { add as storeAdd } from '@web3-storage/access/capabilities/store' +import * as StoreCapabilities from '@web3-storage/access/capabilities/store' import * as Store from '../src/store.js' import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' @@ -23,7 +23,7 @@ describe('Storage', () => { const car = await randomCAR(128) const proofs = [ - await storeAdd.delegate({ + await StoreCapabilities.add.delegate({ issuer: account, audience: id, with: account.did(), @@ -33,12 +33,11 @@ describe('Storage', () => { const service = mockService({ store: { - /** @param {Server.Invocation} invocation */ add(invocation) { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] - assert.equal(invCap.can, 'store/add') + assert.equal(invCap.can, StoreCapabilities.add.can) assert.equal(invCap.with, account.did()) assert.equal(String(invCap.nb.link), car.cid.toString()) return res @@ -71,7 +70,7 @@ describe('Storage', () => { const car = await randomCAR(128) const proofs = [ - await storeAdd.delegate({ + await StoreCapabilities.add.delegate({ issuer: account, audience: id, with: account.did(), @@ -116,7 +115,7 @@ describe('Storage', () => { const car = await randomCAR(128) const proofs = [ - await storeAdd.delegate({ + await StoreCapabilities.add.delegate({ issuer: account, audience: id, with: account.did(), @@ -135,4 +134,108 @@ describe('Storage', () => { { name: 'Error', message: 'upload aborted' } ) }) + + it('lists stored CAR files', async () => { + const car = await randomCAR(128) + const res = { + page: 1, + pageSize: 1000, + count: 1, + results: [ + { + payloadCID: car.cid, + size: 123, + uploadedAt: Date.now(), + }, + ], + } + + const account = await Signer.generate() + const issuer = await Signer.generate() + + const proofs = [ + await StoreCapabilities.list.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + store: { + list(invocation) { + assert.equal(invocation.issuer.did(), issuer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, StoreCapabilities.list.can) + assert.equal(invCap.with, account.did()) + return res + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + const list = await Store.list({ issuer, proofs }, { connection }) + + assert.equal(list.count, res.count) + assert.equal(list.page, res.page) + assert.equal(list.pageSize, res.pageSize) + assert(list.results) + assert.equal(list.results.length, res.results.length) + list.results.forEach((r, i) => { + assert.equal( + r.payloadCID.toString(), + res.results[i].payloadCID.toString() + ) + assert.equal(r.size, res.results[i].size) + assert.equal(r.uploadedAt, res.results[i].uploadedAt) + }) + }) + + it('removes a stored CAR file', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await StoreCapabilities.remove.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + store: { + remove(invocation) { + assert.equal(invocation.issuer.did(), issuer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, StoreCapabilities.remove.can) + assert.equal(invCap.with, account.did()) + assert.equal(String(invCap.nb.link), car.cid.toString()) + return null + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await Store.remove({ issuer, proofs }, car.cid, { connection }) + }) }) diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index 66e3cb8c7..887e6779c 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -4,7 +4,7 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' -import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' +import * as UploadCapabilities from '@web3-storage/access/capabilities/upload' import * as Upload from '../src/upload.js' import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' @@ -17,7 +17,7 @@ describe('Upload', () => { const car = await randomCAR(128) const proofs = [ - await uploadAdd.delegate({ + await UploadCapabilities.add.delegate({ issuer: account, audience: id, with: account.did(), @@ -27,12 +27,11 @@ describe('Upload', () => { const service = mockService({ upload: { - /** @param {Server.Invocation} invocation */ add: (invocation) => { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] - assert.equal(invCap.can, 'upload/add') + assert.equal(invCap.can, UploadCapabilities.add.can) assert.equal(invCap.with, account.did()) assert.equal(String(invCap.nb.root), car.roots[0].toString()) assert.equal(invCap.nb.shards?.length, 1) @@ -53,4 +52,105 @@ describe('Upload', () => { const root = car.roots[0] await Upload.add({ issuer, proofs }, root, [car.cid], { connection }) }) + + it('lists uploads', async () => { + const car = await randomCAR(128) + const res = { + page: 1, + pageSize: 1000, + count: 1, + results: [ + { + carCID: car.cid, + dataCID: car.roots[0], + uploadedAt: Date.now(), + }, + ], + } + + const account = await Signer.generate() + const issuer = await Signer.generate() + + const proofs = [ + await UploadCapabilities.list.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + upload: { + list(invocation) { + assert.equal(invocation.issuer.did(), issuer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, UploadCapabilities.list.can) + assert.equal(invCap.with, account.did()) + return res + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + const list = await Upload.list({ issuer, proofs }, { connection }) + + assert.equal(list.count, res.count) + assert.equal(list.page, res.page) + assert.equal(list.pageSize, res.pageSize) + assert(list.results) + assert.equal(list.results.length, res.results.length) + list.results.forEach((r, i) => { + assert.equal(r.carCID.toString(), res.results[i].carCID.toString()) + assert.equal(r.dataCID.toString(), res.results[i].dataCID.toString()) + assert.equal(r.uploadedAt, res.results[i].uploadedAt) + }) + }) + + it('removes an upload', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await UploadCapabilities.remove.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + upload: { + remove(invocation) { + assert.equal(invocation.issuer.did(), issuer.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, UploadCapabilities.remove.can) + assert.equal(invCap.with, account.did()) + assert.equal(String(invCap.nb.root), car.roots[0].toString()) + return null + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await Upload.remove({ issuer, proofs }, car.roots[0], { connection }) + }) }) From b48a6d43fbcd1ae8b726ac4234355ae67ae14baa Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Nov 2022 14:36:18 +0000 Subject: [PATCH 24/47] refactor: do not export abilities --- packages/upload-client/src/types.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index ef73851d8..6235d1815 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -10,8 +10,6 @@ import { UploadList, UploadRemove, } from '@web3-storage/access/capabilities/types' -import * as StoreCapabilities from '@web3-storage/access/capabilities/store' -import * as UploadCapabilities from '@web3-storage/access/capabilities/upload' export type { StoreAdd, @@ -35,16 +33,6 @@ export interface Service { } } -export type ServiceAbilities = - | typeof StoreCapabilities.store.can - | typeof StoreCapabilities.add.can - | typeof StoreCapabilities.remove.can - | typeof StoreCapabilities.list.can - | typeof UploadCapabilities.upload.can - | typeof UploadCapabilities.add.can - | typeof UploadCapabilities.remove.can - | typeof UploadCapabilities.list.can - export interface StoreAddResponse { status: string headers: Record From aef55e718eb8756ed95beb237780ef7125b3aace Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Nov 2022 14:46:27 +0000 Subject: [PATCH 25/47] fix: tests --- packages/upload-client/test/sharding.test.js | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index 948d83a3a..e1e3d739a 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -9,6 +9,7 @@ import { createFileEncoderStream } from '../src/unixfs.js' import { ShardingStream, ShardStoringStream } from '../src/sharding.js' import { service as id } from './fixtures.js' import { randomBytes, randomCAR } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' describe('ShardingStream', () => { it('creates shards from blocks', async () => { @@ -59,9 +60,8 @@ describe('ShardStoringStream', () => { }), ] - const service = { + const service = mockService({ store: { - /** @param {Server.Invocation} invocation */ add(invocation) { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) @@ -73,12 +73,7 @@ describe('ShardStoringStream', () => { return res }, }, - upload: { - add: () => { - throw new Server.Failure('not expected to be called') - }, - }, - } + }) const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) const connection = Client.connect({ From 46d1970ff807a8f95064b17ba6628cb5606cca09 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 10:19:53 +0000 Subject: [PATCH 26/47] Update packages/upload-client/README.md Co-authored-by: Oli Evans --- packages/upload-client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index efb986189..6822ac9b1 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -24,7 +24,7 @@ import { add as uploadAdd } from '@web3-storage/access-client/capabilities/uploa const agent = await Agent.create({ store }) -// // Note: you need to create and register an account 1st time: +// Note: you need to create and register an account 1st time: // await agent.createAccount('you@youremail.com') const conf = { From 405b4ec30c26c41d4d7b39e2278ae8ec0544700f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 10:25:27 +0000 Subject: [PATCH 27/47] Update packages/upload-client/README.md Co-authored-by: Oli Evans --- packages/upload-client/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 6822ac9b1..fab7c999d 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -13,9 +13,9 @@ npm install @web3-storage/upload-client [API Reference](#api) -### Step 0 +### Create an Agent -Obtain the invocation configuration. i.e. the issuer (the signing authority) and proofs that the issuer has been delegated the capabilities to store data and register uploads: +An Agent provides an `issuer` (a key linked to your account) and `proofs` to show your `issuer` has been delegated the capabilities to store data and register uploads. ```js import { Agent } from '@web3-storage/access-client' From 792986023d38649daf3499b8a3d468ba7e409a74 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 10:26:14 +0000 Subject: [PATCH 28/47] Update packages/upload-client/README.md Co-authored-by: Oli Evans --- packages/upload-client/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index fab7c999d..04a90401d 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -35,6 +35,10 @@ const conf = { ### Uploading files +Once you have the `issuer` and `proofs`, you can upload a directory of files by passing that invocation config to `uploadDirectory` along with your list of files to upload. + +You can get your list of Files from a [``](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file) element in the browser or using [`files-from-path`](https://npm.im/files-from-path) in Node.js + ```js import { uploadFile } from '@web3-storage/upload-client' @@ -48,9 +52,6 @@ const cid = await uploadDirectory(conf, [ new File(['doc0'], 'doc0.txt'), new File(['doc1'], 'dir/doc1.txt'), ]) - -// Note: you can use https://npm.im/files-from-path to read files from the -// filesystem in Nodejs. ``` ### Advanced usage From fb1082193f384757c234913ae52107e3cf9aa8ce Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 10:26:36 +0000 Subject: [PATCH 29/47] Update packages/upload-client/README.md Co-authored-by: Oli Evans --- packages/upload-client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 04a90401d..6174002dd 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -58,7 +58,7 @@ const cid = await uploadDirectory(conf, [ #### Buffering API -The buffering API loads all data into memory so is suitable only for small files. The root data CID is obtained before any transfer to the service takes place. +The buffering API loads all data into memory so is suitable only for small files. The root data CID is derived from the data before any transfer to the service takes place. ```js import { UnixFS, CAR, Store, Upload } from '@web3-storage/upload-client' From 673690cfdfe2c136a586410e95e163431a1675a0 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 10:27:16 +0000 Subject: [PATCH 30/47] Update packages/upload-client/README.md Co-authored-by: Oli Evans --- packages/upload-client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 6174002dd..bc9525c5d 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -104,7 +104,7 @@ await UnixFS.createFileEncoderStream(file) ) // The last CAR stored contains the root data CID -const rootCID = metadatas[metadatas.length - 1].roots[0] +const rootCID = metadatas.at(-1).roots[0] const carCIDs = metadatas.map((meta) => meta.cid) // Register an "upload" - a root CID contained within the passed CAR file(s) From 3decc9b140f573452cdf374ea19b0cc4221a47b3 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 13:39:14 +0000 Subject: [PATCH 31/47] test: add hundreds and test coverage --- packages/upload-client/package.json | 10 +- packages/upload-client/src/car.js | 4 +- packages/upload-client/src/index.js | 7 +- packages/upload-client/src/store.js | 22 ++- packages/upload-client/src/unixfs.js | 44 ++++-- packages/upload-client/src/upload.js | 33 ++-- packages/upload-client/src/utils.js | 44 +----- packages/upload-client/test/car.test.js | 16 ++ .../{ok-server.js => bucket-server.js} | 5 +- packages/upload-client/test/index.test.js | 4 +- packages/upload-client/test/sharding.test.js | 49 +++++- packages/upload-client/test/store.test.js | 148 +++++++++++++++++- packages/upload-client/test/unixfs.test.js | 14 +- packages/upload-client/test/upload.test.js | 112 ++++++++++++- packages/upload-client/test/utils.test.js | 59 +++++++ pnpm-lock.yaml | 9 ++ 16 files changed, 494 insertions(+), 86 deletions(-) create mode 100644 packages/upload-client/test/car.test.js rename packages/upload-client/test/helpers/{ok-server.js => bucket-server.js} (62%) create mode 100644 packages/upload-client/test/utils.test.js diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index b5b0c8228..fa9fea7ca 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -18,11 +18,14 @@ "build": "run-s build:*", "build:deps": "pnpm -r --filter @web3-storage/access run build", "build:tsc": "tsc --build", - "test": "npm-run-all -p -r mock:bucket test:all", + "test": "npm-run-all -p -r mock test:all", "test:all": "run-s test:browser test:node", - "test:node": "c8 -r html -r text mocha 'test/**/!(*.browser).test.js' -n experimental-vm-modules -n no-warnings", + "test:node": "hundreds -r html -r text mocha 'test/**/!(*.browser).test.js' -n experimental-vm-modules -n no-warnings", "test:browser": "playwright-test 'test/**/!(*.node).test.js'", - "mock:bucket": "node test/helpers/ok-server.js", + "mock": "run-p mock:*", + "mock:bucket-200": "PORT=9200 STATUS=200 node test/helpers/bucket-server.js", + "mock:bucket-401": "PORT=9400 STATUS=400 node test/helpers/bucket-server.js", + "mock:bucket-500": "PORT=9500 STATUS=500 node test/helpers/bucket-server.js", "rc": "npm version prerelease --preid rc" }, "exports": { @@ -79,6 +82,7 @@ "blockstore-core": "^2.0.2", "c8": "^7.12.0", "hd-scripts": "^3.0.2", + "hundreds": "^0.0.9", "ipfs-unixfs-exporter": "^9.0.1", "mocha": "^10.1.0", "npm-run-all": "^4.1.5", diff --git a/packages/upload-client/src/car.js b/packages/upload-client/src/car.js index b60d5cd1d..839df8ee4 100644 --- a/packages/upload-client/src/car.js +++ b/packages/upload-client/src/car.js @@ -1,5 +1,4 @@ import { CarWriter } from '@ipld/car' -import { collect } from './utils.js' /** * @param {Iterable|AsyncIterable} blocks @@ -23,7 +22,8 @@ export async function encode(blocks, root) { await writer.close() } })() - const chunks = await collect(out) + const chunks = [] + for await (const chunk of out) chunks.push(chunk) // @ts-expect-error if (error != null) throw error const roots = root != null ? [root] : [] diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index cc9286301..26810df4b 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -74,8 +74,6 @@ export async function uploadDirectory({ issuer, proofs }, files, options = {}) { * @returns {Promise>} */ async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { - const onStoredShard = options.onStoredShard ?? (() => {}) - /** @type {import('./types').CARLink[]} */ const shards = [] /** @type {import('multiformats').Link?} */ @@ -88,12 +86,13 @@ async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { write(meta) { root = root || meta.roots[0] shards.push(meta.cid) - onStoredShard(meta) + if (options.onStoredShard) options.onStoredShard(meta) }, }) ) - if (root == null) throw new Error('missing root CID') + /* c8 ignore next */ + if (!root) throw new Error('missing root CID') await Upload.add({ issuer, proofs }, root, shards, options) return root diff --git a/packages/upload-client/src/store.js b/packages/upload-client/src/store.js index 6c74028ef..7592d6eeb 100644 --- a/packages/upload-client/src/store.js +++ b/packages/upload-client/src/store.js @@ -30,10 +30,11 @@ export async function add({ issuer, proofs }, car, options = {}) { // TODO: validate blob contains CAR data const bytes = new Uint8Array(await car.arrayBuffer()) const link = await CAR.codec.link(bytes) + /* c8 ignore next */ const conn = options.connection ?? connection const result = await retry( async () => { - const res = await StoreCapabilities.add + return await StoreCapabilities.add .invoke({ issuer, audience: serviceDID, @@ -43,7 +44,6 @@ export async function add({ issuer, proofs }, car, options = {}) { proofs, }) .execute(conn) - return res }, { onFailedAttempt: console.warn, @@ -90,7 +90,7 @@ export async function add({ issuer, proofs }, car, options = {}) { ) if (!res.ok) { - throw new Error('store failed') + throw new Error(`upload failed: ${res.status}`) } return link @@ -113,6 +113,7 @@ export async function add({ issuer, proofs }, car, options = {}) { */ export async function list({ issuer, proofs }, options = {}) { const capability = findCapability(proofs, StoreCapabilities.list.can) + /* c8 ignore next */ const conn = options.connection ?? connection const result = await StoreCapabilities.list @@ -123,7 +124,12 @@ export async function list({ issuer, proofs }, options = {}) { with: capability.with, }) .execute(conn) - if (result.error === true) throw result + + if (result.error) { + throw new Error(`failed ${StoreCapabilities.list.can} invocation`, { + cause: result, + }) + } return result } @@ -146,6 +152,7 @@ export async function list({ issuer, proofs }, options = {}) { */ export async function remove({ issuer, proofs }, link, options = {}) { const capability = findCapability(proofs, StoreCapabilities.remove.can) + /* c8 ignore next */ const conn = options.connection ?? connection const result = await StoreCapabilities.remove @@ -157,5 +164,10 @@ export async function remove({ issuer, proofs }, link, options = {}) { nb: { link }, }) .execute(conn) - if (result?.error === true) throw result + + if (result?.error) { + throw new Error(`failed ${StoreCapabilities.remove.can} invocation`, { + cause: result, + }) + } } diff --git a/packages/upload-client/src/unixfs.js b/packages/upload-client/src/unixfs.js index 44a6bd177..1b573718c 100644 --- a/packages/upload-client/src/unixfs.js +++ b/packages/upload-client/src/unixfs.js @@ -1,6 +1,5 @@ import * as UnixFS from '@ipld/unixfs' import * as raw from 'multiformats/codecs/raw' -import { toIterable, collect } from './utils.js' const queuingStrategy = UnixFS.withCapacity(1_048_576 * 175) @@ -16,10 +15,9 @@ const settings = UnixFS.configure({ */ export async function encodeFile(blob) { const readable = createFileEncoderStream(blob) - const blocks = await collect(toIterable(readable)) - const rootBlock = blocks.at(-1) - if (rootBlock == null) throw new Error('missing root block') - return { cid: rootBlock.cid, blocks } + const blocks = await collect(readable) + // @ts-expect-error There is always a root block + return { cid: blocks.at(-1).cid, blocks } } /** @@ -49,10 +47,13 @@ class UnixFsFileBuilder { /** @param {import('@ipld/unixfs').View} writer */ async finalize(writer) { const unixfsFileWriter = UnixFS.createFileWriter(writer) - const stream = toIterable(this.#file.stream()) - for await (const chunk of stream) { - await unixfsFileWriter.write(chunk) - } + await this.#file.stream().pipeTo( + new WritableStream({ + async write(chunk) { + await unixfsFileWriter.write(chunk) + }, + }) + ) return await unixfsFileWriter.close() } } @@ -78,10 +79,9 @@ class UnixFSDirectoryBuilder { */ export async function encodeDirectory(files) { const readable = createDirectoryEncoderStream(files) - const blocks = await collect(toIterable(readable)) - const rootBlock = blocks.at(-1) - if (rootBlock == null) throw new Error('missing root block') - return { cid: rootBlock.cid, blocks } + const blocks = await collect(readable) + // @ts-expect-error There is always a root block + return { cid: blocks.at(-1).cid, blocks } } /** @@ -124,3 +124,21 @@ export function createDirectoryEncoderStream(files) { return readable } + +/** + * @template T + * @param {ReadableStream} collectable + * @returns {Promise} + */ +async function collect(collectable) { + /** @type {T[]} */ + const chunks = [] + await collectable.pipeTo( + new WritableStream({ + write(chunk) { + chunks.push(chunk) + }, + }) + ) + return chunks +} diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js index 3ae480f16..b521f2ec9 100644 --- a/packages/upload-client/src/upload.js +++ b/packages/upload-client/src/upload.js @@ -26,28 +26,31 @@ import { REQUEST_RETRIES } from './constants.js' */ export async function add({ issuer, proofs }, root, shards, options = {}) { const capability = findCapability(proofs, UploadCapabilities.add.can) + /* c8 ignore next */ const conn = options.connection ?? connection - await retry( + const result = await retry( async () => { - const result = await UploadCapabilities.add + return await UploadCapabilities.add .invoke({ issuer, audience: serviceDID, // @ts-expect-error expects did:${string} but cap with is ${string}:${string} with: capability.with, - nb: { - root, - shards, - }, + nb: { root, shards }, }) .execute(conn) - if (result?.error === true) throw result }, { onFailedAttempt: console.warn, retries: options.retries ?? REQUEST_RETRIES, } ) + + if (result?.error) { + throw new Error(`failed ${UploadCapabilities.add.can} invocation`, { + cause: result, + }) + } } /** @@ -67,6 +70,7 @@ export async function add({ issuer, proofs }, root, shards, options = {}) { */ export async function list({ issuer, proofs }, options = {}) { const capability = findCapability(proofs, UploadCapabilities.list.can) + /* c8 ignore next */ const conn = options.connection ?? connection const result = await UploadCapabilities.list @@ -77,7 +81,12 @@ export async function list({ issuer, proofs }, options = {}) { with: capability.with, }) .execute(conn) - if (result.error === true) throw result + + if (result.error) { + throw new Error(`failed ${UploadCapabilities.list.can} invocation`, { + cause: result, + }) + } return result } @@ -100,6 +109,7 @@ export async function list({ issuer, proofs }, options = {}) { */ export async function remove({ issuer, proofs }, root, options = {}) { const capability = findCapability(proofs, UploadCapabilities.remove.can) + /* c8 ignore next */ const conn = options.connection ?? connection const result = await UploadCapabilities.remove @@ -111,5 +121,10 @@ export async function remove({ issuer, proofs }, root, options = {}) { nb: { root }, }) .execute(conn) - if (result?.error === true) throw result + + if (result?.error) { + throw new Error(`failed ${UploadCapabilities.remove.can} invocation`, { + cause: result, + }) + } } diff --git a/packages/upload-client/src/utils.js b/packages/upload-client/src/utils.js index 387ae7b85..d4c0a82dd 100644 --- a/packages/upload-client/src/utils.js +++ b/packages/upload-client/src/utils.js @@ -1,45 +1,5 @@ import { isDelegation } from '@ucanto/core' -/** - * @template T - * @param {ReadableStream | NodeJS.ReadableStream} readable - * @returns {AsyncIterable} - */ -export function toIterable(readable) { - // @ts-expect-error - if (readable[Symbol.asyncIterator] != null) return readable - - // Browser ReadableStream - if ('getReader' in readable) { - return (async function* () { - const reader = readable.getReader() - - try { - while (true) { - const { done, value } = await reader.read() - if (done) return - yield value - } - } finally { - reader.releaseLock() - } - })() - } - - throw new Error('unknown stream') -} - -/** - * @template T - * @param {AsyncIterable|Iterable} collectable - * @returns {Promise} - */ -export async function collect(collectable) { - const chunks = [] - for await (const chunk of collectable) chunks.push(chunk) - return chunks -} - /** * @param {import('@ucanto/interface').Proof[]} proofs * @param {import('@ucanto/interface').Ability} ability @@ -57,7 +17,9 @@ export function findCapability(proofs, ability, audience) { } if (!capability) { throw new Error( - `Missing proof of delegated capability "${ability}" for audience "${audience}"` + `Missing proof of delegated capability "${ability}"${ + audience ? ` for audience "${audience}"` : '' + }` ) } return capability diff --git a/packages/upload-client/test/car.test.js b/packages/upload-client/test/car.test.js new file mode 100644 index 000000000..369b6a542 --- /dev/null +++ b/packages/upload-client/test/car.test.js @@ -0,0 +1,16 @@ +import assert from 'assert' +import { CID } from 'multiformats' +import { encode } from '../src/car.js' + +describe('CAR.encode', () => { + it('propagates error when source throws', async () => { + // eslint-disable-next-line require-yield + const blocks = (async function* () { + throw new Error('boom') + })() + const root = CID.parse( + 'bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy' + ) + await assert.rejects(encode(blocks, root), { message: 'boom' }) + }) +}) diff --git a/packages/upload-client/test/helpers/ok-server.js b/packages/upload-client/test/helpers/bucket-server.js similarity index 62% rename from packages/upload-client/test/helpers/ok-server.js rename to packages/upload-client/test/helpers/bucket-server.js index ba07494b5..9db6842fd 100644 --- a/packages/upload-client/test/helpers/ok-server.js +++ b/packages/upload-client/test/helpers/bucket-server.js @@ -1,11 +1,14 @@ import { createServer } from 'http' const port = process.env.PORT ?? 9000 +const status = process.env.STATUS ? parseInt(process.env.STATUS) : 200 -const server = createServer((_, res) => { +const server = createServer((req, res) => { res.setHeader('Access-Control-Allow-Origin', '*') res.setHeader('Access-Control-Allow-Methods', '*') res.setHeader('Access-Control-Allow-Headers', '*') + if (req.method === 'OPTIONS') return res.end() + res.statusCode = status res.end() }) diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index f54422025..52430eafc 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -17,7 +17,7 @@ describe('uploadFile', () => { const res = { status: 'upload', headers: { 'x-test': 'true' }, - url: 'http://localhost:9000', + url: 'http://localhost:9200', } const account = await Signer.generate() @@ -90,7 +90,7 @@ describe('uploadDirectory', () => { const res = { status: 'upload', headers: { 'x-test': 'true' }, - url: 'http://localhost:9000', + url: 'http://localhost:9200', } const account = await Signer.generate() diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index e1e3d739a..3a292ec21 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -43,7 +43,7 @@ describe('ShardStoringStream', () => { const res = { status: 'upload', headers: { 'x-test': 'true' }, - url: 'http://localhost:9000', + url: 'http://localhost:9200', } const account = await Signer.generate() @@ -108,4 +108,51 @@ describe('ShardStoringStream', () => { assert.equal(cid.toString(), carCIDs[i].toString()) ) }) + + it('aborts on service failure', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + const cars = await Promise.all([randomCAR(128), randomCAR(128)]) + + const proofs = [ + await storeAdd.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + store: { + add() { + throw new Server.Failure('boom') + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + let pulls = 0 + const carStream = new ReadableStream({ + pull(controller) { + if (pulls >= cars.length) return controller.close() + controller.enqueue(cars[pulls]) + pulls++ + }, + }) + + await assert.rejects( + carStream + .pipeThrough(new ShardStoringStream({ issuer, proofs }, { connection })) + .pipeTo(new WritableStream()), + { message: 'failed store/add invocation' } + ) + }) }) diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index 25fa9a7d2..8ebd74d22 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -10,12 +10,12 @@ import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' -describe('Storage', () => { +describe('Store.add', () => { it('stores a DAG with the service', async () => { const res = { status: 'upload', headers: { 'x-test': 'true' }, - url: 'http://localhost:9000', + url: 'http://localhost:9200', } const account = await Signer.generate() @@ -58,6 +58,76 @@ describe('Storage', () => { assert.equal(carCID.toString(), car.cid.toString()) }) + it('throws for bucket URL client error 4xx', async () => { + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9400', + } + + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await StoreCapabilities.add.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ store: { add: () => res } }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + assert.rejects(Store.add({ issuer, proofs }, car, { connection }), { + message: 'upload failed: 400', + }) + }) + + it('throws for bucket URL server error 5xx', async () => { + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9500', + } + + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await StoreCapabilities.add.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ store: { add: () => res } }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + assert.rejects(Store.add({ issuer, proofs }, car, { connection }), { + message: 'upload failed: 500', + }) + }) + it('skips sending CAR if status = done', async () => { const res = { status: 'done', @@ -134,7 +204,9 @@ describe('Storage', () => { { name: 'Error', message: 'upload aborted' } ) }) +}) +describe('Store.list', () => { it('lists stored CAR files', async () => { const car = await randomCAR(128) const res = { @@ -200,6 +272,42 @@ describe('Storage', () => { }) }) + it('throws on service error', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + + const proofs = [ + await StoreCapabilities.list.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + store: { + list: () => { + throw new Server.Failure('boom') + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await assert.rejects(Store.list({ issuer, proofs }, { connection }), { + message: 'failed store/list invocation', + }) + }) +}) + +describe('Store.remove', () => { it('removes a stored CAR file', async () => { const account = await Signer.generate() const issuer = await Signer.generate() @@ -238,4 +346,40 @@ describe('Storage', () => { await Store.remove({ issuer, proofs }, car.cid, { connection }) }) + + it('throws on service error', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await StoreCapabilities.remove.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + store: { + remove: () => { + throw new Server.Failure('boom') + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await assert.rejects( + Store.remove({ issuer, proofs }, car.cid, { connection }), + { message: 'failed store/remove invocation' } + ) + }) }) diff --git a/packages/upload-client/test/unixfs.test.js b/packages/upload-client/test/unixfs.test.js index a3a5b5b33..7f197c9eb 100644 --- a/packages/upload-client/test/unixfs.test.js +++ b/packages/upload-client/test/unixfs.test.js @@ -4,7 +4,6 @@ import { MemoryBlockstore } from 'blockstore-core/memory' import * as raw from 'multiformats/codecs/raw' import path from 'path' import { encodeFile, encodeDirectory } from '../src/unixfs.js' -import { collect } from '../src/utils.js' import { File } from './helpers/shims.js' /** @param {import('ipfs-unixfs-exporter').UnixFSDirectory} dir */ @@ -37,7 +36,9 @@ describe('UnixFS', () => { const { cid, blocks } = await encodeFile(file) const blockstore = await blocksToBlockstore(blocks) const entry = await exporter(cid.toString(), blockstore) - const out = new Blob(await collect(entry.content())) + const chunks = [] + for await (const chunk of entry.content()) chunks.push(chunk) + const out = new Blob(chunks) assert.equal(await out.text(), await file.text()) }) @@ -66,6 +67,15 @@ describe('UnixFS', () => { expectedPaths.forEach((p) => assert(actualPaths.includes(p))) }) + it('throws then treating a file as a directory', () => + assert.rejects( + encodeDirectory([ + new File(['a file, not a directory'], 'file.txt'), + new File(['a file in a file!!!'], 'file.txt/another.txt'), + ]), + { message: '"file.txt" cannot be a file and a directory' } + )) + it('configured to use raw leaves', async () => { const file = new Blob(['test']) const { cid } = await encodeFile(file) diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index 887e6779c..96ce60667 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -10,7 +10,7 @@ import { service as id } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' -describe('Upload', () => { +describe('Upload.add', () => { it('registers an upload with the service', async () => { const account = await Signer.generate() const issuer = await Signer.generate() @@ -53,6 +53,44 @@ describe('Upload', () => { await Upload.add({ issuer, proofs }, root, [car.cid], { connection }) }) + it('throws on service error', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await UploadCapabilities.add.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + upload: { + add: () => { + throw new Server.Failure('boom') + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await assert.rejects( + Upload.add({ issuer, proofs }, car.roots[0], [car.cid], { connection }), + { message: 'failed upload/add invocation' } + ) + }) +}) + +describe('Upload.list', () => { it('lists uploads', async () => { const car = await randomCAR(128) const res = { @@ -115,6 +153,42 @@ describe('Upload', () => { }) }) + it('throws on service error', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + + const proofs = [ + await UploadCapabilities.list.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + upload: { + list: () => { + throw new Server.Failure('boom') + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await assert.rejects(Upload.list({ issuer, proofs }, { connection }), { + message: 'failed upload/list invocation', + }) + }) +}) + +describe('Upload.remove', () => { it('removes an upload', async () => { const account = await Signer.generate() const issuer = await Signer.generate() @@ -153,4 +227,40 @@ describe('Upload', () => { await Upload.remove({ issuer, proofs }, car.roots[0], { connection }) }) + + it('throws on service error', async () => { + const account = await Signer.generate() + const issuer = await Signer.generate() + const car = await randomCAR(128) + + const proofs = [ + await UploadCapabilities.remove.delegate({ + issuer: account, + audience: id, + with: account.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + upload: { + remove: () => { + throw new Server.Failure('boom') + }, + }, + }) + + const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const connection = Client.connect({ + id, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + await assert.rejects( + Upload.remove({ issuer, proofs }, car.roots[0], { connection }), + { message: 'failed upload/remove invocation' } + ) + }) }) diff --git a/packages/upload-client/test/utils.test.js b/packages/upload-client/test/utils.test.js new file mode 100644 index 000000000..486ede123 --- /dev/null +++ b/packages/upload-client/test/utils.test.js @@ -0,0 +1,59 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as StoreCapabilities from '@web3-storage/access/capabilities/store' +import { service as id } from './fixtures.js' +import { findCapability } from '../src/utils.js' + +describe('findCapability', () => { + it('throws when capability is not found', () => { + assert.throws(() => findCapability([], 'store/add'), { + message: 'Missing proof of delegated capability "store/add"', + }) + }) + + it('throws for mismatched audience', async () => { + const issuer = await Signer.generate() + const proofs = [ + await StoreCapabilities.add.delegate({ + issuer, + audience: id, + with: issuer.did(), + expiration: Infinity, + }), + ] + + assert.throws(() => findCapability(proofs, 'store/add', issuer.did()), { + message: `Missing proof of delegated capability "store/add" for audience "${issuer.did()}"`, + }) + }) + + it('matches wildcard capability', async () => { + const issuer = await Signer.generate() + const proofs = [ + await StoreCapabilities.store.delegate({ + issuer, + audience: id, + with: issuer.did(), + expiration: Infinity, + }), + ] + + const cap = findCapability(proofs, 'store/add') + assert.equal(cap.can, 'store/*') + }) + + it('ignores non-delegation proofs', async () => { + const issuer = await Signer.generate() + const delegation = await StoreCapabilities.store.delegate({ + issuer, + audience: id, + with: issuer.did(), + expiration: Infinity, + }) + const proofs = [delegation.cid] + + assert.throws(() => findCapability(proofs, 'store/add'), { + message: 'Missing proof of delegated capability "store/add"', + }) + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cab3a4955..d26bb9719 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -320,6 +320,7 @@ importers: blockstore-core: ^2.0.2 c8: ^7.12.0 hd-scripts: ^3.0.2 + hundreds: ^0.0.9 ipfs-unixfs-exporter: ^9.0.1 mocha: ^10.1.0 multiformats: ^10.0.2 @@ -350,6 +351,7 @@ importers: blockstore-core: 2.0.2 c8: 7.12.0 hd-scripts: 3.0.2 + hundreds: 0.0.9 ipfs-unixfs-exporter: 9.0.1 mocha: 10.1.0 npm-run-all: 4.1.5 @@ -4522,6 +4524,13 @@ packages: engines: {node: '>=12.20.0'} dev: true + /hundreds/0.0.9: + resolution: {integrity: sha512-4pHIJQl4SiVeMeg00w6WypEjVrSgVUxQS8YlOWW3KehCdlz/PoEhKlJY2DYeR56lPoF5KA+YjHKgnwhCsKJ7IQ==} + hasBin: true + dependencies: + c8: 7.12.0 + dev: true + /iconv-lite/0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} From b541b959e3d5e63343ee4cac60212d9a7a3bbbf2 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 13:42:25 +0000 Subject: [PATCH 32/47] docs: move top level API to the top of the API listing --- packages/upload-client/README.md | 100 +++++++++++++++---------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index bc9525c5d..0f1d9737c 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -15,7 +15,7 @@ npm install @web3-storage/upload-client ### Create an Agent -An Agent provides an `issuer` (a key linked to your account) and `proofs` to show your `issuer` has been delegated the capabilities to store data and register uploads. +An Agent provides an `issuer` (a key linked to your account) and `proofs` to show your `issuer` has been delegated the capabilities to store data and register uploads. ```js import { Agent } from '@web3-storage/access-client' @@ -35,7 +35,7 @@ const conf = { ### Uploading files -Once you have the `issuer` and `proofs`, you can upload a directory of files by passing that invocation config to `uploadDirectory` along with your list of files to upload. +Once you have the `issuer` and `proofs`, you can upload a directory of files by passing that invocation config to `uploadDirectory` along with your list of files to upload. You can get your list of Files from a [``](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file) element in the browser or using [`files-from-path`](https://npm.im/files-from-path) in Node.js @@ -113,6 +113,8 @@ await Upload.add(conf, rootCID, carCIDs) ## API +- [`uploadDirectory`](#uploaddirectory) +- [`uploadFile`](#uploadfile) - `CAR` - [`encode`](#carencode) - [`ShardingStream`](#shardingstream) @@ -130,11 +132,55 @@ await Upload.add(conf, rootCID, carCIDs) - [`add`](#uploadadd) - [`list`](#uploadlist) - [`remove`](#uploadremove) -- [`uploadDirectory`](#uploaddirectory) -- [`uploadFile`](#uploadfile) --- +### `uploadDirectory` + +```ts +function uploadDirectory( + conf: InvocationConfig, + files: File[], + options: { + retries?: number + signal?: AbortSignal + onShardStored: ShardStoredCallback + } = {} +): Promise +``` + +Uploads a directory of files to the service and returns the root data CID for the generated DAG. All files are added to a container directory, with paths in file names preserved. + +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `store/add`, `upload/add` + +### `uploadFile` + +```ts +function uploadFile( + conf: InvocationConfig, + file: Blob, + options: { + retries?: number + signal?: AbortSignal + onShardStored: ShardStoredCallback + } = {} +): Promise +``` + +Uploads a file to the service and returns the root data CID for the generated DAG. + +Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. + +Required delegated capability proofs: `store/add`, `upload/add` + ### `CAR.encode` ```ts @@ -354,52 +400,6 @@ Note: `InvocationConfig` is configuration for the UCAN invocation. It's values c Required delegated capability proofs: `upload/remove` -### `uploadDirectory` - -```ts -function uploadDirectory( - conf: InvocationConfig, - files: File[], - options: { - retries?: number - signal?: AbortSignal - onShardStored: ShardStoredCallback - } = {} -): Promise -``` - -Uploads a directory of files to the service and returns the root data CID for the generated DAG. All files are added to a container directory, with paths in file names preserved. - -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - -Required delegated capability proofs: `store/add`, `upload/add` - -### `uploadFile` - -```ts -function uploadFile( - conf: InvocationConfig, - file: Blob, - options: { - retries?: number - signal?: AbortSignal - onShardStored: ShardStoredCallback - } = {} -): Promise -``` - -Uploads a file to the service and returns the root data CID for the generated DAG. - -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - -Required delegated capability proofs: `store/add`, `upload/add` - ## Contributing Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! From 69dc7ae9c2b4fa27abf763017aa0e929252a9fcf Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Nov 2022 13:49:58 +0000 Subject: [PATCH 33/47] docs: more DRY --- packages/upload-client/README.md | 81 +++++++++++++------------------- 1 file changed, 33 insertions(+), 48 deletions(-) diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 0f1d9737c..945bcb0f0 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -151,13 +151,10 @@ function uploadDirectory( Uploads a directory of files to the service and returns the root data CID for the generated DAG. All files are added to a container directory, with paths in file names preserved. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - Required delegated capability proofs: `store/add`, `upload/add` +More information: [`InvocationConfig`](#invocationconfig) + ### `uploadFile` ```ts @@ -174,13 +171,10 @@ function uploadFile( Uploads a file to the service and returns the root data CID for the generated DAG. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - Required delegated capability proofs: `store/add`, `upload/add` +More information: [`InvocationConfig`](#invocationconfig) + ### `CAR.encode` ```ts @@ -189,11 +183,7 @@ function encode(blocks: Iterable, root?: CID): Promise Encode a DAG as a CAR file. -Note: `CARFile` is just a `Blob` with two extra properties: - -```ts -type CARFile = Blob & { version: 1; roots: CID[] } -``` +More information: [`CARFile`](#carfile) Example: @@ -210,11 +200,7 @@ class ShardingStream extends TransformStream Shard a set of blocks into a set of CAR files. The last block written to the stream is assumed to be the DAG root and becomes the CAR root CID for the last CAR output. -Note: `CARFile` is just a `Blob` with two extra properties: - -```ts -type CARFile = Blob & { version: 1; roots: CID[] } -``` +More information: [`CARFile`](#carfile) ### `ShardStoringStream` @@ -240,13 +226,10 @@ function add( Store a CAR file to the service. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - Required delegated capability proofs: `store/add` +More information: [`InvocationConfig`](#invocationconfig) + ### `Store.list` ```ts @@ -258,13 +241,10 @@ function list( List CAR files stored by the issuer. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - Required delegated capability proofs: `store/list` +More information: [`InvocationConfig`](#invocationconfig) + ### `Store.remove` ```ts @@ -277,13 +257,10 @@ function remove( Remove a stored CAR file by CAR CID. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - Required delegated capability proofs: `store/remove` +More information: [`InvocationConfig`](#invocationconfig) + ### `UnixFS.createDirectoryEncoderStream` ```ts @@ -356,13 +333,10 @@ function add( Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - Required delegated capability proofs: `upload/add` +More information: [`InvocationConfig`](#invocationconfig) + ### `Upload.list` ```ts @@ -374,13 +348,10 @@ function list( List uploads created by the issuer. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. - Required delegated capability proofs: `upload/list` +More information: [`InvocationConfig`](#invocationconfig) + ### `Upload.remove` ```ts @@ -393,13 +364,27 @@ function remove( Remove a upload by root data CID. -Note: `InvocationConfig` is configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Step 0](#step-0) for an example. It is an object with `issuer` and `proofs`: +Required delegated capability proofs: `upload/remove` + +More information: [`InvocationConfig`](#invocationconfig) + +## Types + +### `CARFile` + +A `Blob` with two extra properties: + +```ts +type CARFile = Blob & { version: 1; roots: CID[] } +``` + +### `InvocationConfig` + +This is the configuration for the UCAN invocation. It's values can be obtained from an `Agent`. See [Create an Agent](#create-an-agent) for an example. It is an object with `issuer` and `proofs`: - The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. - The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. -Required delegated capability proofs: `upload/remove` - ## Contributing Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! From 39045e6b742d8e33c5f56946e73f4e1314840cc6 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 12:15:16 +0000 Subject: [PATCH 34/47] fix: send CAR size in bytes --- packages/upload-client/src/store.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/src/store.js b/packages/upload-client/src/store.js index 7592d6eeb..d6b2f2253 100644 --- a/packages/upload-client/src/store.js +++ b/packages/upload-client/src/store.js @@ -40,7 +40,7 @@ export async function add({ issuer, proofs }, car, options = {}) { audience: serviceDID, // @ts-expect-error expects did:${string} but cap with is ${string}:${string} with: capability.with, - nb: { link }, + nb: { link, size: car.size }, proofs, }) .execute(conn) From 374b33fa48ac64ccf779875ab3a44185d25bc5f8 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 13:01:09 +0000 Subject: [PATCH 35/47] refactor: use longer name for clarity --- packages/upload-client/test/fixtures.js | 2 +- packages/upload-client/test/index.test.js | 28 +++-- packages/upload-client/test/sharding.test.js | 24 +++-- packages/upload-client/test/store.test.js | 101 ++++++++++++++----- packages/upload-client/test/upload.test.js | 68 +++++++++---- packages/upload-client/test/utils.test.js | 8 +- 6 files changed, 163 insertions(+), 68 deletions(-) diff --git a/packages/upload-client/test/fixtures.js b/packages/upload-client/test/fixtures.js index 03ee05d2f..50a464a0a 100644 --- a/packages/upload-client/test/fixtures.js +++ b/packages/upload-client/test/fixtures.js @@ -1,6 +1,6 @@ import * as ed25519 from '@ucanto/principal/ed25519' /** did:key:z6MkrZ1r5XBFZjBU34qyD8fueMbMRkKw17BZaq2ivKFjnz2z */ -export const service = ed25519.parse( +export const serviceSigner = ed25519.parse( 'MgCYKXoHVy7Vk4/QjcEGi+MCqjntUiasxXJ8uJKY0qh11e+0Bs8WsdqGK7xothgrDzzWD0ME7ynPjz2okXDh8537lId8=' ) diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 52430eafc..007a52449 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -7,7 +7,7 @@ import * as Signer from '@ucanto/principal/ed25519' import { add as storeAdd } from '@web3-storage/access/capabilities/store' import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' import { uploadFile, uploadDirectory } from '../src/index.js' -import { service as id } from './fixtures.js' +import { serviceSigner } from './fixtures.js' import { randomBytes } from './helpers/random.js' import { File } from './helpers/shims.js' import { mockService } from './helpers/mocks.js' @@ -29,13 +29,13 @@ describe('uploadFile', () => { const proofs = await Promise.all([ storeAdd.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), uploadAdd.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -66,9 +66,14 @@ describe('uploadFile', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -105,13 +110,13 @@ describe('uploadDirectory', () => { const proofs = await Promise.all([ storeAdd.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), uploadAdd.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -142,9 +147,14 @@ describe('uploadDirectory', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index 3a292ec21..16ca3aecd 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -7,7 +7,7 @@ import * as Signer from '@ucanto/principal/ed25519' import { add as storeAdd } from '@web3-storage/access/capabilities/store' import { createFileEncoderStream } from '../src/unixfs.js' import { ShardingStream, ShardStoringStream } from '../src/sharding.js' -import { service as id } from './fixtures.js' +import { serviceSigner } from './fixtures.js' import { randomBytes, randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -54,7 +54,7 @@ describe('ShardStoringStream', () => { const proofs = [ await storeAdd.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -75,9 +75,14 @@ describe('ShardStoringStream', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -117,7 +122,7 @@ describe('ShardStoringStream', () => { const proofs = [ await storeAdd.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -131,9 +136,14 @@ describe('ShardStoringStream', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index 8ebd74d22..01188e8d2 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -6,7 +6,7 @@ import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' import * as StoreCapabilities from '@web3-storage/access/capabilities/store' import * as Store from '../src/store.js' -import { service as id } from './fixtures.js' +import { serviceSigner } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -25,7 +25,7 @@ describe('Store.add', () => { const proofs = [ await StoreCapabilities.add.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -45,9 +45,14 @@ describe('Store.add', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -72,7 +77,7 @@ describe('Store.add', () => { const proofs = [ await StoreCapabilities.add.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -80,9 +85,14 @@ describe('Store.add', () => { const service = mockService({ store: { add: () => res } }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -107,7 +117,7 @@ describe('Store.add', () => { const proofs = [ await StoreCapabilities.add.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -115,9 +125,14 @@ describe('Store.add', () => { const service = mockService({ store: { add: () => res } }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -142,7 +157,7 @@ describe('Store.add', () => { const proofs = [ await StoreCapabilities.add.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -150,9 +165,14 @@ describe('Store.add', () => { const service = mockService({ store: { add: () => res } }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -172,9 +192,14 @@ describe('Store.add', () => { const service = mockService({ store: { add: () => res } }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -187,7 +212,7 @@ describe('Store.add', () => { const proofs = [ await StoreCapabilities.add.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -228,7 +253,7 @@ describe('Store.list', () => { const proofs = [ await StoreCapabilities.list.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -247,9 +272,14 @@ describe('Store.list', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -279,7 +309,7 @@ describe('Store.list', () => { const proofs = [ await StoreCapabilities.list.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -293,9 +323,14 @@ describe('Store.list', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -316,7 +351,7 @@ describe('Store.remove', () => { const proofs = [ await StoreCapabilities.remove.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -336,9 +371,14 @@ describe('Store.remove', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -355,7 +395,7 @@ describe('Store.remove', () => { const proofs = [ await StoreCapabilities.remove.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -369,9 +409,14 @@ describe('Store.remove', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index 96ce60667..c555342ef 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -6,7 +6,7 @@ import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' import * as UploadCapabilities from '@web3-storage/access/capabilities/upload' import * as Upload from '../src/upload.js' -import { service as id } from './fixtures.js' +import { serviceSigner } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -19,7 +19,7 @@ describe('Upload.add', () => { const proofs = [ await UploadCapabilities.add.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -41,9 +41,14 @@ describe('Upload.add', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -61,7 +66,7 @@ describe('Upload.add', () => { const proofs = [ await UploadCapabilities.add.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -75,9 +80,14 @@ describe('Upload.add', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -112,7 +122,7 @@ describe('Upload.list', () => { const proofs = [ await UploadCapabilities.list.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -131,9 +141,14 @@ describe('Upload.list', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -160,7 +175,7 @@ describe('Upload.list', () => { const proofs = [ await UploadCapabilities.list.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -174,9 +189,14 @@ describe('Upload.list', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -197,7 +217,7 @@ describe('Upload.remove', () => { const proofs = [ await UploadCapabilities.remove.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -217,9 +237,14 @@ describe('Upload.remove', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, @@ -236,7 +261,7 @@ describe('Upload.remove', () => { const proofs = [ await UploadCapabilities.remove.delegate({ issuer: account, - audience: id, + audience: serviceSigner, with: account.did(), expiration: Infinity, }), @@ -250,9 +275,14 @@ describe('Upload.remove', () => { }, }) - const server = Server.create({ id, service, decoder: CAR, encoder: CBOR }) + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) const connection = Client.connect({ - id, + id: serviceSigner, encoder: CAR, decoder: CBOR, channel: server, diff --git a/packages/upload-client/test/utils.test.js b/packages/upload-client/test/utils.test.js index 486ede123..31ef8228f 100644 --- a/packages/upload-client/test/utils.test.js +++ b/packages/upload-client/test/utils.test.js @@ -1,7 +1,7 @@ import assert from 'assert' import * as Signer from '@ucanto/principal/ed25519' import * as StoreCapabilities from '@web3-storage/access/capabilities/store' -import { service as id } from './fixtures.js' +import { serviceSigner } from './fixtures.js' import { findCapability } from '../src/utils.js' describe('findCapability', () => { @@ -16,7 +16,7 @@ describe('findCapability', () => { const proofs = [ await StoreCapabilities.add.delegate({ issuer, - audience: id, + audience: serviceSigner, with: issuer.did(), expiration: Infinity, }), @@ -32,7 +32,7 @@ describe('findCapability', () => { const proofs = [ await StoreCapabilities.store.delegate({ issuer, - audience: id, + audience: serviceSigner, with: issuer.did(), expiration: Infinity, }), @@ -46,7 +46,7 @@ describe('findCapability', () => { const issuer = await Signer.generate() const delegation = await StoreCapabilities.store.delegate({ issuer, - audience: id, + audience: serviceSigner, with: issuer.did(), expiration: Infinity, }) From 5daf40766a90d3e76338ab084349002b44f93120 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 13:03:59 +0000 Subject: [PATCH 36/47] Update packages/upload-client/test/store.test.js Co-authored-by: Oli Evans --- packages/upload-client/test/store.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index 01188e8d2..18b678759 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -67,7 +67,7 @@ describe('Store.add', () => { const res = { status: 'upload', headers: { 'x-test': 'true' }, - url: 'http://localhost:9400', + url: 'http://localhost:9400', // this bucket always returns a 400 } const account = await Signer.generate() From 6b210406d45f71c26581c60008c2c13c89791e49 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 13:04:07 +0000 Subject: [PATCH 37/47] Update packages/upload-client/test/store.test.js Co-authored-by: Oli Evans --- packages/upload-client/test/store.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index 18b678759..510b533d7 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -107,7 +107,7 @@ describe('Store.add', () => { const res = { status: 'upload', headers: { 'x-test': 'true' }, - url: 'http://localhost:9500', + url: 'http://localhost:9500', // this bucket always returns a 500 } const account = await Signer.generate() From 56931c00abc9278189f30000029f140a61fdd3b7 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 13:49:37 +0000 Subject: [PATCH 38/47] test: assert that service functions were called --- packages/upload-client/test/helpers/mocks.js | 29 +++++++++++++++----- packages/upload-client/test/index.test.js | 26 ++++++++++++------ packages/upload-client/test/sharding.test.js | 3 ++ packages/upload-client/test/store.test.js | 14 ++++++++++ packages/upload-client/test/upload.test.js | 9 ++++++ 5 files changed, 66 insertions(+), 15 deletions(-) diff --git a/packages/upload-client/test/helpers/mocks.js b/packages/upload-client/test/helpers/mocks.js index 9a5e53b51..4d1386aae 100644 --- a/packages/upload-client/test/helpers/mocks.js +++ b/packages/upload-client/test/helpers/mocks.js @@ -9,19 +9,34 @@ const notImplemented = () => { * store: Partial * upload: Partial * }>} impl - * @returns {import('../../src/types').Service} */ export function mockService(impl) { return { store: { - add: impl.store?.add ?? notImplemented, - list: impl.store?.list ?? notImplemented, - remove: impl.store?.remove ?? notImplemented, + add: withCallCount(impl.store?.add ?? notImplemented), + list: withCallCount(impl.store?.list ?? notImplemented), + remove: withCallCount(impl.store?.remove ?? notImplemented), }, upload: { - add: impl.upload?.add ?? notImplemented, - list: impl.upload?.list ?? notImplemented, - remove: impl.upload?.remove ?? notImplemented, + add: withCallCount(impl.upload?.add ?? notImplemented), + list: withCallCount(impl.upload?.list ?? notImplemented), + remove: withCallCount(impl.upload?.remove ?? notImplemented), }, } } + +/** + * @template {Function} T + * @param {T} fn + */ +function withCallCount(fn) { + /** @param {T extends (...args: infer A) => any ? A : never} args */ + const countedFn = (...args) => { + countedFn.called = true + countedFn.callCount++ + return fn(...args) + } + countedFn.called = false + countedFn.callCount = 0 + return countedFn +} diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 007a52449..63235fbe0 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -4,8 +4,8 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as CBOR from '@ucanto/transport/cbor' import * as Signer from '@ucanto/principal/ed25519' -import { add as storeAdd } from '@web3-storage/access/capabilities/store' -import { add as uploadAdd } from '@web3-storage/access/capabilities/upload' +import * as StoreCapabilities from '@web3-storage/access/capabilities/store' +import * as UploadCapabilities from '@web3-storage/access/capabilities/upload' import { uploadFile, uploadDirectory } from '../src/index.js' import { serviceSigner } from './fixtures.js' import { randomBytes } from './helpers/random.js' @@ -27,13 +27,13 @@ describe('uploadFile', () => { let carCID const proofs = await Promise.all([ - storeAdd.delegate({ + StoreCapabilities.add.delegate({ issuer: account, audience: serviceSigner, with: account.did(), expiration: Infinity, }), - uploadAdd.delegate({ + UploadCapabilities.add.delegate({ issuer: account, audience: serviceSigner, with: account.did(), @@ -47,7 +47,7 @@ describe('uploadFile', () => { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] - assert.equal(invCap.can, 'store/add') + assert.equal(invCap.can, StoreCapabilities.add.can) assert.equal(invCap.with, account.did()) return res }, @@ -57,7 +57,7 @@ describe('uploadFile', () => { assert.equal(invocation.issuer.did(), issuer.did()) assert.equal(invocation.capabilities.length, 1) const invCap = invocation.capabilities[0] - assert.equal(invCap.can, 'upload/add') + assert.equal(invCap.can, UploadCapabilities.add.can) assert.equal(invCap.with, account.did()) assert.equal(invCap.nb.shards?.length, 1) assert.equal(String(invCap.nb.shards?.[0]), carCID?.toString()) @@ -85,6 +85,11 @@ describe('uploadFile', () => { }, }) + assert(service.store.add.called) + assert.equal(service.store.add.callCount, 1) + assert(service.upload.add.called) + assert.equal(service.upload.add.callCount, 1) + assert(carCID) assert(dataCID) }) @@ -108,13 +113,13 @@ describe('uploadDirectory', () => { let carCID = null const proofs = await Promise.all([ - storeAdd.delegate({ + StoreCapabilities.add.delegate({ issuer: account, audience: serviceSigner, with: account.did(), expiration: Infinity, }), - uploadAdd.delegate({ + UploadCapabilities.add.delegate({ issuer: account, audience: serviceSigner, with: account.did(), @@ -166,6 +171,11 @@ describe('uploadDirectory', () => { }, }) + assert(service.store.add.called) + assert.equal(service.store.add.callCount, 1) + assert(service.upload.add.called) + assert.equal(service.upload.add.callCount, 1) + assert(carCID) assert(dataCID) }) diff --git a/packages/upload-client/test/sharding.test.js b/packages/upload-client/test/sharding.test.js index 16ca3aecd..106412ae2 100644 --- a/packages/upload-client/test/sharding.test.js +++ b/packages/upload-client/test/sharding.test.js @@ -112,6 +112,9 @@ describe('ShardStoringStream', () => { cars.forEach(({ cid }, i) => assert.equal(cid.toString(), carCIDs[i].toString()) ) + + assert(service.store.add.called) + assert.equal(service.store.add.callCount, 2) }) it('aborts on service failure', async () => { diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index 01188e8d2..767bcab29 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -59,6 +59,10 @@ describe('Store.add', () => { }) const carCID = await Store.add({ issuer, proofs }, car, { connection }) + + assert(service.store.add.called) + assert.equal(service.store.add.callCount, 1) + assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -179,6 +183,10 @@ describe('Store.add', () => { }) const carCID = await Store.add({ issuer, proofs }, car, { connection }) + + assert(service.store.add.called) + assert.equal(service.store.add.callCount, 1) + assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) }) @@ -287,6 +295,9 @@ describe('Store.list', () => { const list = await Store.list({ issuer, proofs }, { connection }) + assert(service.store.list.called) + assert.equal(service.store.list.callCount, 1) + assert.equal(list.count, res.count) assert.equal(list.page, res.page) assert.equal(list.pageSize, res.pageSize) @@ -385,6 +396,9 @@ describe('Store.remove', () => { }) await Store.remove({ issuer, proofs }, car.cid, { connection }) + + assert(service.store.remove.called) + assert.equal(service.store.remove.callCount, 1) }) it('throws on service error', async () => { diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index c555342ef..3727bac1e 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -56,6 +56,9 @@ describe('Upload.add', () => { const root = car.roots[0] await Upload.add({ issuer, proofs }, root, [car.cid], { connection }) + + assert(service.upload.add.called) + assert.equal(service.upload.add.callCount, 1) }) it('throws on service error', async () => { @@ -156,6 +159,9 @@ describe('Upload.list', () => { const list = await Upload.list({ issuer, proofs }, { connection }) + assert(service.upload.list.called) + assert.equal(service.upload.list.callCount, 1) + assert.equal(list.count, res.count) assert.equal(list.page, res.page) assert.equal(list.pageSize, res.pageSize) @@ -251,6 +257,9 @@ describe('Upload.remove', () => { }) await Upload.remove({ issuer, proofs }, car.roots[0], { connection }) + + assert(service.upload.remove.called) + assert.equal(service.upload.remove.callCount, 1) }) it('throws on service error', async () => { From 63673ff659d6305e25ff2dbbaf225df742345374 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 13:50:32 +0000 Subject: [PATCH 39/47] Update packages/upload-client/test/utils.test.js Co-authored-by: Oli Evans --- packages/upload-client/test/utils.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/upload-client/test/utils.test.js b/packages/upload-client/test/utils.test.js index 31ef8228f..89e290779 100644 --- a/packages/upload-client/test/utils.test.js +++ b/packages/upload-client/test/utils.test.js @@ -22,6 +22,7 @@ describe('findCapability', () => { }), ] + // we match on `audience`. Passing in the issuer or any other DID here should fail. assert.throws(() => findCapability(proofs, 'store/add', issuer.did()), { message: `Missing proof of delegated capability "store/add" for audience "${issuer.did()}"`, }) From e1339c3b722d5007ed6d898dd6fd37474333f7b4 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 13:50:42 +0000 Subject: [PATCH 40/47] Update packages/upload-client/test/index.test.js Co-authored-by: Oli Evans --- packages/upload-client/test/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 63235fbe0..ed91b9bb4 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -21,7 +21,7 @@ describe('uploadFile', () => { } const account = await Signer.generate() - const issuer = await Signer.generate() + const issuer = await Signer.generate() // The "user" that will ask the service to accept the upload const file = new Blob([await randomBytes(128)]) /** @type {import('../src/types').CARLink|undefined} */ let carCID From 8b7998b14cafb6329502de86678d5771b8310990 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 15:09:30 +0000 Subject: [PATCH 41/47] test: add test for specific matching and wildcard matching with audience --- packages/upload-client/test/utils.test.js | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/upload-client/test/utils.test.js b/packages/upload-client/test/utils.test.js index 89e290779..ba75167b6 100644 --- a/packages/upload-client/test/utils.test.js +++ b/packages/upload-client/test/utils.test.js @@ -28,6 +28,21 @@ describe('findCapability', () => { }) }) + it('matches capability', async () => { + const issuer = await Signer.generate() + const proofs = [ + await StoreCapabilities.add.delegate({ + issuer, + audience: serviceSigner, + with: issuer.did(), + expiration: Infinity, + }), + ] + + const cap = findCapability(proofs, 'store/add') + assert.equal(cap.can, 'store/add') + }) + it('matches wildcard capability', async () => { const issuer = await Signer.generate() const proofs = [ @@ -43,6 +58,21 @@ describe('findCapability', () => { assert.equal(cap.can, 'store/*') }) + it('matches wildcard capability with audience', async () => { + const issuer = await Signer.generate() + const proofs = [ + await StoreCapabilities.store.delegate({ + issuer, + audience: serviceSigner, + with: issuer.did(), + expiration: Infinity, + }), + ] + + const cap = findCapability(proofs, 'store/add', serviceSigner.did()) + assert.equal(cap.can, 'store/*') + }) + it('ignores non-delegation proofs', async () => { const issuer = await Signer.generate() const delegation = await StoreCapabilities.store.delegate({ From a693fddd55e3030f7e635cb87c4db9090e5dabb5 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 15:11:55 +0000 Subject: [PATCH 42/47] Update packages/upload-client/src/utils.js Co-authored-by: Oli Evans --- packages/upload-client/src/utils.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/upload-client/src/utils.js b/packages/upload-client/src/utils.js index d4c0a82dd..ab4bcf36d 100644 --- a/packages/upload-client/src/utils.js +++ b/packages/upload-client/src/utils.js @@ -30,7 +30,9 @@ export function findCapability(proofs, ability, audience) { * @param {import('@ucanto/interface').Ability} ability */ function capabilityMatches(can, ability) { - return can === ability - ? true - : can.endsWith('*') && ability.startsWith(can.split('*')[0]) + if (can === ability) return true + if (can === '*/*') return true + if (can === '*') return true + if (can.endsWith('*') && ability.startsWith(can.slice(0, -1)) return true + return false } From 7cde4be8fabb905d6ce0832b19777a5fdbe69e36 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 15:42:13 +0000 Subject: [PATCH 43/47] test: more findCapability tests --- packages/upload-client/package.json | 1 + packages/upload-client/src/utils.js | 2 +- packages/upload-client/test/utils.test.js | 37 +++++++++++++++++++++++ pnpm-lock.yaml | 2 ++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index fa9fea7ca..a90881f1c 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -78,6 +78,7 @@ "@types/mocha": "^10.0.0", "@ucanto/principal": "^3.0.0", "@ucanto/server": "^3.0.1", + "@ucanto/validator": "^3.0.1", "assert": "^2.0.0", "blockstore-core": "^2.0.2", "c8": "^7.12.0", diff --git a/packages/upload-client/src/utils.js b/packages/upload-client/src/utils.js index ab4bcf36d..872a3cb9e 100644 --- a/packages/upload-client/src/utils.js +++ b/packages/upload-client/src/utils.js @@ -33,6 +33,6 @@ function capabilityMatches(can, ability) { if (can === ability) return true if (can === '*/*') return true if (can === '*') return true - if (can.endsWith('*') && ability.startsWith(can.slice(0, -1)) return true + if (can.endsWith('*') && ability.startsWith(can.slice(0, -1))) return true return false } diff --git a/packages/upload-client/test/utils.test.js b/packages/upload-client/test/utils.test.js index ba75167b6..9e81ca9c2 100644 --- a/packages/upload-client/test/utils.test.js +++ b/packages/upload-client/test/utils.test.js @@ -1,6 +1,9 @@ import assert from 'assert' import * as Signer from '@ucanto/principal/ed25519' +import { capability, URI } from '@ucanto/validator' +import { any } from '@web3-storage/access/capabilities/any' import * as StoreCapabilities from '@web3-storage/access/capabilities/store' +import { equalWith } from '@web3-storage/access/capabilities/utils' import { serviceSigner } from './fixtures.js' import { findCapability } from '../src/utils.js' @@ -43,6 +46,40 @@ describe('findCapability', () => { assert.equal(cap.can, 'store/add') }) + it('matches any wildcard capability', async () => { + const issuer = await Signer.generate() + const proofs = [ + await any.delegate({ + issuer, + audience: serviceSigner, + with: issuer.did(), + expiration: Infinity, + }), + ] + + const cap = findCapability(proofs, 'store/add') + assert.equal(cap.can, '*') + }) + + it('matches top wildcard capability', async () => { + const issuer = await Signer.generate() + const proofs = [ + await capability({ + can: '*/*', + with: URI.match({ protocol: 'did:' }), + derives: equalWith, + }).delegate({ + issuer, + audience: serviceSigner, + with: issuer.did(), + expiration: Infinity, + }), + ] + + const cap = findCapability(proofs, 'store/add') + assert.equal(cap.can, '*/*') + }) + it('matches wildcard capability', async () => { const issuer = await Signer.generate() const proofs = [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d26bb9719..c9ceaceb5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -315,6 +315,7 @@ importers: '@ucanto/principal': ^3.0.0 '@ucanto/server': ^3.0.1 '@ucanto/transport': ^3.0.1 + '@ucanto/validator': ^3.0.1 '@web3-storage/access': workspace:^ assert: ^2.0.0 blockstore-core: ^2.0.2 @@ -347,6 +348,7 @@ importers: '@types/mocha': 10.0.0 '@ucanto/principal': 3.0.0 '@ucanto/server': 3.0.1 + '@ucanto/validator': 3.0.1 assert: 2.0.0 blockstore-core: 2.0.2 c8: 7.12.0 From d50e1ff692913f2b51a6122d945989c9eea1413b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 16:06:35 +0000 Subject: [PATCH 44/47] fix: use default capacity --- packages/upload-client/src/unixfs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-client/src/unixfs.js b/packages/upload-client/src/unixfs.js index 1b573718c..07e3c8f1a 100644 --- a/packages/upload-client/src/unixfs.js +++ b/packages/upload-client/src/unixfs.js @@ -1,7 +1,7 @@ import * as UnixFS from '@ipld/unixfs' import * as raw from 'multiformats/codecs/raw' -const queuingStrategy = UnixFS.withCapacity(1_048_576 * 175) +const queuingStrategy = UnixFS.withCapacity() // TODO: configure chunk size and max children https://github.com/ipld/js-unixfs/issues/36 const settings = UnixFS.configure({ From 3e0e07f3c5951afb6357bf63e70efa26c7b23291 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 16:08:59 +0000 Subject: [PATCH 45/47] refactor: default shard size --- packages/upload-client/src/sharding.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/upload-client/src/sharding.js b/packages/upload-client/src/sharding.js index 92b3bef16..155fc41c9 100644 --- a/packages/upload-client/src/sharding.js +++ b/packages/upload-client/src/sharding.js @@ -2,8 +2,7 @@ import Queue from 'p-queue' import { encode } from './car.js' import { add } from './store.js' -// most thing are < 30MB -const SHARD_SIZE = 1024 * 1024 * 30 +const SHARD_SIZE = 1024 * 1024 * 100 const CONCURRENT_UPLOADS = 3 /** From 622c1144dc225ea8c27b4bb051bd5e9a0bbe194c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 16:15:29 +0000 Subject: [PATCH 46/47] refactor: defaine an AnyLink type --- packages/upload-client/src/car.js | 2 +- packages/upload-client/src/index.js | 4 ++-- packages/upload-client/src/types.ts | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/upload-client/src/car.js b/packages/upload-client/src/car.js index 839df8ee4..f1042c303 100644 --- a/packages/upload-client/src/car.js +++ b/packages/upload-client/src/car.js @@ -2,7 +2,7 @@ import { CarWriter } from '@ipld/car' /** * @param {Iterable|AsyncIterable} blocks - * @param {import('multiformats').Link} [root] + * @param {import('./types').AnyLink} [root] * @returns {Promise} */ export async function encode(blocks, root) { diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 26810df4b..eeeb6778a 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -71,12 +71,12 @@ export async function uploadDirectory({ issuer, proofs }, files, options = {}) { * @param {import('./types').InvocationConfig} invocationConfig * @param {ReadableStream} blocks * @param {UploadOptions} [options] - * @returns {Promise>} + * @returns {Promise} */ async function uploadBlockStream({ issuer, proofs }, blocks, options = {}) { /** @type {import('./types').CARLink[]} */ const shards = [] - /** @type {import('multiformats').Link?} */ + /** @type {import('./types').AnyLink?} */ let root = null await blocks .pipeThrough(new ShardingStream()) diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 6235d1815..f63a38cde 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -104,6 +104,11 @@ export interface CARFile extends CARHeaderInfo, Blob {} */ export type CARLink = Link +/** + * Any IPLD link. + */ +export type AnyLink = Link + /** * Metadata pertaining to a CAR file. */ From 777327609cd2c96b2be4e5e8d48e9a45f17b9a7b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 16 Nov 2022 16:20:40 +0000 Subject: [PATCH 47/47] fix: lockfile --- pnpm-lock.yaml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 042240a73..08833bd9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -292,10 +292,10 @@ importers: '@ipld/car': 5.0.0 '@ipld/dag-ucan': 2.0.1 '@ipld/unixfs': 2.0.0 - '@ucanto/client': 3.0.1 - '@ucanto/core': 3.0.1 - '@ucanto/interface': 3.0.0 - '@ucanto/transport': 3.0.1 + '@ucanto/client': 3.0.2 + '@ucanto/core': 3.0.2 + '@ucanto/interface': 3.0.1 + '@ucanto/transport': 3.0.2 '@web3-storage/access': link:../access multiformats: 10.0.2 p-queue: 7.3.0 @@ -303,9 +303,9 @@ importers: devDependencies: '@types/assert': 1.5.6 '@types/mocha': 10.0.0 - '@ucanto/principal': 3.0.0 - '@ucanto/server': 3.0.1 - '@ucanto/validator': 3.0.1 + '@ucanto/principal': 3.0.1 + '@ucanto/server': 3.0.4 + '@ucanto/validator': 3.0.4 assert: 2.0.0 blockstore-core: 2.0.2 c8: 7.12.0 @@ -1507,7 +1507,6 @@ packages: '@ucanto/core': 3.0.2 '@ucanto/interface': 3.0.1 '@ucanto/validator': 3.0.4 - dev: false /@ucanto/transport/3.0.2: resolution: {integrity: sha512-IyfI26VWPxCL2jnGiGP1i6mZblk8QORHzEVt5t+7Pic2k7pANQHoqbveQveRb9a8z6D/UdFogSPQxWYYtKaxWQ==} @@ -5997,10 +5996,6 @@ packages: util: 0.10.4 dev: true - /pathval/1.1.1: - resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} - dev: true - /picocolors/1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} dev: false