From 8b31255521e6fd875fe043b9c09b347759ebf315 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 11 Jan 2023 10:39:31 +0000 Subject: [PATCH] feat: add CAR upload method (#72) --- README.md | 23 ++++++++++++ package-lock.json | 90 ++++++++++++++++++++++----------------------- package.json | 2 +- src/client.js | 20 +++++++++- src/types.ts | 1 + test/client.test.js | 65 +++++++++++++++++++++++++++++++- 6 files changed, 153 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index b4e589e1d..d21fd1b35 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ In the example above, `directoryCid` resolves to an IPFS directory with the foll - `Client` - [`uploadDirectory`](#uploaddirectory) - [`uploadFile`](#uploadfile) + - [`uploadCAR`](#uploadcar) - [`agent`](#agent) - [`currentSpace`](#currentspace) - [`setCurrentSpace`](#setcurrentspace) @@ -215,6 +216,7 @@ function uploadDirectory ( signal?: AbortSignal onShardStored?: ShardStoredCallback shardSize?: number + concurrentRequests?: number } = {} ): Promise ``` @@ -233,6 +235,7 @@ function uploadFile ( signal?: AbortSignal onShardStored?: ShardStoredCallback shardSize?: number + concurrentRequests?: number } = {} ): Promise ``` @@ -241,6 +244,26 @@ Uploads a file to the service and returns the root data CID for the generated DA More information: [`ShardStoredCallback`](#shardstoredcallback) +### `uploadCAR` + +```ts +function uploadCAR ( + car: Blob, + options: { + retries?: number + signal?: AbortSignal + onShardStored?: ShardStoredCallback + shardSize?: number + concurrentRequests?: number + rootCID?: CID + } = {} +): Promise +``` + +Uploads a CAR file to the service. The difference between this function and [capability.store.add](#capabilitystoreadd) is that the CAR file is automatically sharded and an "upload" is registered (see [`capability.upload.add`](#capabilityuploadadd)), linking the individual shards. Use the `onShardStored` callback to obtain the CIDs of the CAR file shards. + +More information: [`ShardStoredCallback`](#shardstoredcallback) + ### `agent` ```ts diff --git a/package-lock.json b/package-lock.json index 7cd4491be..bd51737fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@ucanto/transport": "^4.0.2", "@web3-storage/access": "^9.1.1", "@web3-storage/capabilities": "^2.0.0", - "@web3-storage/upload-client": "^5.2.0" + "@web3-storage/upload-client": "^5.3.0" }, "devDependencies": { "@ucanto/server": "^4.0.2", @@ -394,30 +394,30 @@ "integrity": "sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==" }, "node_modules/@ucanto/client": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/client/-/client-4.0.2.tgz", - "integrity": "sha512-kSAlNlk8lpK2eShsXe9cE2I4iP1a7vq8wIFpzLR8Jjvh0YN4oI3zYQ6grrKJGHrork1mubkqIimzZerHCzFiwQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/client/-/client-4.0.3.tgz", + "integrity": "sha512-Kr+6A9VB/m2sFEatEKWzJk7Ccwg7AURdqdFyzhzUGU6YvUe4Z/wcly+yz1hswHmGc77dVPo+b19k2U/jjwnSKA==", "dependencies": { - "@ucanto/interface": "^4.0.2", + "@ucanto/interface": "^4.0.3", "multiformats": "^10.0.2" } }, "node_modules/@ucanto/core": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-4.0.2.tgz", - "integrity": "sha512-FxR6o4HsJepiYCj21j0F7D2vSPV4z6gk4489oo9NP/uF7+YJdZBSTxPytvBd/Ir7+2xlyFvBXP8Uxx/JRQu9HA==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-4.0.3.tgz", + "integrity": "sha512-5Uc6vdmKZzlA9NFvAN6BC1Tp1Npz0sepp2up1ZWU4BqArQ0w4U0YMtL9KPdBnL3TDAyDNgS9PgK+vHpjcSoeiQ==", "dependencies": { "@ipld/car": "^5.0.0", "@ipld/dag-cbor": "^8.0.0", "@ipld/dag-ucan": "^3.0.1", - "@ucanto/interface": "^4.0.2", + "@ucanto/interface": "^4.0.3", "multiformats": "^10.0.2" } }, "node_modules/@ucanto/interface": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-4.0.2.tgz", - "integrity": "sha512-3EPO9LRJy9ENWNBLk/x5XOx6ALCzgMkndvdRHJi8VTMKm4XroSnUYLarC3pPLdAWsF7NlmFN4g6aLz4mS9bHUQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-4.0.3.tgz", + "integrity": "sha512-ip1ZziMUhi9nFm9jPLEDLs8zX4HleYsuHHITH5w8GjST7chbRz1LBSq43A3nMUgea17cuIp+rr7i4QcOSFgXHw==", "dependencies": { "@ipld/dag-ucan": "^3.0.1", "multiformats": "^10.0.2" @@ -447,14 +447,14 @@ } }, "node_modules/@ucanto/transport": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/transport/-/transport-4.0.2.tgz", - "integrity": "sha512-KRWUcmmu6tvVKGO+Q0FnnUcEnIgFNGsQg5Udma+WQrQjJLcd+cDYrz5EhGItMmsC/gtM9Cm2+YMSZL21S/yf9A==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/transport/-/transport-4.0.3.tgz", + "integrity": "sha512-yrJoqoxmMCpPElR+iEb2AKIjUEmM+JGCcM1TZLXVbMlzaAt6ndYDMPajfnh3PBQMk7edIodZi+UxCLKvc8yelg==", "dependencies": { "@ipld/car": "^5.0.0", "@ipld/dag-cbor": "^8.0.0", - "@ucanto/core": "^4.0.2", - "@ucanto/interface": "^4.0.2", + "@ucanto/core": "^4.0.3", + "@ucanto/interface": "^4.0.3", "multiformats": "^10.0.2" } }, @@ -538,16 +538,16 @@ } }, "node_modules/@web3-storage/upload-client": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-5.2.0.tgz", - "integrity": "sha512-rHx8g56IqGuEdnJW44/SVD8AV1kVE1nzt+gKp4f1hFMMe2FT5eUROk6tM+gQzudrYDkUhhwpzQFmo1FsgohbJQ==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-5.3.0.tgz", + "integrity": "sha512-3vLErJTiigQ/cSlg8mCrOz3ZV1AZeGgeKDvmDhOBSaIY5rBvKzW2Bjcu4Vqv1cLdWOz2qkg6Tp6X+0qRV0cfPA==", "dependencies": { "@ipld/car": "^5.0.0", "@ipld/dag-ucan": "^3.0.1", "@ipld/unixfs": "^2.0.0", - "@ucanto/client": "^4.0.2", - "@ucanto/interface": "^4.0.2", - "@ucanto/transport": "^4.0.2", + "@ucanto/client": "^4.0.3", + "@ucanto/interface": "^4.0.3", + "@ucanto/transport": "^4.0.3", "@web3-storage/capabilities": "^2.1.0", "multiformats": "^10.0.2", "p-queue": "^7.3.0", @@ -7250,30 +7250,30 @@ "integrity": "sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==" }, "@ucanto/client": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/client/-/client-4.0.2.tgz", - "integrity": "sha512-kSAlNlk8lpK2eShsXe9cE2I4iP1a7vq8wIFpzLR8Jjvh0YN4oI3zYQ6grrKJGHrork1mubkqIimzZerHCzFiwQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/client/-/client-4.0.3.tgz", + "integrity": "sha512-Kr+6A9VB/m2sFEatEKWzJk7Ccwg7AURdqdFyzhzUGU6YvUe4Z/wcly+yz1hswHmGc77dVPo+b19k2U/jjwnSKA==", "requires": { - "@ucanto/interface": "^4.0.2", + "@ucanto/interface": "^4.0.3", "multiformats": "^10.0.2" } }, "@ucanto/core": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-4.0.2.tgz", - "integrity": "sha512-FxR6o4HsJepiYCj21j0F7D2vSPV4z6gk4489oo9NP/uF7+YJdZBSTxPytvBd/Ir7+2xlyFvBXP8Uxx/JRQu9HA==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-4.0.3.tgz", + "integrity": "sha512-5Uc6vdmKZzlA9NFvAN6BC1Tp1Npz0sepp2up1ZWU4BqArQ0w4U0YMtL9KPdBnL3TDAyDNgS9PgK+vHpjcSoeiQ==", "requires": { "@ipld/car": "^5.0.0", "@ipld/dag-cbor": "^8.0.0", "@ipld/dag-ucan": "^3.0.1", - "@ucanto/interface": "^4.0.2", + "@ucanto/interface": "^4.0.3", "multiformats": "^10.0.2" } }, "@ucanto/interface": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-4.0.2.tgz", - "integrity": "sha512-3EPO9LRJy9ENWNBLk/x5XOx6ALCzgMkndvdRHJi8VTMKm4XroSnUYLarC3pPLdAWsF7NlmFN4g6aLz4mS9bHUQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-4.0.3.tgz", + "integrity": "sha512-ip1ZziMUhi9nFm9jPLEDLs8zX4HleYsuHHITH5w8GjST7chbRz1LBSq43A3nMUgea17cuIp+rr7i4QcOSFgXHw==", "requires": { "@ipld/dag-ucan": "^3.0.1", "multiformats": "^10.0.2" @@ -7303,14 +7303,14 @@ } }, "@ucanto/transport": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@ucanto/transport/-/transport-4.0.2.tgz", - "integrity": "sha512-KRWUcmmu6tvVKGO+Q0FnnUcEnIgFNGsQg5Udma+WQrQjJLcd+cDYrz5EhGItMmsC/gtM9Cm2+YMSZL21S/yf9A==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@ucanto/transport/-/transport-4.0.3.tgz", + "integrity": "sha512-yrJoqoxmMCpPElR+iEb2AKIjUEmM+JGCcM1TZLXVbMlzaAt6ndYDMPajfnh3PBQMk7edIodZi+UxCLKvc8yelg==", "requires": { "@ipld/car": "^5.0.0", "@ipld/dag-cbor": "^8.0.0", - "@ucanto/core": "^4.0.2", - "@ucanto/interface": "^4.0.2", + "@ucanto/core": "^4.0.3", + "@ucanto/interface": "^4.0.3", "multiformats": "^10.0.2" } }, @@ -7386,16 +7386,16 @@ } }, "@web3-storage/upload-client": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-5.2.0.tgz", - "integrity": "sha512-rHx8g56IqGuEdnJW44/SVD8AV1kVE1nzt+gKp4f1hFMMe2FT5eUROk6tM+gQzudrYDkUhhwpzQFmo1FsgohbJQ==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-5.3.0.tgz", + "integrity": "sha512-3vLErJTiigQ/cSlg8mCrOz3ZV1AZeGgeKDvmDhOBSaIY5rBvKzW2Bjcu4Vqv1cLdWOz2qkg6Tp6X+0qRV0cfPA==", "requires": { "@ipld/car": "^5.0.0", "@ipld/dag-ucan": "^3.0.1", "@ipld/unixfs": "^2.0.0", - "@ucanto/client": "^4.0.2", - "@ucanto/interface": "^4.0.2", - "@ucanto/transport": "^4.0.2", + "@ucanto/client": "^4.0.3", + "@ucanto/interface": "^4.0.3", + "@ucanto/transport": "^4.0.3", "@web3-storage/capabilities": "^2.1.0", "multiformats": "^10.0.2", "p-queue": "^7.3.0", diff --git a/package.json b/package.json index 06281b3d4..4b78b026e 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,7 @@ "@ucanto/transport": "^4.0.2", "@web3-storage/access": "^9.1.1", "@web3-storage/capabilities": "^2.0.0", - "@web3-storage/upload-client": "^5.2.0" + "@web3-storage/upload-client": "^5.3.0" }, "devDependencies": { "@ucanto/server": "^4.0.2", diff --git a/src/client.js b/src/client.js index c7b1233a7..592f9ab0a 100644 --- a/src/client.js +++ b/src/client.js @@ -1,4 +1,4 @@ -import { uploadFile, uploadDirectory } from '@web3-storage/upload-client' +import { uploadFile, uploadDirectory, uploadCAR } from '@web3-storage/upload-client' import { Store as StoreCapabilities, Upload as UploadCapabilities } from '@web3-storage/capabilities' import { Base } from './base.js' import { Space } from './space.js' @@ -49,6 +49,24 @@ export class Client extends Base { return uploadDirectory(conf, files, options) } + /** + * Uploads a CAR file to the service. + * + * The difference between this function and `capability.store.add` is that the + * CAR file is automatically sharded and an "upload" is registered, linking + * the individual shards (see `capability.upload.add`). + * + * Use the `onShardStored` callback to obtain the CIDs of the CAR file shards. + * + * @param {import('./types').BlobLike} car CAR file. + * @param {import('./types').UploadOptions} [options] + */ + async uploadCAR (car, options = {}) { + const conf = await this._invocationConfig([StoreCapabilities.add.can, UploadCapabilities.add.can]) + options.connection = this._serviceConf.upload + return uploadCAR(conf, car, options) + } + /** * The current user agent (this device). */ diff --git a/src/types.ts b/src/types.ts index 556fe3d53..433b0c7d2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -75,6 +75,7 @@ export type { RequestOptions, ListRequestOptions, ShardingOptions, + ShardStoringOptions, UploadOptions, FileLike, BlobLike diff --git a/test/client.test.js b/test/client.test.js index 977634b41..2b745d3b2 100644 --- a/test/client.test.js +++ b/test/client.test.js @@ -6,7 +6,7 @@ import * as Signer from '@ucanto/principal/ed25519' import * as StoreCapabilities from '@web3-storage/capabilities/store' import * as UploadCapabilities from '@web3-storage/capabilities/upload' import { AgentData } from '@web3-storage/access/agent' -import { randomBytes } from './helpers/random.js' +import { randomBytes, randomCAR } from './helpers/random.js' import { toCAR } from './helpers/car.js' import { mockService, mockServiceConf } from './helpers/mocks.js' import { File } from './helpers/shims.js' @@ -160,6 +160,69 @@ describe('Client', () => { }) }) + describe('uploadCAR', () => { + it('uploads a CAR file to the service', async () => { + const car = await randomCAR(32) + + /** @type {import('../src/types').CARLink?} */ + let carCID + + const service = mockService({ + store: { + add: provide(StoreCapabilities.add, ({ invocation }) => { + assert.equal(invocation.issuer.did(), alice.agent().did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, StoreCapabilities.add.can) + assert.equal(invCap.with, space.did()) + return { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9200' + } + }) + }, + upload: { + add: provide(UploadCapabilities.add, ({ invocation }) => { + assert.equal(invocation.issuer.did(), alice.agent().did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, UploadCapabilities.add.can) + assert.equal(invCap.with, space.did()) + if (!invCap.nb) throw new Error('nb must be present') + assert.equal(invCap.nb.shards?.length, 1) + assert.equal(invCap.nb.shards[0].toString(), carCID.toString()) + return invCap.nb + }) + } + }) + + const server = createServer({ + id: await Signer.generate(), + service, + decoder: CAR, + encoder: CBOR + }) + + const alice = new Client( + await AgentData.create(), + { serviceConf: await mockServiceConf(server) } + ) + + const space = await alice.createSpace() + await alice.setCurrentSpace(space.did()) + await alice.uploadCAR(car, { onShardStored: meta => { carCID = meta.cid } }) + + assert(service.store.add.called) + assert.equal(service.store.add.callCount, 1) + assert(service.upload.add.called) + assert.equal(service.upload.add.callCount, 1) + + assert(carCID) + assert.equal(carCID.toString(), car.cid.toString()) + }) + }) + describe('currentSpace', () => { it('should return undefined or space', async () => { const alice = new Client(await AgentData.create())