diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index bed958a45..b761102fb 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -347,7 +347,7 @@ function add( root: CID, shards: CID[], options: { retries?: number; signal?: AbortSignal } = {} -): Promise +): Promise ``` Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload. diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 2ae6bc9f3..af48b72a7 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -34,7 +34,7 @@ export interface Service { remove: ServiceMethod } upload: { - add: ServiceMethod + add: ServiceMethod list: ServiceMethod, never> remove: ServiceMethod } @@ -46,6 +46,11 @@ export interface StoreAddResponse { url: string } +export interface UploadAddResponse { + root: AnyLink + shards?: CARLink[] +} + export interface ListResponse { cursor?: string size: number @@ -53,18 +58,12 @@ export interface ListResponse { } export interface StoreListResult { - payloadCID: string - origin?: string + link: CARLink size: number - uploadedAt: string + origin?: CARLink } -export interface UploadListResult { - uploaderDID: string - dataCID: string - carCID: string - uploadedAt: string -} +export interface UploadListResult extends UploadAddResponse {} export interface InvocationConfig { /** diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js index 4555af574..42c7d663c 100644 --- a/packages/upload-client/src/upload.js +++ b/packages/upload-client/src/upload.js @@ -25,6 +25,7 @@ import { REQUEST_RETRIES } from './constants.js' * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG. * @param {import('./types').RequestOptions} [options] + * @returns {Promise} */ export async function add( { issuer, with: resource, proofs, audience = servicePrincipal }, @@ -57,6 +58,8 @@ export async function add( cause: result, }) } + + return result } /** diff --git a/packages/upload-client/test/helpers/car.js b/packages/upload-client/test/helpers/car.js new file mode 100644 index 000000000..a280fbc30 --- /dev/null +++ b/packages/upload-client/test/helpers/car.js @@ -0,0 +1,26 @@ +import { CarWriter } from '@ipld/car' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' +import * as CAR from '@ucanto/transport/car' + +/** + * @param {Uint8Array} bytes + **/ +export async function toCAR(bytes) { + const hash = await sha256.digest(bytes) + const root = CID.create(1, raw.code, hash) + + const { writer, out } = CarWriter.create(root) + writer.put({ cid: root, bytes }) + writer.close() + + const chunks = [] + for await (const chunk of out) { + chunks.push(chunk) + } + const blob = new Blob(chunks) + const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) + + return Object.assign(blob, { cid, roots: [root] }) +} diff --git a/packages/upload-client/test/helpers/random.js b/packages/upload-client/test/helpers/random.js index 1b0360864..9a14b5252 100644 --- a/packages/upload-client/test/helpers/random.js +++ b/packages/upload-client/test/helpers/random.js @@ -1,8 +1,4 @@ -import { CarWriter } from '@ipld/car' -import { CID } from 'multiformats/cid' -import * as raw from 'multiformats/codecs/raw' -import { sha256 } from 'multiformats/hashes/sha2' -import * as CAR from '@ucanto/transport/car' +import { toCAR } from './car.js' /** @param {number} size */ export async function randomBytes(size) { @@ -31,19 +27,5 @@ export async function randomBytes(size) { /** @param {number} size */ export async function randomCAR(size) { const bytes = await randomBytes(size) - const hash = await sha256.digest(bytes) - const root = CID.create(1, raw.code, hash) - - const { writer, out } = CarWriter.create(root) - writer.put({ cid: root, bytes }) - writer.close() - - const chunks = [] - for await (const chunk of out) { - chunks.push(chunk) - } - const blob = new Blob(chunks) - const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) - - return Object.assign(blob, { cid, roots: [root] }) + return toCAR(bytes) } diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 80122137d..1b4cf38b8 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -10,6 +10,7 @@ import * as UploadCapabilities from '@web3-storage/capabilities/upload' import { uploadFile, uploadDirectory } from '../src/index.js' import { serviceSigner } from './fixtures.js' import { randomBytes } from './helpers/random.js' +import { toCAR } from './helpers/car.js' import { File } from './helpers/shims.js' import { mockService } from './helpers/mocks.js' @@ -23,7 +24,10 @@ describe('uploadFile', () => { const space = await Signer.generate() const agent = await Signer.generate() // The "user" that will ask the service to accept the upload - const file = new Blob([await randomBytes(128)]) + const bytes = await randomBytes(128) + const file = new Blob([bytes]) + const expectedCar = await toCAR(bytes) + /** @type {import('../src/types').CARLink|undefined} */ let carCID @@ -62,7 +66,10 @@ describe('uploadFile', () => { assert.equal(invCap.with, space.did()) assert.equal(invCap.nb?.shards?.length, 1) assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString()) - return null + return { + root: expectedCar.roots[0], + shards: [expectedCar.cid], + } }), }, }) @@ -95,8 +102,8 @@ describe('uploadFile', () => { assert(service.upload.add.called) assert.equal(service.upload.add.callCount, 1) - assert(carCID) - assert(dataCID) + assert.equal(carCID?.toString(), expectedCar.cid.toString()) + assert.equal(dataCID.toString(), expectedCar.roots[0].toString()) }) it('allows custom shard size to be set', async () => { @@ -129,7 +136,12 @@ describe('uploadFile', () => { const service = mockService({ store: { add: provide(StoreCapabilities.add, () => res) }, - upload: { add: provide(UploadCapabilities.add, () => null) }, + upload: { + add: provide(UploadCapabilities.add, ({ capability }) => { + if (!capability.nb) throw new Error('nb must be present') + return capability.nb + }), + }, }) const server = Server.create({ @@ -210,7 +222,8 @@ describe('uploadDirectory', () => { assert.equal(invCap.with, space.did()) assert.equal(invCap.nb?.shards?.length, 1) assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString()) - return null + if (!invCap.nb) throw new Error('nb must be present') + return invCap.nb }), }, }) @@ -277,7 +290,12 @@ describe('uploadDirectory', () => { const service = mockService({ store: { add: provide(StoreCapabilities.add, () => res) }, - upload: { add: provide(UploadCapabilities.add, () => null) }, + upload: { + add: provide(UploadCapabilities.add, ({ capability }) => { + if (!capability.nb) throw new Error('nb must be present') + return capability.nb + }), + }, }) const server = Server.create({ diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index 8a97e3450..ad12b54ce 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -294,9 +294,8 @@ describe('Store.list', () => { size: 1000, results: [ { - payloadCID: car.cid.toString(), + link: car.cid, size: 123, - uploadedAt: new Date().toISOString(), }, ], } @@ -352,9 +351,8 @@ describe('Store.list', () => { assert(list.results) assert.equal(list.results.length, res.results.length) list.results.forEach((r, i) => { - assert.equal(r.payloadCID, res.results[i].payloadCID) - assert.equal(r.size, res.results[i].size) - assert.equal(r.uploadedAt, res.results[i].uploadedAt) + assert.deepEqual(r.link, res.results[i].link) + assert.deepEqual(r.size, res.results[i].size) }) }) @@ -365,9 +363,8 @@ describe('Store.list', () => { size: 1, results: [ { - payloadCID: (await randomCAR(128)).cid.toString(), + link: (await randomCAR(128)).cid, size: 123, - uploadedAt: new Date().toISOString(), }, ], } @@ -375,9 +372,8 @@ describe('Store.list', () => { size: 1, results: [ { - payloadCID: (await randomCAR(128)).cid.toString(), + link: (await randomCAR(128)).cid, size: 123, - uploadedAt: new Date().toISOString(), }, ], } @@ -437,18 +433,16 @@ describe('Store.list', () => { assert(results0.results) assert.equal(results0.results.length, page0.results.length) results0.results.forEach((r, i) => { - assert.equal(r.payloadCID, page0.results[i].payloadCID) + assert.equal(r.link.toString(), page0.results[i].link.toString()) assert.equal(r.size, page0.results[i].size) - assert.equal(r.uploadedAt, page0.results[i].uploadedAt) }) assert(results1.results) assert.equal(results1.cursor, undefined) assert.equal(results1.results.length, page1.results.length) results1.results.forEach((r, i) => { - assert.equal(r.payloadCID, page1.results[i].payloadCID) + assert.equal(r.link.toString(), page1.results[i].link.toString()) assert.equal(r.size, page1.results[i].size) - assert.equal(r.uploadedAt, page1.results[i].uploadedAt) }) }) diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index 724b08878..2ff27dd30 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -17,6 +17,11 @@ describe('Upload.add', () => { const agent = await Signer.generate() const car = await randomCAR(128) + const res = { + root: car.roots[0], + shards: [car.cid], + } + const proofs = [ await UploadCapabilities.add.delegate({ issuer: space, @@ -37,7 +42,7 @@ describe('Upload.add', () => { assert.equal(String(invCap.nb?.root), car.roots[0].toString()) assert.equal(invCap.nb?.shards?.length, 1) assert.equal(String(invCap.nb?.shards?.[0]), car.cid.toString()) - return null + return res }), }, }) @@ -56,7 +61,7 @@ describe('Upload.add', () => { }) const root = car.roots[0] - await Upload.add( + const actual = await Upload.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, root, [car.cid], @@ -65,6 +70,11 @@ describe('Upload.add', () => { assert(service.upload.add.called) assert.equal(service.upload.add.callCount, 1) + assert.equal(actual.root.toString(), res.root.toString()) + assert.deepEqual( + new Set(actual.shards?.map((s) => s.toString())), + new Set(res.shards.map((s) => s.toString())) + ) }) it('throws on service error', async () => { @@ -127,10 +137,8 @@ describe('Upload.list', () => { size: 1000, results: [ { - uploaderDID: agent.did(), - carCID: car.cid.toString(), - dataCID: car.roots[0].toString(), - uploadedAt: new Date().toISOString(), + root: car.roots[0], + shards: [car.cid], }, ], } @@ -182,9 +190,11 @@ describe('Upload.list', () => { assert(list.results) assert.equal(list.results.length, res.results.length) list.results.forEach((r, i) => { - assert.equal(r.carCID.toString(), res.results[i].carCID.toString()) - assert.equal(r.dataCID.toString(), res.results[i].dataCID.toString()) - assert.equal(r.uploadedAt, res.results[i].uploadedAt) + assert.equal(r.root.toString(), res.results[i].root.toString()) + assert.deepStrictEqual( + new Set(r.shards?.map((s) => s.toString())), + new Set(res.results[i].shards.map((s) => s.toString())) + ) }) }) @@ -199,10 +209,8 @@ describe('Upload.list', () => { size: 1, results: [ { - uploaderDID: agent.did(), - carCID: car0.cid.toString(), - dataCID: car0.roots[0].toString(), - uploadedAt: new Date().toISOString(), + root: car0.roots[0], + shards: [car0.cid], }, ], } @@ -211,10 +219,8 @@ describe('Upload.list', () => { size: 1, results: [ { - uploaderDID: agent.did(), - carCID: car1.cid.toString(), - dataCID: car1.roots[0].toString(), - uploadedAt: new Date().toISOString(), + root: car1.roots[0], + shards: [car1.cid], }, ], } @@ -271,9 +277,11 @@ describe('Upload.list', () => { assert(results0.results) assert.equal(results0.results.length, page0.results.length) results0.results.forEach((r, i) => { - assert.equal(r.carCID.toString(), page0.results[i].carCID.toString()) - assert.equal(r.dataCID.toString(), page0.results[i].dataCID.toString()) - assert.equal(r.uploadedAt, page0.results[i].uploadedAt) + assert.equal(r.root.toString(), page0.results[i].root.toString()) + assert.deepStrictEqual( + new Set(r.shards?.map((s) => s.toString())), + new Set(page0.results[i].shards.map((s) => s.toString())) + ) }) assert.equal(results1.cursor, undefined) @@ -281,9 +289,11 @@ describe('Upload.list', () => { assert(results1.results) assert.equal(results1.results.length, page1.results.length) results1.results.forEach((r, i) => { - assert.equal(r.carCID.toString(), page1.results[i].carCID.toString()) - assert.equal(r.dataCID.toString(), page1.results[i].dataCID.toString()) - assert.equal(r.uploadedAt, page1.results[i].uploadedAt) + assert.equal(r.root.toString(), page1.results[i].root.toString()) + assert.deepStrictEqual( + new Set(r.shards?.map((s) => s.toString())), + new Set(page1.results[i].shards.map((s) => s.toString())) + ) }) })