feat!: upload/list items have a shards array property (#229)

- update tests and types to handle upload/list items with a shards array
property.
- update upload/add fn to return an UploadAddResponse as shards out may
be more than shards in.
- update README

see also: storacha/w3infra#56

Signed-off-by: Oli Evans <oli@protocol.ai>

packages/upload-client/README.md

@@ -347,7 +347,7 @@ function add(
  root: CID,
  shards: CID[],
  options: { retries?: number; signal?: AbortSignal } = {}
-): Promise<void>
+): Promise<UploadAddResponse>
 ```
 
 Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload.

packages/upload-client/src/types.ts

@@ -34,7 +34,7 @@ export interface Service {
  remove: ServiceMethod<StoreRemove, null, never>
  }
  upload: {
- add: ServiceMethod<UploadAdd, null, never>
+ add: ServiceMethod<UploadAdd, UploadAddResponse, never>
  list: ServiceMethod<UploadList, ListResponse<UploadListResult>, never>
  remove: ServiceMethod<UploadRemove, null, never>
  }
@@ -46,25 +46,24 @@ export interface StoreAddResponse {
  url: string
 }
 
+export interface UploadAddResponse {
+ root: AnyLink
+ shards?: CARLink[]
+}
+
 export interface ListResponse<R> {
  cursor?: string
  size: number
  results: R[]
 }
 
 export interface StoreListResult {
- payloadCID: string
- origin?: string
+ link: CARLink
  size: number
- uploadedAt: string
+ origin?: CARLink
 }
 
-export interface UploadListResult {
- uploaderDID: string
- dataCID: string
- carCID: string
- uploadedAt: string
-}
+export interface UploadListResult extends UploadAddResponse {}
 
 export interface InvocationConfig {
  /**

packages/upload-client/src/upload.js

@@ -25,6 +25,7 @@ import { REQUEST_RETRIES } from './constants.js'
  * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored.
  * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG.
  * @param {import('./types').RequestOptions} [options]
+ * @returns {Promise<import('./types').UploadAddResponse>}
  */
 export async function add(
  { issuer, with: resource, proofs, audience = servicePrincipal },
@@ -57,6 +58,8 @@ export async function add(
  cause: result,
  })
  }
+
+ return result
 }
 
 /**

packages/upload-client/test/helpers/car.js

@@ -0,0 +1,26 @@
+import { CarWriter } from '@ipld/car'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as CAR from '@ucanto/transport/car'
+
+/**
+ * @param {Uint8Array} bytes
+ **/
+export async function toCAR(bytes) {
+ const hash = await sha256.digest(bytes)
+ const root = CID.create(1, raw.code, hash)
+
+ const { writer, out } = CarWriter.create(root)
+ writer.put({ cid: root, bytes })
+ writer.close()
+
+ const chunks = []
+ for await (const chunk of out) {
+ chunks.push(chunk)
+ }
+ const blob = new Blob(chunks)
+ const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer()))
+
+ return Object.assign(blob, { cid, roots: [root] })
+}

packages/upload-client/test/helpers/random.js

@@ -1,8 +1,4 @@
-import { CarWriter } from '@ipld/car'
-import { CID } from 'multiformats/cid'
-import * as raw from 'multiformats/codecs/raw'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as CAR from '@ucanto/transport/car'
+import { toCAR } from './car.js'
 
 /** @param {number} size */
 export async function randomBytes(size) {
@@ -31,19 +27,5 @@ export async function randomBytes(size) {
 /** @param {number} size */
 export async function randomCAR(size) {
  const bytes = await randomBytes(size)
- const hash = await sha256.digest(bytes)
- const root = CID.create(1, raw.code, hash)
-
- const { writer, out } = CarWriter.create(root)
- writer.put({ cid: root, bytes })
- writer.close()
-
- const chunks = []
- for await (const chunk of out) {
- chunks.push(chunk)
- }
- const blob = new Blob(chunks)
- const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer()))
-
- return Object.assign(blob, { cid, roots: [root] })
+ return toCAR(bytes)
 }

packages/upload-client/test/index.test.js

@@ -10,6 +10,7 @@ import * as UploadCapabilities from '@web3-storage/capabilities/upload'
 import { uploadFile, uploadDirectory } from '../src/index.js'
 import { serviceSigner } from './fixtures.js'
 import { randomBytes } from './helpers/random.js'
+import { toCAR } from './helpers/car.js'
 import { File } from './helpers/shims.js'
 import { mockService } from './helpers/mocks.js'
 
@@ -23,7 +24,10 @@ describe('uploadFile', () => {
 
  const space = await Signer.generate()
  const agent = await Signer.generate() // The "user" that will ask the service to accept the upload
- const file = new Blob([await randomBytes(128)])
+ const bytes = await randomBytes(128)
+ const file = new Blob([bytes])
+ const expectedCar = await toCAR(bytes)
+
  /** @type {import('../src/types').CARLink|undefined} */
  let carCID
 
@@ -62,7 +66,10 @@ describe('uploadFile', () => {
  assert.equal(invCap.with, space.did())
  assert.equal(invCap.nb?.shards?.length, 1)
  assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString())
- return null
+ return {
+ root: expectedCar.roots[0],
+ shards: [expectedCar.cid],
+ }
  }),
  },
  })
@@ -95,8 +102,8 @@ describe('uploadFile', () => {
  assert(service.upload.add.called)
  assert.equal(service.upload.add.callCount, 1)
 
- assert(carCID)
- assert(dataCID)
+ assert.equal(carCID?.toString(), expectedCar.cid.toString())
+ assert.equal(dataCID.toString(), expectedCar.roots[0].toString())
  })
 
  it('allows custom shard size to be set', async () => {
@@ -129,7 +136,12 @@ describe('uploadFile', () => {
 
  const service = mockService({
  store: { add: provide(StoreCapabilities.add, () => res) },
- upload: { add: provide(UploadCapabilities.add, () => null) },
+ upload: {
+ add: provide(UploadCapabilities.add, ({ capability }) => {
+ if (!capability.nb) throw new Error('nb must be present')
+ return capability.nb
+ }),
+ },
  })
 
  const server = Server.create({
@@ -210,7 +222,8 @@ describe('uploadDirectory', () => {
  assert.equal(invCap.with, space.did())
  assert.equal(invCap.nb?.shards?.length, 1)
  assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString())
- return null
+ if (!invCap.nb) throw new Error('nb must be present')
+ return invCap.nb
  }),
  },
  })
@@ -277,7 +290,12 @@ describe('uploadDirectory', () => {
 
  const service = mockService({
  store: { add: provide(StoreCapabilities.add, () => res) },
- upload: { add: provide(UploadCapabilities.add, () => null) },
+ upload: {
+ add: provide(UploadCapabilities.add, ({ capability }) => {
+ if (!capability.nb) throw new Error('nb must be present')
+ return capability.nb
+ }),
+ },
  })
 
  const server = Server.create({

packages/upload-client/test/store.test.js

@@ -294,9 +294,8 @@ describe('Store.list', () => {
  size: 1000,
  results: [
  {
- payloadCID: car.cid.toString(),
+ link: car.cid,
  size: 123,
- uploadedAt: new Date().toISOString(),
  },
  ],
  }
@@ -352,9 +351,8 @@ describe('Store.list', () => {
  assert(list.results)
  assert.equal(list.results.length, res.results.length)
  list.results.forEach((r, i) => {
- assert.equal(r.payloadCID, res.results[i].payloadCID)
- assert.equal(r.size, res.results[i].size)
- assert.equal(r.uploadedAt, res.results[i].uploadedAt)
+ assert.deepEqual(r.link, res.results[i].link)
+ assert.deepEqual(r.size, res.results[i].size)
  })
  })
 
@@ -365,19 +363,17 @@ describe('Store.list', () => {
  size: 1,
  results: [
  {
- payloadCID: (await randomCAR(128)).cid.toString(),
+ link: (await randomCAR(128)).cid,
  size: 123,
- uploadedAt: new Date().toISOString(),
  },
  ],
  }
  const page1 = {
  size: 1,
  results: [
  {
- payloadCID: (await randomCAR(128)).cid.toString(),
+ link: (await randomCAR(128)).cid,
  size: 123,
- uploadedAt: new Date().toISOString(),
  },
  ],
  }
@@ -437,18 +433,16 @@ describe('Store.list', () => {
  assert(results0.results)
  assert.equal(results0.results.length, page0.results.length)
  results0.results.forEach((r, i) => {
- assert.equal(r.payloadCID, page0.results[i].payloadCID)
+ assert.equal(r.link.toString(), page0.results[i].link.toString())
  assert.equal(r.size, page0.results[i].size)
- assert.equal(r.uploadedAt, page0.results[i].uploadedAt)
  })
 
  assert(results1.results)
  assert.equal(results1.cursor, undefined)
  assert.equal(results1.results.length, page1.results.length)
  results1.results.forEach((r, i) => {
- assert.equal(r.payloadCID, page1.results[i].payloadCID)
+ assert.equal(r.link.toString(), page1.results[i].link.toString())
  assert.equal(r.size, page1.results[i].size)
- assert.equal(r.uploadedAt, page1.results[i].uploadedAt)
  })
  })
 

packages/upload-client/test/upload.test.js

@@ -17,6 +17,11 @@ describe('Upload.add', () => {
  const agent = await Signer.generate()
  const car = await randomCAR(128)
 
+ const res = {
+ root: car.roots[0],
+ shards: [car.cid],
+ }
+
  const proofs = [
  await UploadCapabilities.add.delegate({
  issuer: space,
@@ -37,7 +42,7 @@ describe('Upload.add', () => {
  assert.equal(String(invCap.nb?.root), car.roots[0].toString())
  assert.equal(invCap.nb?.shards?.length, 1)
  assert.equal(String(invCap.nb?.shards?.[0]), car.cid.toString())
- return null
+ return res
  }),
  },
  })
@@ -56,7 +61,7 @@ describe('Upload.add', () => {
  })
 
  const root = car.roots[0]
- await Upload.add(
+ const actual = await Upload.add(
  { issuer: agent, with: space.did(), proofs, audience: serviceSigner },
  root,
  [car.cid],
@@ -65,6 +70,11 @@ describe('Upload.add', () => {
 
  assert(service.upload.add.called)
  assert.equal(service.upload.add.callCount, 1)
+ assert.equal(actual.root.toString(), res.root.toString())
+ assert.deepEqual(
+ new Set(actual.shards?.map((s) => s.toString())),
+ new Set(res.shards.map((s) => s.toString()))
+ )
  })
 
  it('throws on service error', async () => {
@@ -127,10 +137,8 @@ describe('Upload.list', () => {
  size: 1000,
  results: [
  {
- uploaderDID: agent.did(),
- carCID: car.cid.toString(),
- dataCID: car.roots[0].toString(),
- uploadedAt: new Date().toISOString(),
+ root: car.roots[0],
+ shards: [car.cid],
  },
  ],
  }
@@ -182,9 +190,11 @@ describe('Upload.list', () => {
  assert(list.results)
  assert.equal(list.results.length, res.results.length)
  list.results.forEach((r, i) => {
- assert.equal(r.carCID.toString(), res.results[i].carCID.toString())
- assert.equal(r.dataCID.toString(), res.results[i].dataCID.toString())
- assert.equal(r.uploadedAt, res.results[i].uploadedAt)
+ assert.equal(r.root.toString(), res.results[i].root.toString())
+ assert.deepStrictEqual(
+ new Set(r.shards?.map((s) => s.toString())),
+ new Set(res.results[i].shards.map((s) => s.toString()))
+ )
  })
  })
 
@@ -199,10 +209,8 @@ describe('Upload.list', () => {
  size: 1,
  results: [
  {
- uploaderDID: agent.did(),
- carCID: car0.cid.toString(),
- dataCID: car0.roots[0].toString(),
- uploadedAt: new Date().toISOString(),
+ root: car0.roots[0],
+ shards: [car0.cid],
  },
  ],
  }
@@ -211,10 +219,8 @@ describe('Upload.list', () => {
  size: 1,
  results: [
  {
- uploaderDID: agent.did(),
- carCID: car1.cid.toString(),
- dataCID: car1.roots[0].toString(),
- uploadedAt: new Date().toISOString(),
+ root: car1.roots[0],
+ shards: [car1.cid],
  },
  ],
  }
@@ -271,19 +277,23 @@ describe('Upload.list', () => {
  assert(results0.results)
  assert.equal(results0.results.length, page0.results.length)
  results0.results.forEach((r, i) => {
- assert.equal(r.carCID.toString(), page0.results[i].carCID.toString())
- assert.equal(r.dataCID.toString(), page0.results[i].dataCID.toString())
- assert.equal(r.uploadedAt, page0.results[i].uploadedAt)
+ assert.equal(r.root.toString(), page0.results[i].root.toString())
+ assert.deepStrictEqual(
+ new Set(r.shards?.map((s) => s.toString())),
+ new Set(page0.results[i].shards.map((s) => s.toString()))
+ )
  })
 
  assert.equal(results1.cursor, undefined)
  assert.equal(results1.size, page1.size)
  assert(results1.results)
  assert.equal(results1.results.length, page1.results.length)
  results1.results.forEach((r, i) => {
- assert.equal(r.carCID.toString(), page1.results[i].carCID.toString())
- assert.equal(r.dataCID.toString(), page1.results[i].dataCID.toString())
- assert.equal(r.uploadedAt, page1.results[i].uploadedAt)
+ assert.equal(r.root.toString(), page1.results[i].root.toString())
+ assert.deepStrictEqual(
+ new Set(r.shards?.map((s) => s.toString())),
+ new Set(page1.results[i].shards.map((s) => s.toString()))
+ )
  })
  })