feat: content claims reads by default with fallback for old index sou…

…rces (#93)

Makes content claims used by default. If no index was found from content
claims, the handler is returned wrapped with the old
`withIndexerSources` and `withDagula` (old)

Alternatively, we could add a new middleware and keep there the old
ones. We could inspect in these middlewares if we have a dagula instance
already, but I don't think that is a good way to go as we in the future
can just drop this fallback

src/index.js

@@ -16,8 +16,7 @@ import {
   handleCar
 } from '@web3-storage/gateway-lib/handlers'
 import {
-  withDagula,
-  withIndexSources,
+  withContentClaimsDagula,
   withHttpRangeUnsupported,
   withVersionHeader,
   withCarHandler
@@ -44,8 +43,7 @@ export default {
       withCarHandler,
       withHttpRangeUnsupported,
       withHttpGet,
-      withIndexSources,
-      withDagula,
+      withContentClaimsDagula,
       withFixedLengthStream
     )
     return middleware(handler)(request, env, ctx)

src/middleware.js

@@ -1,5 +1,6 @@
 /* eslint-env browser */
 import { Dagula } from 'dagula'
+import { composeMiddleware } from '@web3-storage/gateway-lib/middleware'
 import { CarReader } from '@ipld/car'
 import { parseCid, HttpError, toIterable } from '@web3-storage/gateway-lib/util'
 import { base32 } from 'multiformats/bases/base32'
@@ -51,6 +52,32 @@ export function withCarHandler (handler) {
   }
 }
 
+/**
+ * Creates a dagula instance backed by the R2 blockstore backed by content claims.
+ *
+ * @type {import('@web3-storage/gateway-lib').Middleware<DagulaContext & IndexSourcesContext & IpfsUrlContext, IndexSourcesContext & IpfsUrlContext, Environment>}
+ */
+export function withContentClaimsDagula (handler) {
+  return async (request, env, ctx) => {
+    const { dataCid } = ctx
+    const index = new ContentClaimsIndex(asSimpleBucket(env.CARPARK), {
+      serviceURL: env.CONTENT_CLAIMS_SERVICE_URL ? new URL(env.CONTENT_CLAIMS_SERVICE_URL) : undefined
+    })
+    const found = await index.get(dataCid)
+    if (!found) {
+      // fallback to old index sources and dagula fallback
+      return composeMiddleware(
+        withIndexSources,
+        withDagulaFallback
+      )(handler)(request, env, ctx)
+    }
+    const blockstore = new BatchingR2Blockstore(env.CARPARK, index)
+
+    const dagula = new Dagula(blockstore)
+    return handler(request, env, { ...ctx, dagula })
+  }
+}
+
 /**
  * Extracts a set of index sources from search params from the URL querystring
  * or DUDEWHERE bucket.
@@ -132,12 +159,12 @@ export function withIndexSources (handler) {
 }
 
 /**
- * Creates a dagula instance backed by the R2 blockstore.
+ * Creates a dagula instance backed by the R2 blockstore fallback with index sources.
  * @type {import('@web3-storage/gateway-lib').Middleware<DagulaContext & IndexSourcesContext & IpfsUrlContext, IndexSourcesContext & IpfsUrlContext, Environment>}
  */
-export function withDagula (handler) {
+export function withDagulaFallback (handler) {
   return async (request, env, ctx) => {
-    const { indexSources, searchParams, dataCid } = ctx
+    const { indexSources, searchParams } = ctx
     if (!indexSources) throw new Error('missing index sources in context')
     if (!searchParams) throw new Error('missing URL search params in context')
 
@@ -163,12 +190,7 @@ export function withDagula (handler) {
         blockstore = new BatchingR2Blockstore(env.CARPARK, index)
       }
     } else {
-      const index = new ContentClaimsIndex(asSimpleBucket(env.CARPARK), {
-        serviceURL: env.CONTENT_CLAIMS_SERVICE_URL ? new URL(env.CONTENT_CLAIMS_SERVICE_URL) : undefined
-      })
-      const found = await index.get(dataCid)
-      if (!found) throw new HttpError('missing index', { status: 404 })
-      blockstore = new BatchingR2Blockstore(env.CARPARK, index)
+      throw new HttpError('missing index', { status: 404 })
     }
 
     const dagula = new Dagula(blockstore)

test/helpers/content-claims.js

@@ -19,7 +19,7 @@ import { CAR_CODE } from '../../src/constants.js'
 /**
  * @typedef {import('carstream/api').Block & { children: import('multiformats').UnknownLink[] }} RelationIndexData
  * @typedef {Map<import('multiformats').UnknownLink, import('carstream/api').Block[]>} Claims
- * @typedef {{ setClaims: (c: Claims) => void, close: () => void, port: number, signer: import('@ucanto/interface').Signer }} MockClaimsService
+ * @typedef {{ setClaims: (c: Claims) => void, close: () => void, port: number, signer: import('@ucanto/interface').Signer, getCallCount: () => number, resetCallCount: () => void }} MockClaimsService
  */
 
 const Decoders = {
@@ -129,11 +129,18 @@ const encode = async invocation => {
 }
 
 export const mockClaimsService = async () => {
+  let callCount = 0
   /** @type {Claims} */
   let claims = new LinkMap()
   /** @param {Claims} s */
   const setClaims = s => { claims = s }
+  const getCallCount = () => callCount
+  const resetCallCount = () => {
+    callCount = 0
+  }
+
   const server = http.createServer(async (req, res) => {
+    callCount++
     const content = Link.parse(String(req.url?.split('/')[2]))
     const blocks = claims.get(content) ?? []
     const readable = new ReadableStream({
@@ -154,5 +161,5 @@ export const mockClaimsService = async () => {
   }
   // @ts-expect-error
   const { port } = server.address()
-  return { setClaims, close, port, signer: await ed25519.generate() }
+  return { setClaims, close, port, signer: await ed25519.generate(), getCallCount, resetCallCount }
 }

test/index.spec.js

@@ -1,4 +1,4 @@
-import { describe, before, after, it } from 'node:test'
+import { describe, before, beforeEach, after, it } from 'node:test'
 import assert from 'node:assert'
 import { randomBytes } from 'node:crypto'
 import { Miniflare } from 'miniflare'
@@ -44,6 +44,10 @@ describe('freeway', () => {
     builder = new Builder(buckets[0], buckets[1], buckets[2])
   })
 
+  beforeEach(() => {
+    claimsService.resetCallCount()
+  })
+
   after(() => claimsService.close())
 
   it('should get a file', async () => {
@@ -190,6 +194,30 @@ describe('freeway', () => {
     assert(equals(input[0].content, output))
   })
 
+  it('should use content claims by default', async () => {
+    const input = [{ path: 'sargo.tar.xz', content: randomBytes(MAX_CAR_BYTES_IN_MEMORY + 1) }]
+    // no dudewhere or satnav so only content claims can satisfy the request
+    const { dataCid, carCids, indexes } = await builder.add(input, {
+      dudewhere: true,
+      satnav: true
+    })
+
+    const carpark = await miniflare.getR2Bucket('CARPARK')
+    const res = await carpark.get(`${carCids[0]}/${carCids[0]}.car`)
+    assert(res)
+
+    // @ts-expect-error nodejs ReadableStream does not implement ReadableStream interface correctly
+    const claims = await generateClaims(claimsService.signer, dataCid, carCids[0], res.body, indexes[0].cid, indexes[0].carCid)
+    claimsService.setClaims(claims)
+
+    const res1 = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${dataCid}/${input[0].path}`)
+    if (!res1.ok) assert.fail(`unexpected response: ${await res1.text()}`)
+
+    const output = new Uint8Array(await res1.arrayBuffer())
+    assert(equals(input[0].content, output))
+    assert.equal(claimsService.getCallCount(), 2)
+  })
+
   it('should GET a CAR by CAR CID', async () => {
     const input = [{ path: 'sargo.tar.xz', content: randomBytes(10) }]
     const { dataCid, carCids } = await builder.add(input, { wrapWithDirectory: false })