feat: support requesting raw IPNS records in @helia/verified-fetch (#443)

Adds support for the application/vnd.ipfs.ipns-record Accept header to return raw IPNS records.

Author: achingbrain

README.md

@@ -351,7 +351,7 @@ console.info(obj) // ...
 
 The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected.
 
-If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptible](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned:
+If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptable](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned:
 
 ```typescript
 import { verifiedFetch } from '@helia/verified-fetch'

package.json

@@ -152,17 +152,20 @@
     "@ipld/dag-json": "^10.2.0",
     "@ipld/dag-pb": "^4.1.0",
     "@libp2p/interface": "^1.1.2",
+    "@libp2p/kad-dht": "^12.0.7",
     "@libp2p/peer-id": "^4.0.5",
     "cborg": "^4.0.9",
     "hashlru": "^2.3.0",
     "interface-blockstore": "^5.2.10",
+    "interface-datastore": "^8.2.11",
     "ipfs-unixfs-exporter": "^13.5.0",
     "it-map": "^3.0.5",
     "it-pipe": "^3.0.1",
     "it-tar": "^6.0.4",
     "it-to-browser-readablestream": "^2.0.6",
     "multiformats": "^13.1.0",
-    "progress-events": "^1.0.0"
+    "progress-events": "^1.0.0",
+    "uint8arrays": "^5.0.2"
   },
   "devDependencies": {
     "@helia/car": "^3.0.0",
@@ -188,8 +191,7 @@
     "magic-bytes.js": "^1.8.0",
     "p-defer": "^4.0.0",
     "sinon": "^17.0.1",
-    "sinon-ts": "^2.0.0",
-    "uint8arrays": "^5.0.1"
+    "sinon-ts": "^2.0.0"
   },
   "sideEffects": false
 }

src/utils/responses.ts

@@ -20,3 +20,10 @@ export function notAcceptableResponse (body?: BodyInit | null): Response {
     statusText: 'Not Acceptable'
   })
 }
+
+export function badRequestResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 400,
+    statusText: 'Bad Request'
+  })
+}

src/verified-fetch.ts

@@ -5,24 +5,30 @@ import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } f
 import * as ipldDagCbor from '@ipld/dag-cbor'
 import * as ipldDagJson from '@ipld/dag-json'
 import { code as dagPbCode } from '@ipld/dag-pb'
+import { Record as DHTRecord } from '@libp2p/kad-dht'
+import { peerIdFromString } from '@libp2p/peer-id'
+import { Key } from 'interface-datastore'
 import toBrowserReadableStream from 'it-to-browser-readablestream'
 import { code as jsonCode } from 'multiformats/codecs/json'
 import { code as rawCode } from 'multiformats/codecs/raw'
 import { identity } from 'multiformats/hashes/identity'
 import { CustomProgressEvent } from 'progress-events'
+import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
 import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js'
 import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { tarStream } from './utils/get-tar-stream.js'
 import { parseResource } from './utils/parse-resource.js'
-import { notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
+import { badRequestResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
 import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
 import { walkPath } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { RequestFormatShorthand } from './types.js'
 import type { Helia } from '@helia/interface'
-import type { AbortOptions, Logger } from '@libp2p/interface'
+import type { AbortOptions, Logger, PeerId } from '@libp2p/interface'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 
@@ -49,6 +55,11 @@ interface FetchHandlerFunctionArg {
    * content cannot be represented in this format a 406 should be returned
    */
   accept?: string
+
+  /**
+   * The originally requested resource
+   */
+  resource: string
 }
 
 interface FetchHandlerFunction {
@@ -129,8 +140,36 @@ export class VerifiedFetch {
    * Accepts an `ipns://...` URL as a string and returns a `Response` containing
    * a raw IPNS record.
    */
-  private async handleIPNSRecord (resource: string, opts?: VerifiedFetchOptions): Promise<Response> {
-    return notSupportedResponse('vnd.ipfs.ipns-record support is not implemented')
+  private async handleIPNSRecord ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    if (path !== '' || !resource.startsWith('ipns://')) {
+      return badRequestResponse('Invalid IPNS name')
+    }
+
+    let peerId: PeerId
+
+    try {
+      peerId = peerIdFromString(resource.replace('ipns://', ''))
+    } catch (err) {
+      this.log.error('could not parse peer id from IPNS url %s', resource)
+
+      return badRequestResponse('Invalid IPNS name')
+    }
+
+    // since this call happens after parseResource, we've already resolved the
+    // IPNS name so a local copy should be in the helia datastore, so we can
+    // just read it out..
+    const routingKey = uint8ArrayConcat([
+      uint8ArrayFromString('/ipns/'),
+      peerId.toBytes()
+    ])
+    const datastoreKey = new Key('/dht/record/' + uint8ArrayToString(routingKey, 'base32'), false)
+    const buf = await this.helia.datastore.get(datastoreKey, options)
+    const record = DHTRecord.deserialize(buf)
+
+    const response = okResponse(record.value)
+    response.headers.set('content-type', 'application/vnd.ipfs.ipns-record')
+
+    return response
   }
 
   /**
@@ -384,28 +423,30 @@ export class VerifiedFetch {
     let response: Response
     let reqFormat: RequestFormatShorthand | undefined
 
+    const handlerArgs = { resource: resource.toString(), cid, path, accept, options }
+
     if (accept === 'application/vnd.ipfs.ipns-record') {
       // the user requested a raw IPNS record
       reqFormat = 'ipns-record'
-      response = await this.handleIPNSRecord(resource.toString(), options)
+      response = await this.handleIPNSRecord(handlerArgs)
     } else if (accept === 'application/vnd.ipld.car') {
       // the user requested a CAR file
       reqFormat = 'car'
       query.download = true
       query.filename = query.filename ?? `${cid.toString()}.car`
-      response = await this.handleCar({ cid, path, options })
+      response = await this.handleCar(handlerArgs)
     } else if (accept === 'application/vnd.ipld.raw') {
       // the user requested a raw block
       reqFormat = 'raw'
       query.download = true
       query.filename = query.filename ?? `${cid.toString()}.bin`
-      response = await this.handleRaw({ cid, path, options })
+      response = await this.handleRaw(handlerArgs)
     } else if (accept === 'application/x-tar') {
       // the user requested a TAR file
       reqFormat = 'tar'
       query.download = true
       query.filename = query.filename ?? `${cid.toString()}.tar`
-      response = await this.handleTar({ cid, path, options })
+      response = await this.handleTar(handlerArgs)
     } else {
       // derive the handler from the CID type
       const codecHandler = this.codecHandlers[cid.code]
@@ -414,7 +455,7 @@ export class VerifiedFetch {
         return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`)
       }
 
-      response = await codecHandler.call(this, { cid, path, accept, options })
+      response = await codecHandler.call(this, handlerArgs)
     }
 
     response.headers.set('etag', getETag({ cid, reqFormat, weak: false }))

test/ipns-record.spec.ts

@@ -0,0 +1,89 @@
+import { dagCbor } from '@helia/dag-cbor'
+import { ipns } from '@helia/ipns'
+import { stop } from '@libp2p/interface'
+import { createEd25519PeerId } from '@libp2p/peer-id-factory'
+import { expect } from 'aegir/chai'
+import { marshal, unmarshal } from 'ipns'
+import { VerifiedFetch } from '../src/verified-fetch.js'
+import { createHelia } from './fixtures/create-offline-helia.js'
+import type { Helia } from '@helia/interface'
+import type { IPNS } from '@helia/ipns'
+
+describe('ipns records', () => {
+  let helia: Helia
+  let name: IPNS
+  let verifiedFetch: VerifiedFetch
+
+  beforeEach(async () => {
+    helia = await createHelia()
+    name = ipns(helia)
+    verifiedFetch = new VerifiedFetch({
+      helia
+    })
+  })
+
+  afterEach(async () => {
+    await stop(helia, verifiedFetch)
+  })
+
+  it('should support fetching a raw IPNS record', async () => {
+    const obj = {
+      hello: 'world'
+    }
+    const c = dagCbor(helia)
+    const cid = await c.add(obj)
+
+    const peerId = await createEd25519PeerId()
+    const record = await name.publish(peerId, cid)
+
+    const resp = await verifiedFetch.fetch(`ipns://${peerId}`, {
+      headers: {
+        accept: 'application/vnd.ipfs.ipns-record'
+      }
+    })
+    expect(resp.status).to.equal(200)
+    expect(resp.headers.get('content-type')).to.equal('application/vnd.ipfs.ipns-record')
+
+    const buf = new Uint8Array(await resp.arrayBuffer())
+    expect(marshal(record)).to.equalBytes(buf)
+
+    const output = unmarshal(buf)
+    expect(output.value).to.deep.equal(`/ipfs/${cid}`)
+  })
+
+  it('should reject a request for non-IPNS url', async () => {
+    const resp = await verifiedFetch.fetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv', {
+      headers: {
+        accept: 'application/vnd.ipfs.ipns-record'
+      }
+    })
+    expect(resp.status).to.equal(400)
+  })
+
+  it('should reject a request for a DNSLink url', async () => {
+    const resp = await verifiedFetch.fetch('ipns://ipfs.io', {
+      headers: {
+        accept: 'application/vnd.ipfs.ipns-record'
+      }
+    })
+    expect(resp.status).to.equal(400)
+  })
+
+  it('should reject a request for a url with a path component', async () => {
+    const obj = {
+      hello: 'world'
+    }
+    const c = dagCbor(helia)
+    const cid = await c.add(obj)
+
+    const peerId = await createEd25519PeerId()
+    await name.publish(peerId, cid)
+
+    const resp = await verifiedFetch.fetch(`ipns://${peerId}/hello`, {
+      headers: {
+        accept: 'application/vnd.ipfs.ipns-record'
+      }
+    })
+    expect(resp.status).to.equal(400)
+  })
+})

test/verified-fetch.spec.ts

@@ -1,8 +1,7 @@
 import { dagCbor } from '@helia/dag-cbor'
 import { dagJson } from '@helia/dag-json'
-import { type IPNS } from '@helia/ipns'
 import { json } from '@helia/json'
-import { unixfs, type UnixFS } from '@helia/unixfs'
+import { unixfs } from '@helia/unixfs'
 import * as ipldDagCbor from '@ipld/dag-cbor'
 import * as ipldDagJson from '@ipld/dag-json'
 import { stop } from '@libp2p/interface'
@@ -19,7 +18,6 @@ import Sinon from 'sinon'
 import { stubInterface } from 'sinon-ts'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { VerifiedFetch } from '../src/verified-fetch.js'
-import { cids } from './fixtures/cids.js'
 import { createHelia } from './fixtures/create-offline-helia.js'
 import type { Helia } from '@helia/interface'
 
@@ -54,52 +52,6 @@ describe('@helia/verifed-fetch', () => {
     expect(helia.start.callCount).to.equal(1)
   })
 
-  describe('format not implemented', () => {
-    let verifiedFetch: VerifiedFetch
-
-    before(async () => {
-      verifiedFetch = new VerifiedFetch({
-        helia: stubInterface<Helia>({
-          logger: defaultLogger()
-        }),
-        ipns: stubInterface<IPNS>({
-          resolveDns: async (dnsLink: string) => {
-            expect(dnsLink).to.equal('mydomain.com')
-            return {
-              cid: cids.file,
-              path: ''
-            }
-          }
-        }),
-        unixfs: stubInterface<UnixFS>()
-      })
-    })
-
-    after(async () => {
-      await verifiedFetch.stop()
-    })
-
-    const formatsAndAcceptHeaders = [
-      ['ipns-record', 'application/vnd.ipfs.ipns-record']
-    ]
-
-    for (const [format, acceptHeader] of formatsAndAcceptHeaders) {
-      // eslint-disable-next-line no-loop-func
-      it(`returns 501 for ${acceptHeader}`, async () => {
-        const resp = await verifiedFetch.fetch(`ipns://mydomain.com?format=${format}`)
-        expect(resp).to.be.ok()
-        expect(resp.status).to.equal(501)
-        const resp2 = await verifiedFetch.fetch(cids.file, {
-          headers: {
-            accept: acceptHeader
-          }
-        })
-        expect(resp2).to.be.ok()
-        expect(resp2.status).to.equal(501)
-      })
-    }
-  })
-
   describe('implicit format', () => {
     let verifiedFetch: VerifiedFetch