feat: implement new ipns record&answer properties

packages/verified-fetch/src/utils/parse-resource.ts

@@ -32,8 +32,9 @@ export async function parseResource (resource: Resource, { ipns, logger }: Parse
       cid,
       protocol: 'ipfs',
       path: '',
-      query: {}
-    }
+      query: {},
+      ttl: 29030400 // 1 year for ipfs content
+    } satisfies ParsedUrlStringResults
   }
 
   throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`)

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -2,11 +2,11 @@ import { peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
 import { TLRU } from './tlru.js'
 import type { RequestFormatShorthand } from '../types.js'
-import type { IPNS, ResolveDNSLinkProgressEvents, ResolveResult } from '@helia/ipns'
+import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, ResolveDNSLinkProgressEvents, ResolveResult } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
 
-const ipnsCache = new TLRU<ResolveResult>(1000)
+const ipnsCache = new TLRU<DNSLinkResolveResult | IPNSResolveResult>(1000)
 
 export interface ParseUrlStringInput {
   urlString: string
@@ -23,24 +23,32 @@ export interface ParsedUrlQuery extends Record<string, string | unknown> {
   filename?: string
 }
 
-export interface ParsedUrlStringResults {
-  protocol: string
-  path: string
-  cid: CID
+interface ParsedUrlStringResultsBase extends ResolveResult {
+  protocol: 'ipfs' | 'ipns'
   query: ParsedUrlQuery
+  ttl: number
 }
 
+export type ParsedUrlStringResults = ParsedUrlStringResultsBase // | DNSLinkResolveResult | IPNSResolveResult
+
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
 const PATH_REGEX = /^\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
 const PATH_GATEWAY_REGEX = /^https?:\/\/(.*[^/])\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
 const SUBDOMAIN_GATEWAY_REGEX = /^https?:\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\.(?<protocol>ip[fn]s)\.([^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
 
-function matchURLString (urlString: string): Record<string, string> {
+interface MatchUrlGroups {
+  protocol: 'ipfs' | 'ipns'
+  cidOrPeerIdOrDnsLink: string
+  path?: string
+  queryString?: string
+
+}
+function matchURLString (urlString: string): MatchUrlGroups {
   for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) {
     const match = urlString.match(pattern)
 
     if (match?.groups != null) {
-      return match.groups
+      return match.groups as unknown as MatchUrlGroups // force cast to MatchUrlGroups, because if it matches, it has to contain this structure.
     }
   }
 
@@ -89,30 +97,43 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
   let cid: CID | undefined
   let resolvedPath: string | undefined
   const errors: Error[] = []
+  let resolveResult: IPNSResolveResult | DNSLinkResolveResult | undefined
+  let ttl: number = 5 * 60 // 5 minutes based on https://github.com/ipfs/boxo/issues/329#issuecomment-1995236409
 
   if (protocol === 'ipfs') {
     try {
       cid = CID.parse(cidOrPeerIdOrDnsLink)
+      ttl = 29030400 // 1 year for ipfs content
     } catch (err) {
       log.error(err)
       errors.push(new TypeError('Invalid CID for ipfs://<cid> URL'))
     }
   } else {
-    let resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink)
+    // protocol is ipns
+    resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink)
 
     if (resolveResult != null) {
       cid = resolveResult.cid
       resolvedPath = resolveResult.path
+      const answerTtl = (resolveResult as DNSLinkResolveResult).answer?.TTL
+      const recordTtl = (resolveResult as IPNSResolveResult).record?.ttl
+      ttl = Number(answerTtl ?? recordTtl ?? ttl)
       log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid)
     } else {
-      // protocol is ipns
       log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
       let peerId = null
       try {
         peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
         resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress })
         cid = resolveResult?.cid
         resolvedPath = resolveResult?.path
+        /**
+         * For TTL nuances, see
+         *
+         * @see https://github.com/ipfs/js-ipns/blob/16e0e10682fa9a663e0bb493a44d3e99a5200944/src/index.ts#L200
+         * @see https://github.com/ipfs/js-ipns/pull/308
+         */
+        ttl = Number(resolveResult.record.ttl)
         log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
         ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
       } catch (err) {
@@ -137,6 +158,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
           resolveResult = await ipns.resolveDNSLink(decodedDnsLinkLabel, { onProgress: options?.onProgress })
           cid = resolveResult?.cid
           resolvedPath = resolveResult?.path
+          ttl = resolveResult.answer.TTL
           log.trace('resolved %s to %c', decodedDnsLinkLabel, cid)
           ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
         } catch (err: any) {
@@ -177,9 +199,10 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
   return {
     protocol,
     cid,
-    path: joinPaths(resolvedPath, urlPath),
-    query
-  }
+    path: joinPaths(resolvedPath, urlPath ?? ''),
+    query,
+    ttl
+  } satisfies ParsedUrlStringResults
 }
 
 /**

packages/verified-fetch/src/utils/response-headers.ts

@@ -0,0 +1,31 @@
+interface CacheControlHeaderOptions {
+  ttl?: number
+  protocol: 'ipfs' | 'ipns'
+  response: Response
+}
+/**
+ * Implementations may place an upper bound on any TTL received, as noted in Section 8 of [rfc2181].
+ * If TTL value is unknown, implementations should not send a Cache-Control
+ * No matter if TTL value is known or not, implementations should always send a Last-Modified header with the timestamp of the record resolution.
+ *
+ * @see https://specs.ipfs.tech/http-gateways/path-gateway/#cache-control-response-header
+ */
+export function setCacheControlHeader ({ ttl, protocol, response }: CacheControlHeaderOptions): void {
+  let headerValue: string
+  if (protocol === 'ipfs') {
+    headerValue = 'public, max-age=29030400, immutable'
+  } else if (ttl == null) {
+    /**
+     * default limit for unknown TTL: "use 5 minute as default fallback when it is not available."
+     *
+     * @see https://github.com/ipfs/boxo/issues/329#issuecomment-1995236409
+     */
+    headerValue = 'public, max-age=300'
+  } else {
+    headerValue = `public, max-age=${ttl}`
+  }
+
+  if (headerValue != null) {
+    response.headers.set('cache-control', headerValue)
+  }
+}

packages/verified-fetch/src/verified-fetch.ts

@@ -21,11 +21,13 @@
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { tarStream } from './utils/get-tar-stream.js'
 import { parseResource } from './utils/parse-resource.js'
+import { setCacheControlHeader } from './utils/response-headers.js'
 import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
 import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
 import { walkPath } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { RequestFormatShorthand } from './types.js'
+import type { ParsedUrlStringResults } from './utils/parse-url-string'
 import type { Helia } from '@helia/interface'
 import type { AbortOptions, Logger, PeerId } from '@libp2p/interface'
 import type { DNSResolver } from '@multiformats/dns/resolvers'
@@ -408,7 +410,23 @@
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { resource }))
 
     // resolve the CID/path from the requested resource
-    const { path, query, cid, protocol } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
+    let cid: ParsedUrlStringResults['cid']
+    let path: ParsedUrlStringResults['path']
+    let query: ParsedUrlStringResults['query']
+    let ttl: ParsedUrlStringResults['ttl']
+    let protocol: ParsedUrlStringResults['protocol']
+    try {
+      const result = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
+      cid = result.cid
+      path = result.path
+      query = result.query
+      ttl = result.ttl
+      protocol = result.protocol
+    } catch (err) {
+      this.log.error('error parsing resource %s', resource, err)
+
+      return badRequestResponse('Invalid resource')
+    }
 
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:resolve', { cid, path }))
 
@@ -473,9 +491,7 @@
 
     response.headers.set('etag', getETag({ cid, reqFormat, weak: false }))
 
-    if (protocol === 'ipfs') {
-      response.headers.set('cache-control', 'public, max-age=29030400, immutable')
-    }
+    setCacheControlHeader({ response, ttl, protocol })
     // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
     response.headers.set('X-Ipfs-Path', resource.toString())
 

packages/verified-fetch/test/cache-control-header.spec.ts

@@ -2,20 +2,41 @@ import { dagCbor } from '@helia/dag-cbor'
 import { ipns } from '@helia/ipns'
 import { stop } from '@libp2p/interface'
 import { createEd25519PeerId } from '@libp2p/peer-id-factory'
+import { dns } from '@multiformats/dns'
 import { expect } from 'aegir/chai'
 import Sinon from 'sinon'
+import { stubInterface } from 'sinon-ts'
 import { VerifiedFetch } from '../src/verified-fetch.js'
 import { createHelia } from './fixtures/create-offline-helia.js'
 import type { Helia } from '@helia/interface'
 import type { IPNS } from '@helia/ipns'
-
+import type { DNSResponse } from '@multiformats/dns'
+
+function answerFake (data: string, TTL: number, name: string, type: number): DNSResponse {
+  const fake = stubInterface<DNSResponse>()
+  fake.Answer = [{
+    data,
+    TTL,
+    name,
+    type
+  }]
+  return fake
+}
 describe('cache-control header', () => {
   let helia: Helia
   let name: IPNS
   let verifiedFetch: VerifiedFetch
+  let customDnsResolver: Sinon.SinonStub<any[], Promise<DNSResponse>>
 
   beforeEach(async () => {
-    helia = await createHelia()
+    customDnsResolver = Sinon.stub()
+    helia = await createHelia({
+      dns: dns({
+        resolvers: {
+          '.': customDnsResolver
+        }
+      })
+    })
     name = ipns(helia)
     verifiedFetch = new VerifiedFetch({
       helia
@@ -60,7 +81,7 @@ describe('cache-control header', () => {
     expect(resp.headers.get('Cache-Control')).to.not.containIgnoreCase('immutable')
   })
 
-  it.skip('should return the correct max-age in the cache-control header for an IPNS name', async () => {
+  it('should return the correct max-age in the cache-control header for an IPNS name', async () => {
     const obj = {
       hello: 'world'
     }
@@ -70,19 +91,24 @@ describe('cache-control header', () => {
     const oneHourInMs = 1000 * 60 * 60
     const peerId = await createEd25519PeerId()
 
-    // ipns currently only allows customising the lifetime which is also used as the TTL
-    await name.publish(peerId, cid, { lifetime: oneHourInMs })
+    /**
+     * ipns currently only allows customising the lifetime which is also used as the TTL
+     *
+     * lifetime is coming back as 100000 times larger than expected
+     *
+     * @see https://github.com/ipfs/js-ipns/blob/16e0e10682fa9a663e0bb493a44d3e99a5200944/src/index.ts#L200
+     * @see https://github.com/ipfs/js-ipns/pull/308
+     */
+    await name.publish(peerId, cid, { lifetime: oneHourInMs / 100000 })
 
     const resp = await verifiedFetch.fetch(`ipns://${peerId}`)
     expect(resp).to.be.ok()
     expect(resp.status).to.equal(200)
 
-    expect(resp.headers.get('Cache-Control')).to.equal(`public, max-age=${oneHourInMs.toString()}`)
+    expect(resp.headers.get('Cache-Control')).to.equal(`public, max-age=${oneHourInMs}`)
   })
 
   it('should not contain immutable in the cache-control header for a DNSLink name', async () => {
-    const customDnsResolver = Sinon.stub()
-
     verifiedFetch = new VerifiedFetch({
       helia
     }, {
@@ -94,12 +120,12 @@ describe('cache-control header', () => {
     }
     const c = dagCbor(helia)
     const cid = await c.add(obj)
-    customDnsResolver.returns(Promise.resolve(`/ipfs/${cid.toString()}`))
+    customDnsResolver.withArgs('_dnslink.example-domain.com').resolves(answerFake(`dnslink=/ipfs/${cid}`, 666, '_dnslink.example-domain.com', 16))
 
     const resp = await verifiedFetch.fetch('ipns://example-domain.com')
     expect(resp).to.be.ok()
     expect(resp.status).to.equal(200)
 
-    expect(resp.headers.get('Cache-Control')).to.not.containIgnoreCase('immutable')
+    expect(resp.headers.get('Cache-Control')).to.equal('public, max-age=666')
   })
 })

packages/verified-fetch/test/custom-dns-resolvers.spec.ts

@@ -19,9 +19,11 @@ describe('custom dns-resolvers', () => {
   })
 
   it('is used when passed to createVerifiedFetch', async () => {
-    const customDnsResolver = Sinon.stub()
-
-    customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg'))
+    const customDnsResolver = Sinon.stub().withArgs('_dnslink.some-non-cached-domain.com').resolves({
+      Answer: [{
+        data: 'dnslink=/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg'
+      }]
+    })
 
     const fetch = await createVerifiedFetch({
       gateways: ['http://127.0.0.1:8080'],

packages/verified-fetch/test/utils/parse-url-string.spec.ts

@@ -75,8 +75,9 @@ describe('parseUrlString', () => {
           ipns,
           logger
         })
-      ).to.eventually.be.rejected
-        .with.property('message', 'Could not parse PeerId in ipns url "mydomain.com", Non-base64 character')
+      ).to.eventually.be.rejected.and.to.have.nested.property('errors[0]').to.deep.equal(
+        new TypeError('Could not parse PeerId in ipns url "mydomain.com", Non-base64 character')
+      )
     })
   })