diff --git a/packages/verified-fetch/src/types.ts b/packages/verified-fetch/src/types.ts new file mode 100644 index 00000000..4a235e1a --- /dev/null +++ b/packages/verified-fetch/src/types.ts @@ -0,0 +1 @@ +export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor' diff --git a/packages/verified-fetch/src/utils/get-e-tag.ts b/packages/verified-fetch/src/utils/get-e-tag.ts new file mode 100644 index 00000000..74ab7303 --- /dev/null +++ b/packages/verified-fetch/src/utils/get-e-tag.ts @@ -0,0 +1,36 @@ +import type { RequestFormatShorthand } from '../types.js' +import type { CID } from 'multiformats/cid' + +interface GetETagArg { + cid: CID + reqFormat?: RequestFormatShorthand + rangeStart?: number + rangeEnd?: number + /** + * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content). + * Some examples: + * - IPNS requests + * - CAR streamed with blocks in non-deterministic order + * - TAR streamed with files in non-deterministic order + */ + weak?: boolean +} + +/** + * etag + * you need to wrap cid with "" + * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML). + * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway + * + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag + * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header + */ +export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string { + const prefix = weak === true ? 'W/' : '' + let suffix = reqFormat == null ? '' : `.${reqFormat}` + if (rangeStart != null || rangeEnd != null) { + suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}` + } + + return `${prefix}"${cid.toString()}${suffix}"` +} diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 5846c413..4c078973 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -1,6 +1,7 @@ import { peerIdFromString } from '@libp2p/peer-id' import { CID } from 'multiformats/cid' import { TLRU } from './tlru.js' +import type { RequestFormatShorthand } from '../types.js' import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' import type { ComponentLogger } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events' @@ -16,11 +17,15 @@ export interface ParseUrlStringOptions extends ProgressOptions { + format?: RequestFormatShorthand +} + export interface ParsedUrlStringResults { protocol: string path: string cid: CID - query: Record + query: ParsedUrlQuery } const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/$?]+)\/?(?[^$?]*)\??(?.*)$/ diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index a8894ecf..9f319ad5 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -9,10 +9,12 @@ import { code as rawCode } from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { CustomProgressEvent } from 'progress-events' import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' +import { getETag } from './utils/get-e-tag.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { parseResource } from './utils/parse-resource.js' import { walkPath, type PathWalkerFn } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' +import type { RequestFormatShorthand } from './types.js' import type { Helia } from '@helia/interface' import type { AbortOptions, Logger } from '@libp2p/interface' import type { UnixFSEntry } from 'ipfs-unixfs-exporter' @@ -231,8 +233,8 @@ export class VerifiedFetch { * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter * @default 'raw' */ - private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: string | null }): string | null { - const formatMap: Record = { + private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: RequestFormatShorthand | null }): RequestFormatShorthand | null { + const formatMap: Record = { 'vnd.ipld.raw': 'raw', 'vnd.ipld.car': 'car', 'application/x-tar': 'tar', @@ -323,7 +325,7 @@ export class VerifiedFetch { } } - response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header + response.headers.set('etag', getETag({ cid, reqFormat: format ?? undefined, weak: false })) response.headers.set('cache-control', 'public, max-age=29030400, immutable') response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header diff --git a/packages/verified-fetch/test/get-e-tag.spec.ts b/packages/verified-fetch/test/get-e-tag.spec.ts new file mode 100644 index 00000000..a0908d58 --- /dev/null +++ b/packages/verified-fetch/test/get-e-tag.spec.ts @@ -0,0 +1,33 @@ +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { getETag } from '../src/utils/get-e-tag.js' + +const cidString = 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr' +const testCID = CID.parse(cidString) + +describe('getETag', () => { + it('CID eTag', () => { + expect(getETag({ cid: testCID, weak: true })).to.equal(`W/"${cidString}"`) + expect(getETag({ cid: testCID, weak: false })).to.equal(`"${cidString}"`) + }) + + it('should return ETag with CID and format suffix', () => { + expect(getETag({ cid: testCID, reqFormat: 'raw' })).to.equal(`"${cidString}.raw"`) + expect(getETag({ cid: testCID, reqFormat: 'json' })).to.equal(`"${cidString}.json"`) + }) + + it('should return ETag with CID and range suffix', () => { + expect(getETag({ cid: testCID, weak: true, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`W/"${cidString}.car.10-20"`) + expect(getETag({ cid: testCID, weak: false, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.car.10-20"`) + }) + + it('should return ETag with CID, format and range suffix', () => { + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.raw.10-20"`) + }) + + it('should handle undefined rangeStart and rangeEnd', () => { + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: undefined })).to.equal(`"${cidString}.raw"`) + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 55, rangeEnd: undefined })).to.equal(`"${cidString}.raw.55-N"`) + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: 77 })).to.equal(`"${cidString}.raw.0-77"`) + }) +})