diff --git a/packages/interop/CHANGELOG.md b/packages/interop/CHANGELOG.md index 745ac92..1c81d4f 100644 --- a/packages/interop/CHANGELOG.md +++ b/packages/interop/CHANGELOG.md @@ -1,3 +1,11 @@ +## @helia/verified-fetch-interop [1.7.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.6.0...@helia/verified-fetch-interop-1.7.0) (2024-03-15) + + + +### Dependencies + +* **@helia/verified-fetch:** upgraded to 1.2.0 + ## @helia/verified-fetch-interop [1.6.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.5.1...@helia/verified-fetch-interop-1.6.0) (2024-03-14) diff --git a/packages/interop/package.json b/packages/interop/package.json index 86d8115..0c974d9 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -1,6 +1,6 @@ { "name": "@helia/verified-fetch-interop", - "version": "1.6.0", + "version": "1.7.0", "description": "Interop tests for @helia/verified-fetch", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-verified-fetch/tree/main/packages/interop#readme", @@ -57,7 +57,7 @@ "test:electron-main": "aegir test -t electron-main" }, "dependencies": { - "@helia/verified-fetch": "1.1.3", + "@helia/verified-fetch": "1.2.0", "aegir": "^42.2.5", "ipfsd-ctl": "^13.0.0", "it-drain": "^3.0.5", diff --git a/packages/verified-fetch/CHANGELOG.md b/packages/verified-fetch/CHANGELOG.md index 548a42e..155d533 100644 --- a/packages/verified-fetch/CHANGELOG.md +++ b/packages/verified-fetch/CHANGELOG.md @@ -1,3 +1,15 @@ +## @helia/verified-fetch [1.2.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.1.3...@helia/verified-fetch-1.2.0) (2024-03-15) + + +### Features + +* support http range header ([#10](https://github.com/ipfs/helia-verified-fetch/issues/10)) ([9f5078a](https://github.com/ipfs/helia-verified-fetch/commit/9f5078a09846ba6569d637ea1dd90a6d8fb4e629)) + + +### Trivial Changes + +* fix build ([#22](https://github.com/ipfs/helia-verified-fetch/issues/22)) ([01261fe](https://github.com/ipfs/helia-verified-fetch/commit/01261feabd4397c10446609b072a7cb97fb81911)) + ## @helia/verified-fetch [1.1.3](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.1.2...@helia/verified-fetch-1.1.3) (2024-03-14) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 63f99e1..88d5bc1 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -1,6 +1,6 @@ { "name": "@helia/verified-fetch", - "version": "1.1.3", + "version": "1.2.0", "description": "A fetch-like API for obtaining verified & trustless IPFS content on the web", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-verified-fetch/tree/main/packages/verified-fetch#readme", diff --git a/packages/verified-fetch/src/types.ts b/packages/verified-fetch/src/types.ts index 4a235e1..f46515d 100644 --- a/packages/verified-fetch/src/types.ts +++ b/packages/verified-fetch/src/types.ts @@ -1 +1,3 @@ export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor' + +export type SupportedBodyTypes = string | ArrayBuffer | Blob | ReadableStream | null diff --git a/packages/verified-fetch/src/utils/byte-range-context.ts b/packages/verified-fetch/src/utils/byte-range-context.ts new file mode 100644 index 0000000..54df115 --- /dev/null +++ b/packages/verified-fetch/src/utils/byte-range-context.ts @@ -0,0 +1,303 @@ +import { calculateByteRangeIndexes, getHeader } from './request-headers.js' +import { getContentRangeHeader } from './response-headers.js' +import type { SupportedBodyTypes } from '../types.js' +import type { ComponentLogger, Logger } from '@libp2p/interface' + +type SliceableBody = Exclude | null> + +/** + * Gets the body size of a given body if it's possible to calculate it synchronously. + */ +function getBodySizeSync (body: SupportedBodyTypes): number | null { + if (typeof body === 'string') { + return body.length + } + if (body instanceof ArrayBuffer || body instanceof Uint8Array) { + return body.byteLength + } + if (body instanceof Blob) { + return body.size + } + + if (body instanceof ReadableStream) { + return null + } + + return null +} + +function getByteRangeFromHeader (rangeHeader: string): { start: string, end: string } { + /** + * Range: bytes=- | bytes=- | bytes=- + */ + const match = rangeHeader.match(/^bytes=(?\d+)?-(?\d+)?$/) + if (match?.groups == null) { + throw new Error('Invalid range request') + } + + const { start, end } = match.groups + + return { start, end } +} + +export class ByteRangeContext { + public readonly isRangeRequest: boolean + + /** + * This property is purposefully only set in `set fileSize` and should not be set directly. + */ + private _fileSize: number | null | undefined + private _body: SupportedBodyTypes = null + private readonly rangeRequestHeader: string | undefined + private readonly log: Logger + private readonly requestRangeStart: number | null + private readonly requestRangeEnd: number | null + private byteStart: number | undefined + private byteEnd: number | undefined + private byteSize: number | undefined + + constructor (logger: ComponentLogger, private readonly headers?: HeadersInit) { + this.log = logger.forComponent('helia:verified-fetch:byte-range-context') + this.rangeRequestHeader = getHeader(this.headers, 'Range') + if (this.rangeRequestHeader != null) { + this.isRangeRequest = true + this.log.trace('range request detected') + try { + const { start, end } = getByteRangeFromHeader(this.rangeRequestHeader) + this.requestRangeStart = start != null ? parseInt(start) : null + this.requestRangeEnd = end != null ? parseInt(end) : null + } catch (e) { + this.log.error('error parsing range request header: %o', e) + this.requestRangeStart = null + this.requestRangeEnd = null + } + + this.setOffsetDetails() + } else { + this.log.trace('no range request detected') + this.isRangeRequest = false + this.requestRangeStart = null + this.requestRangeEnd = null + } + } + + public setBody (body: SupportedBodyTypes): void { + this._body = body + // if fileSize was already set, don't recalculate it + this.setFileSize(this._fileSize ?? getBodySizeSync(body)) + + this.log.trace('set request body with fileSize %o', this._fileSize) + } + + public getBody (): SupportedBodyTypes { + const body = this._body + if (body == null) { + this.log.trace('body is null') + return body + } + if (!this.isRangeRequest || !this.isValidRangeRequest) { + this.log.trace('returning body unmodified for non-range, or invalid range, request') + return body + } + const byteStart = this.byteStart + const byteEnd = this.byteEnd + const byteSize = this.byteSize + if (byteStart != null || byteEnd != null) { + this.log.trace('returning body with byteStart=%o, byteEnd=%o, byteSize=%o', byteStart, byteEnd, byteSize) + if (body instanceof ReadableStream) { + // stream should already be spliced by `unixfs.cat` + return body + } + return this.getSlicedBody(body) + } + + // we should not reach this point, but return body untouched. + this.log.error('returning unmodified body for valid range request') + return body + } + + private getSlicedBody (body: T): SliceableBody { + if (this.isPrefixLengthRequest) { + this.log.trace('sliced body with byteStart %o', this.byteStart) + return body.slice(this.offset) satisfies SliceableBody + } + if (this.isSuffixLengthRequest && this.length != null) { + this.log.trace('sliced body with length %o', -this.length) + return body.slice(-this.length) satisfies SliceableBody + } + const offset = this.byteStart ?? 0 + const length = this.byteEnd == null ? undefined : this.byteEnd + 1 + this.log.trace('returning body with offset %o and length %o', offset, length) + + return body.slice(offset, length) satisfies SliceableBody + } + + private get isSuffixLengthRequest (): boolean { + return this.requestRangeStart == null && this.requestRangeEnd != null + } + + private get isPrefixLengthRequest (): boolean { + return this.requestRangeStart != null && this.requestRangeEnd == null + } + + /** + * Sometimes, we need to set the fileSize explicitly because we can't calculate + * the size of the body (e.g. for unixfs content where we call .stat). + * + * This fileSize should otherwise only be called from `setBody`. + */ + public setFileSize (size: number | bigint | null): void { + this._fileSize = size != null ? Number(size) : null + this.log.trace('set _fileSize to %o', this._fileSize) + // when fileSize changes, we need to recalculate the offset details + this.setOffsetDetails() + } + + public getFileSize (): number | null | undefined { + return this._fileSize + } + + private isValidByteStart (): boolean { + if (this.byteStart != null) { + if (this.byteStart < 0) { + return false + } + if (this._fileSize != null && this.byteStart > this._fileSize) { + return false + } + } + return true + } + + private isValidByteEnd (): boolean { + if (this.byteEnd != null) { + if (this.byteEnd < 0) { + return false + } + if (this._fileSize != null && this.byteEnd > this._fileSize) { + return false + } + } + return true + } + + /** + * We may get the values required to determine if this is a valid range request at different times + * so we need to calculate it when asked. + */ + public get isValidRangeRequest (): boolean { + if (!this.isRangeRequest) { + return false + } + if (this.requestRangeStart == null && this.requestRangeEnd == null) { + this.log.trace('invalid range request, range request values not provided') + return false + } + if (!this.isValidByteStart()) { + this.log.trace('invalid range request, byteStart is less than 0 or greater than fileSize') + return false + } + if (!this.isValidByteEnd()) { + this.log.trace('invalid range request, byteEnd is less than 0 or greater than fileSize') + return false + } + if (this.requestRangeEnd != null && this.requestRangeStart != null) { + // we may not have enough info.. base check on requested bytes + if (this.requestRangeStart > this.requestRangeEnd) { + this.log.trace('invalid range request, start is greater than end') + return false + } else if (this.requestRangeStart < 0) { + this.log.trace('invalid range request, start is less than 0') + return false + } else if (this.requestRangeEnd < 0) { + this.log.trace('invalid range request, end is less than 0') + return false + } + } + + return true + } + + /** + * Given all the information we have, this function returns the offset that will be used when: + * 1. calling unixfs.cat + * 2. slicing the body + */ + public get offset (): number { + if (this.byteStart === 0) { + return 0 + } + if (this.isPrefixLengthRequest || this.isSuffixLengthRequest) { + if (this.byteStart != null) { + // we have to subtract by 1 because the offset is inclusive + return this.byteStart - 1 + } + } + + return this.byteStart ?? 0 + } + + /** + * Given all the information we have, this function returns the length that will be used when: + * 1. calling unixfs.cat + * 2. slicing the body + */ + public get length (): number | undefined { + return this.byteSize ?? undefined + } + + /** + * Converts a range request header into helia/unixfs supported range options + * Note that the gateway specification says we "MAY" support multiple ranges (https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header) but we don't + * + * Also note that @helia/unixfs and ipfs-unixfs-exporter expect length and offset to be numbers, the range header is a string, and the size of the resource is likely a bigint. + * + * SUPPORTED: + * Range: bytes=- + * Range: bytes=- + * Range: bytes=- // must pass size so we can calculate the offset. suffix-length is the number of bytes from the end of the file. + * + * NOT SUPPORTED: + * Range: bytes=-, - + * Range: bytes=-, -, - + * + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range#directives + */ + private setOffsetDetails (): void { + if (this.requestRangeStart == null && this.requestRangeEnd == null) { + this.log.trace('requestRangeStart and requestRangeEnd are null') + return + } + + const { start, end, byteSize } = calculateByteRangeIndexes(this.requestRangeStart ?? undefined, this.requestRangeEnd ?? undefined, this._fileSize ?? undefined) + this.log.trace('set byteStart to %o, byteEnd to %o, byteSize to %o', start, end, byteSize) + this.byteStart = start + this.byteEnd = end + this.byteSize = byteSize + } + + /** + * This function returns the value of the "content-range" header. + * + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range + * + * Returns a string representing the following content ranges: + * + * @example + * - Content-Range: -/ + * - Content-Range: -/* + */ + // - Content-Range: */ // this is purposefully not in jsdoc block + public get contentRangeHeaderValue (): string { + if (!this.isValidRangeRequest) { + this.log.error('cannot get contentRangeHeaderValue for invalid range request') + throw new Error('Invalid range request') + } + + return getContentRangeHeader({ + byteStart: this.byteStart, + byteEnd: this.byteEnd, + byteSize: this._fileSize ?? undefined + }) + } +} diff --git a/packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts b/packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts index c9266e2..02342d5 100644 --- a/packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts +++ b/packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts @@ -11,7 +11,7 @@ export async function getStreamFromAsyncIterable (iterator: AsyncIterable(1000) +const ipnsCache = new TLRU(1000) export interface ParseUrlStringInput { urlString: string @@ -23,30 +23,66 @@ export interface ParsedUrlQuery extends Record { filename?: string } -export interface ParsedUrlStringResults { - protocol: string - path: string - cid: CID +interface ParsedUrlStringResultsBase extends ResolveResult { + protocol: 'ipfs' | 'ipns' query: ParsedUrlQuery + + /** + * seconds as a number + */ + ttl?: number } +export type ParsedUrlStringResults = ParsedUrlStringResultsBase + const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/?]+)\/?(?[^?]*)\??(?.*)$/ const PATH_REGEX = /^\/(?ip[fn]s)\/(?[^/?]+)\/?(?[^?]*)\??(?.*)$/ const PATH_GATEWAY_REGEX = /^https?:\/\/(.*[^/])\/(?ip[fn]s)\/(?[^/?]+)\/?(?[^?]*)\??(?.*)$/ const SUBDOMAIN_GATEWAY_REGEX = /^https?:\/\/(?[^/?]+)\.(?ip[fn]s)\.([^/?]+)\/?(?[^?]*)\??(?.*)$/ -function matchURLString (urlString: string): Record { +interface MatchUrlGroups { + protocol: 'ipfs' | 'ipns' + cidOrPeerIdOrDnsLink: string + path?: string + queryString?: string + +} +function matchURLString (urlString: string): MatchUrlGroups { for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) { const match = urlString.match(pattern) if (match?.groups != null) { - return match.groups + return match.groups as unknown as MatchUrlGroups // force cast to MatchUrlGroups, because if it matches, it has to contain this structure. } } throw new TypeError(`Invalid URL: ${urlString}, please use ipfs://, ipns://, or gateway URLs only`) } +/** + * determines the TTL for the resolved resource that will be used for the `Cache-Control` header's `max-age` directive. + * max-age is in seconds + * + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#response_directives + * + * If we have ipnsTtlNs, it will be a BigInt representing "nanoseconds". We need to convert it back to seconds. + * + * For more TTL nuances: + * + * @see https://github.com/ipfs/js-ipns/blob/16e0e10682fa9a663e0bb493a44d3e99a5200944/src/index.ts#L200 + * @see https://github.com/ipfs/js-ipns/pull/308 + */ +function calculateTtl (resolveResult?: IPNSResolveResult | DNSLinkResolveResult): number | undefined { + if (resolveResult == null) { + return undefined + } + const dnsLinkTtl = (resolveResult as DNSLinkResolveResult).answer?.TTL + const ipnsTtlNs = (resolveResult as IPNSResolveResult).record?.ttl + // For some reason, ipns "nanoseconds" are 1e-8 of a second, instead of 1e-9. + const ipnsTtl = ipnsTtlNs != null ? Number(ipnsTtlNs / BigInt(1e8)) : undefined + return dnsLinkTtl ?? ipnsTtl +} + /** * For dnslinks see https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header * DNSLink names include . which means they must be inlined into a single DNS label to provide unique origin and work with wildcard TLS certificates. @@ -89,43 +125,48 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin let cid: CID | undefined let resolvedPath: string | undefined const errors: Error[] = [] + let resolveResult: IPNSResolveResult | DNSLinkResolveResult | undefined if (protocol === 'ipfs') { try { cid = CID.parse(cidOrPeerIdOrDnsLink) + /** + * no ttl set. @link {setCacheControlHeader} + */ } catch (err) { log.error(err) errors.push(new TypeError('Invalid CID for ipfs:// URL')) } } else { - let resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink) + // protocol is ipns + resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink) if (resolveResult != null) { cid = resolveResult.cid resolvedPath = resolveResult.path log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid) } else { - // protocol is ipns log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink) let peerId = null try { + // try resolving as an IPNS name peerId = peerIdFromString(cidOrPeerIdOrDnsLink) resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress }) - cid = resolveResult?.cid - resolvedPath = resolveResult?.path + cid = resolveResult.cid + resolvedPath = resolveResult.path log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid) - ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) } catch (err) { if (peerId == null) { - log.error('Could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err) + log.error('could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err) errors.push(new TypeError(`Could not parse PeerId in ipns url "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`)) } else { - log.error('Could not resolve PeerId %c', peerId, err) + log.error('could not resolve PeerId %c', peerId, err) errors.push(new TypeError(`Could not resolve PeerId "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`)) } } if (cid == null) { + // cid is still null, try resolving as a DNSLink let decodedDnsLinkLabel = cidOrPeerIdOrDnsLink if (isInlinedDnsLink(cidOrPeerIdOrDnsLink)) { decodedDnsLinkLabel = dnsLinkLabelDecoder(cidOrPeerIdOrDnsLink) @@ -138,9 +179,8 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin cid = resolveResult?.cid resolvedPath = resolveResult?.path log.trace('resolved %s to %c', decodedDnsLinkLabel, cid) - ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) } catch (err: any) { - log.error('Could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err) + log.error('could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err) errors.push(err) } } @@ -155,6 +195,13 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin throw new AggregateError(errors, `Invalid resource. Cannot determine CID from URL "${urlString}"`) } + const ttl = calculateTtl(resolveResult) + + if (resolveResult != null) { + // use the ttl for the resolved resouce for the cache, but fallback to 2 minutes if not available + ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, ttl ?? 60 * 1000 * 2) + } + // parse query string const query: Record = {} @@ -177,9 +224,10 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin return { protocol, cid, - path: joinPaths(resolvedPath, urlPath), - query - } + path: joinPaths(resolvedPath, urlPath ?? ''), + query, + ttl + } satisfies ParsedUrlStringResults } /** diff --git a/packages/verified-fetch/src/utils/request-headers.ts b/packages/verified-fetch/src/utils/request-headers.ts new file mode 100644 index 0000000..3eed63a --- /dev/null +++ b/packages/verified-fetch/src/utils/request-headers.ts @@ -0,0 +1,51 @@ +export function getHeader (headers: HeadersInit | undefined, header: string): string | undefined { + if (headers == null) { + return undefined + } + if (headers instanceof Headers) { + return headers.get(header) ?? undefined + } + if (Array.isArray(headers)) { + const entry = headers.find(([key]) => key.toLowerCase() === header.toLowerCase()) + return entry?.[1] + } + const key = Object.keys(headers).find(k => k.toLowerCase() === header.toLowerCase()) + if (key == null) { + return undefined + } + + return headers[key] +} + +/** + * Given two ints from a Range header, and potential fileSize, returns: + * 1. number of bytes the response should contain. + * 2. the start index of the range. // inclusive + * 3. the end index of the range. // inclusive + */ +export function calculateByteRangeIndexes (start: number | undefined, end: number | undefined, fileSize?: number): { byteSize?: number, start?: number, end?: number } { + if (start != null && end != null) { + if (start > end) { + throw new Error('Invalid range') + } + + return { byteSize: end - start + 1, start, end } + } else if (start == null && end != null) { + // suffix byte range requested + if (fileSize == null) { + return { end } + } + const result = { byteSize: end, start: fileSize - end + 1, end: fileSize } + return result + } else if (start != null && end == null) { + if (fileSize == null) { + return { start } + } + const byteSize = fileSize - start + 1 + const end = fileSize + return { byteSize, start, end } + } + + // both start and end are undefined + return { byteSize: fileSize } +} diff --git a/packages/verified-fetch/src/utils/response-headers.ts b/packages/verified-fetch/src/utils/response-headers.ts new file mode 100644 index 0000000..47574dc --- /dev/null +++ b/packages/verified-fetch/src/utils/response-headers.ts @@ -0,0 +1,70 @@ +interface CacheControlHeaderOptions { + /** + * This should be seconds as a number. + * + * See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#response_directives + */ + ttl?: number + protocol: 'ipfs' | 'ipns' + response: Response +} + +/** + * Implementations may place an upper bound on any TTL received, as noted in Section 8 of [rfc2181]. + * If TTL value is unknown, implementations should not send a Cache-Control + * No matter if TTL value is known or not, implementations should always send a Last-Modified header with the timestamp of the record resolution. + * + * @see https://specs.ipfs.tech/http-gateways/path-gateway/#cache-control-response-header + */ +export function setCacheControlHeader ({ ttl, protocol, response }: CacheControlHeaderOptions): void { + let headerValue: string + if (protocol === 'ipfs') { + headerValue = 'public, max-age=29030400, immutable' + } else if (ttl == null) { + /** + * default limit for unknown TTL: "use 5 minute as default fallback when it is not available." + * + * @see https://github.com/ipfs/boxo/issues/329#issuecomment-1995236409 + */ + headerValue = 'public, max-age=300' + } else { + headerValue = `public, max-age=${ttl}` + } + + if (headerValue != null) { + response.headers.set('cache-control', headerValue) + } +} + +/** + * This function returns the value of the `Content-Range` header for a given range. + * If you know the total size of the body, pass it as `byteSize` + * + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range + */ +export function getContentRangeHeader ({ byteStart, byteEnd, byteSize }: { byteStart: number | undefined, byteEnd: number | undefined, byteSize: number | undefined }): string { + const total = byteSize ?? '*' // if we don't know the total size, we should use * + + if (byteStart != null && byteEnd == null) { + // only byteStart in range + if (byteSize == null) { + return `bytes */${total}` + } + return `bytes ${byteStart}-${byteSize}/${byteSize}` + } + + if (byteStart == null && byteEnd != null) { + // only byteEnd in range + if (byteSize == null) { + return `bytes */${total}` + } + return `bytes ${byteSize - byteEnd + 1}-${byteSize}/${byteSize}` + } + + if (byteStart == null && byteEnd == null) { + // neither are provided, we can't return a valid range. + return `bytes */${total}` + } + + return `bytes ${byteStart}-${byteEnd}/${total}` +} diff --git a/packages/verified-fetch/src/utils/responses.ts b/packages/verified-fetch/src/utils/responses.ts index a596370..667318c 100644 --- a/packages/verified-fetch/src/utils/responses.ts +++ b/packages/verified-fetch/src/utils/responses.ts @@ -1,3 +1,7 @@ +import type { ByteRangeContext } from './byte-range-context' +import type { SupportedBodyTypes } from '../types.js' +import type { Logger } from '@libp2p/interface' + function setField (response: Response, name: string, value: string | boolean): void { Object.defineProperty(response, name, { enumerable: true, @@ -23,7 +27,7 @@ export interface ResponseOptions extends ResponseInit { redirected?: boolean } -export function okResponse (url: string, body?: BodyInit | null, init?: ResponseOptions): Response { +export function okResponse (url: string, body?: SupportedBodyTypes, init?: ResponseOptions): Response { const response = new Response(body, { ...(init ?? {}), status: 200, @@ -34,13 +38,27 @@ export function okResponse (url: string, body?: BodyInit | null, init?: Response setRedirected(response) } + setType(response, 'basic') + setUrl(response, url) + response.headers.set('Accept-Ranges', 'bytes') + + return response +} + +export function badGatewayResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response { + const response = new Response(body, { + ...(init ?? {}), + status: 502, + statusText: 'Bad Gateway' + }) + setType(response, 'basic') setUrl(response, url) return response } -export function notSupportedResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response { +export function notSupportedResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response { const response = new Response(body, { ...(init ?? {}), status: 501, @@ -54,7 +72,7 @@ export function notSupportedResponse (url: string, body?: BodyInit | null, init? return response } -export function notAcceptableResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response { +export function notAcceptableResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response { const response = new Response(body, { ...(init ?? {}), status: 406, @@ -67,7 +85,7 @@ export function notAcceptableResponse (url: string, body?: BodyInit | null, init return response } -export function badRequestResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response { +export function badRequestResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response { const response = new Response(body, { ...(init ?? {}), status: 400, @@ -96,3 +114,63 @@ export function movedPermanentlyResponse (url: string, location: string, init?: return response } + +interface RangeOptions { + byteRangeContext: ByteRangeContext + log?: Logger +} + +export function okRangeResponse (url: string, body: SupportedBodyTypes, { byteRangeContext, log }: RangeOptions, init?: ResponseOptions): Response { + if (!byteRangeContext.isRangeRequest) { + return okResponse(url, body, init) + } + + if (!byteRangeContext.isValidRangeRequest) { + return badRangeResponse(url, body, init) + } + + let response: Response + try { + response = new Response(body, { + ...(init ?? {}), + status: 206, + statusText: 'Partial Content', + headers: { + ...(init?.headers ?? {}), + 'content-range': byteRangeContext.contentRangeHeaderValue + } + }) + } catch (e: any) { + log?.error('failed to create range response', e) + return badRangeResponse(url, body, init) + } + + if (init?.redirected === true) { + setRedirected(response) + } + + setType(response, 'basic') + setUrl(response, url) + response.headers.set('Accept-Ranges', 'bytes') + + return response +} + +/** + * We likely need to catch errors handled by upstream helia libraries if range-request throws an error. Some examples: + * * The range is out of bounds + * * The range is invalid + * * The range is not supported for the given type + */ +export function badRangeResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response { + const response = new Response(body, { + ...(init ?? {}), + status: 416, + statusText: 'Requested Range Not Satisfiable' + }) + + setType(response, 'basic') + setUrl(response, url) + + return response +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index a8354cc..15280d9 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,6 +1,6 @@ import { car } from '@helia/car' import { ipns as heliaIpns, type IPNS } from '@helia/ipns' -import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' +import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs' import * as ipldDagCbor from '@ipld/dag-cbor' import * as ipldDagJson from '@ipld/dag-json' import { code as dagPbCode } from '@ipld/dag-pb' @@ -15,17 +15,20 @@ import { CustomProgressEvent } from 'progress-events' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { ByteRangeContext } from './utils/byte-range-context.js' import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js' import { getETag } from './utils/get-e-tag.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { tarStream } from './utils/get-tar-stream.js' import { parseResource } from './utils/parse-resource.js' -import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js' +import { setCacheControlHeader } from './utils/response-headers.js' +import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse } from './utils/responses.js' import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js' import { walkPath } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' import type { RequestFormatShorthand } from './types.js' +import type { ParsedUrlStringResults } from './utils/parse-url-string' import type { Helia } from '@helia/interface' import type { AbortOptions, Logger, PeerId } from '@libp2p/interface' import type { DNSResolver } from '@multiformats/dns/resolvers' @@ -275,17 +278,19 @@ export class VerifiedFetch { let terminalElement: UnixFSEntry | undefined let ipfsRoots: CID[] | undefined let redirected = false + const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers) try { const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options) ipfsRoots = pathDetails.ipfsRoots terminalElement = pathDetails.terminalElement } catch (err) { - this.log.error('Error walking path %s', path, err) + this.log.error('error walking path %s', path, err) + + return badGatewayResponse('Error walking path') } let resolvedCID = terminalElement?.cid ?? cid - let stat: UnixFSStats if (terminalElement?.type === 'directory') { const dirCid = terminalElement.cid @@ -307,7 +312,7 @@ export class VerifiedFetch { const rootFilePath = 'index.html' try { this.log.trace('found directory at %c/%s, looking for index.html', cid, path) - stat = await this.unixfs.stat(dirCid, { + const stat = await this.unixfs.stat(dirCid, { path: rootFilePath, signal: options?.signal, onProgress: options?.onProgress @@ -323,30 +328,56 @@ export class VerifiedFetch { } } + // we have a validRangeRequest & terminalElement is a file, we know the size and should set it + if (byteRangeContext.isRangeRequest && byteRangeContext.isValidRangeRequest && terminalElement.type === 'file') { + byteRangeContext.setFileSize(terminalElement.unixfs.fileSize()) + + this.log.trace('fileSize for rangeRequest %d', byteRangeContext.getFileSize()) + } + const offset = byteRangeContext.offset + const length = byteRangeContext.length + this.log.trace('calling unixfs.cat for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length) const asyncIter = this.unixfs.cat(resolvedCID, { signal: options?.signal, - onProgress: options?.onProgress + onProgress: options?.onProgress, + offset, + length }) this.log('got async iterator for %c/%s', cid, path) - const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, { - onProgress: options?.onProgress - }) - const response = okResponse(resource, stream, { - redirected - }) - await this.setContentType(firstChunk, path, response) + try { + const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, { + onProgress: options?.onProgress + }) + byteRangeContext.setBody(stream) + // if not a valid range request, okRangeRequest will call okResponse + const response = okRangeResponse(resource, byteRangeContext.getBody(), { byteRangeContext, log: this.log }, { + redirected + }) + + await this.setContentType(firstChunk, path, response) + + if (ipfsRoots != null) { + response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header + } - if (ipfsRoots != null) { - response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header + return response + } catch (err: any) { + this.log.error('error streaming %c/%s', cid, path, err) + if (byteRangeContext.isRangeRequest && err.code === 'ERR_INVALID_PARAMS') { + return badRangeResponse(resource) + } + return badGatewayResponse('Unable to stream content') } - - return response } private async handleRaw ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise { + const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers) const result = await this.helia.blockstore.get(cid, options) - const response = okResponse(resource, result) + byteRangeContext.setBody(result) + const response = okRangeResponse(resource, byteRangeContext.getBody(), { byteRangeContext, log: this.log }, { + redirected: false + }) // if the user has specified an `Accept` header that corresponds to a raw // type, honour that header, so for example they don't request @@ -380,10 +411,10 @@ export class VerifiedFetch { contentType = parsed } } catch (err) { - this.log.error('Error parsing content type', err) + this.log.error('error parsing content type', err) } } - + this.log.trace('setting content type to "%s"', contentType) response.headers.set('content-type', contentType) } @@ -408,7 +439,23 @@ export class VerifiedFetch { options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { resource })) // resolve the CID/path from the requested resource - const { path, query, cid, protocol } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) + let cid: ParsedUrlStringResults['cid'] + let path: ParsedUrlStringResults['path'] + let query: ParsedUrlStringResults['query'] + let ttl: ParsedUrlStringResults['ttl'] + let protocol: ParsedUrlStringResults['protocol'] + try { + const result = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) + cid = result.cid + path = result.path + query = result.query + ttl = result.ttl + protocol = result.protocol + } catch (err) { + this.log.error('error parsing resource %s', resource, err) + + return badRequestResponse('Invalid resource') + } options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:resolve', { cid, path })) @@ -461,21 +508,21 @@ export class VerifiedFetch { query.filename = query.filename ?? `${cid.toString()}.tar` response = await this.handleTar(handlerArgs) } else { + this.log.trace('finding handler for cid code "%s" and output type "%s"', cid.code, accept) // derive the handler from the CID type const codecHandler = this.codecHandlers[cid.code] if (codecHandler == null) { return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`) } + this.log.trace('calling handler "%s"', codecHandler.name) response = await codecHandler.call(this, handlerArgs) } response.headers.set('etag', getETag({ cid, reqFormat, weak: false })) - if (protocol === 'ipfs') { - response.headers.set('cache-control', 'public, max-age=29030400, immutable') - } + setCacheControlHeader({ response, ttl, protocol }) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header response.headers.set('X-Ipfs-Path', resource.toString()) diff --git a/packages/verified-fetch/test/cache-control-header.spec.ts b/packages/verified-fetch/test/cache-control-header.spec.ts index bae8847..0cfe126 100644 --- a/packages/verified-fetch/test/cache-control-header.spec.ts +++ b/packages/verified-fetch/test/cache-control-header.spec.ts @@ -2,20 +2,41 @@ import { dagCbor } from '@helia/dag-cbor' import { ipns } from '@helia/ipns' import { stop } from '@libp2p/interface' import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { dns } from '@multiformats/dns' import { expect } from 'aegir/chai' import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' import { VerifiedFetch } from '../src/verified-fetch.js' import { createHelia } from './fixtures/create-offline-helia.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' - +import type { DNSResponse } from '@multiformats/dns' + +function answerFake (data: string, TTL: number, name: string, type: number): DNSResponse { + const fake = stubInterface() + fake.Answer = [{ + data, + TTL, + name, + type + }] + return fake +} describe('cache-control header', () => { let helia: Helia let name: IPNS let verifiedFetch: VerifiedFetch + let customDnsResolver: Sinon.SinonStub> beforeEach(async () => { - helia = await createHelia() + customDnsResolver = Sinon.stub() + helia = await createHelia({ + dns: dns({ + resolvers: { + '.': customDnsResolver + } + }) + }) name = ipns(helia) verifiedFetch = new VerifiedFetch({ helia @@ -60,29 +81,34 @@ describe('cache-control header', () => { expect(resp.headers.get('Cache-Control')).to.not.containIgnoreCase('immutable') }) - it.skip('should return the correct max-age in the cache-control header for an IPNS name', async () => { + it('should return the correct max-age in the cache-control header for an IPNS name', async () => { const obj = { hello: 'world' } const c = dagCbor(helia) const cid = await c.add(obj) - const oneHourInMs = 1000 * 60 * 60 + const oneHourInSeconds = 60 * 60 const peerId = await createEd25519PeerId() - // ipns currently only allows customising the lifetime which is also used as the TTL - await name.publish(peerId, cid, { lifetime: oneHourInMs }) + /** + * ipns currently only allows customising the lifetime which is also used as the TTL + * + * lifetime is coming back as 100000 times larger than expected + * + * @see https://github.com/ipfs/js-ipns/blob/16e0e10682fa9a663e0bb493a44d3e99a5200944/src/index.ts#L200 + * @see https://github.com/ipfs/js-ipns/pull/308 + */ + await name.publish(peerId, cid, { lifetime: oneHourInSeconds * 1000 }) // pass to ipns as milliseconds const resp = await verifiedFetch.fetch(`ipns://${peerId}`) expect(resp).to.be.ok() expect(resp.status).to.equal(200) - expect(resp.headers.get('Cache-Control')).to.equal(`public, max-age=${oneHourInMs.toString()}`) + expect(resp.headers.get('Cache-Control')).to.equal(`public, max-age=${oneHourInSeconds}`) }) it('should not contain immutable in the cache-control header for a DNSLink name', async () => { - const customDnsResolver = Sinon.stub() - verifiedFetch = new VerifiedFetch({ helia }, { @@ -94,12 +120,12 @@ describe('cache-control header', () => { } const c = dagCbor(helia) const cid = await c.add(obj) - customDnsResolver.returns(Promise.resolve(`/ipfs/${cid.toString()}`)) + customDnsResolver.withArgs('_dnslink.example-domain.com').resolves(answerFake(`dnslink=/ipfs/${cid}`, 666, '_dnslink.example-domain.com', 16)) const resp = await verifiedFetch.fetch('ipns://example-domain.com') expect(resp).to.be.ok() expect(resp.status).to.equal(200) - expect(resp.headers.get('Cache-Control')).to.not.containIgnoreCase('immutable') + expect(resp.headers.get('Cache-Control')).to.equal('public, max-age=666') }) }) diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts index ae44cde..d6bdce6 100644 --- a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts +++ b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts @@ -19,16 +19,19 @@ describe('custom dns-resolvers', () => { }) it('is used when passed to createVerifiedFetch', async () => { - const customDnsResolver = Sinon.stub() - - customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) + const customDnsResolver = Sinon.stub().withArgs('_dnslink.some-non-cached-domain.com').resolves({ + Answer: [{ + data: 'dnslink=/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg' + }] + }) const fetch = await createVerifiedFetch({ gateways: ['http://127.0.0.1:8080'], dnsResolvers: [customDnsResolver] }) - // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors') + const response = await fetch('ipns://some-non-cached-domain.com') + expect(response.status).to.equal(502) + expect(response.statusText).to.equal('Bad Gateway') expect(customDnsResolver.callCount).to.equal(1) expect(customDnsResolver.getCall(0).args).to.deep.equal(['_dnslink.some-non-cached-domain.com', { @@ -58,8 +61,10 @@ describe('custom dns-resolvers', () => { const verifiedFetch = new VerifiedFetch({ helia }) - // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(verifiedFetch.fetch('ipns://some-non-cached-domain2.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) + + const response = await verifiedFetch.fetch('ipns://some-non-cached-domain2.com') + expect(response.status).to.equal(502) + expect(response.statusText).to.equal('Bad Gateway') expect(customDnsResolver.callCount).to.equal(1) expect(customDnsResolver.getCall(0).args).to.deep.equal(['_dnslink.some-non-cached-domain2.com', { diff --git a/packages/verified-fetch/test/fixtures/ipns-stubs.ts b/packages/verified-fetch/test/fixtures/ipns-stubs.ts new file mode 100644 index 0000000..759790c --- /dev/null +++ b/packages/verified-fetch/test/fixtures/ipns-stubs.ts @@ -0,0 +1,19 @@ +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import type { PeerId } from '@libp2p/interface' +import type { IPNSRecord } from 'ipns' + +export interface IpnsRecordStubOptions { + peerId: PeerId + ttl?: bigint +} + +/** + * When stubbing an IPNSRecord, we need to provide a PeerId and some ttl value or else we will get + * "SyntaxError: Cannot convert stub to a BigInt" when parse-url-string.ts calls `calculateTtl` + */ +export function ipnsRecordStub ({ peerId, ttl }: IpnsRecordStubOptions): StubbedInstance { + return stubInterface({ + value: peerId.toString(), + ttl + }) +} diff --git a/packages/verified-fetch/test/range-requests.spec.ts b/packages/verified-fetch/test/range-requests.spec.ts new file mode 100644 index 0000000..489ce3c --- /dev/null +++ b/packages/verified-fetch/test/range-requests.spec.ts @@ -0,0 +1,162 @@ +import { unixfs } from '@helia/unixfs' +import { stop } from '@libp2p/interface' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' +import { VerifiedFetch } from '../src/verified-fetch.js' +import { createHelia } from './fixtures/create-offline-helia.js' +import type { Helia } from '@helia/interface' + +/** + * Range request headers for IPFS gateways only support raw and unixfs + */ +describe('range requests', () => { + let helia: Helia + let verifiedFetch: VerifiedFetch + const content = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + beforeEach(async () => { + helia = await createHelia() + verifiedFetch = new VerifiedFetch({ + helia + }) + }) + + afterEach(async () => { + await stop(helia, verifiedFetch) + }) + + interface SuccessfulTestExpectation { + contentRange: string + bytes: Uint8Array + } + async function testRange (cid: CID, headerRange: string, expected: SuccessfulTestExpectation): Promise { + const response = await verifiedFetch.fetch(cid, { + headers: { + Range: headerRange + } + }) + + expect(response.status).to.equal(206) + expect(response.statusText).to.equal('Partial Content') + + expect(response).to.have.property('headers') + const contentRange = response.headers.get('content-range') + expect(contentRange).to.be.ok() + expect(contentRange).to.equal(expected.contentRange) // the response should include the range that was requested + + const responseContent = await response.arrayBuffer() + expect(new Uint8Array(responseContent)).to.deep.equal(expected.bytes) + } + + async function assertFailingRange (response: Promise): Promise { + await expect(response).to.eventually.have.property('status', 416) + await expect(response).to.eventually.have.property('statusText', 'Requested Range Not Satisfiable') + } + + function runTests (description: string, getCid: () => Promise): void { + describe(description, () => { + let cid: CID + beforeEach(async () => { + cid = await getCid() + }) + const validTestCases = [ + { + byteSize: 6, + contentRange: 'bytes 0-5/11', + rangeHeader: 'bytes=0-5', + bytes: new Uint8Array([0, 1, 2, 3, 4, 5]) + }, + { + byteSize: 8, + contentRange: 'bytes 4-11/11', + rangeHeader: 'bytes=4-', + bytes: new Uint8Array([3, 4, 5, 6, 7, 8, 9, 10]) + }, + { + byteSize: 9, + contentRange: 'bytes 3-11/11', + rangeHeader: 'bytes=-9', + bytes: new Uint8Array([2, 3, 4, 5, 6, 7, 8, 9, 10]) + } + ] + validTestCases.forEach(({ bytes, contentRange, rangeHeader }) => { + // if these fail, check response-headers.spec.ts first + it(`should return correct 206 Partial Content response for ${rangeHeader}`, async () => { + const expected: SuccessfulTestExpectation = { + bytes, + contentRange + } + await testRange(cid, rangeHeader, expected) + }) + }) + + it('should return 416 Range Not Satisfiable when the range is invalid', async () => { + await assertFailingRange(verifiedFetch.fetch(cid, { + headers: { + Range: 'bytes=-0-' + } + })) + await assertFailingRange(verifiedFetch.fetch(cid, { + headers: { + Range: 'bytes=foobar' + } + })) + }) + + it('should return 416 Range Not Satisfiable when the range offset is larger than content', async () => { + await assertFailingRange(verifiedFetch.fetch(cid, { + headers: { + Range: 'bytes=50-' + } + })) + }) + + it('should return 416 Range Not Satisfiable when the suffix-length is larger than content', async () => { + await assertFailingRange(verifiedFetch.fetch(cid, { + headers: { + Range: 'bytes=-50' + } + })) + }) + + it('should return 416 Range Not Satisfiable when the range is out of bounds', async () => { + await assertFailingRange(verifiedFetch.fetch(cid, { + headers: { + Range: 'bytes=0-900' + } + })) + }) + + it('should return 416 Range Not Satisfiable when passed multiple ranges', async () => { + await assertFailingRange(verifiedFetch.fetch(cid, { + headers: { + Range: 'bytes=0-2,3-5' + } + })) + }) + }) + } + + const testTuples = [ + ['unixfs', async () => { + return unixfs(helia).addFile({ + content + }, { + rawLeaves: false, + leafType: 'file' + }) + }], + ['raw', async () => { + const buf = raw.encode(content) + const cid = CID.createV1(raw.code, await sha256.digest(buf)) + await helia.blockstore.put(cid, buf) + return cid + }] + ] as const + + testTuples.forEach(([name, fn]) => { + runTests(name, fn) + }) +}) diff --git a/packages/verified-fetch/test/utils/byte-range-context.spec.ts b/packages/verified-fetch/test/utils/byte-range-context.spec.ts new file mode 100644 index 0000000..4f95a5b --- /dev/null +++ b/packages/verified-fetch/test/utils/byte-range-context.spec.ts @@ -0,0 +1,150 @@ +import { unixfs, type UnixFS } from '@helia/unixfs' +import { stop } from '@libp2p/interface' +import { defaultLogger, prefixLogger } from '@libp2p/logger' +import { expect } from 'aegir/chai' +import { ByteRangeContext } from '../../src/utils/byte-range-context.js' +import { getStreamFromAsyncIterable } from '../../src/utils/get-stream-from-async-iterable.js' +import { createHelia } from '../fixtures/create-offline-helia.js' +import type { Helia } from 'helia' +import type { CID } from 'multiformats/cid' + +describe('ByteRangeContext', () => { + const logger = prefixLogger('test') + + it('should correctly detect range request', () => { + const context = new ByteRangeContext(logger, { Range: 'bytes=0-2' }) + expect(context.isRangeRequest).to.be.true() + }) + + it('should correctly detect non-range request', () => { + const context = new ByteRangeContext(logger, {}) + expect(context.isRangeRequest).to.be.false() + }) + + it('should correctly set body and calculate fileSize', () => { + const context = new ByteRangeContext(logger, {}) + const body = new Uint8Array([1, 2, 3, 4, 5]) + context.setBody(body) + expect(context.getBody()).to.equal(body) + expect(context.getFileSize()).to.equal(body.length) + }) + + it('should correctly handle invalid range request', () => { + const invalidRanges = [ + 'bytes=f', + 'bytes=0-foobar', + 'bytes=f-0', + 'byte=0-2' + ] + invalidRanges.forEach(range => { + const context = new ByteRangeContext(logger, { Range: range }) + expect(context.isValidRangeRequest).to.be.false() + }) + }) + + const array = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + const uint8arrayRangeTests = [ + // full ranges: + { type: 'Uint8Array', range: 'bytes=0-11', contentRange: 'bytes 0-11/11', body: new Uint8Array(array), expected: new Uint8Array(array) }, + { type: 'Uint8Array', range: 'bytes=-11', contentRange: 'bytes 1-11/11', body: new Uint8Array(array), expected: new Uint8Array(array) }, + { type: 'Uint8Array', range: 'bytes=0-', contentRange: 'bytes 0-11/11', body: new Uint8Array(array), expected: new Uint8Array(array) }, + + // partial ranges: + { type: 'Uint8Array', range: 'bytes=0-1', contentRange: 'bytes 0-1/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2]) }, + { type: 'Uint8Array', range: 'bytes=0-2', contentRange: 'bytes 0-2/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3]) }, + { type: 'Uint8Array', range: 'bytes=0-3', contentRange: 'bytes 0-3/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4]) }, + { type: 'Uint8Array', range: 'bytes=0-4', contentRange: 'bytes 0-4/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5]) }, + { type: 'Uint8Array', range: 'bytes=0-5', contentRange: 'bytes 0-5/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5, 6]) }, + { type: 'Uint8Array', range: 'bytes=0-6', contentRange: 'bytes 0-6/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5, 6, 7]) }, + { type: 'Uint8Array', range: 'bytes=0-7', contentRange: 'bytes 0-7/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]) }, + { type: 'Uint8Array', range: 'bytes=0-8', contentRange: 'bytes 0-8/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9]) }, + { type: 'Uint8Array', range: 'bytes=0-9', contentRange: 'bytes 0-9/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) }, + { type: 'Uint8Array', range: 'bytes=0-10', contentRange: 'bytes 0-10/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) }, + { type: 'Uint8Array', range: 'bytes=1-', contentRange: 'bytes 1-11/11', body: new Uint8Array(array), expected: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) }, + { type: 'Uint8Array', range: 'bytes=2-', contentRange: 'bytes 2-11/11', body: new Uint8Array(array), expected: new Uint8Array([2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) }, + { type: 'Uint8Array', range: 'bytes=-2', contentRange: 'bytes 10-11/11', body: new Uint8Array(array), expected: new Uint8Array(array.slice(-2)) }, + + // single byte ranges: + { type: 'Uint8Array', range: 'bytes=1-1', contentRange: 'bytes 1-1/11', body: new Uint8Array(array), expected: new Uint8Array(array.slice(1, 2)) }, + { type: 'Uint8Array', range: 'bytes=-1', contentRange: 'bytes 11-11/11', body: new Uint8Array(array), expected: new Uint8Array(array.slice(-1)) } + + ] + const validRanges = [ + ...uint8arrayRangeTests, + ...uint8arrayRangeTests.map(({ range, contentRange, body, expected }) => ({ + type: 'ArrayBuffer', + range, + contentRange, + body: body.buffer, + expected: expected.buffer + })), + ...uint8arrayRangeTests.map(({ range, contentRange, body, expected }) => ({ + type: 'Blob', + range, + contentRange, + body: new Blob([body]), + expected: new Blob([expected]) + })) + ] + validRanges.forEach(({ type, range, expected, body, contentRange }) => { + it(`should correctly slice ${type} body with range ${range}`, async () => { + const context = new ByteRangeContext(logger, { Range: range }) + + context.setBody(body) + const actualBody = context.getBody() + + if (actualBody instanceof Blob || type === 'Blob') { + const bodyAsUint8Array = new Uint8Array(await (actualBody as Blob).arrayBuffer()) + const expectedAsUint8Array = new Uint8Array(await (expected as Blob).arrayBuffer()) + // loop through the bytes and compare them + for (let i = 0; i < bodyAsUint8Array.length; i++) { + expect(bodyAsUint8Array[i]).to.equal(expectedAsUint8Array[i]) + } + } else { + expect(actualBody).to.deep.equal(expected) + } + + expect(context.contentRangeHeaderValue).to.equal(contentRange) + }) + }) + + describe('handling ReadableStreams', () => { + let helia: Helia + let fs: UnixFS + let cid: CID + const getBodyStream = async (offset?: number, length?: number): Promise> => { + const iter = fs.cat(cid, { offset, length }) + const { stream } = await getStreamFromAsyncIterable(iter, 'test.txt', defaultLogger()) + return stream + } + + before(async () => { + helia = await createHelia() + fs = unixfs(helia) + }) + + after(async () => { + await stop(helia) + }) + + uint8arrayRangeTests.forEach(({ range, expected, body, contentRange }) => { + it(`should correctly slice Stream with range ${range}`, async () => { + const context = new ByteRangeContext(logger, { Range: range }) + cid = await fs.addFile({ + content: body + }, { + rawLeaves: false, + leafType: 'file' + }) + const stat = await fs.stat(cid) + context.setFileSize(stat.fileSize) + + context.setBody(await getBodyStream(context.offset, context.length)) + const response = new Response(context.getBody()) + const bodyResult = await response.arrayBuffer() + expect(new Uint8Array(bodyResult)).to.deep.equal(expected) + expect(context.contentRangeHeaderValue).to.equal(contentRange) + }) + }) + }) +}) diff --git a/packages/verified-fetch/test/utils/parse-url-string.spec.ts b/packages/verified-fetch/test/utils/parse-url-string.spec.ts index cd91d8c..fbf2341 100644 --- a/packages/verified-fetch/test/utils/parse-url-string.spec.ts +++ b/packages/verified-fetch/test/utils/parse-url-string.spec.ts @@ -3,11 +3,11 @@ import { defaultLogger } from '@libp2p/logger' import { createEd25519PeerId } from '@libp2p/peer-id-factory' import { type Answer } from '@multiformats/dns' import { expect } from 'aegir/chai' -import { type IPNSRecord } from 'ipns' import { CID } from 'multiformats/cid' import { match } from 'sinon' import { stubInterface } from 'sinon-ts' import { parseUrlString } from '../../src/utils/parse-url-string.js' +import { ipnsRecordStub } from '../fixtures/ipns-stubs.js' import type { IPNS } from '@helia/ipns' import type { ComponentLogger, PeerId } from '@libp2p/interface' import type { StubbedInstance } from 'sinon-ts' @@ -75,8 +75,7 @@ describe('parseUrlString', () => { ipns, logger }) - ).to.eventually.be.rejected - .with.property('message', 'Could not parse PeerId in ipns url "mydomain.com", Non-base64 character') + ).to.eventually.be.rejected.with.property('message', 'Could not parse PeerId in ipns url "mydomain.com", Non-base64 character') }) }) @@ -441,7 +440,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -458,7 +457,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -475,7 +474,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -492,7 +491,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -511,7 +510,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -535,7 +534,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ cid, path: recordPath, - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -557,7 +556,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ cid, path: recordPath, - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -579,7 +578,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ cid, path: recordPath, - record: stubInterface() + record: ipnsRecordStub({ peerId: testPeerId }) }) await assertMatchUrl( @@ -603,7 +602,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ cid, path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId }) }) }) @@ -691,7 +690,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ cid, path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId }) }) }) @@ -793,7 +792,7 @@ describe('parseUrlString', () => { ipns.resolve.withArgs(matchPeerId(value as PeerId)).resolves({ cid, path: '', - record: stubInterface() + record: ipnsRecordStub({ peerId: value as PeerId }) }) } else if (type === 'dnslink-encoded') { const matchValue = (value as string).replace(/-/g, '.') diff --git a/packages/verified-fetch/test/utils/request-headers.spec.ts b/packages/verified-fetch/test/utils/request-headers.spec.ts new file mode 100644 index 0000000..77c1697 --- /dev/null +++ b/packages/verified-fetch/test/utils/request-headers.spec.ts @@ -0,0 +1,61 @@ +import { expect } from 'aegir/chai' +import { getHeader, calculateByteRangeIndexes } from '../../src/utils/request-headers.js' + +describe('request-headers', () => { + describe('getHeader', () => { + it('should return undefined when headers are undefined', () => { + expect(getHeader(undefined, 'dummy')).to.be.undefined() + expect(getHeader(new Headers(), 'dummy')).to.be.undefined() + expect(getHeader({}, 'dummy')).to.be.undefined() + expect(getHeader([], 'dummy')).to.be.undefined() + }) + + it('should return correct header value for Headers instance', () => { + const headers = new Headers({ Dummy: 'value' }) + expect(getHeader(headers, 'Dummy')).to.equal('value') + expect(getHeader(headers, 'dummy')).to.equal('value') + }) + + it('should return correct header value for array of tuples', () => { + const headers: Array<[string, string]> = [['Dummy', 'value']] + expect(getHeader(headers, 'Dummy')).to.equal('value') + expect(getHeader(headers, 'dummy')).to.equal('value') + }) + + it('should return correct header value for record', () => { + const headers: Record = { Dummy: 'value' } + expect(getHeader(headers, 'Dummy')).to.equal('value') + expect(getHeader(headers, 'dummy')).to.equal('value') + }) + }) + + describe('calculateByteRangeIndexes', () => { + const testCases = [ + // Range: bytes=5- + { start: 5, end: undefined, fileSize: 10, expected: { byteSize: 6, start: 5, end: 10 } }, + // Range: bytes=-5 + { start: undefined, end: 5, fileSize: 10, expected: { byteSize: 5, start: 6, end: 10 } }, + // Range: bytes=0-0 + { start: 0, end: 0, fileSize: 10, expected: { byteSize: 1, start: 0, end: 0 } }, + // Range: bytes=5- with unknown filesize + { start: 5, end: undefined, fileSize: undefined, expected: { start: 5 } }, + // Range: bytes=-5 with unknown filesize + { start: undefined, end: 5, fileSize: undefined, expected: { end: 5 } }, + // Range: bytes=0-0 with unknown filesize + { start: 0, end: 0, fileSize: undefined, expected: { byteSize: 1, start: 0, end: 0 } }, + // Range: bytes=-9 & fileSize=11 + { start: undefined, end: 9, fileSize: 11, expected: { byteSize: 9, start: 3, end: 11 } }, + // Range: bytes=0-11 & fileSize=11 + { start: 0, end: 11, fileSize: 11, expected: { byteSize: 12, start: 0, end: 11 } } + ] + testCases.forEach(({ start, end, fileSize, expected }) => { + it(`should return expected result for bytes=${start ?? ''}-${end ?? ''} and fileSize=${fileSize}`, () => { + const result = calculateByteRangeIndexes(start, end, fileSize) + expect(result).to.deep.equal(expected) + }) + }) + it('throws error for invalid range', () => { + expect(() => calculateByteRangeIndexes(5, 4, 10)).to.throw('Invalid range') + }) + }) +}) diff --git a/packages/verified-fetch/test/utils/response-headers.spec.ts b/packages/verified-fetch/test/utils/response-headers.spec.ts new file mode 100644 index 0000000..6197450 --- /dev/null +++ b/packages/verified-fetch/test/utils/response-headers.spec.ts @@ -0,0 +1,33 @@ +import { expect } from 'aegir/chai' +import { getContentRangeHeader } from '../../src/utils/response-headers.js' + +describe('response-headers', () => { + describe('getContentRangeHeader', () => { + it('should return correct content range header when all options are set', () => { + const byteStart = 0 + const byteEnd = 500 + const byteSize = 1000 + expect(getContentRangeHeader({ byteStart, byteEnd, byteSize })).to.equal(`bytes ${byteStart}-${byteEnd}/${byteSize}`) + }) + + it('should return correct content range header when only byteEnd and byteSize are provided', () => { + expect(getContentRangeHeader({ byteStart: undefined, byteEnd: 9, byteSize: 11 })).to.equal('bytes 3-11/11') + }) + + it('should return correct content range header when only byteStart and byteSize are provided', () => { + expect(getContentRangeHeader({ byteStart: 5, byteEnd: undefined, byteSize: 11 })).to.equal('bytes 5-11/11') + }) + + it('should return correct content range header when only byteStart is provided', () => { + expect(getContentRangeHeader({ byteStart: 500, byteEnd: undefined, byteSize: undefined })).to.equal('bytes */*') + }) + + it('should return correct content range header when only byteEnd is provided', () => { + expect(getContentRangeHeader({ byteStart: undefined, byteEnd: 500, byteSize: undefined })).to.equal('bytes */*') + }) + + it('should return content range header with when only byteSize is provided', () => { + expect(getContentRangeHeader({ byteStart: undefined, byteEnd: undefined, byteSize: 50 })).to.equal('bytes */50') + }) + }) +})