-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This PR adds a new handler that allows freeway to serve CAR files directly from R2 (CARPARK). Requesting a CID from the gateway with the CAR codec `0x0202` will invoke this new handler. Obviously CARs are not served in the same way as regular IPFS data, since we can serve the data directly from an R2 bucket. To deal with this difference, there's a new middlewares that detects the CAR codec and calls the correct handler, passing through to the regular handlers if the CAR codec is not detected. The handler supports `HEAD` requests, allowing you to get the file size of the item you're requesting. It also supports the HTTP `Range` header, allowing you to extract byte ranges from within a CAR, which you might do if you fetch the index for the CAR. The idea is that we can use these handlers to build a system that runs on content-claims. The flow might be something like: 1. Get [partition claim](https://github.com/web3-storage/content-claims#partition-claim) for a given content CID 2. Get [inclusion claims](https://github.com/web3-storage/content-claims#inclusion-claim) for each CAR `part` in the partition claim 3. Fetch each CARv2 index from the gateway for each `includes` CID specified in the inclusion claims (You must first fetch partition claims for each index CID to find out which CAR file they can be found in). 4. Use HTTP `Range` requests to export the whole DAG or a sub-DAG directly from the `parts`, using the index data as a guide for which blocks to extract. Note: if exporting a full DAG you might want to just stop after (1) and import all data in all CARs into IPFS.
- Loading branch information
Alan Shaw
authored
Jul 31, 2023
1 parent
86d1f7f
commit 0b95438
Showing
14 changed files
with
441 additions
and
76 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* eslint-env browser */ | ||
| /* global FixedLengthStream */ | ||
| import { HttpError } from '@web3-storage/gateway-lib/util' | ||
| import { CAR_CODE } from '../constants.js' | ||
| import * as Http from '../lib/http.js' | ||
|
|
||
| /** @typedef {import('@web3-storage/gateway-lib').IpfsUrlContext} CarBlockHandlerContext */ | ||
|
|
||
| /** | ||
| * Handler that serves CAR files directly from R2. | ||
| * | ||
| * @type {import('@web3-storage/gateway-lib').Handler<CarBlockHandlerContext, import('../bindings').Environment>} | ||
| */ | ||
| export async function handleCarBlock (request, env, ctx) { | ||
| const { searchParams, dataCid } = ctx | ||
| if (!dataCid) throw new Error('missing data CID') | ||
| if (!searchParams) throw new Error('missing URL search params') | ||
|
|
||
| if (request.method !== 'HEAD' && request.method !== 'GET') { | ||
| throw new HttpError('method not allowed', { status: 405 }) | ||
| } | ||
| if (dataCid.code !== CAR_CODE) { | ||
| throw new HttpError('not a CAR CID', { status: 400 }) | ||
| } | ||
|
|
||
| const etag = `"${dataCid}"` | ||
| if (request.headers.get('If-None-Match') === etag) { | ||
| return new Response(null, { status: 304 }) | ||
| } | ||
|
|
||
| if (request.method === 'HEAD') { | ||
| const obj = await env.CARPARK.head(`${dataCid}/${dataCid}.car`) | ||
| if (!obj) throw new HttpError('CAR not found', { status: 404 }) | ||
| return new Response(undefined, { | ||
| headers: { | ||
| 'Accept-Ranges': 'bytes', | ||
| 'Content-Length': obj.size.toString(), | ||
| Etag: etag | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| /** @type {import('../lib/http').Range|undefined} */ | ||
| let range | ||
| if (request.headers.has('range')) { | ||
| try { | ||
| range = Http.parseRange(request.headers.get('range') ?? '') | ||
| } catch (err) { | ||
| throw new HttpError('invalid range', { status: 400, cause: err }) | ||
| } | ||
| } | ||
|
|
||
| const obj = await env.CARPARK.get(`${dataCid}/${dataCid}.car`, { range }) | ||
| if (!obj) throw new HttpError('CAR not found', { status: 404 }) | ||
|
|
||
| const status = range ? 206 : 200 | ||
| const headers = new Headers({ | ||
| 'Content-Type': 'application/vnd.ipld.car; version=1;', | ||
| 'X-Content-Type-Options': 'nosniff', | ||
| 'Cache-Control': 'public, max-age=29030400, immutable', | ||
| 'Content-Disposition': `attachment; filename="${dataCid}.car"`, | ||
| Etag: etag | ||
| }) | ||
|
|
||
| let contentLength = obj.size | ||
| if (range) { | ||
| let first, last | ||
| if ('suffix' in range) { | ||
| first = obj.size - range.suffix | ||
| last = obj.size - 1 | ||
| } else { | ||
| first = range.offset || 0 | ||
| last = range.length != null ? first + range.length - 1 : obj.size - 1 | ||
| } | ||
| headers.set('Content-Range', `bytes ${first}-${last}/${obj.size}`) | ||
| contentLength = last - first | ||
| } | ||
| headers.set('Content-Length', contentLength.toString()) | ||
|
|
||
| // @ts-expect-error ReadableStream types incompatible | ||
| return new Response(obj.body.pipeThrough(new FixedLengthStream(contentLength)), { status, headers }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| import { CarReader } from '@ipld/car' | ||
| import { CAR_CODE } from '../constants.js' | ||
|
|
||
| export const code = CAR_CODE | ||
|
|
||
| /** | ||
| * @param {Uint8Array} bytes | ||
| * @returns {Promise<Array<{ cid: import('multiformats').UnknownLink, bytes: Uint8Array }>>} | ||
| */ | ||
| export async function decode (bytes) { | ||
| const reader = await CarReader.fromBytes(bytes) | ||
| const blocks = [] | ||
| for await (const b of reader.blocks()) { | ||
| blocks.push({ | ||
| cid: /** @type {import('multiformats').UnknownLink} */ (b.cid), | ||
| bytes: b.bytes | ||
| }) | ||
| } | ||
| return blocks | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| // @ts-expect-error no types | ||
| import httpRangeParse from 'http-range-parse' | ||
|
|
||
| /** @typedef {{ offset: number, length?: number } | { offset?: number, length: number } | { suffix: number }} Range */ | ||
|
|
||
| /** | ||
| * Convert a HTTP Range header to a range object. | ||
| * @param {string} value | ||
| * @returns {Range} | ||
| */ | ||
| export function parseRange (value) { | ||
| const result = httpRangeParse(value) | ||
| if (result.ranges) throw new Error('Multipart ranges not supported') | ||
| const { unit, first, last, suffix } = result | ||
| if (unit !== 'bytes') throw new Error(`Unsupported range unit: ${unit}`) | ||
| return suffix != null | ||
| ? { suffix } | ||
| : { offset: first, length: last != null ? last - first + 1 : undefined } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.