From 2a66379785f4bf0d983efdeb4aa3b14ba13b321f Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 17 Jan 2024 15:40:46 -0800
Subject: [PATCH 001/104] feat: create @helia/verified-fetch
---
packages/verified-fetch/.aegir.js | 49 +++++
packages/verified-fetch/LICENSE | 4 +
packages/verified-fetch/LICENSE-APACHE | 5 +
packages/verified-fetch/LICENSE-MIT | 19 ++
packages/verified-fetch/README.md | 204 ++++++++++++++++++
packages/verified-fetch/package.json | 100 +++++++++
packages/verified-fetch/src/index.ts | 101 +++++++++
packages/verified-fetch/src/interface.ts | 22 ++
.../src/utils/get-content-type.ts | 55 +++++
packages/verified-fetch/src/verified-fetch.ts | 153 +++++++++++++
packages/verified-fetch/tsconfig.json | 33 +++
packages/verified-fetch/typedoc.json | 5 +
12 files changed, 750 insertions(+)
create mode 100644 packages/verified-fetch/.aegir.js
create mode 100644 packages/verified-fetch/LICENSE
create mode 100644 packages/verified-fetch/LICENSE-APACHE
create mode 100644 packages/verified-fetch/LICENSE-MIT
create mode 100644 packages/verified-fetch/README.md
create mode 100644 packages/verified-fetch/package.json
create mode 100644 packages/verified-fetch/src/index.ts
create mode 100644 packages/verified-fetch/src/interface.ts
create mode 100644 packages/verified-fetch/src/utils/get-content-type.ts
create mode 100644 packages/verified-fetch/src/verified-fetch.ts
create mode 100644 packages/verified-fetch/tsconfig.json
create mode 100644 packages/verified-fetch/typedoc.json
diff --git a/packages/verified-fetch/.aegir.js b/packages/verified-fetch/.aegir.js
new file mode 100644
index 00000000..65eb2562
--- /dev/null
+++ b/packages/verified-fetch/.aegir.js
@@ -0,0 +1,49 @@
+import getPort from 'aegir/get-port'
+import { createServer } from 'ipfsd-ctl'
+import * as kuboRpcClient from 'kubo-rpc-client'
+
+/** @type {import('aegir').PartialOptions} */
+export default {
+ build: {
+ bundlesizeMax: '10kB',
+ },
+ test: {
+ files: './dist/src/*.spec.js',
+ before: async (options) => {
+ if (options.runner !== 'node') {
+ const ipfsdPort = await getPort()
+ const ipfsdServer = await createServer({
+ host: '127.0.0.1',
+ port: ipfsdPort
+ }, {
+ ipfsBin: (await import('kubo')).default.path(),
+ kuboRpcModule: kuboRpcClient,
+ ipfsOptions: {
+ config: {
+ Addresses: {
+ Swarm: [
+ "/ip4/0.0.0.0/tcp/0",
+ "/ip4/0.0.0.0/tcp/0/ws"
+ ]
+ }
+ }
+ }
+ }).start()
+
+ return {
+ env: {
+ IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}`
+ },
+ ipfsdServer
+ }
+ }
+
+ return {}
+ },
+ after: async (options, beforeResult) => {
+ if (options.runner !== 'node') {
+ await beforeResult.ipfsdServer.stop()
+ }
+ }
+ }
+}
diff --git a/packages/verified-fetch/LICENSE b/packages/verified-fetch/LICENSE
new file mode 100644
index 00000000..20ce483c
--- /dev/null
+++ b/packages/verified-fetch/LICENSE
@@ -0,0 +1,4 @@
+This project is dual licensed under MIT and Apache-2.0.
+
+MIT: https://www.opensource.org/licenses/mit
+Apache-2.0: https://www.apache.org/licenses/license-2.0
diff --git a/packages/verified-fetch/LICENSE-APACHE b/packages/verified-fetch/LICENSE-APACHE
new file mode 100644
index 00000000..14478a3b
--- /dev/null
+++ b/packages/verified-fetch/LICENSE-APACHE
@@ -0,0 +1,5 @@
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
diff --git a/packages/verified-fetch/LICENSE-MIT b/packages/verified-fetch/LICENSE-MIT
new file mode 100644
index 00000000..72dc60d8
--- /dev/null
+++ b/packages/verified-fetch/LICENSE-MIT
@@ -0,0 +1,19 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md
new file mode 100644
index 00000000..ca1c057d
--- /dev/null
+++ b/packages/verified-fetch/README.md
@@ -0,0 +1,204 @@
+
+
+
+
+
+
+[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech)
+[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech)
+[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia)
+[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain)
+
+> A fetch-like API for IPFS content on the web.
+
+# About
+
+`@helia/verified-fetch`` is a library that provides a fetch-like API for fetching content from IPFS. This library should act as a replacement for the `fetch()` API for fetching content from IPFS, and will return a [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) object that can be used in a similar manner to the `fetch()` API. This means browser and HTTP caching inside browser main threads, web-workers, and service workers, as well as other features of the `fetch()` API should work in a way familiar to developers.
+
+## Example
+
+```ts
+import { createVerifiedFetch } from '@helia/verified-fetch'
+
+const verifiedFetch = await createVerifiedFetch({
+ gateways: ['mygateway.info', 'trustless-gateway.link']
+})
+
+const resp = await verifiedFetch('ipfs://bafy...')
+
+const json = await resp.json()
+```
+
+# Install
+
+```console
+$ npm i @helia/verified-fetch
+```
+
+## Browser `
+```
+
+### Configuration
+
+#### Usage with customized Helia
+
+You can see variations of Helia and js-libp2p configuration options at https://helia.io/interfaces/helia.index.HeliaInit.html.
+
+The `@helia/http` module is currently in-progress, but the init options should be a subset of the `helia` module's init options. See https://github.com/ipfs/helia/issues/289 for more information.
+
+```ts
+import { createVerifiedFetch } from '@helia/verified-fetch'
+import { CreateHelia as CreateHeliaHttpOnly } from '@helia/http'
+
+const verifiedFetch = await createVerifiedFetch(
+ CreateHeliaHttpOnly({
+ gateways: ['mygateway.info', 'trustless-gateway.link'],
+ routers: ['delegated-ipfs.dev'],
+ })
+)
+
+const resp = await verifiedFetch('ipfs://bafy...')
+
+const json = await resp.json()
+```
+
+### Comparison to fetch
+
+First, this library will require instantiation in order to configure the gateways and delegated routers, or potentially a custom Helia instance. Secondly, once your verified-fetch method is created, it will act as similar to the `fetch()` API as possible.
+
+[The `fetch()` API](https://developer.mozilla.org/en-US/docs/Web/API/fetch) takes two parameters:
+
+1. A [resource](https://developer.mozilla.org/en-US/docs/Web/API/fetch#resource)
+2. An [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options)
+
+#### Resource argument
+
+This library intends to support the following methods of fetching web3 content from IPFS:
+
+1. IPFS protocol: `ipfs://` & `ipfs://`
+2. IPNS protocol: `ipns://` & `ipns://` & `ipns://`
+3. CID instances: An actual CID instance `CID.parse('bafy...')`
+4. CID strings: A CID string `bafy...`
+
+As well as support for pathing & params for all of the above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453).
+
+#### Options argument
+
+This library will not plan to support the exact Fetch API options object, as some of the arguments don't make sense. Instead, it will only support options necessary to meet [IPFS specs](https://specs.ipfs.tech/) related to specifying the resultant shape of desired content.
+
+Some of those header specifications are:
+
+1. https://specs.ipfs.tech/http-gateways/path-gateway/#request-headers
+2. https://specs.ipfs.tech/http-gateways/trustless-gateway/#request-headers
+3. https://specs.ipfs.tech/http-gateways/subdomain-gateway/#request-headers
+
+Where possible, options and Helia internals will be automatically configured to the appropriate codec & content type based on the `verified-fetch` configuration and `options` argument passed.
+
+Known Fetch API options that will be supported:
+
+1. `signal` - An AbortSignal that a user can use to abort the request.
+2. `redirect` - A string that specifies the redirect type. One of `follow`, `error`, or `manual`. Defaults to `follow`. Best effort to adhere to the [Fetch API redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) parameter.
+3. `headers` - An object of headers to be sent with the request. Best effort to adhere to the [Fetch API headers](https://developer.mozilla.org/en-US/docs/Web/API/fetch#headers) parameter.
+ - `accept` - A string that specifies the accept header. Relevant values:
+ - [`vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw). (default)
+ - [`vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car)
+ - [`vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record)
+4. `method` - A string that specifies the HTTP method to use for the request. Defaults to `GET`. Best effort to adhere to the [Fetch API method](https://developer.mozilla.org/en-US/docs/Web/API/fetch#method) parameter.
+5. `body` - An object that specifies the body of the request. Best effort to adhere to the [Fetch API body](https://developer.mozilla.org/en-US/docs/Web/API/fetch#body) parameter.
+6. `cache` - Will basically act as `force-cache` for the request. Best effort to adhere to the [Fetch API cache](https://developer.mozilla.org/en-US/docs/Web/API/fetch#cache) parameter.
+
+
+Non-Fetch API options that will be supported:
+
+1. `onProgress` - Similar to Helia `onProgress` options, this will be a function that will be called with a progress event. Supported progress events are:
+ - `helia:verified-fetch:error` - An error occurred during the request.
+ - `helia:verified-fetch:request:start` - The request has been sent
+ - `helia:verified-fetch:request:complete` - The request has been sent
+ - `helia:verified-fetch:request:error` - An error occurred during the request.
+ - `helia:verified-fetch:request:abort` - The request was aborted prior to completion.
+ - `helia:verified-fetch:response:start` - The initial HTTP Response headers have been set, and response stream is started.
+ - `helia:verified-fetch:response:complete` - The response stream has completed.
+ - `helia:verified-fetch:response:error` - An error occurred while building the response.
+
+Some in-flight specs (IPIPs) that will affect the options object this library supports in the future can be seen at https://specs.ipfs.tech/ipips, but a few that I'm aware of are:
+
+1. [IPIP-0412: Signaling Block Order in CARs on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0412/)
+2. [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/)
+3. [IPIP-0386: Subdomain Gateway Interop with _redirects](https://specs.ipfs.tech/ipips/ipip-0386/)
+4. [IPIP-0328: JSON and CBOR Response Formats on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0328/)
+5. [IPIP-0288: TAR Response Format on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0288/)
+
+#### Response types
+
+This library's purpose is to return reasonably representable content from IPFS. In other words, fetching content is intended for leaf-node content -- such as images/videos/audio & other assets, or other IPLD content (with link) -- that can be represented by https://developer.mozilla.org/en-US/docs/Web/API/Response#instance_methods. The content type you receive back will depend upon the CID you request as well as the `Accept` header value you provide.
+
+All content we retrieve from the IPFS network is obtained via an AsyncIterable, and will be set as the [body of the HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response#body) via a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream) or other efficient method that avoids loading the entire response into memory or getting the entire response from the network before returning a response to the user.
+
+If your content doesn't have a mime-type or an [IPFS spec](https://specs.ipfs.tech), this library will not support it, but you can use the [`helia`](https://github.com/ipfs/helia) library directly for those use cases. See [Unsupported response types](#unsupported-response-types) for more information.
+
+##### Handling response types
+
+For handling responses we want to follow conventions/abstractions from Fetch API where possible:
+
+- For JSON, assuming you abstract any differences between dag-json/dag-cbor/json/and json-file-on-unixfs, you would call `.json()` to get a JSON object.
+- For images (or other web-relevant asset) you want to add to the DOM, use `.blob()` or `.arrayBuffer()` to get the raw bytes.
+- For plain text in utf-8, you would call `.text()`
+- For streaming response data, use something like `response.body.getReader()` to get a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream).
+
+##### Unsupported response types
+
+* Returning IPLD nodes or DAGs as JS objects is not supported, as there is no currently well-defined structure for representing this data in an [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response). Instead, users should request `aplication/vnd.ipld.car` or use the [`helia`](https://github.com/ipfs/helia) library directly for this use case.
+* Others? Open an issue or PR!
+
+#### Response headers
+
+This library will set the [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) headers to the appropriate values for the content type according to the appropriate [IPFS Specifications](https://specs.ipfs.tech/).
+
+Some known header specifications:
+
+* https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers
+* https://specs.ipfs.tech/http-gateways/trustless-gateway/#response-headers
+* https://specs.ipfs.tech/http-gateways/subdomain-gateway/#response-headers
+
+#### Possible Scenarios that could cause confusion
+
+##### Attempting to fetch the CID for content that does not make sense
+
+If you request `bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze`, which points to the root of the en.wikipedia.org mirror, a response object does not make sense.
+
+#### Errors
+
+Known Errors that can be thrown:
+
+1. `TypeError` - If the resource argument is not a string, CID, or CID string.
+2. `TypeError` - If the options argument is passed and not an object.
+3. `TypeError` - If the options argument is passed and is malformed.
+4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal.
+
+# API Docs
+
+-
+
+# License
+
+Licensed under either of
+
+- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / )
+- MIT ([LICENSE-MIT](LICENSE-MIT) / )
+
+# Contribute
+
+Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues).
+
+Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general.
+
+Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md).
+
+Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions.
+
+[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md)
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
new file mode 100644
index 00000000..4650fae3
--- /dev/null
+++ b/packages/verified-fetch/package.json
@@ -0,0 +1,100 @@
+{
+ "name": "@helia/verified-fetch",
+ "version": "0.0.0",
+ "description": "A fetch-like API for IPFS content on the web.",
+ "license": "Apache-2.0 OR MIT",
+ "homepage": "https://github.com/ipfs/helia/tree/main/packages/verified-fetch#readme",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/ipfs/helia.git"
+ },
+ "bugs": {
+ "url": "https://github.com/ipfs/helia/issues"
+ },
+ "publishConfig": {
+ "access": "public",
+ "provenance": true
+ },
+ "keywords": [
+ "IPFS",
+ "fetch",
+ "helia"
+ ],
+ "type": "module",
+ "types": "./dist/src/index.d.ts",
+ "files": [
+ "src",
+ "dist",
+ "!dist/test",
+ "!**/*.tsbuildinfo"
+ ],
+ "exports": {
+ ".": {
+ "types": "./dist/src/index.d.ts",
+ "import": "./dist/src/index.js"
+ }
+ },
+ "eslintConfig": {
+ "extends": "ipfs",
+ "parserOptions": {
+ "project": true,
+ "sourceType": "module"
+ }
+ },
+ "scripts": {
+ "clean": "aegir clean",
+ "lint": "aegir lint",
+ "dep-check": "aegir dep-check",
+ "build": "aegir build",
+ "test": "aegir test",
+ "test:chrome": "aegir test -t browser --cov",
+ "test:chrome-webworker": "aegir test -t webworker",
+ "test:firefox": "aegir test -t browser -- --browser firefox",
+ "test:firefox-webworker": "aegir test -t webworker -- --browser firefox",
+ "test:node": "aegir test -t node --cov",
+ "test:electron-main": "aegir test -t electron-main"
+ },
+ "dependencies": {
+ "@helia/http": "next",
+ "@helia/interface": "^3.0.1",
+ "@ipld/dag-cbor": "^9.0.7",
+ "@ipld/dag-json": "^10.1.5",
+ "@ipld/dag-pb": "^4.0.6",
+ "@libp2p/interface": "^1.1.1",
+ "@libp2p/logger": "^4.0.4",
+ "@libp2p/peer-collections": "^5.1.4",
+ "@libp2p/utils": "^5.2.0",
+ "any-signal": "^4.1.1",
+ "cborg": "^4.0.3",
+ "file-type": "^19.0.0",
+ "interface-blockstore": "^5.2.7",
+ "interface-datastore": "^8.2.9",
+ "interface-store": "^5.1.5",
+ "it-drain": "^3.0.5",
+ "it-filter": "^3.0.4",
+ "it-foreach": "^2.0.6",
+ "it-merge": "^3.0.3",
+ "mime-type": "^4.0.0",
+ "mime-types": "^2.1.35",
+ "mortice": "^3.0.1",
+ "multiformats": "^13.0.0",
+ "progress-events": "^1.0.0",
+ "uint8arrays": "^5.0.1"
+ },
+ "devDependencies": {
+ "@helia/ipns": "^4.0.0",
+ "@types/mime-types": "^2.1.4",
+ "@types/sinon": "^17.0.2",
+ "aegir": "^42.1.0",
+ "blockstore-core": "^4.3.10",
+ "datastore-core": "^9.2.7",
+ "delay": "^6.0.0",
+ "it-all": "^3.0.4",
+ "sinon": "^17.0.1",
+ "sinon-ts": "^2.0.0"
+ },
+ "browser": {
+ "node:buffer": false,
+ "node:stream": false
+ }
+}
diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts
new file mode 100644
index 00000000..809eec47
--- /dev/null
+++ b/packages/verified-fetch/src/index.ts
@@ -0,0 +1,101 @@
+/**
+ * @packageDocumentation
+ *
+ * Exports a `createVerifiedFetch` function that returns a `fetch()` like API method {@link Helia} for fetching IPFS content.
+ *
+ * You may use any supported resource argument to fetch content:
+ *
+ * - CID string
+ * - CID instance
+ * - IPFS URL
+ * - IPNS URL
+ *
+ * @example Use a CID string to fetch a text file
+ *
+ * ```typescript
+ * import { createVerifiedFetch } from '@helia/verified-fetch'
+ *
+ * const verifiedFetch = await createVerifiedFetch({
+ * gateways: ['mygateway.info', 'trustless-gateway.link']
+ * })
+ *
+ * const response = await verifiedFetch('bafyFoo') // CID for some text file
+ * // OR const response = await verifiedFetch('ipfs://bafy...')
+ * // OR const response = await verifiedFetch('ipns://mydomain.com/path/to/file')
+ * // OR const response = await verifiedFetch('https://mygateway.info/ipfs/bafyFoo')
+ * const text = await response.text()
+ * ```
+ *
+ * @example Using a CID instance to fetch JSON
+ *
+ * ```typescript
+ * import { createVerifiedFetch } from '@helia/verified-fetch'
+ * import { CID } from 'multiformats/cid'
+ *
+ * const verifiedFetch = await createVerifiedFetch({
+ * gateways: ['mygateway.info', 'trustless-gateway.link']
+ * })
+ *
+ * const cid = CID.parse('bafyFoo') // some image file
+ * const response = await verifiedFetch(cid)
+ * const json = await response.json()
+ * ```
+ *
+ * @example Using ipfs protocol to fetch an image
+ *
+ * ```typescript
+ * import { createVerifiedFetch } from '@helia/verified-fetch'
+ *
+ * const verifiedFetch = await createVerifiedFetch({
+ * gateways: ['mygateway.info', 'trustless-gateway.link']
+ * })
+ * const response = await verifiedFetch('ipfs://bafyFoo') // CID for some image file
+ * const blob = await response.blob()
+ * ```
+ *
+ * @example Using ipns protocol to fetch a video
+ *
+ * ```typescript
+ * import { createVerifiedFetch } from '@helia/verified-fetch'
+ *
+ * const verifiedFetch = await createVerifiedFetch({
+ * gateways: ['mygateway.info', 'trustless-gateway.link']
+ * })
+ * const response = await verifiedFetch('ipns://mydomain.com/path/to/video.mp4')
+ * const videoStreamReader = await response.body.getReader()
+ */
+
+import type { Helia, Routing } from '@helia/interface'
+import { createHeliaHTTP } from '@helia/http'
+import { trustlessGateway } from '@helia/block-brokers'
+import { VerifiedFetch } from './verified-fetch.js'
+import type { CreateVerifiedFetchWithOptions } from './interface.js'
+import { delegatedHTTPRouting } from '@helia/routers'
+
+/**
+ * Create and return a Helia node
+ */
+export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWithOptions): Promise['fetch']> {
+ let heliaInstance: null | Helia = null
+ if ((init as CreateVerifiedFetchWithOptions).gateways == null) {
+ heliaInstance = init as Helia
+ } else {
+ const config = init as CreateVerifiedFetchWithOptions
+ let routers: undefined | Array> = undefined
+ if (config.routers != null) {
+ routers = config.routers.map((routerUrl) => delegatedHTTPRouting(routerUrl))
+ }
+ heliaInstance = await createHeliaHTTP({
+ blockBrokers: [
+ trustlessGateway({
+ gateways: config.gateways,
+ }),
+ ],
+ routers,
+ })
+ }
+
+ const verifiedFetchInstance = new VerifiedFetch(heliaInstance)
+
+ return verifiedFetchInstance.fetch.bind(verifiedFetchInstance)
+}
diff --git a/packages/verified-fetch/src/interface.ts b/packages/verified-fetch/src/interface.ts
new file mode 100644
index 00000000..3d9ee7e6
--- /dev/null
+++ b/packages/verified-fetch/src/interface.ts
@@ -0,0 +1,22 @@
+import type { CID } from 'multiformats/cid'
+
+/**
+ * Instead of passing a Helia instance, you can pass a list of gateways and routers, and a Helia instance will be created for you.
+ */
+export interface CreateVerifiedFetchWithOptions {
+ gateways: string[]
+ routers?: string[]
+}
+
+/**
+ * The types for the first argument of the `verifiedFetch` function.
+ */
+export type ResourceType = string | CID
+
+/**
+ * The second argument of the `verifiedFetch` function.
+ */
+export interface VerifiedFetchOptions extends RequestInit {
+ signal?: AbortSignal
+}
+
diff --git a/packages/verified-fetch/src/utils/get-content-type.ts b/packages/verified-fetch/src/utils/get-content-type.ts
new file mode 100644
index 00000000..c2b6fea3
--- /dev/null
+++ b/packages/verified-fetch/src/utils/get-content-type.ts
@@ -0,0 +1,55 @@
+// currently getting error from esbuild when trying to import file-type
+// import { fileTypeFromBuffer } from 'file-type';
+import mime from 'mime-types'
+
+interface testInput {
+ bytes: Uint8Array
+ path: string
+}
+
+type testOutput = Promise
+
+export const DEFAULT_MIME_TYPE = 'text/html'
+
+const xmlRegex = /^(<\?xml[^>]+>)?[^<^\w]+ testOutput> = [
+ // svg
+ async ({ bytes }): testOutput => xmlRegex.test(new TextDecoder().decode(bytes.slice(0, 64)))
+ ? 'image/svg+xml'
+ : undefined,
+ // testing file-type from buffer
+ // async ({ bytes }): testOutput => (await fileTypeFromBuffer(bytes))?.mime,
+ // testing file-type from path
+ async ({ path }): testOutput => mime.lookup(path) || undefined,
+ // default
+ async (): Promise => DEFAULT_MIME_TYPE
+]
+
+const overrides: Record = {
+ 'video/quicktime': 'video/mp4'
+}
+
+/**
+ * Override the content type based on overrides.
+ */
+function overrideContentType (type: string): string {
+ return overrides[type] ?? type
+}
+
+/**
+ * Get the content type from the input based on the tests.
+ */
+export async function getContentType (input: testInput): Promise {
+ for (const test of tests) {
+ const type = await test(input)
+ if (type !== undefined) {
+ return overrideContentType(type)
+ }
+ }
+ return DEFAULT_MIME_TYPE
+}
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
new file mode 100644
index 00000000..e931b9a2
--- /dev/null
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -0,0 +1,153 @@
+import { CID } from 'multiformats/cid';
+import type { ResourceType, VerifiedFetchOptions } from './interface.js';
+import type { Helia } from '@helia/interface';
+import { ipns, type IPNS } from '@helia/ipns'
+import {unixfs, type UnixFS} from '@helia/unixfs'
+import { peerIdFromString } from '@libp2p/peer-id'
+import { getContentType } from './utils/get-content-type.js';
+
+export class VerifiedFetch {
+ // @ts-expect-error - currently unused.
+ private readonly helia: Helia;
+ private readonly ipns: IPNS;
+ private readonly unixfs: UnixFS;
+ constructor (heliaInstance: Helia) {
+ this.helia = heliaInstance
+ this.ipns = ipns(heliaInstance)
+ this.unixfs = unixfs(heliaInstance)
+ }
+
+ /**
+ * Handles the different use cases for the `resource` argument.
+ * The resource can represent an IPFS path, IPNS path, or CID.
+ * If the resource represents an IPNS path, we need to resolve it to a CID.
+ */
+ private async parseResource (resource: ResourceType): Promise<{ cid: CID, path: string, protocol?: string }> {
+ if (typeof resource === 'string') {
+ // either an `ipfs://` or `ipns://` URL
+ const url = new URL(resource)
+ const protocol = url.protocol.slice(0, -1)
+ const urlPathParts = url.pathname.slice(2).split('/')
+ const cidOrPeerIdOrDnsLink = urlPathParts[0]
+ const path = urlPathParts.slice(1).join('/')
+ try {
+ const cid = CID.parse(cidOrPeerIdOrDnsLink)
+ return {
+ cid,
+ path,
+ protocol,
+ }
+ } catch {
+ // ignore non-CID
+ }
+
+ try {
+ const cid = await this.ipns.resolveDns(cidOrPeerIdOrDnsLink)
+ return {
+ cid,
+ path,
+ protocol,
+ }
+ } catch {
+ // ignore non DNSLink
+ }
+
+ try {
+ const peerId = await peerIdFromString(cidOrPeerIdOrDnsLink)
+ const cid = await this.ipns.resolve(peerId)
+ return {
+ cid,
+ path,
+ protocol,
+ }
+ } catch {
+ // ignore non PeerId
+ }
+ throw new Error(`Invalid resource. Cannot determine CID from resource: ${resource}`)
+ }
+
+ // an actual CID
+ return {
+ cid: resource,
+ protocol: 'ipfs',
+ path: ''
+ }
+ }
+
+ private async getStreamAndContentType (iterator: AsyncIterable, path: string): Promise<{ contentType: string, stream: ReadableStream }> {
+ const reader = iterator[Symbol.asyncIterator]()
+ const { value, done } = await reader.next()
+ if (done) {
+ throw new Error('No content found')
+ }
+
+ const contentType = await getContentType({ bytes: value, path })
+ const stream = new ReadableStream({
+ async pull (controller) {
+ const { value, done } = await reader.next()
+ if (done) {
+ controller.close()
+ return
+ }
+ controller.enqueue(value)
+ }
+ })
+
+ return { contentType, stream }
+ }
+
+
+ // handle vnd.ipfs.ipns-record
+ private async handleIPNSRecord ({cid, path, options}: {cid: CID, path: string, options?: VerifiedFetchOptions}): Promise {
+ return new Response('TODO: handleIPNSRecord', { status: 500 })
+ }
+
+ // handle vnd.ipld.car
+ private async handleIPLDCar ({cid, path, options}: {cid: CID, path: string, options?: VerifiedFetchOptions}): Promise {
+ return new Response('TODO: handleIPLDCar', { status: 500 })
+ }
+
+ /**
+ * handle vnd.ipld.raw
+ * This is the default method for fetched content.
+ */
+ private async handleIPLDRaw ({cid, path, options}: {cid: CID, path: string, options?: VerifiedFetchOptions}): Promise {
+ const asyncIter = await this.unixfs.cat(cid, { path, signal: options?.signal })
+ const { contentType, stream } = await this.getStreamAndContentType(asyncIter, path)
+
+ const response = new Response(stream, { status: 200 })
+ response.headers.set('content-type', contentType)
+
+ return response;
+ }
+
+
+ async fetch (resource: ResourceType, options?: VerifiedFetchOptions): Promise {
+ const { cid, path } = await this.parseResource(resource)
+ let response: Response | undefined
+ if (options?.headers != null) {
+ const contentType = new Headers(options.headers).get('content-type')
+ if (contentType != null) {
+ if (contentType.includes('vnd.ipld.car')) {
+ response = await this.handleIPLDCar({cid, path, options})
+
+ } else if (contentType.includes('vnd.ipfs.ipns-record')) {
+ response = await this.handleIPNSRecord({cid, path, options})
+ }
+ }
+ }
+
+ if (response == null) {
+ response = await this.handleIPLDRaw({cid, path, options})
+ }
+
+ response.headers.set('etag', cid.toString())
+ // response.headers.set('cache-cotrol', 'public, max-age=29030400, immutable')
+ response.headers.set('cache-cotrol', 'no-cache') // disable caching when debugging
+ response.headers.set('x-ipfs-path', path)
+ response.headers.set('x-ipfs-cid', cid.toString())
+ response.headers.set('x-ipfs-protocol', 'ipfs')
+
+ return response
+ }
+}
diff --git a/packages/verified-fetch/tsconfig.json b/packages/verified-fetch/tsconfig.json
new file mode 100644
index 00000000..f800762a
--- /dev/null
+++ b/packages/verified-fetch/tsconfig.json
@@ -0,0 +1,33 @@
+{
+ "extends": "aegir/src/config/tsconfig.aegir.json",
+ "compilerOptions": {
+ "outDir": "dist"
+ },
+ "include": [
+ "src",
+ "test"
+ ],
+ "references": [
+ {
+ "path": "../interface"
+ },
+ {
+ "path": "../utils"
+ },
+ {
+ "path": "../http"
+ },
+ {
+ "path": "../routers"
+ },
+ {
+ "path": "../block-brokers"
+ },
+ {
+ "path": "../ipns"
+ },
+ {
+ "path": "../unixfs"
+ }
+ ]
+}
diff --git a/packages/verified-fetch/typedoc.json b/packages/verified-fetch/typedoc.json
new file mode 100644
index 00000000..f599dc72
--- /dev/null
+++ b/packages/verified-fetch/typedoc.json
@@ -0,0 +1,5 @@
+{
+ "entryPoints": [
+ "./src/index.ts"
+ ]
+}
From ab0f99406b667a81d42c80d18939c6257aa326ed Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 17 Jan 2024 16:50:44 -0800
Subject: [PATCH 002/104] fix: dep-check passes with no warnings
---
packages/verified-fetch/package.json | 39 +++++++---------------------
1 file changed, 10 insertions(+), 29 deletions(-)
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index 4650fae3..eb63e96f 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -55,43 +55,24 @@
"test:electron-main": "aegir test -t electron-main"
},
"dependencies": {
+ "@helia/block-brokers": "^1.0.0-163df38",
"@helia/http": "next",
"@helia/interface": "^3.0.1",
- "@ipld/dag-cbor": "^9.0.7",
- "@ipld/dag-json": "^10.1.5",
- "@ipld/dag-pb": "^4.0.6",
- "@libp2p/interface": "^1.1.1",
- "@libp2p/logger": "^4.0.4",
- "@libp2p/peer-collections": "^5.1.4",
- "@libp2p/utils": "^5.2.0",
- "any-signal": "^4.1.1",
- "cborg": "^4.0.3",
- "file-type": "^19.0.0",
- "interface-blockstore": "^5.2.7",
- "interface-datastore": "^8.2.9",
- "interface-store": "^5.1.5",
- "it-drain": "^3.0.5",
- "it-filter": "^3.0.4",
- "it-foreach": "^2.0.6",
- "it-merge": "^3.0.3",
- "mime-type": "^4.0.0",
+ "@helia/ipns": "^4.0.0",
+ "@helia/routers": "^0.0.0-163df38",
+ "@helia/unixfs": "^2.0.1",
+ "@libp2p/peer-id": "^4.0.5",
"mime-types": "^2.1.35",
- "mortice": "^3.0.1",
- "multiformats": "^13.0.0",
- "progress-events": "^1.0.0",
- "uint8arrays": "^5.0.1"
+ "multiformats": "^13.0.0"
},
"devDependencies": {
- "@helia/ipns": "^4.0.0",
"@types/mime-types": "^2.1.4",
"@types/sinon": "^17.0.2",
"aegir": "^42.1.0",
- "blockstore-core": "^4.3.10",
- "datastore-core": "^9.2.7",
- "delay": "^6.0.0",
- "it-all": "^3.0.4",
- "sinon": "^17.0.1",
- "sinon-ts": "^2.0.0"
+ "helia": "^3.0.1",
+ "ipfsd-ctl": "^13.0.0",
+ "kubo": "^0.25.0",
+ "kubo-rpc-client": "^3.0.2"
},
"browser": {
"node:buffer": false,
From 7a5dc751f81495dc5adcdf8e80d0239cf851649d Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 17 Jan 2024 18:38:37 -0800
Subject: [PATCH 003/104] test: adding some tests
---
packages/verified-fetch/.aegir.js | 54 +++++++++---------
packages/verified-fetch/package.json | 13 +++--
packages/verified-fetch/src/index.ts | 9 ++-
packages/verified-fetch/src/verified-fetch.ts | 29 ++++++++--
packages/verified-fetch/test/index.spec.ts | 55 +++++++++++++++++++
5 files changed, 119 insertions(+), 41 deletions(-)
create mode 100644 packages/verified-fetch/test/index.spec.ts
diff --git a/packages/verified-fetch/.aegir.js b/packages/verified-fetch/.aegir.js
index 65eb2562..d4ca8b17 100644
--- a/packages/verified-fetch/.aegir.js
+++ b/packages/verified-fetch/.aegir.js
@@ -8,42 +8,38 @@ export default {
bundlesizeMax: '10kB',
},
test: {
- files: './dist/src/*.spec.js',
+ files: './dist/test/*.spec.js',
before: async (options) => {
- if (options.runner !== 'node') {
- const ipfsdPort = await getPort()
- const ipfsdServer = await createServer({
- host: '127.0.0.1',
- port: ipfsdPort
- }, {
- ipfsBin: (await import('kubo')).default.path(),
- kuboRpcModule: kuboRpcClient,
- ipfsOptions: {
- config: {
- Addresses: {
- Swarm: [
- "/ip4/0.0.0.0/tcp/0",
- "/ip4/0.0.0.0/tcp/0/ws"
- ]
- }
+ const ipfsdPort = await getPort()
+ const ipfsdServer = await createServer({
+ host: '127.0.0.1',
+ port: ipfsdPort
+ }, {
+ ipfsBin: (await import('kubo')).default.path(),
+ kuboRpcModule: kuboRpcClient,
+ ipfsOptions: {
+ // TODO: enable delegated routing
+ // TODO: enable trustless-gateway
+ config: {
+ Addresses: {
+ Swarm: [
+ "/ip4/0.0.0.0/tcp/0",
+ "/ip4/0.0.0.0/tcp/0/ws"
+ ]
}
}
- }).start()
-
- return {
- env: {
- IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}`
- },
- ipfsdServer
}
- }
+ }).start()
- return {}
+ return {
+ env: {
+ IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}`,
+ },
+ ipfsdServer
+ }
},
after: async (options, beforeResult) => {
- if (options.runner !== 'node') {
- await beforeResult.ipfsdServer.stop()
- }
+ await beforeResult.ipfsdServer.stop()
}
}
}
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index eb63e96f..a1acc289 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -55,12 +55,12 @@
"test:electron-main": "aegir test -t electron-main"
},
"dependencies": {
- "@helia/block-brokers": "^1.0.0-163df38",
+ "@helia/block-brokers": "next",
"@helia/http": "next",
- "@helia/interface": "^3.0.1",
- "@helia/ipns": "^4.0.0",
- "@helia/routers": "^0.0.0-163df38",
- "@helia/unixfs": "^2.0.1",
+ "@helia/interface": "next",
+ "@helia/ipns": "next",
+ "@helia/routers": "next",
+ "@helia/unixfs": "next",
"@libp2p/peer-id": "^4.0.5",
"mime-types": "^2.1.35",
"multiformats": "^13.0.0"
@@ -69,7 +69,8 @@
"@types/mime-types": "^2.1.4",
"@types/sinon": "^17.0.2",
"aegir": "^42.1.0",
- "helia": "^3.0.1",
+ "helia": "next",
+ "ipfs-unixfs": "^11.1.2",
"ipfsd-ctl": "^13.0.0",
"kubo": "^0.25.0",
"kubo-rpc-client": "^3.0.2"
diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts
index 809eec47..ad865f2d 100644
--- a/packages/verified-fetch/src/index.ts
+++ b/packages/verified-fetch/src/index.ts
@@ -75,7 +75,7 @@ import { delegatedHTTPRouting } from '@helia/routers'
/**
* Create and return a Helia node
*/
-export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWithOptions): Promise['fetch']> {
+export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWithOptions): Promise['fetch'] & { start: InstanceType['start'], stop: InstanceType['stop'] }> {
let heliaInstance: null | Helia = null
if ((init as CreateVerifiedFetchWithOptions).gateways == null) {
heliaInstance = init as Helia
@@ -96,6 +96,11 @@ export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWith
}
const verifiedFetchInstance = new VerifiedFetch(heliaInstance)
+ async function verifiedFetch(...args: Parameters): ReturnType {
+ return verifiedFetchInstance.fetch(...args)
+ }
+ verifiedFetch.stop = verifiedFetchInstance.stop.bind(verifiedFetchInstance)
+ verifiedFetch.start = verifiedFetchInstance.start.bind(verifiedFetchInstance)
- return verifiedFetchInstance.fetch.bind(verifiedFetchInstance)
+ return verifiedFetch
}
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index e931b9a2..d8682314 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -7,7 +7,6 @@ import { peerIdFromString } from '@libp2p/peer-id'
import { getContentType } from './utils/get-content-type.js';
export class VerifiedFetch {
- // @ts-expect-error - currently unused.
private readonly helia: Helia;
private readonly ipns: IPNS;
private readonly unixfs: UnixFS;
@@ -37,7 +36,8 @@ export class VerifiedFetch {
path,
protocol,
}
- } catch {
+ } catch (err) {
+ console.error(err)
// ignore non-CID
}
@@ -48,7 +48,8 @@ export class VerifiedFetch {
path,
protocol,
}
- } catch {
+ } catch (err) {
+ console.error(err)
// ignore non DNSLink
}
@@ -60,7 +61,8 @@ export class VerifiedFetch {
path,
protocol,
}
- } catch {
+ } catch (err) {
+ console.error(err)
// ignore non PeerId
}
throw new Error(`Invalid resource. Cannot determine CID from resource: ${resource}`)
@@ -78,11 +80,16 @@ export class VerifiedFetch {
const reader = iterator[Symbol.asyncIterator]()
const { value, done } = await reader.next()
if (done) {
+ console.error('No content found')
throw new Error('No content found')
}
const contentType = await getContentType({ bytes: value, path })
const stream = new ReadableStream({
+ async start (controller) {
+ // the initial value is already available
+ controller.enqueue(value)
+ },
async pull (controller) {
const { value, done } = await reader.next()
if (done) {
@@ -150,4 +157,18 @@ export class VerifiedFetch {
return response
}
+
+ /**
+ * Start the Helia instance
+ */
+ async start (): Promise {
+ await this.helia.start()
+ }
+
+ /**
+ * Shut down the Helia instance
+ */
+ async stop (): Promise {
+ await this.helia.stop()
+ }
}
diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts
new file mode 100644
index 00000000..c8f26a06
--- /dev/null
+++ b/packages/verified-fetch/test/index.spec.ts
@@ -0,0 +1,55 @@
+/* eslint-env mocha */
+import { createHeliaHTTP } from '@helia/http'
+import { createVerifiedFetch } from '../src/index.js'
+import { expect } from 'aegir/chai'
+
+describe('createVerifiedFetch', () => {
+ it('Can be constructed with a HeliaHttp instance', async () => {
+ const heliaHttp = await createHeliaHTTP()
+ const verifiedFetch = await createVerifiedFetch(heliaHttp)
+
+ expect(verifiedFetch).to.be.ok()
+ await verifiedFetch.stop()
+ })
+
+ /**
+ * Currently erroring:
+ *
+ * Error: Package subpath './peer-job-queue' is not defined by "exports" in /Users/sgtpooki/code/work/protocol.ai/ipfs/helia/node_modules/@libp2p/utils/package.json imported from /Users/sgtpooki/code/work/protocol.ai/ipfs/helia/node_modules/@libp2p/circuit-relay-v2/dist/src/transport/reservation-store.js
+ * at new NodeError (node:internal/errors:406:5)
+ * at exportsNotFound (node:internal/modules/esm/resolve:268:10)
+ * at packageExportsResolve (node:internal/modules/esm/resolve:598:9)
+ * at packageResolve (node:internal/modules/esm/resolve:772:14)
+ * at moduleResolve (node:internal/modules/esm/resolve:838:20)
+ * at defaultResolve (node:internal/modules/esm/resolve:1043:11)
+ * at ModuleLoader.defaultResolve (node:internal/modules/esm/loader:383:12)
+ * at ModuleLoader.resolve (node:internal/modules/esm/loader:352:25)
+ * at ModuleLoader.getModuleJob (node:internal/modules/esm/loader:228:38)
+ * at ModuleWrap. (node:internal/modules/esm/module_job:85:39)
+ * at link (node:internal/modules/esm/module_job:84:36)
+ */
+ // it('Can be constructed with a HeliaP2P instance', async () => {
+ // const heliaP2P = await createHelia()
+ // const verifiedFetch = await createVerifiedFetch(heliaP2P)
+
+ // expect(verifiedFetch).to.be.ok()
+ // await heliaP2P.stop()
+ // })
+
+ it('Can be constructed with gateways', async () => {
+ const verifiedFetch = await createVerifiedFetch({
+ gateways: ['https://127.0.0.1']
+ })
+ expect(verifiedFetch).to.be.ok()
+ await verifiedFetch.stop()
+ })
+
+ it('Can be constructed with gateways & routers', async () => {
+ const verifiedFetch = await createVerifiedFetch({
+ gateways: ['https://127.0.0.1'],
+ routers: ['https://127.0.0.1']
+ })
+ expect(verifiedFetch).to.be.ok()
+ await verifiedFetch.stop()
+ })
+})
From 8c15f0518263d75d1e6f02e180f28961efe399ca Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 17 Jan 2024 18:54:19 -0800
Subject: [PATCH 004/104] test: adding gateway test
---
.../test/fixtures/add-content-to-kubo-node.ts | 8 ++++
.../test/fixtures/create-kubo.ts | 23 +++++++++++
packages/verified-fetch/test/gateways.spec.ts | 39 +++++++++++++++++++
3 files changed, 70 insertions(+)
create mode 100644 packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts
create mode 100644 packages/verified-fetch/test/fixtures/create-kubo.ts
create mode 100644 packages/verified-fetch/test/gateways.spec.ts
diff --git a/packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts b/packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts
new file mode 100644
index 00000000..b7bbfe41
--- /dev/null
+++ b/packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts
@@ -0,0 +1,8 @@
+import type { Controller } from 'ipfsd-ctl'
+
+export async function addContentToKuboNode (kuboNode: Controller<'go'>, content: any) {
+ return await kuboNode.api.add(content, {
+ cidVersion: 1,
+ pin: false
+ })
+}
diff --git a/packages/verified-fetch/test/fixtures/create-kubo.ts b/packages/verified-fetch/test/fixtures/create-kubo.ts
new file mode 100644
index 00000000..18e51720
--- /dev/null
+++ b/packages/verified-fetch/test/fixtures/create-kubo.ts
@@ -0,0 +1,23 @@
+import { type Controller, createController } from 'ipfsd-ctl'
+import * as kuboRpcClient from 'kubo-rpc-client'
+import { path as kuboPath } from 'kubo'
+
+export async function createKuboNode (): Promise {
+ return createController({
+ kuboRpcModule: kuboRpcClient,
+ ipfsBin: kuboPath(),
+ test: true,
+ endpoint: process.env.IPFSD_SERVER,
+ ipfsOptions: {
+ config: {
+ Addresses: {
+ Swarm: [
+ '/ip4/0.0.0.0/tcp/0',
+ '/ip4/0.0.0.0/tcp/0/ws'
+ ]
+ }
+ }
+ },
+ args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
+ })
+}
diff --git a/packages/verified-fetch/test/gateways.spec.ts b/packages/verified-fetch/test/gateways.spec.ts
new file mode 100644
index 00000000..b368c474
--- /dev/null
+++ b/packages/verified-fetch/test/gateways.spec.ts
@@ -0,0 +1,39 @@
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { createVerifiedFetch } from '../src/index.js'
+import { expect } from 'aegir/chai'
+import type { Controller } from 'ipfsd-ctl'
+import { addContentToKuboNode } from './fixtures/add-content-to-kubo-node.js'
+import { UnixFS } from 'ipfs-unixfs'
+
+describe('verified-fetch gateways', () => {
+ let controller: Controller<'go'>
+ beforeEach(async () => {
+ controller = await createKuboNode()
+ await controller.start()
+ })
+
+ afterEach(async () => {
+ await controller.stop()
+ })
+
+ it('Uses the provided gateway', async () => {
+ const verifiedFetch = await createVerifiedFetch({
+ gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
+ })
+ const givenString = 'hello sgtpooki from verified-fetch test'
+ const content = new UnixFS({ type: 'raw', data: Buffer.from(givenString) })
+ const {cid} = await addContentToKuboNode(controller, content.marshal())
+ expect(cid).to.be.ok()
+ // @ts-expect-error - todo fix types
+ const resp = await verifiedFetch(cid)
+ expect(resp).to.be.ok()
+ const text = await resp.text() // this currently has UnixFS data in it, and should not when returned from verified-fetch
+
+ // the below commented lines will get the test to pass, but we need to move this into verified fetch
+ // const marshalledResponseData = await resp.arrayBuffer()
+ // const encodedText = UnixFS.unmarshal(new Uint8Array(marshalledResponseData)).data
+ // const text = textDecoder.decode(encodedText)
+
+ expect(text).to.equal(givenString)
+ })
+})
From 94cf631f4d0c73e28991b26349098719fe9ab0b1 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:43:17 -0800
Subject: [PATCH 005/104] chore: lint succeeds
---
packages/verified-fetch/package.json | 1 +
packages/verified-fetch/src/index.ts | 16 ++---
packages/verified-fetch/src/interface.ts | 1 -
.../src/utils/get-content-type.ts | 8 ++-
packages/verified-fetch/src/verified-fetch.ts | 66 +++++++++++--------
.../test/fixtures/add-content-to-kubo-node.ts | 4 +-
.../test/fixtures/create-kubo.ts | 2 +-
packages/verified-fetch/test/gateways.spec.ts | 12 ++--
packages/verified-fetch/test/index.spec.ts | 24 +++----
9 files changed, 75 insertions(+), 59 deletions(-)
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index a1acc289..4fa61c9f 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -61,6 +61,7 @@
"@helia/ipns": "next",
"@helia/routers": "next",
"@helia/unixfs": "next",
+ "@libp2p/logger": "^4.0.5",
"@libp2p/peer-id": "^4.0.5",
"mime-types": "^2.1.35",
"multiformats": "^13.0.0"
diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts
index ad865f2d..049dae7f 100644
--- a/packages/verified-fetch/src/index.ts
+++ b/packages/verified-fetch/src/index.ts
@@ -65,12 +65,12 @@
* const videoStreamReader = await response.body.getReader()
*/
-import type { Helia, Routing } from '@helia/interface'
-import { createHeliaHTTP } from '@helia/http'
import { trustlessGateway } from '@helia/block-brokers'
+import { createHeliaHTTP } from '@helia/http'
+import { delegatedHTTPRouting } from '@helia/routers'
import { VerifiedFetch } from './verified-fetch.js'
import type { CreateVerifiedFetchWithOptions } from './interface.js'
-import { delegatedHTTPRouting } from '@helia/routers'
+import type { Helia, Routing } from '@helia/interface'
/**
* Create and return a Helia node
@@ -81,22 +81,22 @@ export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWith
heliaInstance = init as Helia
} else {
const config = init as CreateVerifiedFetchWithOptions
- let routers: undefined | Array> = undefined
+ let routers: undefined | Array>
if (config.routers != null) {
routers = config.routers.map((routerUrl) => delegatedHTTPRouting(routerUrl))
}
heliaInstance = await createHeliaHTTP({
blockBrokers: [
trustlessGateway({
- gateways: config.gateways,
- }),
+ gateways: config.gateways
+ })
],
- routers,
+ routers
})
}
const verifiedFetchInstance = new VerifiedFetch(heliaInstance)
- async function verifiedFetch(...args: Parameters): ReturnType {
+ async function verifiedFetch (...args: Parameters): ReturnType {
return verifiedFetchInstance.fetch(...args)
}
verifiedFetch.stop = verifiedFetchInstance.stop.bind(verifiedFetchInstance)
diff --git a/packages/verified-fetch/src/interface.ts b/packages/verified-fetch/src/interface.ts
index 3d9ee7e6..ec1caa8d 100644
--- a/packages/verified-fetch/src/interface.ts
+++ b/packages/verified-fetch/src/interface.ts
@@ -19,4 +19,3 @@ export type ResourceType = string | CID
export interface VerifiedFetchOptions extends RequestInit {
signal?: AbortSignal
}
-
diff --git a/packages/verified-fetch/src/utils/get-content-type.ts b/packages/verified-fetch/src/utils/get-content-type.ts
index c2b6fea3..31c3f47d 100644
--- a/packages/verified-fetch/src/utils/get-content-type.ts
+++ b/packages/verified-fetch/src/utils/get-content-type.ts
@@ -25,7 +25,13 @@ const tests: Array<(input: testInput) => testOutput> = [
// testing file-type from buffer
// async ({ bytes }): testOutput => (await fileTypeFromBuffer(bytes))?.mime,
// testing file-type from path
- async ({ path }): testOutput => mime.lookup(path) || undefined,
+ async ({ path }): testOutput => {
+ const mimeType = mime.lookup(path)
+ if (mimeType !== false) {
+ return mimeType
+ }
+ return undefined
+ },
// default
async (): Promise => DEFAULT_MIME_TYPE
]
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index d8682314..1495e2a1 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -1,15 +1,18 @@
-import { CID } from 'multiformats/cid';
-import type { ResourceType, VerifiedFetchOptions } from './interface.js';
-import type { Helia } from '@helia/interface';
import { ipns, type IPNS } from '@helia/ipns'
-import {unixfs, type UnixFS} from '@helia/unixfs'
+import { unixfs, type UnixFS } from '@helia/unixfs'
+import { logger } from '@libp2p/logger'
import { peerIdFromString } from '@libp2p/peer-id'
-import { getContentType } from './utils/get-content-type.js';
+import { CID } from 'multiformats/cid'
+import { getContentType } from './utils/get-content-type.js'
+import type { ResourceType, VerifiedFetchOptions } from './interface.js'
+import type { Helia } from '@helia/interface'
+
+const log = logger('helia:verified-fetch')
export class VerifiedFetch {
- private readonly helia: Helia;
- private readonly ipns: IPNS;
- private readonly unixfs: UnixFS;
+ private readonly helia: Helia
+ private readonly ipns: IPNS
+ private readonly unixfs: UnixFS
constructor (heliaInstance: Helia) {
this.helia = heliaInstance
this.ipns = ipns(heliaInstance)
@@ -34,10 +37,10 @@ export class VerifiedFetch {
return {
cid,
path,
- protocol,
+ protocol
}
} catch (err) {
- console.error(err)
+ log.error(err)
// ignore non-CID
}
@@ -46,23 +49,23 @@ export class VerifiedFetch {
return {
cid,
path,
- protocol,
+ protocol
}
} catch (err) {
- console.error(err)
+ log.error(err)
// ignore non DNSLink
}
try {
- const peerId = await peerIdFromString(cidOrPeerIdOrDnsLink)
+ const peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
const cid = await this.ipns.resolve(peerId)
return {
cid,
path,
- protocol,
+ protocol
}
} catch (err) {
- console.error(err)
+ log.error(err)
// ignore non PeerId
}
throw new Error(`Invalid resource. Cannot determine CID from resource: ${resource}`)
@@ -79,8 +82,8 @@ export class VerifiedFetch {
private async getStreamAndContentType (iterator: AsyncIterable, path: string): Promise<{ contentType: string, stream: ReadableStream }> {
const reader = iterator[Symbol.asyncIterator]()
const { value, done } = await reader.next()
- if (done) {
- console.error('No content found')
+ if (done === true) {
+ log.error('No content found')
throw new Error('No content found')
}
@@ -92,7 +95,7 @@ export class VerifiedFetch {
},
async pull (controller) {
const { value, done } = await reader.next()
- if (done) {
+ if (done === true) {
controller.close()
return
}
@@ -103,14 +106,22 @@ export class VerifiedFetch {
return { contentType, stream }
}
+ // private async getHeliaModuleForCID (cid: CID) {
+ // switch (cid.code) {
+ // case 112: // unixfs
+ // return this.unixfs
+ // default:
+ // return this.helia
+ // }
+ // }
// handle vnd.ipfs.ipns-record
- private async handleIPNSRecord ({cid, path, options}: {cid: CID, path: string, options?: VerifiedFetchOptions}): Promise {
+ private async handleIPNSRecord ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
return new Response('TODO: handleIPNSRecord', { status: 500 })
}
// handle vnd.ipld.car
- private async handleIPLDCar ({cid, path, options}: {cid: CID, path: string, options?: VerifiedFetchOptions}): Promise {
+ private async handleIPLDCar ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
return new Response('TODO: handleIPLDCar', { status: 500 })
}
@@ -118,17 +129,17 @@ export class VerifiedFetch {
* handle vnd.ipld.raw
* This is the default method for fetched content.
*/
- private async handleIPLDRaw ({cid, path, options}: {cid: CID, path: string, options?: VerifiedFetchOptions}): Promise {
- const asyncIter = await this.unixfs.cat(cid, { path, signal: options?.signal })
+ private async handleIPLDRaw ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
+ const asyncIter = this.unixfs.cat(cid, { path, signal: options?.signal })
+ // const asyncIter = await this.helia.blockstore.get(cid, { signal: options?.signal })
const { contentType, stream } = await this.getStreamAndContentType(asyncIter, path)
const response = new Response(stream, { status: 200 })
response.headers.set('content-type', contentType)
- return response;
+ return response
}
-
async fetch (resource: ResourceType, options?: VerifiedFetchOptions): Promise {
const { cid, path } = await this.parseResource(resource)
let response: Response | undefined
@@ -136,16 +147,15 @@ export class VerifiedFetch {
const contentType = new Headers(options.headers).get('content-type')
if (contentType != null) {
if (contentType.includes('vnd.ipld.car')) {
- response = await this.handleIPLDCar({cid, path, options})
-
+ response = await this.handleIPLDCar({ cid, path, options })
} else if (contentType.includes('vnd.ipfs.ipns-record')) {
- response = await this.handleIPNSRecord({cid, path, options})
+ response = await this.handleIPNSRecord({ cid, path, options })
}
}
}
if (response == null) {
- response = await this.handleIPLDRaw({cid, path, options})
+ response = await this.handleIPLDRaw({ cid, path, options })
}
response.headers.set('etag', cid.toString())
diff --git a/packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts b/packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts
index b7bbfe41..82ceaecf 100644
--- a/packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts
+++ b/packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts
@@ -1,7 +1,7 @@
import type { Controller } from 'ipfsd-ctl'
-export async function addContentToKuboNode (kuboNode: Controller<'go'>, content: any) {
- return await kuboNode.api.add(content, {
+export async function addContentToKuboNode (kuboNode: Controller<'go'>, content: any): Promise> {
+ return kuboNode.api.add(content, {
cidVersion: 1,
pin: false
})
diff --git a/packages/verified-fetch/test/fixtures/create-kubo.ts b/packages/verified-fetch/test/fixtures/create-kubo.ts
index 18e51720..124b9769 100644
--- a/packages/verified-fetch/test/fixtures/create-kubo.ts
+++ b/packages/verified-fetch/test/fixtures/create-kubo.ts
@@ -1,6 +1,6 @@
import { type Controller, createController } from 'ipfsd-ctl'
-import * as kuboRpcClient from 'kubo-rpc-client'
import { path as kuboPath } from 'kubo'
+import * as kuboRpcClient from 'kubo-rpc-client'
export async function createKuboNode (): Promise {
return createController({
diff --git a/packages/verified-fetch/test/gateways.spec.ts b/packages/verified-fetch/test/gateways.spec.ts
index b368c474..85564c98 100644
--- a/packages/verified-fetch/test/gateways.spec.ts
+++ b/packages/verified-fetch/test/gateways.spec.ts
@@ -1,9 +1,9 @@
-import { createKuboNode } from './fixtures/create-kubo.js'
-import { createVerifiedFetch } from '../src/index.js'
import { expect } from 'aegir/chai'
-import type { Controller } from 'ipfsd-ctl'
-import { addContentToKuboNode } from './fixtures/add-content-to-kubo-node.js'
import { UnixFS } from 'ipfs-unixfs'
+import { createVerifiedFetch } from '../src/index.js'
+import { addContentToKuboNode } from './fixtures/add-content-to-kubo-node.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { Controller } from 'ipfsd-ctl'
describe('verified-fetch gateways', () => {
let controller: Controller<'go'>
@@ -18,11 +18,11 @@ describe('verified-fetch gateways', () => {
it('Uses the provided gateway', async () => {
const verifiedFetch = await createVerifiedFetch({
- gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
+ gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
})
const givenString = 'hello sgtpooki from verified-fetch test'
const content = new UnixFS({ type: 'raw', data: Buffer.from(givenString) })
- const {cid} = await addContentToKuboNode(controller, content.marshal())
+ const { cid } = await addContentToKuboNode(controller, content.marshal())
expect(cid).to.be.ok()
// @ts-expect-error - todo fix types
const resp = await verifiedFetch(cid)
diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts
index c8f26a06..2c04d885 100644
--- a/packages/verified-fetch/test/index.spec.ts
+++ b/packages/verified-fetch/test/index.spec.ts
@@ -1,7 +1,7 @@
/* eslint-env mocha */
import { createHeliaHTTP } from '@helia/http'
-import { createVerifiedFetch } from '../src/index.js'
import { expect } from 'aegir/chai'
+import { createVerifiedFetch } from '../src/index.js'
describe('createVerifiedFetch', () => {
it('Can be constructed with a HeliaHttp instance', async () => {
@@ -16,17 +16,17 @@ describe('createVerifiedFetch', () => {
* Currently erroring:
*
* Error: Package subpath './peer-job-queue' is not defined by "exports" in /Users/sgtpooki/code/work/protocol.ai/ipfs/helia/node_modules/@libp2p/utils/package.json imported from /Users/sgtpooki/code/work/protocol.ai/ipfs/helia/node_modules/@libp2p/circuit-relay-v2/dist/src/transport/reservation-store.js
- * at new NodeError (node:internal/errors:406:5)
- * at exportsNotFound (node:internal/modules/esm/resolve:268:10)
- * at packageExportsResolve (node:internal/modules/esm/resolve:598:9)
- * at packageResolve (node:internal/modules/esm/resolve:772:14)
- * at moduleResolve (node:internal/modules/esm/resolve:838:20)
- * at defaultResolve (node:internal/modules/esm/resolve:1043:11)
- * at ModuleLoader.defaultResolve (node:internal/modules/esm/loader:383:12)
- * at ModuleLoader.resolve (node:internal/modules/esm/loader:352:25)
- * at ModuleLoader.getModuleJob (node:internal/modules/esm/loader:228:38)
- * at ModuleWrap. (node:internal/modules/esm/module_job:85:39)
- * at link (node:internal/modules/esm/module_job:84:36)
+ * at new NodeError (node:internal/errors:406:5)
+ * at exportsNotFound (node:internal/modules/esm/resolve:268:10)
+ * at packageExportsResolve (node:internal/modules/esm/resolve:598:9)
+ * at packageResolve (node:internal/modules/esm/resolve:772:14)
+ * at moduleResolve (node:internal/modules/esm/resolve:838:20)
+ * at defaultResolve (node:internal/modules/esm/resolve:1043:11)
+ * at ModuleLoader.defaultResolve (node:internal/modules/esm/loader:383:12)
+ * at ModuleLoader.resolve (node:internal/modules/esm/loader:352:25)
+ * at ModuleLoader.getModuleJob (node:internal/modules/esm/loader:228:38)
+ * at ModuleWrap. (node:internal/modules/esm/module_job:85:39)
+ * at link (node:internal/modules/esm/module_job:84:36)
*/
// it('Can be constructed with a HeliaP2P instance', async () => {
// const heliaP2P = await createHelia()
From 56008e8708fe523a9ad640268dc04c8b5787c2d9 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:48:40 -0800
Subject: [PATCH 006/104] chore: fix dep-check
---
packages/verified-fetch/package.json | 1 -
1 file changed, 1 deletion(-)
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index 4fa61c9f..e206564c 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -70,7 +70,6 @@
"@types/mime-types": "^2.1.4",
"@types/sinon": "^17.0.2",
"aegir": "^42.1.0",
- "helia": "next",
"ipfs-unixfs": "^11.1.2",
"ipfsd-ctl": "^13.0.0",
"kubo": "^0.25.0",
From b8a5f67b3c0965ea0fa76d3f56fba88d535c923f Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:55:24 -0800
Subject: [PATCH 007/104] docs: update custom helia example
---
packages/verified-fetch/README.md | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md
index ca1c057d..7f626603 100644
--- a/packages/verified-fetch/README.md
+++ b/packages/verified-fetch/README.md
@@ -52,14 +52,20 @@ You can see variations of Helia and js-libp2p configuration options at https://h
The `@helia/http` module is currently in-progress, but the init options should be a subset of the `helia` module's init options. See https://github.com/ipfs/helia/issues/289 for more information.
```ts
+import { trustlessGateway } from '@helia/block-brokers'
+import { createHeliaHTTP } from '@helia/http'
+import { delegatedHTTPRouting } from '@helia/routers'
import { createVerifiedFetch } from '@helia/verified-fetch'
-import { CreateHelia as CreateHeliaHttpOnly } from '@helia/http'
const verifiedFetch = await createVerifiedFetch(
- CreateHeliaHttpOnly({
- gateways: ['mygateway.info', 'trustless-gateway.link'],
- routers: ['delegated-ipfs.dev'],
- })
+ await createHeliaHTTP({
+ blockBrokers: [
+ trustlessGateway({
+ gateways: ['http://mygateway.info', 'http://trustless-gateway.link']
+ })
+ ],
+ routers: ['http://delegated-ipfs.dev'].map((routerUrl) => delegatedHTTPRouting(routerUrl))
+ })
)
const resp = await verifiedFetch('ipfs://bafy...')
From 84adcbba219422966339fa074acb5c66cb9d31e2 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:56:59 -0800
Subject: [PATCH 008/104] Update packages/verified-fetch/src/interface.ts
---
packages/verified-fetch/src/interface.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/verified-fetch/src/interface.ts b/packages/verified-fetch/src/interface.ts
index ec1caa8d..6f32e4a8 100644
--- a/packages/verified-fetch/src/interface.ts
+++ b/packages/verified-fetch/src/interface.ts
@@ -1,7 +1,7 @@
import type { CID } from 'multiformats/cid'
/**
- * Instead of passing a Helia instance, you can pass a list of gateways and routers, and a Helia instance will be created for you.
+ * Instead of passing a Helia instance, you can pass a list of gateways and routers, and a HeliaHTTP instance will be created for you.
*/
export interface CreateVerifiedFetchWithOptions {
gateways: string[]
From 603fb7f502c4b49ff48f3f7ad2bc68ed50e525aa Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 11:14:32 -0800
Subject: [PATCH 009/104] docs: Update packages/verified-fetch/README.md
Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>
---
packages/verified-fetch/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md
index 7f626603..361ea7d5 100644
--- a/packages/verified-fetch/README.md
+++ b/packages/verified-fetch/README.md
@@ -13,7 +13,7 @@
# About
-`@helia/verified-fetch`` is a library that provides a fetch-like API for fetching content from IPFS. This library should act as a replacement for the `fetch()` API for fetching content from IPFS, and will return a [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) object that can be used in a similar manner to the `fetch()` API. This means browser and HTTP caching inside browser main threads, web-workers, and service workers, as well as other features of the `fetch()` API should work in a way familiar to developers.
+`@helia/verified-fetch` is a library that provides a fetch-like API for fetching content from IPFS. This library should act as a replacement for the `fetch()` API for fetching content from IPFS, and will return a [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) object that can be used in a similar manner to the `fetch()` API. This means browser and HTTP caching inside browser main threads, web-workers, and service workers, as well as other features of the `fetch()` API should work in a way familiar to developers.
## Example
From a49d3531f2209f127f457a637fd4be65fe19e728 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 10:52:06 -0800
Subject: [PATCH 010/104] docs: remove support for CID string
---
packages/verified-fetch/README.md | 5 ++--
packages/verified-fetch/src/index.ts | 23 +++----------------
packages/verified-fetch/src/verified-fetch.ts | 10 --------
3 files changed, 6 insertions(+), 32 deletions(-)
diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md
index 361ea7d5..856f393a 100644
--- a/packages/verified-fetch/README.md
+++ b/packages/verified-fetch/README.md
@@ -89,9 +89,10 @@ This library intends to support the following methods of fetching web3 content f
1. IPFS protocol: `ipfs://` & `ipfs://`
2. IPNS protocol: `ipns://` & `ipns://` & `ipns://`
3. CID instances: An actual CID instance `CID.parse('bafy...')`
-4. CID strings: A CID string `bafy...`
-As well as support for pathing & params for all of the above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453).
+As well as support for pathing & params for item 1&2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453).
+
+If you pass a CID instance, we assume you want the content for that specific CID only, and do not support pathing or params for that CID.
#### Options argument
diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts
index 049dae7f..08a47809 100644
--- a/packages/verified-fetch/src/index.ts
+++ b/packages/verified-fetch/src/index.ts
@@ -5,27 +5,10 @@
*
* You may use any supported resource argument to fetch content:
*
- * - CID string
* - CID instance
* - IPFS URL
* - IPNS URL
*
- * @example Use a CID string to fetch a text file
- *
- * ```typescript
- * import { createVerifiedFetch } from '@helia/verified-fetch'
- *
- * const verifiedFetch = await createVerifiedFetch({
- * gateways: ['mygateway.info', 'trustless-gateway.link']
- * })
- *
- * const response = await verifiedFetch('bafyFoo') // CID for some text file
- * // OR const response = await verifiedFetch('ipfs://bafy...')
- * // OR const response = await verifiedFetch('ipns://mydomain.com/path/to/file')
- * // OR const response = await verifiedFetch('https://mygateway.info/ipfs/bafyFoo')
- * const text = await response.text()
- * ```
- *
* @example Using a CID instance to fetch JSON
*
* ```typescript
@@ -33,7 +16,7 @@
* import { CID } from 'multiformats/cid'
*
* const verifiedFetch = await createVerifiedFetch({
- * gateways: ['mygateway.info', 'trustless-gateway.link']
+ * gateways: ['http://mygateway.info', 'http://trustless-gateway.link']
* })
*
* const cid = CID.parse('bafyFoo') // some image file
@@ -47,7 +30,7 @@
* import { createVerifiedFetch } from '@helia/verified-fetch'
*
* const verifiedFetch = await createVerifiedFetch({
- * gateways: ['mygateway.info', 'trustless-gateway.link']
+ * gateways: ['http://mygateway.info', 'http://trustless-gateway.link']
* })
* const response = await verifiedFetch('ipfs://bafyFoo') // CID for some image file
* const blob = await response.blob()
@@ -59,7 +42,7 @@
* import { createVerifiedFetch } from '@helia/verified-fetch'
*
* const verifiedFetch = await createVerifiedFetch({
- * gateways: ['mygateway.info', 'trustless-gateway.link']
+ * gateways: ['http://mygateway.info', 'http://trustless-gateway.link']
* })
* const response = await verifiedFetch('ipns://mydomain.com/path/to/video.mp4')
* const videoStreamReader = await response.body.getReader()
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index 1495e2a1..f2b98084 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -106,15 +106,6 @@ export class VerifiedFetch {
return { contentType, stream }
}
- // private async getHeliaModuleForCID (cid: CID) {
- // switch (cid.code) {
- // case 112: // unixfs
- // return this.unixfs
- // default:
- // return this.helia
- // }
- // }
-
// handle vnd.ipfs.ipns-record
private async handleIPNSRecord ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
return new Response('TODO: handleIPNSRecord', { status: 500 })
@@ -131,7 +122,6 @@ export class VerifiedFetch {
*/
private async handleIPLDRaw ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
const asyncIter = this.unixfs.cat(cid, { path, signal: options?.signal })
- // const asyncIter = await this.helia.blockstore.get(cid, { signal: options?.signal })
const { contentType, stream } = await this.getStreamAndContentType(asyncIter, path)
const response = new Response(stream, { status: 200 })
From 2444ca38e863924ed5ea8dd7047d5ecb55df9a6a Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 10:54:08 -0800
Subject: [PATCH 011/104] chore: change default mime-type
---
packages/verified-fetch/src/utils/get-content-type.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/verified-fetch/src/utils/get-content-type.ts b/packages/verified-fetch/src/utils/get-content-type.ts
index 31c3f47d..76a71f82 100644
--- a/packages/verified-fetch/src/utils/get-content-type.ts
+++ b/packages/verified-fetch/src/utils/get-content-type.ts
@@ -9,7 +9,7 @@ interface testInput {
type testOutput = Promise
-export const DEFAULT_MIME_TYPE = 'text/html'
+export const DEFAULT_MIME_TYPE = 'application/octet-stream'
const xmlRegex = /^(<\?xml[^>]+>)?[^<^\w]+
Date: Thu, 18 Jan 2024 10:55:47 -0800
Subject: [PATCH 012/104] chore: car & ipns-record return 501 not implemented
---
packages/verified-fetch/src/verified-fetch.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index f2b98084..d3161df8 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -108,12 +108,12 @@ export class VerifiedFetch {
// handle vnd.ipfs.ipns-record
private async handleIPNSRecord ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
- return new Response('TODO: handleIPNSRecord', { status: 500 })
+ return new Response('vnd.ipfs.ipns-record support is not implemented', { status: 501 })
}
// handle vnd.ipld.car
private async handleIPLDCar ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
- return new Response('TODO: handleIPLDCar', { status: 500 })
+ return new Response('vnd.ipld.car support is not implemented', { status: 501 })
}
/**
From 6b42abe19ece54fdabe4e4b71fd2bf93851105ab Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 11:14:25 -0800
Subject: [PATCH 013/104] chore: address header comments
---
packages/verified-fetch/src/verified-fetch.ts | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index d3161df8..7b4c4a31 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -108,12 +108,16 @@ export class VerifiedFetch {
// handle vnd.ipfs.ipns-record
private async handleIPNSRecord ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
- return new Response('vnd.ipfs.ipns-record support is not implemented', { status: 501 })
+ const response = new Response('vnd.ipfs.ipns-record support is not implemented', { status: 501 })
+ response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+ return response
}
// handle vnd.ipld.car
private async handleIPLDCar ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
- return new Response('vnd.ipld.car support is not implemented', { status: 501 })
+ const response = new Response('vnd.ipld.car support is not implemented', { status: 501 })
+ response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+ return response
}
/**
@@ -148,12 +152,12 @@ export class VerifiedFetch {
response = await this.handleIPLDRaw({ cid, path, options })
}
- response.headers.set('etag', cid.toString())
+ response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
// response.headers.set('cache-cotrol', 'public, max-age=29030400, immutable')
response.headers.set('cache-cotrol', 'no-cache') // disable caching when debugging
- response.headers.set('x-ipfs-path', path)
- response.headers.set('x-ipfs-cid', cid.toString())
- response.headers.set('x-ipfs-protocol', 'ipfs')
+ response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
+ // response.headers.set('X-Ipfs-Roots', 'TODO') // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+ // response.headers.set('Content-Disposition', `TODO`) // https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header
return response
}
From 2dd264f62ebc1f4d2df93b7b0031fd96ba530610 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 12:18:10 -0800
Subject: [PATCH 014/104] test: add fixture for importing CID content from the
network
---
packages/verified-fetch/test/fixtures/create-kubo.ts | 4 ++--
.../test/fixtures/import-content-to-kubo-node.ts | 7 +++++++
2 files changed, 9 insertions(+), 2 deletions(-)
create mode 100644 packages/verified-fetch/test/fixtures/import-content-to-kubo-node.ts
diff --git a/packages/verified-fetch/test/fixtures/create-kubo.ts b/packages/verified-fetch/test/fixtures/create-kubo.ts
index 124b9769..8573382d 100644
--- a/packages/verified-fetch/test/fixtures/create-kubo.ts
+++ b/packages/verified-fetch/test/fixtures/create-kubo.ts
@@ -1,12 +1,11 @@
import { type Controller, createController } from 'ipfsd-ctl'
-import { path as kuboPath } from 'kubo'
import * as kuboRpcClient from 'kubo-rpc-client'
export async function createKuboNode (): Promise {
return createController({
kuboRpcModule: kuboRpcClient,
- ipfsBin: kuboPath(),
test: true,
+ remote: true,
endpoint: process.env.IPFSD_SERVER,
ipfsOptions: {
config: {
@@ -18,6 +17,7 @@ export async function createKuboNode (): Promise {
}
}
},
+ // TODO: enable delegated routing
args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
})
}
diff --git a/packages/verified-fetch/test/fixtures/import-content-to-kubo-node.ts b/packages/verified-fetch/test/fixtures/import-content-to-kubo-node.ts
new file mode 100644
index 00000000..fcdad2b1
--- /dev/null
+++ b/packages/verified-fetch/test/fixtures/import-content-to-kubo-node.ts
@@ -0,0 +1,7 @@
+import type { Controller } from 'ipfsd-ctl'
+
+export async function importContentToKuboNode (kuboNode: Controller<'go'>, path: Parameters['api']['refs']>[0]): Promise['api']['refs']>> {
+ return kuboNode.api.refs(path, {
+ recursive: true
+ })
+}
From 516d896bc4bdb86facac0a4e6149fe4f88e657d5 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 12:18:42 -0800
Subject: [PATCH 015/104] test: add test for multiblock-json unixfs content
---
...nd.ipld.raw_unixfs_multiblock-json.spec.ts | 45 +++++++++++++++++++
1 file changed, 45 insertions(+)
create mode 100644 packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
diff --git a/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
new file mode 100644
index 00000000..4b985ccb
--- /dev/null
+++ b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
@@ -0,0 +1,45 @@
+import { expect } from 'aegir/chai'
+import drain from 'it-drain'
+import { CID } from 'multiformats/cid'
+import { createVerifiedFetch } from '../src/index.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { importContentToKuboNode } from './fixtures/import-content-to-kubo-node.js'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('vnd.ipld.raw - unixfs - multiblock-json', () => {
+ let controller: Controller<'go'>
+
+ beforeEach(async () => {
+ controller = await createKuboNode()
+ await controller.start()
+ })
+
+ afterEach(async () => {
+ await controller.stop()
+ })
+
+ // As of 2024-01-18, https://cloudflare-ipfs.com/ipns/tokens.uniswap.org resolves to:
+ // root: QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr
+ // child1: QmNik5N4ryNwzzXYq5hCYKGcRjAf9QtigxtiJh9o8aXXbG // partial JSON
+ // child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON
+ it('handles uniswap tokens list json', async () => {
+ // add the root node to the kubo node
+ await drain(await importContentToKuboNode(controller, '/ipfs/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'))
+
+ const verifiedFetch = await createVerifiedFetch({
+ gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+ })
+
+ const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'))
+ expect(resp).to.be.ok()
+ const jsonObj = await resp.json()
+ expect(jsonObj).to.be.ok()
+ expect(jsonObj).to.have.property('name').equal('Uniswap Labs Default')
+ expect(jsonObj).to.have.property('timestamp').equal('2023-12-13T18:25:25.830Z')
+ expect(jsonObj).to.have.property('version').to.deep.equal({ major: 11, minor: 11, patch: 0 })
+ expect(jsonObj).to.have.property('tags')
+ expect(jsonObj).to.have.property('logoURI').equal('ipfs://QmNa8mQkrNKp1WEEeGjFezDmDeodkWRevGFN8JCV7b4Xir')
+ expect(jsonObj).to.have.property('keywords').to.deep.equal(['uniswap', 'default'])
+ expect(jsonObj.tokens).to.be.an('array').of.length(767)
+ })
+})
From ccb1739b6acd91c51a9e6c0c0aa77839a2c70969 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 12:20:41 -0800
Subject: [PATCH 016/104] chore: fix types and tests
---
packages/verified-fetch/.aegir.js | 2 +-
packages/verified-fetch/package.json | 1 +
packages/verified-fetch/src/index.ts | 7 ++++++-
packages/verified-fetch/test/gateways.spec.ts | 9 +++++----
.../test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts | 1 +
5 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/packages/verified-fetch/.aegir.js b/packages/verified-fetch/.aegir.js
index d4ca8b17..48bed3fc 100644
--- a/packages/verified-fetch/.aegir.js
+++ b/packages/verified-fetch/.aegir.js
@@ -8,7 +8,7 @@ export default {
bundlesizeMax: '10kB',
},
test: {
- files: './dist/test/*.spec.js',
+ files: './dist/test/**/*.spec.js',
before: async (options) => {
const ipfsdPort = await getPort()
const ipfsdServer = await createServer({
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index e206564c..d6722b78 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -72,6 +72,7 @@
"aegir": "^42.1.0",
"ipfs-unixfs": "^11.1.2",
"ipfsd-ctl": "^13.0.0",
+ "it-drain": "^3.0.5",
"kubo": "^0.25.0",
"kubo-rpc-client": "^3.0.2"
},
diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts
index 08a47809..339d77e3 100644
--- a/packages/verified-fetch/src/index.ts
+++ b/packages/verified-fetch/src/index.ts
@@ -55,10 +55,15 @@ import { VerifiedFetch } from './verified-fetch.js'
import type { CreateVerifiedFetchWithOptions } from './interface.js'
import type { Helia, Routing } from '@helia/interface'
+export type VerifiedFetchMethod = InstanceType['fetch'] & {
+ start: InstanceType['start']
+ stop: InstanceType['stop']
+}
+
/**
* Create and return a Helia node
*/
-export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWithOptions): Promise['fetch'] & { start: InstanceType['start'], stop: InstanceType['stop'] }> {
+export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWithOptions): Promise {
let heliaInstance: null | Helia = null
if ((init as CreateVerifiedFetchWithOptions).gateways == null) {
heliaInstance = init as Helia
diff --git a/packages/verified-fetch/test/gateways.spec.ts b/packages/verified-fetch/test/gateways.spec.ts
index 85564c98..3d857340 100644
--- a/packages/verified-fetch/test/gateways.spec.ts
+++ b/packages/verified-fetch/test/gateways.spec.ts
@@ -4,6 +4,7 @@ import { createVerifiedFetch } from '../src/index.js'
import { addContentToKuboNode } from './fixtures/add-content-to-kubo-node.js'
import { createKuboNode } from './fixtures/create-kubo.js'
import type { Controller } from 'ipfsd-ctl'
+import type { CID } from 'multiformats/cid'
describe('verified-fetch gateways', () => {
let controller: Controller<'go'>
@@ -21,10 +22,9 @@ describe('verified-fetch gateways', () => {
gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
})
const givenString = 'hello sgtpooki from verified-fetch test'
- const content = new UnixFS({ type: 'raw', data: Buffer.from(givenString) })
- const { cid } = await addContentToKuboNode(controller, content.marshal())
+ const content = new UnixFS({ type: 'raw', data: (new TextEncoder()).encode(givenString) })
+ const { cid } = await addContentToKuboNode(controller, content.marshal()) as { cid: CID }
expect(cid).to.be.ok()
- // @ts-expect-error - todo fix types
const resp = await verifiedFetch(cid)
expect(resp).to.be.ok()
const text = await resp.text() // this currently has UnixFS data in it, and should not when returned from verified-fetch
@@ -32,8 +32,9 @@ describe('verified-fetch gateways', () => {
// the below commented lines will get the test to pass, but we need to move this into verified fetch
// const marshalledResponseData = await resp.arrayBuffer()
// const encodedText = UnixFS.unmarshal(new Uint8Array(marshalledResponseData)).data
- // const text = textDecoder.decode(encodedText)
+ // const text = (new TextDecoder()).decode(encodedText)
expect(text).to.equal(givenString)
+ await verifiedFetch.stop()
})
})
diff --git a/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
index 4b985ccb..a129f017 100644
--- a/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
+++ b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
@@ -41,5 +41,6 @@ describe('vnd.ipld.raw - unixfs - multiblock-json', () => {
expect(jsonObj).to.have.property('logoURI').equal('ipfs://QmNa8mQkrNKp1WEEeGjFezDmDeodkWRevGFN8JCV7b4Xir')
expect(jsonObj).to.have.property('keywords').to.deep.equal(['uniswap', 'default'])
expect(jsonObj.tokens).to.be.an('array').of.length(767)
+ await verifiedFetch.stop()
})
})
From 44c4c204a6ed56eb72bf5ae056778cfb27907c81 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 12:39:04 -0800
Subject: [PATCH 017/104] fix: unmarshal unixfs data with transform stream
---
.../src/utils/get-unixfs-transform-stream.ts | 17 +++++++++++++++++
packages/verified-fetch/src/verified-fetch.ts | 12 +++++++++---
...ec.ts => vnd.ipld.raw_unixfs_string.spec.ts} | 12 +++---------
3 files changed, 29 insertions(+), 12 deletions(-)
create mode 100644 packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts
rename packages/verified-fetch/test/{gateways.spec.ts => vnd.ipld.raw_unixfs_string.spec.ts} (67%)
diff --git a/packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts b/packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts
new file mode 100644
index 00000000..c1e057ba
--- /dev/null
+++ b/packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts
@@ -0,0 +1,17 @@
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+
+const log = logger('helia:verified-fetch:transform-streams:unixfs')
+
+export const getUnixFsTransformStream = (): TransformStream => new TransformStream({
+ async transform (chunk, controller) {
+ try {
+ const unmarshalled = UnixFS.unmarshal(chunk)
+ controller.enqueue(unmarshalled.data)
+ } catch (e) {
+ log.error(e)
+ // unmarshalling failed, so just pass the chunk through
+ controller.enqueue(chunk)
+ }
+ }
+})
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index 7b4c4a31..3e97c0de 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -1,9 +1,10 @@
import { ipns, type IPNS } from '@helia/ipns'
-import { unixfs, type UnixFS } from '@helia/unixfs'
+import { unixfs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
import { logger } from '@libp2p/logger'
import { peerIdFromString } from '@libp2p/peer-id'
import { CID } from 'multiformats/cid'
import { getContentType } from './utils/get-content-type.js'
+import { getUnixFsTransformStream } from './utils/get-unixfs-transform-stream.js'
import type { ResourceType, VerifiedFetchOptions } from './interface.js'
import type { Helia } from '@helia/interface'
@@ -12,7 +13,7 @@ const log = logger('helia:verified-fetch')
export class VerifiedFetch {
private readonly helia: Helia
private readonly ipns: IPNS
- private readonly unixfs: UnixFS
+ private readonly unixfs: HeliaUnixFs
constructor (heliaInstance: Helia) {
this.helia = heliaInstance
this.ipns = ipns(heliaInstance)
@@ -125,10 +126,15 @@ export class VerifiedFetch {
* This is the default method for fetched content.
*/
private async handleIPLDRaw ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
+ // const finalFileStat = await this.unixfs.stat(cid, { path, signal: options?.signal })
+
const asyncIter = this.unixfs.cat(cid, { path, signal: options?.signal })
const { contentType, stream } = await this.getStreamAndContentType(asyncIter, path)
+ // now we need to pipe the stream through a transform to unmarshal unixfs data
+
+ const readable = stream.pipeThrough(getUnixFsTransformStream())
- const response = new Response(stream, { status: 200 })
+ const response = new Response(readable, { status: 200 })
response.headers.set('content-type', contentType)
return response
diff --git a/packages/verified-fetch/test/gateways.spec.ts b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts
similarity index 67%
rename from packages/verified-fetch/test/gateways.spec.ts
rename to packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts
index 3d857340..6d3319eb 100644
--- a/packages/verified-fetch/test/gateways.spec.ts
+++ b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts
@@ -6,7 +6,7 @@ import { createKuboNode } from './fixtures/create-kubo.js'
import type { Controller } from 'ipfsd-ctl'
import type { CID } from 'multiformats/cid'
-describe('verified-fetch gateways', () => {
+describe('vnd.ipld.raw - unixfs - string', () => {
let controller: Controller<'go'>
beforeEach(async () => {
controller = await createKuboNode()
@@ -17,7 +17,7 @@ describe('verified-fetch gateways', () => {
await controller.stop()
})
- it('Uses the provided gateway', async () => {
+ it('Can return a raw unixfs string', async () => {
const verifiedFetch = await createVerifiedFetch({
gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
})
@@ -27,13 +27,7 @@ describe('verified-fetch gateways', () => {
expect(cid).to.be.ok()
const resp = await verifiedFetch(cid)
expect(resp).to.be.ok()
- const text = await resp.text() // this currently has UnixFS data in it, and should not when returned from verified-fetch
-
- // the below commented lines will get the test to pass, but we need to move this into verified fetch
- // const marshalledResponseData = await resp.arrayBuffer()
- // const encodedText = UnixFS.unmarshal(new Uint8Array(marshalledResponseData)).data
- // const text = (new TextDecoder()).decode(encodedText)
-
+ const text = await resp.text()
expect(text).to.equal(givenString)
await verifiedFetch.stop()
})
From 22cadfeb7c143c351df6e3244959d377866338b4 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 14:52:29 -0800
Subject: [PATCH 018/104] chore: many updates
- add parseUrlString function and tests
- stat cid+path then cat final CID
---
packages/verified-fetch/package.json | 5 +-
.../src/utils/parse-url-string.ts | 74 +++++++++++++++++++
packages/verified-fetch/src/verified-fetch.ts | 70 ++++--------------
.../test/fixtures/create-kubo.ts | 3 +
.../test/parse-url-string.spec.ts | 29 ++++++++
...nd.ipld.raw_unixfs_multiblock-json.spec.ts | 14 ++--
.../test/vnd.ipld.raw_unixfs_string.spec.ts | 25 +++++--
7 files changed, 150 insertions(+), 70 deletions(-)
create mode 100644 packages/verified-fetch/src/utils/parse-url-string.ts
create mode 100644 packages/verified-fetch/test/parse-url-string.spec.ts
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index d6722b78..c697ce36 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -63,6 +63,7 @@
"@helia/unixfs": "next",
"@libp2p/logger": "^4.0.5",
"@libp2p/peer-id": "^4.0.5",
+ "ipfs-unixfs": "^11.1.2",
"mime-types": "^2.1.35",
"multiformats": "^13.0.0"
},
@@ -70,11 +71,11 @@
"@types/mime-types": "^2.1.4",
"@types/sinon": "^17.0.2",
"aegir": "^42.1.0",
- "ipfs-unixfs": "^11.1.2",
"ipfsd-ctl": "^13.0.0",
"it-drain": "^3.0.5",
"kubo": "^0.25.0",
- "kubo-rpc-client": "^3.0.2"
+ "kubo-rpc-client": "^3.0.2",
+ "sinon-ts": "^2.0.0"
},
"browser": {
"node:buffer": false,
diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts
new file mode 100644
index 00000000..647a1449
--- /dev/null
+++ b/packages/verified-fetch/src/utils/parse-url-string.ts
@@ -0,0 +1,74 @@
+import { type IPNS } from '@helia/ipns'
+import { logger } from '@libp2p/logger'
+import { peerIdFromString } from '@libp2p/peer-id'
+import { CID } from 'multiformats/cid'
+
+const log = logger('helia:verified-fetch:parse-url-string')
+
+export interface ParsedUrlStringResults {
+ protocol: string
+ path: string
+ cid: CID
+}
+
+export interface ParseUrlStringOptions {
+ urlString: string
+ ipns: IPNS
+}
+
+/**
+ * A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties.
+ */
+export async function parseUrlString ({ urlString, ipns }: ParseUrlStringOptions): Promise {
+ const url = new URL(urlString)
+ const protocol = url.protocol.slice(0, -1)
+ let hostnameRecognized = true
+ if (url.pathname.slice(0, 2) === '//') {
+ // Browser and NodeJS URL parser handles `ipfs://` URL hostnames differently.
+ hostnameRecognized = false
+ }
+ const urlPathParts = hostnameRecognized ? url.pathname.slice(1).split('/') : url.pathname.slice(2).split('/')
+ const cidOrPeerIdOrDnsLink = hostnameRecognized ? url.hostname : urlPathParts.shift() as string
+
+ const remainderPath = urlPathParts.map(decodeURIComponent).join('/')
+ const path = remainderPath.length > 0 ? remainderPath : ''
+
+ let cid: CID | null = null
+ if (protocol === 'ipfs') {
+ try {
+ cid = CID.parse(cidOrPeerIdOrDnsLink)
+ } catch (err) {
+ log.error(err)
+ throw new TypeError('Invalid CID for ipfs:// URL')
+ }
+ } else if (protocol === 'ipns') {
+ if (cidOrPeerIdOrDnsLink.includes('.')) {
+ try {
+ cid = await ipns.resolveDns(cidOrPeerIdOrDnsLink)
+ } catch (err) {
+ log.error(err)
+ throw new TypeError('Invalid DNSLink for ipns:// URL')
+ }
+ }
+
+ try {
+ const peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
+ cid = await ipns.resolve(peerId)
+ } catch (err) {
+ log.error(err)
+ // ignore non PeerId
+ }
+ } else {
+ throw new TypeError('Invalid protocol for URL. Please use ipfs:// or ipns:// URLs only.')
+ }
+
+ if (cid == null) {
+ throw new TypeError(`Invalid resource. Cannot determine CID from URL: ${urlString}`)
+ }
+
+ return {
+ protocol,
+ cid,
+ path
+ }
+}
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index 3e97c0de..d9e6cd63 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -1,10 +1,10 @@
import { ipns, type IPNS } from '@helia/ipns'
import { unixfs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
import { logger } from '@libp2p/logger'
-import { peerIdFromString } from '@libp2p/peer-id'
import { CID } from 'multiformats/cid'
import { getContentType } from './utils/get-content-type.js'
import { getUnixFsTransformStream } from './utils/get-unixfs-transform-stream.js'
+import { parseUrlString } from './utils/parse-url-string.js'
import type { ResourceType, VerifiedFetchOptions } from './interface.js'
import type { Helia } from '@helia/interface'
@@ -27,57 +27,18 @@ export class VerifiedFetch {
*/
private async parseResource (resource: ResourceType): Promise<{ cid: CID, path: string, protocol?: string }> {
if (typeof resource === 'string') {
- // either an `ipfs://` or `ipns://` URL
- const url = new URL(resource)
- const protocol = url.protocol.slice(0, -1)
- const urlPathParts = url.pathname.slice(2).split('/')
- const cidOrPeerIdOrDnsLink = urlPathParts[0]
- const path = urlPathParts.slice(1).join('/')
- try {
- const cid = CID.parse(cidOrPeerIdOrDnsLink)
- return {
- cid,
- path,
- protocol
- }
- } catch (err) {
- log.error(err)
- // ignore non-CID
- }
-
- try {
- const cid = await this.ipns.resolveDns(cidOrPeerIdOrDnsLink)
- return {
- cid,
- path,
- protocol
- }
- } catch (err) {
- log.error(err)
- // ignore non DNSLink
- }
-
- try {
- const peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
- const cid = await this.ipns.resolve(peerId)
- return {
- cid,
- path,
- protocol
- }
- } catch (err) {
- log.error(err)
- // ignore non PeerId
- }
- throw new Error(`Invalid resource. Cannot determine CID from resource: ${resource}`)
+ return parseUrlString({ urlString: resource, ipns: this.ipns })
}
-
- // an actual CID
- return {
- cid: resource,
- protocol: 'ipfs',
- path: ''
+ const cid = CID.asCID(resource)
+ if (cid != null) {
+ // an actual CID
+ return {
+ cid,
+ protocol: 'ipfs',
+ path: ''
+ }
}
+ throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`)
}
private async getStreamAndContentType (iterator: AsyncIterable, path: string): Promise<{ contentType: string, stream: ReadableStream }> {
@@ -126,14 +87,13 @@ export class VerifiedFetch {
* This is the default method for fetched content.
*/
private async handleIPLDRaw ({ cid, path, options }: { cid: CID, path: string, options?: VerifiedFetchOptions }): Promise {
- // const finalFileStat = await this.unixfs.stat(cid, { path, signal: options?.signal })
-
- const asyncIter = this.unixfs.cat(cid, { path, signal: options?.signal })
- const { contentType, stream } = await this.getStreamAndContentType(asyncIter, path)
+ // const asyncIter = this.unixfs.cat(cid, { path, signal: options?.signal })
+ const stat = await this.unixfs.stat(cid, { path, signal: options?.signal })
+ const asyncIter = this.unixfs.cat(stat.cid, { signal: options?.signal })
// now we need to pipe the stream through a transform to unmarshal unixfs data
+ const { contentType, stream } = await this.getStreamAndContentType(asyncIter, path)
const readable = stream.pipeThrough(getUnixFsTransformStream())
-
const response = new Response(readable, { status: 200 })
response.headers.set('content-type', contentType)
diff --git a/packages/verified-fetch/test/fixtures/create-kubo.ts b/packages/verified-fetch/test/fixtures/create-kubo.ts
index 8573382d..61177e6a 100644
--- a/packages/verified-fetch/test/fixtures/create-kubo.ts
+++ b/packages/verified-fetch/test/fixtures/create-kubo.ts
@@ -7,6 +7,9 @@ export async function createKuboNode (): Promise {
test: true,
remote: true,
endpoint: process.env.IPFSD_SERVER,
+ // env: {
+ // IPFS_PATH: './tmp/kubo'
+ // },
ipfsOptions: {
config: {
Addresses: {
diff --git a/packages/verified-fetch/test/parse-url-string.spec.ts b/packages/verified-fetch/test/parse-url-string.spec.ts
new file mode 100644
index 00000000..c72491bb
--- /dev/null
+++ b/packages/verified-fetch/test/parse-url-string.spec.ts
@@ -0,0 +1,29 @@
+import { expect } from 'aegir/chai'
+import { stubInterface } from 'sinon-ts'
+import { parseUrlString } from '../src/utils/parse-url-string.js'
+import type { IPNS } from '@helia/ipns'
+
+describe('parseUrlString', () => {
+ describe('ipfs:// URLs', () => {
+ it('can parse a URL with CID only', async () => {
+ const ipns = stubInterface({})
+ const result = await parseUrlString({
+ urlString: 'ipfs://QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr',
+ ipns
+ })
+ expect(result.protocol).to.equal('ipfs')
+ expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ expect(result.path).to.equal('')
+ })
+ it('can parse URL with CID+path', async () => {
+ const ipns = stubInterface({})
+ const result = await parseUrlString({
+ urlString: 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
+ ipns
+ })
+ expect(result.protocol).to.equal('ipfs')
+ expect(result.cid.toString()).to.equal('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm')
+ expect(result.path).to.equal('1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt')
+ })
+ })
+})
diff --git a/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
index a129f017..7929de51 100644
--- a/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
+++ b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_multiblock-json.spec.ts
@@ -8,14 +8,19 @@ import type { Controller } from 'ipfsd-ctl'
describe('vnd.ipld.raw - unixfs - multiblock-json', () => {
let controller: Controller<'go'>
+ let verifiedFetch: Awaited>
- beforeEach(async () => {
+ before(async () => {
controller = await createKuboNode()
await controller.start()
+ verifiedFetch = await createVerifiedFetch({
+ gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+ })
})
- afterEach(async () => {
+ after(async () => {
await controller.stop()
+ await verifiedFetch.stop()
})
// As of 2024-01-18, https://cloudflare-ipfs.com/ipns/tokens.uniswap.org resolves to:
@@ -26,10 +31,6 @@ describe('vnd.ipld.raw - unixfs - multiblock-json', () => {
// add the root node to the kubo node
await drain(await importContentToKuboNode(controller, '/ipfs/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'))
- const verifiedFetch = await createVerifiedFetch({
- gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
- })
-
const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'))
expect(resp).to.be.ok()
const jsonObj = await resp.json()
@@ -41,6 +42,5 @@ describe('vnd.ipld.raw - unixfs - multiblock-json', () => {
expect(jsonObj).to.have.property('logoURI').equal('ipfs://QmNa8mQkrNKp1WEEeGjFezDmDeodkWRevGFN8JCV7b4Xir')
expect(jsonObj).to.have.property('keywords').to.deep.equal(['uniswap', 'default'])
expect(jsonObj.tokens).to.be.an('array').of.length(767)
- await verifiedFetch.stop()
})
})
diff --git a/packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts
index 6d3319eb..685b0ac5 100644
--- a/packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts
+++ b/packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts
@@ -1,26 +1,31 @@
import { expect } from 'aegir/chai'
import { UnixFS } from 'ipfs-unixfs'
+import drain from 'it-drain'
import { createVerifiedFetch } from '../src/index.js'
import { addContentToKuboNode } from './fixtures/add-content-to-kubo-node.js'
import { createKuboNode } from './fixtures/create-kubo.js'
+import { importContentToKuboNode } from './fixtures/import-content-to-kubo-node.js'
import type { Controller } from 'ipfsd-ctl'
import type { CID } from 'multiformats/cid'
describe('vnd.ipld.raw - unixfs - string', () => {
let controller: Controller<'go'>
- beforeEach(async () => {
+ let verifiedFetch: Awaited>
+ before(async () => {
controller = await createKuboNode()
await controller.start()
+
+ verifiedFetch = await createVerifiedFetch({
+ gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+ })
})
- afterEach(async () => {
+ after(async () => {
await controller.stop()
+ await verifiedFetch.stop()
})
it('Can return a raw unixfs string', async () => {
- const verifiedFetch = await createVerifiedFetch({
- gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
- })
const givenString = 'hello sgtpooki from verified-fetch test'
const content = new UnixFS({ type: 'raw', data: (new TextEncoder()).encode(givenString) })
const { cid } = await addContentToKuboNode(controller, content.marshal()) as { cid: CID }
@@ -29,6 +34,14 @@ describe('vnd.ipld.raw - unixfs - string', () => {
expect(resp).to.be.ok()
const text = await resp.text()
expect(text).to.equal(givenString)
- await verifiedFetch.stop()
+ })
+
+ it('Can return a string for unixfs pathed data', async () => {
+ const ipfsUrl = 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt'
+ await drain(await importContentToKuboNode(controller, '/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt'))
+ const resp = await verifiedFetch(ipfsUrl)
+ expect(resp).to.be.ok()
+ const text = await resp.text()
+ expect(text).to.equal('Don\'t we all.')
})
})
From 16a26299fc53ad3c90d9c289d66bdf9ba727b00a Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 18 Jan 2024 16:02:48 -0800
Subject: [PATCH 019/104] fix: url parsing in firefox
---
.../src/utils/parse-url-string.ts | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts
index 647a1449..c2c4d097 100644
--- a/packages/verified-fetch/src/utils/parse-url-string.ts
+++ b/packages/verified-fetch/src/utils/parse-url-string.ts
@@ -16,22 +16,17 @@ export interface ParseUrlStringOptions {
ipns: IPNS
}
+const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/$]+)\/?(?[^$^?]*)/
+
/**
* A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties.
*/
export async function parseUrlString ({ urlString, ipns }: ParseUrlStringOptions): Promise {
- const url = new URL(urlString)
- const protocol = url.protocol.slice(0, -1)
- let hostnameRecognized = true
- if (url.pathname.slice(0, 2) === '//') {
- // Browser and NodeJS URL parser handles `ipfs://` URL hostnames differently.
- hostnameRecognized = false
+ const match = urlString.match(URL_REGEX)
+ if (match == null || match.groups == null) {
+ throw new TypeError(`Invalid URL: ${urlString}`)
}
- const urlPathParts = hostnameRecognized ? url.pathname.slice(1).split('/') : url.pathname.slice(2).split('/')
- const cidOrPeerIdOrDnsLink = hostnameRecognized ? url.hostname : urlPathParts.shift() as string
-
- const remainderPath = urlPathParts.map(decodeURIComponent).join('/')
- const path = remainderPath.length > 0 ? remainderPath : ''
+ const { protocol, cidOrPeerIdOrDnsLink, path } = match.groups
let cid: CID | null = null
if (protocol === 'ipfs') {
From ade40b2bd02a75ce8df80cbe2d25a262281516ac Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 19 Jan 2024 12:48:07 -0800
Subject: [PATCH 020/104] test: test than IPNS:// urls are handled properly
---
.../src/utils/parse-url-string.ts | 19 +++++++-----
.../test/parse-url-string.spec.ts | 30 ++++++++++++++++++-
2 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts
index c2c4d097..75e79030 100644
--- a/packages/verified-fetch/src/utils/parse-url-string.ts
+++ b/packages/verified-fetch/src/utils/parse-url-string.ts
@@ -38,20 +38,23 @@ export async function parseUrlString ({ urlString, ipns }: ParseUrlStringOptions
}
} else if (protocol === 'ipns') {
if (cidOrPeerIdOrDnsLink.includes('.')) {
+ log.trace('Attempting to resolve DNSLink for %s', cidOrPeerIdOrDnsLink)
try {
cid = await ipns.resolveDns(cidOrPeerIdOrDnsLink)
+ log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
} catch (err) {
log.error(err)
throw new TypeError('Invalid DNSLink for ipns:// URL')
}
- }
-
- try {
- const peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
- cid = await ipns.resolve(peerId)
- } catch (err) {
- log.error(err)
- // ignore non PeerId
+ } else {
+ log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
+ try {
+ const peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
+ cid = await ipns.resolve(peerId)
+ log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
+ } catch (err) {
+ log.error(err)
+ }
}
} else {
throw new TypeError('Invalid protocol for URL. Please use ipfs:// or ipns:// URLs only.')
diff --git a/packages/verified-fetch/test/parse-url-string.spec.ts b/packages/verified-fetch/test/parse-url-string.spec.ts
index c72491bb..832d7b97 100644
--- a/packages/verified-fetch/test/parse-url-string.spec.ts
+++ b/packages/verified-fetch/test/parse-url-string.spec.ts
@@ -1,10 +1,11 @@
import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
import { stubInterface } from 'sinon-ts'
import { parseUrlString } from '../src/utils/parse-url-string.js'
import type { IPNS } from '@helia/ipns'
describe('parseUrlString', () => {
- describe('ipfs:// URLs', () => {
+ describe('ipfs:// URLs', () => {
it('can parse a URL with CID only', async () => {
const ipns = stubInterface({})
const result = await parseUrlString({
@@ -26,4 +27,31 @@ describe('parseUrlString', () => {
expect(result.path).to.equal('1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt')
})
})
+
+ describe('ipns:// URLs', () => {
+ const ipns = stubInterface({
+ resolveDns: async (dnsLink: string) => {
+ expect(dnsLink).to.equal('mydomain.com')
+ return CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ }
+ })
+ it('can parse a URL with DNSLinkDomain only', async () => {
+ const result = await parseUrlString({
+ urlString: 'ipns://mydomain.com',
+ ipns
+ })
+ expect(result.protocol).to.equal('ipns')
+ expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ expect(result.path).to.equal('')
+ })
+ it('can parse a URL with DNSLinkDomain+path', async () => {
+ const result = await parseUrlString({
+ urlString: 'ipns://mydomain.com/some/path/to/file.txt',
+ ipns
+ })
+ expect(result.protocol).to.equal('ipns')
+ expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ expect(result.path).to.equal('some/path/to/file.txt')
+ })
+ })
})
From 4cc599a3c818323bec1a17293fc8a5d95a4e8b7b Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 19 Jan 2024 13:03:15 -0800
Subject: [PATCH 021/104] test: peerId urls
---
.../test/parse-url-string.spec.ts | 49 +++++++++++++++++--
1 file changed, 44 insertions(+), 5 deletions(-)
diff --git a/packages/verified-fetch/test/parse-url-string.spec.ts b/packages/verified-fetch/test/parse-url-string.spec.ts
index 832d7b97..b97184cf 100644
--- a/packages/verified-fetch/test/parse-url-string.spec.ts
+++ b/packages/verified-fetch/test/parse-url-string.spec.ts
@@ -1,3 +1,5 @@
+import { type PeerId } from '@libp2p/interface'
+import { createEd25519PeerId } from '@libp2p/peer-id-factory'
import { expect } from 'aegir/chai'
import { CID } from 'multiformats/cid'
import { stubInterface } from 'sinon-ts'
@@ -29,12 +31,16 @@ describe('parseUrlString', () => {
})
describe('ipns:// URLs', () => {
- const ipns = stubInterface({
- resolveDns: async (dnsLink: string) => {
- expect(dnsLink).to.equal('mydomain.com')
- return CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
- }
+ let ipns: IPNS
+ before(async () => {
+ ipns = stubInterface({
+ resolveDns: async (dnsLink: string) => {
+ expect(dnsLink).to.equal('mydomain.com')
+ return CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ }
+ })
})
+
it('can parse a URL with DNSLinkDomain only', async () => {
const result = await parseUrlString({
urlString: 'ipns://mydomain.com',
@@ -54,4 +60,37 @@ describe('parseUrlString', () => {
expect(result.path).to.equal('some/path/to/file.txt')
})
})
+
+ describe('ipns:// URLs', () => {
+ let ipns: IPNS
+ let testPeerId: PeerId
+ before(async () => {
+ testPeerId = await createEd25519PeerId()
+ ipns = stubInterface({
+ resolve: async (peerId: PeerId) => {
+ expect(peerId.toString()).to.equal(testPeerId.toString())
+ return CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ }
+ })
+ })
+
+ it('can parse a URL with PeerId only', async () => {
+ const result = await parseUrlString({
+ urlString: `ipns://${testPeerId.toString()}`,
+ ipns
+ })
+ expect(result.protocol).to.equal('ipns')
+ expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ expect(result.path).to.equal('')
+ })
+ it('can parse a URL with PeerId+path', async () => {
+ const result = await parseUrlString({
+ urlString: `ipns://${testPeerId.toString()}/some/path/to/file.txt`,
+ ipns
+ })
+ expect(result.protocol).to.equal('ipns')
+ expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')
+ expect(result.path).to.equal('some/path/to/file.txt')
+ })
+ })
})
From ff25ff153cb15a43f596b4cbc998a99733534d2e Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 19 Jan 2024 13:09:08 -0800
Subject: [PATCH 022/104] feat: support custom modules in verifiedFetch
constructor
---
packages/verified-fetch/src/index.ts | 2 +-
packages/verified-fetch/src/verified-fetch.ts | 28 +++++++++++++++----
2 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts
index 339d77e3..d5bfce15 100644
--- a/packages/verified-fetch/src/index.ts
+++ b/packages/verified-fetch/src/index.ts
@@ -83,7 +83,7 @@ export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWith
})
}
- const verifiedFetchInstance = new VerifiedFetch(heliaInstance)
+ const verifiedFetchInstance = new VerifiedFetch({ helia: heliaInstance })
async function verifiedFetch (...args: Parameters): ReturnType {
return verifiedFetchInstance.fetch(...args)
}
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index d9e6cd63..57bdc615 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -1,5 +1,6 @@
-import { ipns, type IPNS } from '@helia/ipns'
-import { unixfs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
+import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
+import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers'
+import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
import { logger } from '@libp2p/logger'
import { CID } from 'multiformats/cid'
import { getContentType } from './utils/get-content-type.js'
@@ -10,14 +11,29 @@ import type { Helia } from '@helia/interface'
const log = logger('helia:verified-fetch')
+interface VerifiedFetchConstructorOptions {
+ helia: Helia
+ ipns?: IPNS
+ unixfs?: HeliaUnixFs
+}
export class VerifiedFetch {
private readonly helia: Helia
private readonly ipns: IPNS
private readonly unixfs: HeliaUnixFs
- constructor (heliaInstance: Helia) {
- this.helia = heliaInstance
- this.ipns = ipns(heliaInstance)
- this.unixfs = unixfs(heliaInstance)
+ constructor ({ helia, ipns, unixfs }: VerifiedFetchConstructorOptions) {
+ this.helia = helia
+ this.ipns = ipns ?? heliaIpns(helia, {
+ resolvers: [
+ dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),
+ dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),
+ dnsOverHttps('https://cloudflare-dns.com/dns-query'),
+ dnsOverHttps('https://dns.google/dns-query'),
+ dnsJsonOverHttps('https://dns.google/resolve'),
+ dnsOverHttps('https://dns.quad9.net/dns-query')
+ ]
+ })
+ this.unixfs = unixfs ?? heliaUnixFs(helia)
+ log.trace('created VerifiedFetch instance')
}
/**
From 3f347d8718cd07e3684134fd36342491670b6780 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 19 Jan 2024 13:11:29 -0800
Subject: [PATCH 023/104] fix: dep-check
---
packages/verified-fetch/package.json | 2 ++
1 file changed, 2 insertions(+)
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index c697ce36..ce1075d1 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -68,6 +68,8 @@
"multiformats": "^13.0.0"
},
"devDependencies": {
+ "@libp2p/interface": "^1.1.2",
+ "@libp2p/peer-id-factory": "^4.0.5",
"@types/mime-types": "^2.1.4",
"@types/sinon": "^17.0.2",
"aegir": "^42.1.0",
From b11db65117719f27e3146bdbd6e70b29cb48c8dc Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Mon, 22 Jan 2024 08:19:16 -0800
Subject: [PATCH 024/104] chore: change unixfs stream log from error to trace
---
.../verified-fetch/src/utils/get-unixfs-transform-stream.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts b/packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts
index c1e057ba..aabff48b 100644
--- a/packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts
+++ b/packages/verified-fetch/src/utils/get-unixfs-transform-stream.ts
@@ -9,7 +9,7 @@ export const getUnixFsTransformStream = (): TransformStream
Date: Mon, 22 Jan 2024 11:13:12 -0800
Subject: [PATCH 025/104] chore: move verified-fetch interop tests
---
...Jp4iAqCr-tokens.uniswap.org-2024-01-18.car | Bin 0 -> 363041 bytes
...xH23mcZURwPHjGv-helia-identify-website.car | Bin 0 -> 8115781 bytes
...nd.ipld.raw_unixfs_multiblock-json.spec.ts | 12 +--
.../vnd.ipld.raw_unixfs_string.spec.ts | 70 ++++++++++++++++++
.../vnd.ipld.raw_unixfs_websites.spec.ts | 70 ++++++++++++++++++
packages/verified-fetch/.aegir.js | 40 +---------
packages/verified-fetch/src/interface.ts | 1 +
packages/verified-fetch/src/verified-fetch.ts | 44 ++++++++++-
.../test/fixtures/add-content-to-kubo-node.ts | 8 --
.../test/fixtures/create-kubo.ts | 26 -------
.../fixtures/import-content-to-kubo-node.ts | 7 --
.../test/vnd.ipld.raw_unixfs_string.spec.ts | 47 ------------
12 files changed, 190 insertions(+), 135 deletions(-)
create mode 100644 packages/interop/src/fixtures/data/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car
create mode 100644 packages/interop/src/fixtures/data/QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car
rename packages/{verified-fetch/test => interop/src/verified-fetch}/vnd.ipld.raw_unixfs_multiblock-json.spec.ts (83%)
create mode 100644 packages/interop/src/verified-fetch/vnd.ipld.raw_unixfs_string.spec.ts
create mode 100644 packages/interop/src/verified-fetch/vnd.ipld.raw_unixfs_websites.spec.ts
delete mode 100644 packages/verified-fetch/test/fixtures/add-content-to-kubo-node.ts
delete mode 100644 packages/verified-fetch/test/fixtures/create-kubo.ts
delete mode 100644 packages/verified-fetch/test/fixtures/import-content-to-kubo-node.ts
delete mode 100644 packages/verified-fetch/test/vnd.ipld.raw_unixfs_string.spec.ts
diff --git a/packages/interop/src/fixtures/data/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car b/packages/interop/src/fixtures/data/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car
new file mode 100644
index 0000000000000000000000000000000000000000..a1279473ea89a937d40fbb77f0ee1ebf77267546
GIT binary patch
literal 363041
zcmd44ON=B*n%}o1Ac9x|(rN(-dl8UmVg){wU2gui)nVb!>YA?CR8{wMuO10=GxMsL
z%FM`#%{J?ce`j|BL^w
z{YU@m>UaP2NdC9~|NbBT)4%=p2fMmlEYJVKpZtZ-
z|KZ>M8^8I*Zy)_j|KOkf@qhhK{v^@=*T4M-f9J3K&gvij#(#eH*WUam|GxU$|HJ?H
z5C6;m=kNdaZ~V^R`J>;u`LjP}|INSh5B~ap{L9r}{lEXmFaPlTAOBB(|F8TX_5bq!
z{s;frzxGf6&42zk|J`r+v;Wb5`j>w5U;fK~@pu08@BZsYfAmMcb@Zoy_b>e;{`Jo;
z^P7M2N5A!#|Ke}+ul(U}9vyvguFmWizjyS-H|LAXAJxUtQ`KA^m9|yar&nM6TkcO>
zEza!al{&j{f6Vv%;8^&_B6uUx-}58>`(+wYLi;eM6-
zU~xYE;6h#Lcl-&DpZ~3!_0-!-_Zx2h>2a`OT{*eDKBMqP_gRM~{Af_FScBKYzb^
z{`H0UMOE3yzpU-A%d^rh&HG=xvOheopQk@7zD=6&yT$6}!QR`CKQ33s{oa4)=0UD+
zzVG_SJeS#gV}AJW-8{wey*(fP{LS~_59)VnasI^ci#A`0&y+DMd+Gl07v2Xk{3kJm
zr&N+QN#eCxoFzf(1)=c0BJljp%KYoy7Kp_;uj1xK@$u!yvu1hfeyMo!y!dMPfw6jV
zrcT{@BJKb9=taG|y1H<0N?l&stIIF-a&i9F>i0|j?(C8Kr^`qDg17eakqE+AKDv5$
zebzi$FMoXHzW0|G=Wl;s#9qqtdVV@AZ9iP`Rxdwkvyjw&5w#|1yrhy*Re51pmbOt|B@-3LQ7uA0kYRiK7fQn@n6r|O
zTC0md`Lb@JB#%5lw3SYk&+j*3>$bIi
z$agIALz^G7(THCyC)It{ZV&P-yZmO&UKO|7F$2SMeRrK;p`F1jO}sUPg{
z>2QLx?AvPie4hu{kHctlfZZb9$;C`V!P)EG!=X-Y64!Yk+9p(O6thp6AKAdybrA-o
zO++FiPfI^ZCmYp0l{>vNPj1`9LPTL0j#-qpDQ>klf=@lf
z%#&NjWi10g3!}no{U~bFNQ6=Ar?Jhv#+QEXi$Z$&?F-F5PmPL7Q;Jraq>d8LXc;ON
zOQiBVuL|;_l5ryB9bPY-d!FJzC4TC~o=kHSlrr)?%@zewY|126EvG6-<5Et(RsSj<
zc%IlP6BaTF{m4k^rBxhPQIN%L*~Uqb6*}|sirpNnwLecQ-JCA<`{5^}GnKy?KHlSz
z^0}fnCrWLWzd2P=n0P@Pg~82By0=@Yj2B7~X+LN^FAKQsuxA-XksnrhQs!mqJX~Og
zZ`HqsQ~l1KeLQ-pJ`O(|y@^|*gNFtO+#p`(TZ*qveiw)QyT07K3@KwN{CKo!_jfV#
zF6~?=Cgx&`MJ!VpM^V<6aq8o9{3Hw_FY$;{qPRSW?Pg`=d4-?)VQ8YF5=oi(RpeForme%eA<78CM480cFpK>tt0JX>OcYt-amS0P^O@WcGkv!y
zRnquD>9a{m7{_r?B{IPA1g-R?3QQW+_@U?^mLf&gdTmZ%lI2-aag@EFXu>EFUR>Dq
zbJS&6jSft2DR%XCIT*vpMygkjUkpFkqm{TTvvYE8^^Hx0Nz9d#csqJHp%NR~IAAyP
zuob=%QWjq9w}Guq>8D&YMXv2Aq3B+LdRtxJ0=v1aXi@VQgbx$;=N^{@=*KcGjMmAME+nHMhg#<>~0YAIv^`{IndbZ;uCG
zU}#;!aC-5!Io092lhE@%@Yk2Wyl`nhmucb=mkjIQ-}+}d>{!K383hKPQ>Ss@lVQfO
zUxaB>hDjsJuq4q8t=)rFHi;Bom2wN=psKozgo#R_JQ-lCgvR8XS&++OvedpzCHCWr
zT1srHJP3j&^wQjhaazkrG$s!@AA!zmvBfcFPhR}N(Q|u6`D%L<%+H67+NYs}Y({rQ
zESA4MaRsK0e-tE_0m0v}@ck`hUg!`6#!Jhza3zP7SEo{3N2KBue--+i_qK4E?lktB
z_>!5HXkFN!;CJJ~E1FuRHjeQn9+`exH+AR{xkxMbk@ndl!5Uf{TdEqqjkPaCq808U
z%%%0qin524{wB@1sc>}}7)kBMCU>lDm>ef>L|hp%Tv2+F7c^-U7Ih-ah;@@%Quiq9
zVQYJ*wN647+k@Gs#bKgZK@}tD;@QmubVeA4WV1;)Jn8+7aHes{NC%#0lO*&?KO>9@
zsp5w2+%|O~bne$u^BSkO^uBzDPdw%;KjyqyFAv;5w0JPbsVUPy
zg=JpT84^NQfgciM3??@+#;bQ$9c8PlMZ3@ob$ax4@vG~FIeKz_Wlv8RZ|%9>UOavJ
zWcYP^yy1MG)TF!NPOW>oV|U3NU#-;n<%L=iqaCj^W@|Q<>naW@);m1U!F&$2<^=Cd
z&&b_9nMy+K%fjeXM@5sSrPs3K6wfBxJh}O12kf@Sxr9MX(veefk4T&o>lyqVB~Py3
zmNr&GSbh6au30gwg;2zFbBb6~Vb4hfl&<4Y5ENAu1kys*oJJdEc{5pRU!F88B69JV
zu1re7L~+l1
z>GWImukwLspso4UB1z&%=af+b6E?;YFA;#ouF+P~T9i?-1;da56Y0M22*Hp&dpUf(
zSDWW7c31e>#j#HzgTHQ!J)%Y7N0fhurSI<=XU2a-62in`qOES~CJw1KR3TL_s%+Bb
z_F?ODn}_Bg_Zt46hD4xkEp1$qAe%CZ1!*f-gCwR+p0+_%mZMCmcUkA}w^m)V=U*SD
z=t|2W-0askY4qXEslRbd*qacNK5=h7_us2|*H{~Pt`wAoI%yi6xHeW>Qbb5o*$`8w
zL^8RT+G+aTJ=OXU&$J2Qk(~Jf8@~>%h}rX+Ha5+0uMQPeSgR6Q1k-NBWTDxPNi?mF
zOaNGdu#h5xhzuEr26{xH#5YYLtfbjEt%FXNnp<)Oy;O@Pt|V`Yik+}{*)&d*q^Jo3
z)7mdmGGjBvo6K}fy;-i*+vWLS3KM`8moptd_aDv_Gkf|6QL2CF>#Q4i6>t_h4WriK{Y`$RDf-lSyihfft+5Ld
zM_WXdPAVea7}!SO6Z=Z~atCowp`p$Nd?)2R6qNzGLpWs{%3M*9$fk+PNQS}IobkdI
zz|{sr7{yU9w};vfMZ}Q@rNN_1@O7t;)P;3uxlLI-Oz4hKQx$tD>Fqh4^qkTis6V#;lU0rVc?<&)958JorU
z^L<7d^vb)cmbzF38>1i|0J&&^6MHp>33Dh2{PFk!5JUAz
z-DfwWs6AnF;n$H@dPP}BwGCp9EG@!DYOq(*r)STFy)UEacXib)u2$Ehg5|{Xg8`xD
zkr!=h;y9!vOBJ?g8y)MG{(ysNQN&l&DY|bxjzC}K!^iXOV)H;>w##eU#8)6+A9a0@
z;n&@N%V%HgKx~RClH3$RiUQ~`00MlFtgDpJR(W)EDZn&%&9xIG3dA%{
z>Yi#HX3h&|~HAb{?O6FEa6r*(sX0m9NU&d`Rz1k=1%#0x_9FLdNjmRYX
z-jZ`GpumA|MHvS&stA(;vWFDj98O8efokTajduSR1V6H?G4W3iE@Y53I`X;2FokZK*
z)nfG;^2_l3$FF63#t?Q#XvdPSC%U>twpcml&BjOqk6b=Xe~N&@3!6&Gn!qw9{;s44
zC>X?wF0zJ}Vj31aBSogebTu>T?HpJeC!q;w;-^W}h9b%m!VTXN7Q&FAp|!2ddVRG#
zQ&*#H9PQSd!(1#Pl4)ne*_&(={PB2PCE@59?|+%hbMmv_R_||jET($KRSmYW
z0+x`(qAIL*weJvALhhF;%I~6h%*{NqLT0dJfIgVID#NTPQZI`F4InxQNJ?-8A~c=5
z+$1^u_j_PylgV6ExCbvSFg|T4t=2NfY0zgAv>zmQo%Dd-_s6_*ra6(|E&ZQwiZ@SQ
zJRkm*eR@&Q`5j64Ei;sZARzL*TnrURF_(wFE8NkE5K6X84}ntI4j-;`(y-Am5;3;j~6aPb3TMS?xfCwI*?EdF;W{?!wJS}-S0lbX_c#WX!#-*>5tj57+
z{73d+2v=b>+C5_GsNR
z8h$?7!Ed+UgAc8@@S6M%ux@`mx9_i(c^4jZS0Z1Ea>oXwac_GL0T&9XO3t$%)!H?m
z;~LbcNTMj^6@`(Cs;C{66Z*IE!Jew^_3`hX4likB%@1t!`n-$e5o^A+kG?&QqmvIm
z7yZj{r+ZlR{uVs%rU4)cpDG~22Co~&Q6`h1wq-%rT&c*bwTxSmjC~ZX{lGfZU64Z?
zBWBBExEw%9>a2ly=Wzp(V5VSsCsl^dAFnf{+1<3z)xzPAQYnBRsbbHjKo3CQB@GvQ
zl4%PXyoe6dp2>3$-Ut}2s80Q?@o3_uRcUCi*|aRlU}zD*8fb>i9Sm?`fycCq^TpM6
z=FfMphkt8ihYvoLbgsJiRDD?3lf~vC#wbv-4!8~DgyT>~nTV*QrCXNES886rM6!
zXm(}C(r=RV=%Peom!Mx1SA!S0&=+JfHwctGz&D49zTx5cG)Ww}&_#7h8|XK53nK`;
z8F<@>qX+z91WoTuGFK42S4X#?8}h7tTpgx!C;~t2E()?=9^%BPmhsYqwJ*|CZT7a`)@W}An%-#2jW>_hkJJ_hR?Idp%5GgMbx$=~k8PRx|uKA)W
zl~E99UP)Jih<)NEXGRpmrk*N-+R;@vsk=7R8Qg#9rYn$0)Ea22r+86V(S+|2cvwLC
z!HBRh#0`12{iYmwsF@wNmHP3^w+plin(Iru(o0Z-=T~b<14_o1S3lCRu~!?-gGXoT
z^2)9rt;wV7-+%eY0sL0>`mFmS-iNv0YcD0=c~wK|x)k2iCIj-GD%BJb0SXbFuBrr7
z1`twOX40Ql-X{)drUgq7eNY~95fm92ICXvliZr7FQ`UiPX)$@!1+qK3;By5E$m&z-
zpb$D-@W8Vtj?9@xmFF^j>yb}VnW^viO0&<37kWlirjP*HO_;03F9m(aq=dE~)`A6N
z6#+cl<*aP8(#a{wGeN-bp~v8e&JlEY;>sY7p*|)_8)X7Q7ymn|77WMl+}v54%d_XB
z!?nkIp6sM6Yc%lSjxRO^!kfsO&TtYCos6>J``_?0?J*TgRBIGLY$Vh
z6&@#s936YH(g)c(4^mnZ0Yy#05X6aukSZah8^3`Ig4gp5+(ja|A=VnU?sB1zvMc9{
zXb#<;KjqnB@+Xp;4W6b^Jj$`^9%ZcLvSzO1wttEYI4Eo@)24yQPII90aaxhBr~xZ_
zl+dM}ra|f7MfRP2+^O(-m}uonh2ZTGh@c&nXum9A(hxg={>yCj3Eg+%EN3SWG9@X>
zWn5)pBug=iXfl7H8&r*wo0(R>AKwl|QQ6P@tKqXq%;74HYP~0xy
zEp4g%=L_!1<=NKUA%9XFrX)j0KklF}&E^5daT*aW;Zit~PdS)ZkwQbkl%QPb2zae$
zsWw2rOVQ#qgfBLgcH#z@y>b&H+`9xMIJHobCFtBHhxJXT
zG$`RyP|Bd(OfwOEE=B%7yoBDrFRpG&b)Fo?p>yl?1+mqWlkYFCzD_rVFtDru-*GaX
zg?@4WgEQN?hNu>##>s8~Q7cotk0g%cl7zon6<$-(Z3UB@O^(jYyyLU66hRv*4wBwx
zQ3J?yWOC7DRPZVQNi_9b^C)e?ZT5V+T}L2)dLS7A-^Ib3nq9-|_dQP!Lu=;QI+8@OL=5E#7vxFMbqbtQLi=Gv*}wm;
z&8xE_#NgzA@-|8#;|5s*s;12S3aAyGr@A#{`vp|O$?@vW{kr%$Gq48FNO7@
zw8}|vA!^9bAbyq9wHWm%M)%R)(7XGuCCvR^`>7?&6Cvb+BkMMhmjhKKfN!IsmL#Jw
zfSnw{K%Zi}D~sQ5W3z9I1ok>ja@c(h7}ZihhNkbLrHM0%lthg4j!Vdllci=_*=KJH
zgaE)&`S2C#85iK=tW^}sU@}%7;v9-bPf&pq+~R?jjq1#H`02>vUgU=X6wsgT0R=gW
zbgeEL9yDHmiiBH*In8_&?Ld#hG!0{RO=gADYN8#+(^1!h2BZ$MV_v}B&N~oD+muBR
zNW$Z~ZMjN$j?53L0B;b{`sqAgFOr(8)p@>F8y$W&-me1{N}L#FpVRONH-e`f2$;Sd
zfQ3;9@&32Qyz*dR6I-XP6Zr(w=aX=9S^4yKA+hI;3WD0FR~p`t&&<3vGO)E=PgG&@
zitMtmAQc<%WvoUP%()_K(f|S{*3oU&O>b^);zaR9RFMa7k20A;5HJ)P;iH5zjO=Gx
zRv=y>z)kCe`*Ix=Q*F~`d6gi`TG5OY!rF?Iosr^HxKA
z^Xm11@&j6;9TKGJpW(}w&mXU)d&r*9l^q`GP?m_o69GzSuDHROSZDUvHv<{|3E@)ZbpDkBGyf(>icGyu!t_Mt?i
zq$8Pv`l3A=v~iWdS`Uh(7$5!KJ{IS$r*ibo>rv@;
z);P~a$wH!7R#E}fFiGHoi3Gee5@P@^oUS;k#1PVgXh-oB6m7y|gbmkU<
zR+lSia8fp9EJ;1!Im3gz-2i90LX#mkD_al=h_S*oqm~2YC>F@}G_7ns1O-vLYwqy5
zv!f_1+ksRlYa-l?z;bP%vQg9vld;6y`z5MX`y)wXtxjY
zwDSyIRRsl13tCr^S%YtmYHGhw+}qfvTjHb<1p41oL*Lg<Ej>laRcGsdg-P>
zqZ_)w&LtlHf>Z|jg$tvcpjV!Gh)Tz00q-%*lmbnXq(;}lCpDRgADnwZ!Yd(}hR#`&
zgwoS3>Pk735o}-$Xe$8oR94!*%WKSy)kIk#J%FleM3q^g8HNOR&D|KH)~~UHN)bo}
z{vD7l_oZej#e^-_D)~WX6S!uGMtepBP507TSP~W>>vTj2H75RjUuGZpz~5v9U#AXj
z9;-S)Nm7%n(A7l}jw9EQQO2VdMsN1^ba`|RQ|t$Oq#R-C=&$`^xjK8dJhgv)FuIZ3
zs-EuBg}J?efl%Ojwc`uM$JiRGp_*I7p$vea>{?%mr;O!X2=bG!wHfQ9CNLL9zwr^ktazD`V>`?Cd{_L
zP>Oep)5Wg`V;S8qUmd28Lbs=*Io>SZC7Q0C@IY1hnZBY$S|=?mbQiP=Xd9r`%FZQO
z%8imh6Q#UuG6n-oE|PuiRqCNLRip6({)CWRG+~kx3PUJ>DieEE2}-1CqiqNHbP3nu
zsJ^KWY~8b`-wq$|F%kv~mvqf9y?%Jk{Mo)bLBaL6TGpIa7viQjm2;b
zKo5#a43x(1hLY$#HK9Pe#aaSB;PaH{`=noZ3&TWzw
zszhsY3Y*%+Z^QgYn*ye9#QtSYTU#?~%OC|`Xv>fZdeh3b(G)&7S!iyyg}4$XgC>S3
zBMT{x)+zdc+Ez4~KnjAQkJs##iPxE|G&_6^_%gA{f>s;Uep5(Guyd7x{=lV6K_~+B
zCB-LFOVg$1mV=y_Mp_98=C@#kSCA=&5}{g9cmcAb0Czpqs$B@!`_E5<^4W=9<`dwf
z!W+d53`52gNLywIg~;Zr0(lDr%lsBuuE|pUvJZ^A$`W2Up@K{(NLm~>s6Iqa-C&u*
zgL=MZh0*#Nd`It9Uc4E}(2iZg`13t_YeYReb=qQiwdi9m;5bO-4-a_%m}|BH#x-Sp
z6BI}fCQWJDHdj$$@*t;QLVMT18bQ0qJ4|Z^lt7dMb_~h5k(Z$D6rf1NNJV%v8aZc7
z*0r*?6$uwBeZ9E4JX*F#->TE=tpJna#kaRm6COA=iCkra$kodq)ahDMc(ZxTF=QMK
zHH+?$?vO~$bG-CR3N#~2iZn2G(6rBk8hK2{Cn40PT^Eo34B6;m4w+NZ2QN$MtDz!A
z=LGLhz0`P2a3ZsV4CiW>qu0eLg)Nb-aWi2kDja0fxS1*J0ow9%f>;q|
zE-!V$B52EinJSq4NQmfdVJQ|XFvgd|-i)^F)%M!f15X&LU|mAGEPnJagU3%MELqC)
z%r=1Z47B9|CvE1gA^46WYVAQEaD(+Em`HR$B_d{_cIqP*NLS4W&dIKIk9EIRc}gh5
zaEO-ND?_OPicl?(Oz^@sD}2ym2=CWiY15_lb#ESUy<
zgR~{eg6T+Rt3TDNj~B497Gq(rv5~wy(7~2o2ief>2{ul6J;e`>-pzh|iY4x!`FceK
z7wqqyXf|fopoUL05mMD7WseOVyrKYz)G8G8Ks})XWyV?H_UE
zp8S%EF+g!A=ga&Ek19-mu(pN5bB5fe6;%oi=?K*c;IIJE_?%y
z#E97kVY=rTJLatzNgh>TePTQd$!$(y4Q~_JwAFClX+Be5nrzp^*yn|bDN}&L00eGZ
zfgSfCq8BOsu71|h@4FfHq}izly|id=4Q?CNyr-un`LBV
z66!vIiMt@9EU-)_^{MCs@W%c3Cka>4|9$s&H~;(TMYPTLPwH6k&&bQDK!yySgIOCG
zBoP4I!&H>W%%JNUq%iTeovQVH{Nc_LI}(@e#7>7)>#J-J^H&;$wHf<
zzv;NO>>or`Vih_e3e|$}MG}9q
zX}d)tDyuakpM8yU2!VQ}($cf^1R}1caUv1iJU)Wjlcjq1ad&voo$pdD9I_5|S0`0J
z_`yl25Cg&m#DSvKB4G&dWV>19UGzO{8jT*~?pD44>hS}Rfm7k>JHTQe{
zglTG-IjtP2bDH*^%ijPlF!vZMWAbBIBw*rqZ&2(}4@(1_r@Xul#%54kI7CK;0%Lt4uIrKz3VJQqBEA-J*<8
zmIsO+=!JB%fQ|Vuks@RcFh&2LBo6>9Qs>0|1qx_U!#iXc90P3_##v`jx$7Fj1`V8v
zm+F1v-Iw~di*$042fz)Q_NF>3wNxam0P#_wODN;esez=21b;M&qlO*Z-c>8BTS(Op
z&D!vBn?&I4Q+PW5Ut}P<;f$G1{m5$Gh|51VD@mxHV+nuuu}R
zT1c5(P==siR3q_NfW<-9=#J}qW}E^}K-b>9}fY?1ARNwon7gCnHZf3b4l?j}q~bLgJQSdd)4kME-Db{?_XE
z%e4@q`=`rCi!=4sUOu7|6?Mth`lF5Qfv2E9A3exd&a*v=tW=qR2s5D?07+FcIRuR)
z!7$@I2mO^m&-X-GPF&^0Bxc$krq=p}YC(J<%Sm1p1}!+SG`D#{Oq?;$o-5s(az!Q!
z&9*&FKnnR9nSrXY=$Q#O+|lX95&R?o{89Q<8sYTGciNZjd0Au7Do#A+6+#q5LIkbm
z6ubb*9e$>9LRTJR1&6q$bN>d)$7-Rkybr_AMkaX69q>>SOv8>%Fsy%Lg*-6v6;emT
zq(xa2p9rAKzx6ZIBUsW%MbGFYb#?crzQQp_ftGAcM+t(u`&UzRHu8|hi
z2vB+db3X63fen&!4N&_H>B%|HekO^9DUdc5fe5fO1v8Fu}K|_Av>o4bseLA_JM!c~Qc1(rL`73Pk4l9$3!s#yfMV
zPSxerb{J;$bd-4P(fzwg)*J7?S$z}SIYj{Uk~X>Cy>omrNU{bFE4unxw=wl+WIB-r
z1xY77+!t6O^rA*VNB0uSqJ%H7j^VK>8UZ59Z6d|^p_9vG)_Qs7D6
zQoaQ91#*yZm&(9l3^UKQkKj_!_$cyI`Gqz`4(TNlTA|EC1|AagL!TT>t9M?`Fx}{>
z+ebN9uMTyeB5AmZEuOx3{Nm)RCy&26`S~~5(0kTFAvk~Qzj5%mS#=>M7gx)Q+s_Oc5JU?i3$9?nJ0YEzy
zsF-yRheiLP-9+GGD$#d*vfzs@ET0)
zV%RW9>0sNnf4y4Xe(Qhp>cw{K-2;U>To<|XlP(nWwO)-3|+heL@u#`49XW(~%JS&2>|FrMqgGI}7Z~r2xhaO7c$!!PT&m _Yd
zFShZlo9Eb=2;Ea)N@%)vdvP$-LzqW}A7~mSDf5^Ti$a+eZH>hQAd%fbPV`Tb@Zn8K
zhAA+4iOG_wZo|3_Cs$ik})ic{ElE^E_eyO4-~aokJ&{aabuu2$l^nG
z5iN};Oe1a^@R`&6qK}Hr;1MyMSWWI
zzygJ`2sB~rjG)Qjgu}YBRx>jU{b5C1&oG`)WQN1teYak@xwcpO-I3Z#=Bca0wBMPR
z(@_A|%iq*FL75?^fQIMrkoS)q=B46Z0Nn)eHhqf(JPpNQklNt#Py<9W8t@2`%E&q1
z5oFGcW=U%T#%Py~wzYG-Mk5hH4;~dOO3}a~psW!%@5AYad6j~(h+~EpRCKkXg5p%z
z#5h4_-Ov&+MG+!nHHpyoWq+1t57D}Xs9e{XHhxPB7VL{EQi18y^YyGoDr(r;E}8`;
z+io|$CkINam@nI5KQ7(kqajG_45QT~6K(o)a(@$?mr#KV4HAa@q1a8#5}7K=sUs!|
zF)2e%fYTBU-OoD^sdG&*OL9qAoK|(5l|Vp>l4d!>vQnUcMVY}OcS22jae*P33oJ4&
zV2r`3c2r2c;N_WD44#EV5d#JBcn}kG%IHxGewrROT(1=T5qT+^wW_KrT_z6gh~DNF
z1W3JHU5%RaLy%m*cy$;DNTsTu8^2!tYgzO~Fa0gQ3w3e|QLYIR;JJG(fw
zEq3zm=&{oYyAct;bu7CxoROaHK5K{+U^-;%#k=Lz^2E(+IQGJ0-#_uZFWr9w?$dRm
zMU@6FLomAa4`xD<#bhD}>OqDWs!vlQ0?zn$!V#y`k62P*DyH!sWM6fH#%LV0C|)M}C!Xc99d3gK#iFA_f$5`2qz28Zb=~1DUFRpNlO4MTj+!A
zhur~k!^@~GdekauzZ_`h4dX@-Jz&r@07xI59S9#tOH;<jab?FcY*1
z7=VuH(Ksa_&Kc{DG$tgZ+mF<2OGoUBM4bqSNgE;O$E%zqQkzNI`l2SI+P(xTbS*pLnADG;N-XszdjH_$HoVg+r
z?T>)OBJ&w+f_Ep7W|{?=b>}dKUIi+?cUsdd^*i*W)8s9hbhlcs+282
zlCCZxe3wMkc1r|BlLjk~fZr-va)qug~>*{`AO$k4$j%=^jH0d}q8VB$5e1
zz6iV%B%#qJ!ClZ~A}c?^;Zl`mfTwFWlM=?^$@!)TLvbD?R!6kGcX1vc=jW~xRl=cC
z1Y!jQrdZ^OVoEZev1GzY3^IQ>lj95U==D3b8eNX#mvx|O0JmUQ4OriTm+lERDH!s*
z2>zs_lyQgn*_)#|L}CmZCt|1(rCw1bRHtC8O8jbw(kGpGm?2(t7d?KSbD$iF7(!yW
z8bE`ffznd%p!|wRE%GuTD6BHMv=3M8CR_BuJ0vxLK17W-Wmxjjj|ND@rMqsUSOmXf1iBQMT6|
zt(U9iFRi{hdJL2E!^h$14ae*8Euxzbr5X^I1RWTSHICOA^Rb)m+EqD3NP-&t1n?`6
zN|YP&hBTyv8EtY-8bM#vaw|u%na=|Sk7`0ix5&G
zq<-17DAFCM4kjXQLweGVvuYAc&o7iVbIk|kyna$?j!0le*u&0DoO#@09HT!yx~d*Z
zMnv$p^Su~Hc3ix8^89gCeEp*HzmRFTP(~KLga4gZ0z(0vLF_?kXmnXRhQOZT(@Uh#
zy`pC1B4&(mpq!rUsS}xN9URxFNRf6U8+Cd$PGch_FpHeTg^MdFmqJ9JF10TwViZH(
zVqMe$g4IRqqlSR+1QP*~@yTQ;96SqJK9xqQ#yExbHwwg$NAifLK=PsUp{Nia@
zKY3n!wXqH?gYqa`rS-eHf20CJkPN*<)~`Y>vS|vdDk9g2@r5br8*@#-I*nCkkU>Ta?F!tvT313DT&tN;6q9MNciw
zBwhug9R(E&Z-96c=A~16KgboBlqJF}MWZ|vxgsF^Kz~Zcv>>kvmX;}iQJFym90hFM
zo?*RqYa;mg>0uHuTmR_{+o&
z$l^6A6CJ?5bBb#2?ziA^zvF4fEg^%0&`px;sNi$sgg3)St~pbX)vAE@LEKC(C1rMW!%
zhMSGqYj)Hm7}?y5@<8b)`FTgih5QtV?rvl735oo}S^)FxqfU`7rLqQ=&@ios>PFbW
z3uhh#^+ejYXr80p&`wjry=@xkw~E><2pMuvzzaU>Qr1I<-U<4_t~4qH4Etv{?ae!T
z^q3iz+ryCq^=&Dp$nFYWFMhI4XgWWh!ZM_oGCb_TZsEv{0Eg$!tzqDpBe0N^+LSS(
zdLa@qC=M|b(T#NMH|oz2ujYoUFld}|5R>1tQsh30Wz6D=BV90YlIDt2ZL?`wOfURt
zWIVUZ3`9|9j=P^2#S*Bh`z@D&qER6==`H>Il)&RK5SG%SNXr~#9ik&0QmbfkH^Qt%
z%1c^@Y=Li)n_6M8plci7(#XEWa^AYLqm<
z#v#xSMmVcxC8|7QJi9_}18fJ1pGb2P`4v%779^iQTet#p?~b0{%%H`fiPuVWS=bf$
zW+<%T9Nn%9&s?V-U98};(T#bE>Dr_8h$*o
z!`tSX``t*Z`aRoY}U_k{Pz0N+*l;78@>be`{B8Eu*i9L87Zs`ReAqLB+VWm-7
zp(#B|()#WDz8dj0M!Pood{lUOAR-;omF%+Wwa)X~kM`W9lcSW9x*Uxoz`nhK?67OI
z{0OOsp%vk}5W>VNk01asUd40ZV6sYhwg8f%`i=``rXNAetybSYCNCR^r~`?=1_
z(aqEu;#cbOU3;^ATLZ-Mi=BE+C1
z42iaA?kO4;l+q!wGGiS8lSmQU2P_HGb!vh8+O5QC>?aNRQ69F6!Knf;j-$tT4IvtU
z831!`8shL8`rTT%GSJG}tKny({eqe`d_Hm>4~{)jK(d?I!(AGezru6yUq`3C5xruP
zJHvhgt}+6tQLFDz`xmug&MZD85;@PLpb3)(s+?XBQmB@pJdhYFv|Fa2!rcxwP4o)N
z7kg5MRm%``a6agGqFEGWpxrR$sL_?(n`TkJ*Dqea8g_HE%ahM{y2Bys#ocYN?!TIo
zulTKNJkSX@+93aro-MPoM;G}d0k6>r5e5+rsJ4w~LuJ8}#tnMtvVix_(0J{{x3{+v
zMn-vmE1Bv4DSPLPGW8P#r-)DMs%C&k>(`v`3I|Dh-(x70cZcxVY-kSkjv8ha11BIk
z*O?-1q5bZohRE~+s{LRjP=^2HPTQA9L8gTlpCWx(Mhq`Y*^m^%b%_Y3!ds5G9Qo8EU5Cp
zpwr6Rws)te*FW4c%XB;DNItnuZ~UOpbo=V0f!CbcxQ!FeT63uAaY&}o2<
zqaxQbQn3*&`G#WbMZYOK7e8X5Gt|4&xTPGeAPVI~`8AVW(4z<53?_qtE~xTOmg<*%
zU_Mw!%*(b3oII2Oia=)-84Wz-6wD;|jK&w!Rc^ObB%Yj|U9Z*WkG3D@XQPkzc$(de
zt}ZfIoOM$K0(ck{Gerdt^0i-#Tq87b@iRsjIuaKjQ!(AZmg>}Lhb;U
z`JmH1FH2L1J^IX+;09)tXX2jarhw)A2}s>=Q@U`
zI-I9RhQikjUDCq1B6P**y^w+c+g1A+Q!fgzn+yQj*Wv11^<%sIrP>NM`B^pso9tx%
z?vm;lWA-DI#b4*oKM{q3+Y8WdENL~lf>7{546a}j6)G8FRWy)ns7mXsP8gU)>oBIr
zl8yiM$?08N@4dlRj8*dTkPw~`m^lyv+su=OdhoVO*_43+CE7KqhGH=f$IZ*bv_22bO4^Cvvn{lZrp4d4JZxV
zaZYAd1L~tmxGpr4$5Pf7BFqvMbIfeP4g1*;r
z16)P?VYg9tV19Oev0xJQu#ux3{^sTI@oqawbl#P>+%KS_)kU(vUl7?ERaPelCCqv0
zG^7hjd`Ky_VtNl+iZZ7OX>;`QEkng~JGvgX+u!{yV_rN2I9AXWwG2VFXe^*&5@j~c
z@}#7s98zwpsO;(LFjZ@Z0;E=kj!(`gR}jreX;Ae;Kd27dphV7$IgItT1O2tN7e~+S
z)sM^7`(bN`9sjyIkSsRNtxHayTtENyH%-(`Jts{Y>SnNEhsV1A-p!3MedtPVx+lYG
zX%W_m%kOmsC|6VZ=R+jv&`Oy0v>y;TbVu#AJ#D|XABS%^e(@t$y3cm|jO!Zy^~p)wdL?yV
z`0J02U3K<$RIEM{mz)m2X!0C}nNDGe2ok5==N#@92dvn5j1*>mI)sK-0ybMM?n(a2Q
zWxr>jeX#y67EcC$PohAlD*p1;WKltY6_Kf&3=h4aLgtMElLlxSfGc7OZiH)V
zkqhd*P51Sm7F$!j$;W&A@+ew#>G`i;y?lLr;w?Y9WnuLbMS=_KAN61sTcsI75T0?<
zc@8)9y!uyTYK4twA*5Aa*sIt*{8+*eYHf$CYsAd4KL_CV@
z6*;HC076O&ZF1x(8K=wyDQ*d9S$EX$XNF%iQsBioaR$tlum$@F6_Hp~Q4dhpP45>wRg_~EkX^0F4BjU&@nOBbfT^bjo2L0eGpFVm1^>ARf`-?xf
z_-5`G+fRHm%aS41(=HTw>t@M{TEh7QOHW)|(njXC4AK^`PkV=f%}WC7H1qOWYB-8{
zBaw3lyA&FD&l%S=x?KY8)d&f`iFI|3*X8eD%seqj9(WC{351@A)ERMZOj+BeVYIRK
z+L96uvb?4nAnAVd&L^go2?wJ^WSOaL3GxeBgiKN10I#R(3+@+qF65Pc5V~P_x}kST
zL?R37+l|dCX=pHP#MO|UNTgFN%J$aZ?GEfytKMGQqh)*aHJ~Z8Ji8nY*XYo__J>^KI*G%JMc0`P9T)}5-mvy0J3_20CJuI+YM`)
z9!x?(b*vJ!-7oBLTg-#|QmSlFPRM;2O)$BUx5U)LA}LWK6Qq
zy7cju_v9GEMF6ilf;o`bET)i(#z9j{8!(3g;4vmuB4^v3yD!FD{9;rR-v0jn-2Cd?
zFSZ}QI&aV8xT18dVGIG4w5@7@_~jWrtXTO5P!&>QbOxKfo!H5VKvgo%=pHlBmSDal
z9SYI_lypqI;-NQ3@j~Uek71{KGoo{QVYk;fpT0Z{vKDSxmxnmtajX|B4Y2Ahh|x}d
zfy4p!C4l(eG!J%IajL*;GdCmxC#yBr3GWN?5JD%&>kw=aWG3Boq6b|dekvQq=ov-<
z03!!S?zMzEIV!&lHKV{MATJ{|c9=vpM)?qyC4d()4>S=I)XRYSAqR~!f)PoL4?)yz
z`r}U*zq)3u!>|`4H}Uf~qvG6lH?hf#K?qneSi3g&y;&OMrtD|p6S>D8>qM0{e<42kl;DKgFvjC`q
zig#A2w%lqIul~g?InMoF`?)%1h2)T^z&Rj=ENf!Q9IhJqUVsduP>2+-^|3%MC3nCE
z=Nd9<8gb&)5F5e-gR&*iC)5lBbn^f$ws{^V#uWQ7WPp&a#bX>{Hetv-2*?=mr~q!L
zMA$wQjLJybX8N6GCYRg7j^4O$T-DKpRyJ~1lh1c{OkK_M@oIT}aT1JO5@G9bBqk>c
zsvxogRBwq0L7suf1H|MZ6Hy>{fckVn;+rq=DPItHnXPKb`tI7}?R3TSi=t@&k
zA#8_MB!C>Qt{i-N%B1psn?6mXW||?iU5_px7)YdDd|Gq-?%(;N-(}Dc1T}G9FaRM%ML9#mjEOasgu5&9kWY!jqr+h8ygDlVHuq%3auf6WkQ|CE6k_h%gwv{!;;LwCPi((G2Y1)Pwz;c`V*uLLreYm)
z(mepq?>g142pANMUKCYPIi@>e$K7}9@lwy!<@-OI1CH(F^=FG?JJ|^h{2sgD9^z!v
zH}6m={FvJ1!}`^;oEEwV&L9~l>A;V9YE?-Ii*u$O0zj%9#cfakQ=nV-9@oBPUOCIR
zrBtY66%0m`0bLDlW`2EAaG~aKyr~P}z11|ko2)a}IU*vOXkFGGLWB`QC}*cuh7OW_;Fi7LadfxoHBEyIwo9!I0zDAVe}%v^F0X
z$~s$qxjg;&cDWS;h?zh6Y#y|el`|!(>xi8>EbGR;9XELsilS2EUg9XUIFv0iyT>)f
zK&z&--Ya@_PK=^T0nLPr(BRrc{Kelxo!^bc`rLm6cfrI!c65v=aKe11SXY(^4+-Z6
znMN1Y1i|P50j_SKzwHm7JF!JV@De5!!VpOVZwVzqG-eW)9)=C4foWkaH3BobTRU5P
zW><@=qhei58$8>{zaGXKIja4HxUvbq`_;3j8!N;Uxz6$^a5+y
zhDc+`*hkVRg{)Ub2K^#-0DB^j423U1S!J6Tf9t7Jm)9%RoZ6#O
zU8zqS-J|1I9;TX(ukT>QXXa`&ErWcC$!^qnZ{FA(XismL_G+=U^Y^TJGai4oo7pk3
zxeL#;!0!={5ezO|;Fu>uu?2$xM-^a82Z<9U3S!=vOA`4xPnGG?yUh-hz)W{UataObPvYsx4K*`I5II)_bwSht7>I)k
zRbyb_;%Rdjo(z)ldJrF+C}hg=Xgnj3#m7ENmDMx+I6c8lgyC7h$5N)$i|2cyQf4EtniAn6Da#3f8S*TQUD!IYGGBn8yluX?-CM^DZH@1NVNqi4(W
z?a<<9Pd?k^+|9&eMBz4-l(WT|`yFH+eQ*Zc4qD~l>knm-E!xqw$K)xb9a_fd;!kU&
z7Yc!}3$-cNoe{i}VDwOzS~XrMs*R0V?AsfG=?%A*s{LDMV73a9U^E%_3b>
z-SbUbyXXiz4D%vv5J+lEB2E;PVC{rPwbMjly3}0RKW5Z`Z>1OnH8`)KYB6}qwPZjE
zGQK3X$a+!`NT(Qk2(D-6(VqAnRUXa{mu%h1h7pv8B
zb%eP7)lx4?``*zyV*T>_};azALB-?g<9tMDa1T)R0-&^&5+^e77k8Ork6xK4lRlXI)gN#8sxLn+>)As
ztC(#z@;f7g`1-R3v8|4|#Ypzg5|A^$*H758Q%%xjreaQ9MX)GD8@EBwrU+!ohIv$(
zpVWSf9ZaspU2H0bJWG^=A^<)pjtk^HgmDVSh*TpR2+tM`saTlX7n*(51CUoSQ7#dL
z57QsS1N1rtE|i(k1yVCH+u$?1>5ir5I#zV-eJ&)376cY0tfmL*MHDG0lvNrMBU9!jBby-OMofdJHHs{V937MX@Y&N_NyUv*|O-a~bCEPDp&*73~-;|;BFF}ijVV$9@
zUt8bXM*%Q(WKwdkMu4yk;>>_cXZR0O%TWY$wYkiO4Ux>gY@*-P<+=JnosP1Gkym^%
z?z3%ib~7k4+=yLI?QFBulqSQCM3rRN%!#pr040k+>94U~tzdiRuGGdjV-OfvEnQ6J
zVv@5C|N3_GJD7z~pK?N?2!+t#*G}^msAEN6gm$-IM+jxr#w)6+C*0Rjb`Uq2qe18a
zxFmTD4QMUI)8HfL2qr^8L0F7F-tcz%T~|nX4IhuT{N-V|H+0N;h3@fu-00W~PDJ>n
z?|m7HE)jM;V&2>+vAO^L&G-8Z<7L*74}Q8)c@0M&?pf``rd=!nL;=*Zhl0c251yS6
z9!Fd7X0!!69XfLEjo*Cc7TNeskkit&uZL})9JQM3)T8)MCEF#Qj9zJd#Ca1ZRtMV2
zGRyV<=4CUlesyiBDvrh{XDwyQ1Y-0fgAY)6t
zitasdEyk;IS2D$_gyJBlI-8ToZ}w2Poxat+T&wk4pN6*!_yFQE9uGi$i9{4KSy9_C
z!?h*9pkX{%UuT(TuWuCICQz%fX}%s+!beAFq_nxyp(ms4l`%HYaKbZmho2(`PIF@d
za}H|veV$z-D0BySR&g-nAi_l~+tOU?VNm+;a;}Amje!G4*>h_JcX8iG%z4?4V>nyTMN<~4&X(56b`?N{r_FB~umlE59~
z6q#M4Epovn10-|}+J;Q9rU4u_sI3A5n)2fI+O!D!ymlOw2#I9u@WA)q
zx|t~|Nj&*`PNR$2r@1?2p=Sx>=zEnQ!6?vrwbt=;a@h1jo;z_S!c{ZVCZaVTwTyd#
z(TJ>!$SQ_tvuH>Z;(KO)Xwq(zpWDl;JIlGxtMO>RosPAjBGaAgh~fhHsEr&Er9&(d
z(9^J)6Bq13P_AIFdq`7ws-P7BTt}ci3M$UH0eqoBS1TO3qscAQZN(S&w~%>gDZsWV
zO0Vz)(MhE50rs4Rc3YH0`aw%Ul!N7_aO}$iPhL(0!JckNoV)i!RZuX=k!h|@93srY
zNK)ePU@wmDr+(AF9@Ufv2mXBg@%9(F)Be-N>uOEs@B2yruEr+
zbBHh=&3VcQG4%y%4D{+Lt)zJgld2U4P41a#FZa{R^
z%5{|r)++9wwqr^n2+RVHb%XIi3TRu#Yd-NcA;}IWuNuFE9D&7wA4h>?Y%Ba>1N=5V`QiN(0@B;pKXO=Ds@M2<;N@J*T_
zd$pVO2VW>dC1W^yy;ZIjZ{btk)pURH_&`R93fCqaUtWxCFoDf8Jm?NKIM0tO#I|h9
z*nuF+xh1Q%l=Kf#voDF3MNy&lsH{t~}wL~}xzvnL~u=uuwUyY|a_31TtZXr+haLqr`X_Q~)Xdb|*F4
zcR<@9jzBGwc_>J(aj6s$)NBqlnOc(&Kk}6>J(G)&pFMoX9s
zl8=G
zfV?F|-9K()UmBlOESkrN&s&nbOSpaQTNL>SQ5We6%KgLLY*T-9*>6z39nCYiQhn
zBN_h!VhGBjgwqCgm;6>SN06Y7%mnIRI~aXuKQGls#=2A=hMx>3@G|>u_;|K&`h20;
zy^ziQb)h!OQ0uINYTM?rxVM`^qKlLRu``6Tp^O@4t$AQzIc(hFj%Ztf)B^%Fe4+jg
zWZxYdD`J259XW5m)*=-2`{uoNHv$h
z5@10BS+S){)U6wk3cz@Bp>}S0onXwqyW6S>%NB-hSTMUAp{fW1W(u&QWFSrvTkA9u
zfIUr?n&}2Vdoo~3_>t$vr1O7;rBGnX1~pg3--K2N2JSa(gHa={w+qyhhK(56MD=pi
z8QkNs08SKno6%+$EPg}&>yRTbLV+jy53`L~xo05|j7*^ah15{N)EdBA%%MZesB8(t
zT3Mn28@Kx~4aOkGD5ur-h#X&=4JKj!$B(NN%Yr=0_hJw%Eu
zQm+V5xW9-n;LR&};zpyKEYD9ro_L*%3BV)bxN(hh2lq0|gqV^CXsf1$QZiVmiV!)6
zHlTU0bXriGr-MZOx4I)UnAZUaVGMnzyTm5inxx;&=45Gcavm9smYo6;DmOEZ5cM~
z`f5Aq`Q_8Yl%QSzunT%l-QpVr519$?$RAGBF5eX|bY6P%u
zV7X_=5yyu)T+C)fxgN~}ZH%8o`{Ry5Mj7xy>KO`Kok>1B5z)v}
z*im!8cN`#7P^Wk|7~IIyzj%5W)kvR-9UI@VIRE-Rb;ZsEX|y}q#PBo+a~?WJTBd++
zhBQ4`sicIDo0`%vFDpkli~1z+2op^E*pp~uwrM~Jq$@$21d$^m(~NXJvj{HGt}{;D
zo5&dW_WlN6bbI?utxmTCk6*rcK0YCD-4ml5%@>SD
zH})hlfe#Hil7mQgpTo`y4U?Z*P~YVBNH_@(a98Q$xw{ws#dbG%FzaAS0MvB{k%5fY
z3b?Bv6h)b59HtD`c$<_tg@`*m)NJc8&~tbn2>~e5R4}DsiYTrR$5eAe?sVFe%$+z4
zeHCICUGF-r3#hX=$TTV-cC}QP5cJ7JlORz<95ezIb;k8_`6Gjoo?7#Ed*}P*i(hO9
zTEzXH?qqii0PK9ng?ohK)Ab{C(Ffjx3c-W!u$gy@$266+EV!zGo&c9Dos5W7)^e_4
z^|OnJo-u-A4|hvwC1{td78ZI@3L*)i>LhK!?x8mmWpzn6&Ei)kOYQ4!Y5c&X;K*y+
z>JsE5KqLNJlM;H4=OoGs3Se4n86*C3wK&^eb$RvV*(iwJezteA4qu`#u_?Q}bgMjl
z@%Y7yZW0Mq)CAY`AnO2FAIxd`w~%tsR)rrYqpSd$ilQYz)rx6Hpo>9Jz!xihv5)(O
z)+R6vXzvN?%(FL%PbER!mlAWnm2k5otA;sBbyLKs
zKC}giSxGJ(Ax8(D+0bGltePyvmcZaTa+nJ_2^
znlHe;ZTakKdGT?$OhzHjtL@`(9@w->e=sS#r$M$dP!`z!V9+SDZy(9Gkk
zQUYG&Vw0q>b;ex+h>Gzd9O%7WD*dgHy?XuR=iA@(=;NKvHl^Ku#87$p{O#A?=3Fz?
zhdLSlXjpd=ZGa$as2(wgLrG*NO`uwGc-q{^f^)s0q#IZR4r2J%w|k`f@7BC@h#-*B
z12mzzbR88l`TyB_*QU6UEYAC@5EHQxyDym5{j%?*l;m+6+Zgbz-w-J&Wy%JMF2KgU
zvETjqWl<`nJOV^fV&>WC-tC=U=ys`eGEbg7=YR19gh@=p(K5nOj?qsB$tS#L|BgJe
zfI^2k+%*rKG8ms33_=*gp^XAoDWwN16!j*ZAJ;qf*F-V8s=;J7poxhLdEn8)M5rw&
z;u`6NNR+t1WL44xmZbiZrlF2{R59P
zdbF!+3?oh2qGn^Qw;sX2lc{aLO58-7bgCFSCrzIqPV#_iGb#xLKj@zEdDo*yQG=8Z
z!z-XOXtm&$hxjrLOSDxAE}8&%0Q*|g^i_gNYguDeHvP6UKdq@@V0XB
z7B+2j+t4%p+^36w2cd2MbMF_%8+~Fy3b@I3+LA+m$nhQ^!{h(K2=0G4@oGhAr4dA7
zR>k1s2cmGwEo5ZkfJo#CmhkW;(Y5jPOIr3AkjN%)bStKPDetc3`BnGh?&N#-=0Qln
zMmqz9ddrjZ|9E>HmWy9mCv*-mR`-o``}o8j03!3+QbSy0K@2
z!;W`{`-6{Zu-!I$vyj|Z9Wd)Sx4!-Hwe;#2-pTv(d6pfo-ZK~o5WM#4+<$b^;4LFR
zQAD8XD_j9I1r=%`@Q|Yxm!Nu*zx8dTT0?A_lb|wp#qec2rUxUsivYd4=>>a$#lQ!J7o+9@-U;J8%b(>~
zLvG*EqYvGpl8lvbXbxm*YnzLJKMO7lXe2ys7>ZQb%MhdPtF~`}7cfJfxcf!#9ZaKAMNtJUE&PkbgnK1$KZHd!%A);7i9uxiE
zlHF`0iXvj<1dKMTV9^S{aX3_iOqOw)!ez7Y0u2Gk_1i4l+8y{wA+hf=UZm5i>BWGK
z;5Q;hD5^%dnao6yxM<(B->o~yugjUiIy6>A`rv$Q!`hThu-X{?sU9RRJhUz2_?m28
zR1C9=4DXD>ZE#wLg&2T7a}&T|QHUL?pOZ|np5DT`plh7uOjAh24DhkeLYx+xbr|P}
zxw_yuQ5(dsWSX8f(>2t3G+qSebRD`DrWV2tB#R^nnfq~>f=m~r1M6Z|YC-y9c77Kh
zZ<{@AaI72Nr$g
zfJCfdbJ|s|RE9v7)#xr}gDoJn1d{D>y^VQy+^~r$(d!hv7NC?o5gv&3xb%|*Jr&y)
zA(R%YKQ|qjhRyYLh35+UBf0)^z1aDF7ZL&DwAQ6pNc_-JizS6hf}05J4~G9v14>j&
z5C=xH%3Nd*HV6J=i)`%o>K57Z75`fA-2Fn0H`3d+Y|?;P#Zn0uj0k{i35pqE4Td)g
z6Pm0keB^hHgw}hZ-;F