feat: create @helia/verified-fetch

packages/verified-fetch/.aegir.js

@@ -0,0 +1,45 @@
+import getPort from 'aegir/get-port'
+import { createServer } from 'ipfsd-ctl'
+import * as kuboRpcClient from 'kubo-rpc-client'
+
+/** @type {import('aegir').PartialOptions} */
+export default {
+  build: {
+    bundlesizeMax: '10kB',
+  },
+  test: {
+    files: './dist/test/*.spec.js',
+    before: async (options) => {
+      const ipfsdPort = await getPort()
+      const ipfsdServer = await createServer({
+        host: '127.0.0.1',
+        port: ipfsdPort
+      }, {
+        ipfsBin: (await import('kubo')).default.path(),
+        kuboRpcModule: kuboRpcClient,
+        ipfsOptions: {
+          // TODO: enable delegated routing
+          // TODO: enable trustless-gateway
+          config: {
+            Addresses: {
+              Swarm: [
+                "/ip4/0.0.0.0/tcp/0",
+                "/ip4/0.0.0.0/tcp/0/ws"
+              ]
+            }
+          }
+        }
+      }).start()
+
+      return {
+        env: {
+          IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}`,
+        },
+        ipfsdServer
+      }
+    },
+    after: async (options, beforeResult) => {
+      await beforeResult.ipfsdServer.stop()
+    }
+  }
+}

packages/verified-fetch/LICENSE

@@ -0,0 +1,4 @@
+This project is dual licensed under MIT and Apache-2.0.
+
+MIT: https://www.opensource.org/licenses/mit
+Apache-2.0: https://www.apache.org/licenses/license-2.0

packages/verified-fetch/LICENSE-APACHE

@@ -0,0 +1,5 @@
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

packages/verified-fetch/LICENSE-MIT

@@ -0,0 +1,19 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

packages/verified-fetch/README.md

@@ -0,0 +1,204 @@
+<p align="center">
+  <a href="https://github.com/ipfs/helia" title="Helia">
+    <img src="https://raw.githubusercontent.com/ipfs/helia/main/assets/helia.png" alt="Helia logo" width="300" />
+  </a>
+</p>
+
+[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech)
+[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech)
+[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia)
+[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain)
+
+> A fetch-like API for IPFS content on the web.
+
+# About
+
+`@helia/verified-fetch`` is a library that provides a fetch-like API for fetching content from IPFS. This library should act as a replacement for the `fetch()` API for fetching content from IPFS, and will return a [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) object that can be used in a similar manner to the `fetch()` API. This means browser and HTTP caching inside browser main threads, web-workers, and service workers, as well as other features of the `fetch()` API should work in a way familiar to developers.
+
+## Example
+
+```ts
+import { createVerifiedFetch } from '@helia/verified-fetch'
+
+const verifiedFetch = await createVerifiedFetch({
+  gateways: ['mygateway.info', 'trustless-gateway.link']
+})
+
+const resp = await verifiedFetch('ipfs://bafy...')
+
+const json = await resp.json()
+```
+
+# Install
+
+```console
+$ npm i @helia/verified-fetch
+```
+
+## Browser `<script>` tag
+
+Loading this module through a script tag will make it's exports available as `HeliaVerifiedFetch` in the global namespace.
+
+```html
+<script src="https://unpkg.com/@helia/verified-fetch/dist/index.min.js"></script>
+```
+
+### Configuration
+
+#### Usage with customized Helia
+
+You can see variations of Helia and js-libp2p configuration options at https://helia.io/interfaces/helia.index.HeliaInit.html.
+
+The `@helia/http` module is currently in-progress, but the init options should be a subset of the `helia` module's init options. See https://github.com/ipfs/helia/issues/289 for more information.
+
+```ts
+import { createVerifiedFetch } from '@helia/verified-fetch'
+import { CreateHelia as CreateHeliaHttpOnly } from '@helia/http'
+
+const verifiedFetch = await createVerifiedFetch(
+  CreateHeliaHttpOnly({
+    gateways: ['mygateway.info', 'trustless-gateway.link'],
+    routers: ['delegated-ipfs.dev'],
+  })
+)
+
+const resp = await verifiedFetch('ipfs://bafy...')
+
+const json = await resp.json()
+```
+
+### Comparison to fetch
+
+First, this library will require instantiation in order to configure the gateways and delegated routers, or potentially a custom Helia instance. Secondly, once your verified-fetch method is created, it will act as similar to the `fetch()` API as possible.
+
+[The `fetch()` API](https://developer.mozilla.org/en-US/docs/Web/API/fetch) takes two parameters:
+
+1. A [resource](https://developer.mozilla.org/en-US/docs/Web/API/fetch#resource)
+2. An [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options)
+
+#### Resource argument
+
+This library intends to support the following methods of fetching web3 content from IPFS:
+
+1. IPFS protocol: `ipfs://<cidv0>` & `ipfs://<cidv0>`
+2. IPNS protocol: `ipns://<peerId>` & `ipns://<publicKey>` & `ipns://<hostUri_Supporting_DnsLink_TxtRecords>`
+3. CID instances: An actual CID instance `CID.parse('bafy...')`
+4. CID strings: A CID string `bafy...`
+
+As well as support for pathing & params for all of the above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453).
+
+#### Options argument
+
+This library will not plan to support the exact Fetch API options object, as some of the arguments don't make sense. Instead, it will only support options necessary to meet [IPFS specs](https://specs.ipfs.tech/) related to specifying the resultant shape of desired content.
+
+Some of those header specifications are:
+
+1. https://specs.ipfs.tech/http-gateways/path-gateway/#request-headers
+2. https://specs.ipfs.tech/http-gateways/trustless-gateway/#request-headers
+3. https://specs.ipfs.tech/http-gateways/subdomain-gateway/#request-headers
+
+Where possible, options and Helia internals will be automatically configured to the appropriate codec & content type based on the `verified-fetch` configuration and `options` argument passed.
+
+Known Fetch API options that will be supported:
+
+1. `signal` - An AbortSignal that a user can use to abort the request.
+2. `redirect` - A string that specifies the redirect type. One of `follow`, `error`, or `manual`. Defaults to `follow`. Best effort to adhere to the [Fetch API redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) parameter.
+3. `headers` - An object of headers to be sent with the request. Best effort to adhere to the [Fetch API headers](https://developer.mozilla.org/en-US/docs/Web/API/fetch#headers) parameter.
+    - `accept` - A string that specifies the accept header. Relevant values:
+        - [`vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw). (default)
+        - [`vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car)
+        - [`vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record)
+4. `method` - A string that specifies the HTTP method to use for the request. Defaults to `GET`. Best effort to adhere to the [Fetch API method](https://developer.mozilla.org/en-US/docs/Web/API/fetch#method) parameter.
+5. `body` - An object that specifies the body of the request. Best effort to adhere to the [Fetch API body](https://developer.mozilla.org/en-US/docs/Web/API/fetch#body) parameter.
+6. `cache` - Will basically act as `force-cache` for the request. Best effort to adhere to the [Fetch API cache](https://developer.mozilla.org/en-US/docs/Web/API/fetch#cache) parameter.
+
+
+Non-Fetch API options that will be supported:
+
+1. `onProgress` - Similar to Helia `onProgress` options, this will be a function that will be called with a progress event. Supported progress events are:
+    - `helia:verified-fetch:error` - An error occurred during the request.
+    - `helia:verified-fetch:request:start` - The request has been sent
+    - `helia:verified-fetch:request:complete` - The request has been sent
+    - `helia:verified-fetch:request:error` - An error occurred during the request.
+    - `helia:verified-fetch:request:abort` - The request was aborted prior to completion.
+    - `helia:verified-fetch:response:start` - The initial HTTP Response headers have been set, and response stream is started.
+    - `helia:verified-fetch:response:complete` - The response stream has completed.
+    - `helia:verified-fetch:response:error` - An error occurred while building the response.
+
+Some in-flight specs (IPIPs) that will affect the options object this library supports in the future can be seen at https://specs.ipfs.tech/ipips, but a few that I'm aware of are:
+
+1. [IPIP-0412: Signaling Block Order in CARs on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0412/)
+2. [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/)
+3. [IPIP-0386: Subdomain Gateway Interop with _redirects](https://specs.ipfs.tech/ipips/ipip-0386/)
+4. [IPIP-0328: JSON and CBOR Response Formats on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0328/)
+5. [IPIP-0288: TAR Response Format on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0288/)
+
+#### Response types
+
+This library's purpose is to return reasonably representable content from IPFS. In other words, fetching content is intended for leaf-node content -- such as images/videos/audio & other assets, or other IPLD content (with link) -- that can be represented by https://developer.mozilla.org/en-US/docs/Web/API/Response#instance_methods. The content type you receive back will depend upon the CID you request as well as the `Accept` header value you provide.
+
+All content we retrieve from the IPFS network is obtained via an AsyncIterable, and will be set as the [body of the HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response#body) via a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream) or other efficient method that avoids loading the entire response into memory or getting the entire response from the network before returning a response to the user.
+
+If your content doesn't have a mime-type or an [IPFS spec](https://specs.ipfs.tech), this library will not support it, but you can use the [`helia`](https://github.com/ipfs/helia) library directly for those use cases. See [Unsupported response types](#unsupported-response-types) for more information.
+
+##### Handling response types
+
+For handling responses we want to follow conventions/abstractions from Fetch API where possible:
+
+- For JSON, assuming you abstract any differences between dag-json/dag-cbor/json/and json-file-on-unixfs, you would call `.json()` to get a JSON object.
+- For images (or other web-relevant asset) you want to add to the DOM, use `.blob()` or `.arrayBuffer()` to get the raw bytes.
+- For plain text in utf-8, you would call `.text()`
+- For streaming response data, use something like `response.body.getReader()` to get a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream).
+
+##### Unsupported response types
+
+* Returning IPLD nodes or DAGs as JS objects is not supported, as there is no currently well-defined structure for representing this data in an [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response). Instead, users should request `aplication/vnd.ipld.car` or use the [`helia`](https://github.com/ipfs/helia) library directly for this use case.
+* Others? Open an issue or PR!
+
+#### Response headers
+
+This library will set the [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) headers to the appropriate values for the content type according to the appropriate [IPFS Specifications](https://specs.ipfs.tech/).
+
+Some known header specifications:
+
+* https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers
+* https://specs.ipfs.tech/http-gateways/trustless-gateway/#response-headers
+* https://specs.ipfs.tech/http-gateways/subdomain-gateway/#response-headers
+
+#### Possible Scenarios that could cause confusion
+
+##### Attempting to fetch the CID for content that does not make sense
+
+If you request `bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze`, which points to the root of the en.wikipedia.org mirror, a response object does not make sense.
+
+#### Errors
+
+Known Errors that can be thrown:
+
+1. `TypeError` - If the resource argument is not a string, CID, or CID string.
+2. `TypeError` - If the options argument is passed and not an object.
+3. `TypeError` - If the options argument is passed and is malformed.
+4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal.
+
+# API Docs
+
+- <https://ipfs.github.io/helia/modules/_helia_verified_fetch.html>
+
+# License
+
+Licensed under either of
+
+- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / <http://www.apache.org/licenses/LICENSE-2.0>)
+- MIT ([LICENSE-MIT](LICENSE-MIT) / <http://opensource.org/licenses/MIT>)
+
+# Contribute
+
+Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues).
+
+Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general.
+
+Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md).
+
+Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions.
+
+[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md)

packages/verified-fetch/package.json

@@ -0,0 +1,82 @@
+{
+  "name": "@helia/verified-fetch",
+  "version": "0.0.0",
+  "description": "A fetch-like API for IPFS content on the web.",
+  "license": "Apache-2.0 OR MIT",
+  "homepage": "https://github.com/ipfs/helia/tree/main/packages/verified-fetch#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ipfs/helia.git"
+  },
+  "bugs": {
+    "url": "https://github.com/ipfs/helia/issues"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "keywords": [
+    "IPFS",
+    "fetch",
+    "helia"
+  ],
+  "type": "module",
+  "types": "./dist/src/index.d.ts",
+  "files": [
+    "src",
+    "dist",
+    "!dist/test",
+    "!**/*.tsbuildinfo"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js"
+    }
+  },
+  "eslintConfig": {
+    "extends": "ipfs",
+    "parserOptions": {
+      "project": true,
+      "sourceType": "module"
+    }
+  },
+  "scripts": {
+    "clean": "aegir clean",
+    "lint": "aegir lint",
+    "dep-check": "aegir dep-check",
+    "build": "aegir build",
+    "test": "aegir test",
+    "test:chrome": "aegir test -t browser --cov",
+    "test:chrome-webworker": "aegir test -t webworker",
+    "test:firefox": "aegir test -t browser -- --browser firefox",
+    "test:firefox-webworker": "aegir test -t webworker -- --browser firefox",
+    "test:node": "aegir test -t node --cov",
+    "test:electron-main": "aegir test -t electron-main"
+  },
+  "dependencies": {
+    "@helia/block-brokers": "next",
+    "@helia/http": "next",
+    "@helia/interface": "next",
+    "@helia/ipns": "next",
+    "@helia/routers": "next",
+    "@helia/unixfs": "next",
+    "@libp2p/peer-id": "^4.0.5",
+    "mime-types": "^2.1.35",
+    "multiformats": "^13.0.0"
+  },
+  "devDependencies": {
+    "@types/mime-types": "^2.1.4",
+    "@types/sinon": "^17.0.2",
+    "aegir": "^42.1.0",
+    "helia": "next",
+    "ipfs-unixfs": "^11.1.2",
+    "ipfsd-ctl": "^13.0.0",
+    "kubo": "^0.25.0",
+    "kubo-rpc-client": "^3.0.2"
+  },
+  "browser": {
+    "node:buffer": false,
+    "node:stream": false
+  }
+}