feat: IPFS retrieval client

cli/main.rs

@@ -1,6 +1,7 @@
 mod args;
 
 use std::rc::Rc;
+use std::sync::Arc;
 use std::time::Duration;
 
 use args::{CliArgs, Commands};
@@ -9,7 +10,9 @@ use clap::Parser;
 use zinnia_runtime::anyhow::{Context, Error, Result};
 use zinnia_runtime::deno_core::error::JsError;
 use zinnia_runtime::fmt_errors::format_js_error;
-use zinnia_runtime::{colors, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter};
+use zinnia_runtime::{
+    colors, lassie, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter,
+};
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() {
@@ -32,9 +35,19 @@ async fn main_impl() -> Result<()> {
                 &file,
                 &std::env::current_dir().context("unable to get current working directory")?,
             )?;
+
+            let lassie_daemon = Arc::new(
+                lassie::Daemon::start(lassie::DaemonConfig {
+                    temp_dir: None, // TODO: Should we use something like ~/.cache/zinnia/lassie?
+                    port: 0,
+                })
+                .context("cannot initialize the IPFS retrieval client Lassie")?,
+            );
+
             let config = BootstrapOptions::new(
                 format!("zinnia/{}", env!("CARGO_PKG_VERSION")),
                 Rc::new(ConsoleReporter::new(Duration::from_millis(500))),
+                lassie_daemon,
                 None,
             );
             run_js_module(&main_module, &config).await?;

daemon/main.rs

@@ -4,14 +4,14 @@ mod station_reporter;
 
 use std::path::PathBuf;
 use std::rc::Rc;
+use std::sync::Arc;
 use std::time::Duration;
 
 use args::CliArgs;
 use clap::Parser;
 
-use log::{error, info};
 use zinnia_runtime::anyhow::{anyhow, Context, Error, Result};
-use zinnia_runtime::{get_module_root, resolve_path, run_js_module, BootstrapOptions};
+use zinnia_runtime::{get_module_root, lassie, resolve_path, run_js_module, BootstrapOptions};
 
 use crate::station_reporter::{log_info_activity, StationReporter};
 
@@ -27,7 +27,7 @@ async fn main() {
 }
 
 async fn run(config: CliArgs) -> Result<()> {
-    info!("Starting zinniad with config {config:?}");
+    log::info!("Starting zinniad with config {config:?}");
 
     if config.files.is_empty() {
         return Err(anyhow!("You must provide at least one module to run."));
@@ -41,6 +41,15 @@ async fn run(config: CliArgs) -> Result<()> {
     let state_file = PathBuf::from(config.state_root).join("state.json");
     log::debug!("Using state file: {}", state_file.display());
 
+    let lassie_config = lassie::DaemonConfig {
+        temp_dir: Some(PathBuf::from(config.cache_root).join("lassie")),
+        port: 0,
+    };
+    let lassie_daemon = Arc::new(
+        lassie::Daemon::start(lassie_config)
+            .context("cannot initialize the IPFS retrieval client Lassie")?,
+    );
+
     log_info_activity("Module Runtime started.");
 
     let file = &config.files[0];
@@ -63,6 +72,7 @@ async fn run(config: CliArgs) -> Result<()> {
             Duration::from_millis(200),
             module_name.into(),
         )),
+        lassie_daemon,
         module_root: Some(module_root),
         no_color: true,
         is_tty: false,
@@ -88,6 +98,6 @@ fn exit_with_error(error: Error) {
     let error_string = format!("{error:?}");
     let error_code = 1;
 
-    error!("{}", error_string.trim_start_matches("error: "));
+    log::error!("{}", error_string.trim_start_matches("error: "));
     std::process::exit(error_code);
 }

docs/building-modules.md

@@ -87,6 +87,7 @@ import * as code from "../../other/code.js";
 - [Web APIs](#web-apis)
 - [Unsupported Web APIs](#unsupported-web-apis)
 - [libp2p](#libp2p)
+- [IPFS retrieval client](#ipfs-retrieval-client)
 
 ### Standard JavaScript APIs
 
@@ -329,6 +330,32 @@ Report that a single job was completed.
 
 Call this function every time your module completes a job. It's ok to call it frequently.
 
+### IPFS Retrieval Client
+
+Zinnia provides a built-in IPFS retrieval client making it easy to fetch content-addressed data from
+IPFS and Filecoin networks. You can retrieve data for a given CID using the web platform API `fetch`
+and using the URL scheme `ipfs://`.
+
+Example:
+
+```js
+const response = await fetch("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni");
+assert(response.ok);
+const data = await response.arrayBuffer();
+// data contains binary data in the CAR format
+```
+
+> Note: At the moment, Zinnia does not provide any tools to interpret the returned CAR data. We are
+> discussing support for reading UnixFS data in
+> [zinnia#245](https://github.com/filecoin-station/zinnia/issues/246).
+
+Under the hood, Zinnia handles `ipfs://bafy...` requests by calling Lassie HTTP API. You can learn
+more about supported parameters (request headers, query string arguments), response headers and
+possible error status codes in
+[Lassie's HTTP Specification](https://github.com/filecoin-project/lassie/blob/main/docs/HTTP_SPEC.md).
+The format of CAR data returned by the retrieval client is described in
+[Lassie's Returned CAR Specification](https://github.com/filecoin-project/lassie/blob/main/docs/CAR.md).
+
 ## Testing Guide
 
 Zinnia provides lightweight tooling for writing and running automated tests.

runtime/Cargo.toml

@@ -21,6 +21,8 @@ deno_fetch = "0.129.0"
 deno_url = "0.105.0"
 deno_web = "0.136.0"
 deno_webidl = "0.105.0"
+lassie = "0.3.0"
+# lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" }
 log.workspace = true
 once_cell = "1.18.0"
 serde.workspace = true

runtime/ext.rs

@@ -52,6 +52,7 @@ deno_core::extension!(
       "90_zinnia_apis.js",
       "98_global_scope.js",
       "internals.js",
+      "fetch.js",
       "test.js",
       "vendored/asserts.bundle.js",
       "99_main.js",

runtime/js/98_global_scope.js

@@ -23,7 +23,7 @@ import * as fileReader from "ext:deno_web/10_filereader.js";
 import * as formData from "ext:deno_fetch/21_formdata.js";
 import * as request from "ext:deno_fetch/23_request.js";
 import * as response from "ext:deno_fetch/23_response.js";
-import * as fetch from "ext:deno_fetch/26_fetch.js";
+import * as fetch from "ext:zinnia_runtime/fetch.js";
 import * as messagePort from "ext:deno_web/13_message_port.js";
 import * as webidl from "ext:deno_webidl/00_webidl.js";
 import DOMException from "ext:deno_web/01_dom_exception.js";

runtime/js/99_main.js

@@ -36,6 +36,7 @@ import {
   mainRuntimeGlobalProperties,
   windowOrWorkerGlobalScope,
 } from "ext:zinnia_runtime/98_global_scope.js";
+import { setLassieUrl } from "ext:zinnia_runtime/fetch.js";
 
 function formatException(error) {
   if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) {
@@ -66,6 +67,8 @@ function runtimeStart(runtimeOptions) {
 
   // deno-lint-ignore prefer-primordials
   Error.prepareStackTrace = core.prepareStackTrace;
+
+  setLassieUrl(runtimeOptions.lassieUrl);
 }
 
 let hasBootstrapped = false;

runtime/js/fetch.js

@@ -0,0 +1,66 @@
+import { fetch as fetchImpl } from "ext:deno_fetch/26_fetch.js";
+import { fromInnerResponse, toInnerResponse } from "ext:deno_fetch/23_response.js";
+import { toInnerRequest, fromInnerRequest, Request } from "ext:deno_fetch/23_request.js";
+import { guardFromHeaders } from "ext:deno_fetch/20_headers.js";
+
+let ipfsScheme = "ipfs://";
+let ipfsBaseUrl = undefined;
+
+export function setLassieUrl(/** @type {string} */ value) {
+  ipfsBaseUrl = value + "ipfs/";
+}
+
+export function fetch(resource, options) {
+  let request = new Request(resource, options);
+  // Fortunately, Request#url is a string, not an instance of URL class
+  // See https://developer.mozilla.org/en-US/docs/Web/API/Request/url
+  if (request.url.startsWith(ipfsScheme)) {
+    return fetchFromIpfs(request);
+  } else {
+    return fetchImpl(request);
+  }
+}
+
+async function fetchFromIpfs(request) {
+  // Rewrite request URL to use Lassie
+  request = buildIpfsRequest(request);
+
+  // Call Deno's `fetch` using the rewritten URL to make the actual HTTP request
+  const response = await fetchImpl(request);
+
+  // Patch the response object to hide the fact that we are calling Lassie
+  // We don't want to leak Lassie's URL
+  return patchIpfsResponse(response);
+}
+
+// Deno's Fetch Request is a thin immutable wrapper around InnerRequest. In order to modify the
+// request URL, we must convert Request to InnerRequest first, make changes on the inner object,
+// and finally convert the InnerRequest back to a new Request instance.
+function buildIpfsRequest(request) {
+  const inner = toInnerRequest(request);
+
+  inner.urlList = /** @type {(() => string)[]}*/ (inner.urlList).map((urlFn) => {
+    const url = urlFn();
+    if (!url.startsWith(ipfsScheme)) return urlFn;
+    const newUrl = ipfsBaseUrl + url.slice(ipfsScheme.length);
+    return () => newUrl;
+  });
+  inner.urlListProcessed = /** @type {string[]} */ (inner.urlListProcessed).map((url) =>
+    url.startsWith(ipfsScheme) ? ipfsBaseUrl + url.slice(ipfsScheme.length) : url,
+  );
+
+  return fromInnerRequest(inner, request.signal, guardFromHeaders(request.headers));
+}
+
+// Deno's Fetch Response is a thin immutable wrapper around InnerResponse. In order to modify the
+// response URL, we must convert Response to InnerResponse first, make changes on the inner object,
+// and finally convert the InnerResponse back to a new Response instance.
+function patchIpfsResponse(response) {
+  const inner = toInnerResponse(response);
+
+  inner.urlList = /** @type {string[])} */ (inner.urlList).map((url) =>
+    url.startsWith(ipfsBaseUrl) ? "ipfs://" + url.slice(ipfsBaseUrl.length) : url,
+  );
+
+  return fromInnerResponse(inner, guardFromHeaders(response.headers));
+}

runtime/lib.rs

@@ -18,4 +18,6 @@ mod reporter;
 pub use console_reporter::*;
 pub use reporter::*;
 
+pub use lassie;
+
 mod ext;

runtime/runtime.rs

@@ -1,5 +1,6 @@
 use std::path::PathBuf;
 use std::rc::Rc;
+use std::sync::Arc;
 
 use deno_core::{located_script_name, serde_json, JsRuntime, ModuleSpecifier, RuntimeOptions};
 
@@ -35,12 +36,17 @@ pub struct BootstrapOptions {
 
     /// Report activities
     pub reporter: Rc<dyn Reporter>,
+
+    /// Lassie daemon to use as the IPFS retrieval client. We must use Arc here to allow sharing of
+    /// the singleton Lassie instance between multiple threads spawned by Rust's test runner.
+    pub lassie_daemon: Arc<lassie::Daemon>,
 }
 
 impl BootstrapOptions {
     pub fn new(
         agent_version: String,
         reporter: Rc<dyn Reporter>,
+        lassie_daemon: Arc<lassie::Daemon>,
         module_root: Option<PathBuf>,
     ) -> Self {
         Self {
@@ -52,6 +58,7 @@ impl BootstrapOptions {
             // See https://lotus.filecoin.io/lotus/manage/manage-fil/#public-key-address
             wallet_address: String::from("t1abjxfbp274xpdqcpuaykwkfb43omjotacm2p3za"),
             reporter,
+            lassie_daemon,
         }
     }
 
@@ -60,6 +67,7 @@ impl BootstrapOptions {
           "noColor": self.no_color,
           "isTty": self.is_tty,
           "walletAddress": self.wallet_address,
+          "lassieUrl": format!("http://127.0.0.1:{}/", self.lassie_daemon.port()),
         });
         serde_json::to_string_pretty(&payload).unwrap()
     }

runtime/tests/fetch_api_tests.rs

@@ -8,8 +8,12 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpListener;
 use zinnia_runtime::{anyhow, deno_core, run_js_module, BootstrapOptions, RecordingReporter};
 
+mod helpers;
+
 #[tokio::test]
 async fn fetch_reports_user_agent() -> Result<()> {
+    let _ = env_logger::builder().is_test(true).try_init();
+
     let user_agent = "zinnia_fetch_api_tests agent/007";
     let server_port = start_echo_server().await?;
 
@@ -29,7 +33,12 @@ assertArrayIncludes(request_lines, ["user-agent: {user_agent}"]);
         &std::env::current_dir().context("unable to get current working directory")?,
     )?;
     let reporter = Rc::new(RecordingReporter::new());
-    let config = BootstrapOptions::new(user_agent.into(), reporter.clone(), None);
+    let config = BootstrapOptions::new(
+        user_agent.into(),
+        reporter.clone(),
+        helpers::lassie_daemon(),
+        None,
+    );
     run_js_module(&main_module, &config).await?;
     // the test passes when the JavaScript code does not throw
     Ok(())

runtime/tests/helpers/mod.rs

@@ -0,0 +1,14 @@
+use std::sync::{Arc, OnceLock};
+
+pub fn lassie_daemon() -> Arc<lassie::Daemon> {
+    static LASSIE_DAEMON: OnceLock<Result<Arc<lassie::Daemon>, lassie::StartError>> =
+        OnceLock::new();
+
+    let result = LASSIE_DAEMON
+        .get_or_init(|| lassie::Daemon::start(lassie::DaemonConfig::default()).map(Arc::new));
+
+    match result {
+        Ok(ptr) => Arc::clone(ptr),
+        Err(err) => panic!("could not start Lassie daemon: {err}"),
+    }
+}