diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bac1c4f6..51aa055d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,6 +31,11 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: stable + cache: false # caching requires a go.sum file, which we don't have in our project + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable @@ -64,6 +69,11 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: stable + cache: false # caching requires a go.sum file, which we don't have in our project + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable @@ -100,6 +110,11 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: stable + cache: false # caching requires a go.sum file, which we don't have in our project + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable @@ -137,6 +152,11 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: stable + cache: false # caching requires a go.sum file, which we don't have in our project + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73985d26..01feb693 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,6 +89,11 @@ jobs: - name: Setup | Checkout uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: stable + cache: false # caching requires a go.sum file, which we don't have in our project + - name: Setup | Rust uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: diff --git a/Cargo.lock b/Cargo.lock index 654db40b..b5ea0850 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2178,6 +2178,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lassie" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5726517411b49924096cc729c41092c134b4e82d710f37af0936db83e02657ff" +dependencies = [ + "cc", + "log", +] + [[package]] name = "lazy-regex" version = "2.5.0" @@ -5682,6 +5692,7 @@ dependencies = [ "deno_web", "deno_webidl", "env_logger", + "lassie", "log", "once_cell", "pretty_assertions", diff --git a/cli/README.md b/cli/README.md index 75d4879c..3ce41d84 100644 --- a/cli/README.md +++ b/cli/README.md @@ -33,6 +33,12 @@ If you have Rust tooling installed on your machine (see [Install Rust](https://www.rust-lang.org/tools/install)), you can build & install Zinnia from the source code. +In addition to the Rust build toolchain, you also need Go installed. See +[Go Downloads](https://go.dev/dl/). + +On Windows, Go uses `gcc` to create C libraries. Go recommends installing +[TDM GCC](https://jmeubank.github.io/tdm-gcc/). + ```sh $ cargo install zinnia ``` diff --git a/cli/main.rs b/cli/main.rs index 0c1dae52..df03f920 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -1,6 +1,7 @@ mod args; use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; use args::{CliArgs, Commands}; @@ -9,7 +10,9 @@ use clap::Parser; use zinnia_runtime::anyhow::{Context, Error, Result}; use zinnia_runtime::deno_core::error::JsError; use zinnia_runtime::fmt_errors::format_js_error; -use zinnia_runtime::{colors, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter}; +use zinnia_runtime::{ + colors, lassie, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter, +}; #[tokio::main(flavor = "current_thread")] async fn main() { @@ -32,9 +35,35 @@ async fn main_impl() -> Result<()> { &file, &std::env::current_dir().context("unable to get current working directory")?, )?; + + let lassie_daemon = Arc::new( + lassie::Daemon::start(lassie::DaemonConfig { + // This configuration applies to `zinnia` CLI only. The `zinniad` daemon running + // inside Station uses a different temp_dir config based on the env var + // `CACHE_ROOT` provided by the Station. + // + // By default, Lassie stores its temporary files in the system temp directory. + // That's good enough for now. We can improve this later based on user feedback, + // for example: + // - we can honour CACHE_ROOT + // - we can default to something like + // `~/.cache/zinnia/lassie` on Unix, + // `%APPLOCALDATA%\zinnia\lassie' on Windows. + // + // Important: if we tell Lassie to use a specific temp dir that's not + // automatically cleaned by the operating system, we will need to clean any + // leftover files ourselves. See the GH issue for deleting leftover files + // when `zinniad` starts: https://github.com/filecoin-station/zinnia/issues/245 + temp_dir: None, + port: 0, + }) + .context("cannot initialize the IPFS retrieval client Lassie")?, + ); + let config = BootstrapOptions::new( format!("zinnia/{}", env!("CARGO_PKG_VERSION")), Rc::new(ConsoleReporter::new(Duration::from_millis(500))), + lassie_daemon, None, ); run_js_module(&main_module, &config).await?; diff --git a/daemon/main.rs b/daemon/main.rs index 10e122f9..6e8e731b 100644 --- a/daemon/main.rs +++ b/daemon/main.rs @@ -4,14 +4,14 @@ mod station_reporter; use std::path::PathBuf; use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; use args::CliArgs; use clap::Parser; -use log::{error, info}; use zinnia_runtime::anyhow::{anyhow, Context, Error, Result}; -use zinnia_runtime::{get_module_root, resolve_path, run_js_module, BootstrapOptions}; +use zinnia_runtime::{get_module_root, lassie, resolve_path, run_js_module, BootstrapOptions}; use crate::station_reporter::{log_info_activity, StationReporter}; @@ -27,7 +27,7 @@ async fn main() { } async fn run(config: CliArgs) -> Result<()> { - info!("Starting zinniad with config {config:?}"); + log::info!("Starting zinniad with config {config:?}"); if config.files.is_empty() { return Err(anyhow!("You must provide at least one module to run.")); @@ -41,6 +41,15 @@ async fn run(config: CliArgs) -> Result<()> { let state_file = PathBuf::from(config.state_root).join("state.json"); log::debug!("Using state file: {}", state_file.display()); + let lassie_config = lassie::DaemonConfig { + temp_dir: Some(PathBuf::from(config.cache_root).join("lassie")), + port: 0, + }; + let lassie_daemon = Arc::new( + lassie::Daemon::start(lassie_config) + .context("cannot initialize the IPFS retrieval client Lassie")?, + ); + log_info_activity("Module Runtime started."); let file = &config.files[0]; @@ -63,6 +72,7 @@ async fn run(config: CliArgs) -> Result<()> { Duration::from_millis(200), module_name.into(), )), + lassie_daemon, module_root: Some(module_root), no_color: true, is_tty: false, @@ -88,6 +98,6 @@ fn exit_with_error(error: Error) { let error_string = format!("{error:?}"); let error_code = 1; - error!("{}", error_string.trim_start_matches("error: ")); + log::error!("{}", error_string.trim_start_matches("error: ")); std::process::exit(error_code); } diff --git a/docs/building-modules.md b/docs/building-modules.md index f5e21cc7..b76d10df 100644 --- a/docs/building-modules.md +++ b/docs/building-modules.md @@ -87,6 +87,7 @@ import * as code from "../../other/code.js"; - [Web APIs](#web-apis) - [Unsupported Web APIs](#unsupported-web-apis) - [libp2p](#libp2p) +- [IPFS retrieval client](#ipfs-retrieval-client) ### Standard JavaScript APIs @@ -329,6 +330,32 @@ Report that a single job was completed. Call this function every time your module completes a job. It's ok to call it frequently. +### IPFS Retrieval Client + +Zinnia provides a built-in IPFS retrieval client making it easy to fetch content-addressed data from +IPFS and Filecoin networks. You can retrieve data for a given CID using the web platform API `fetch` +together with the URL scheme `ipfs://`. + +Example: + +```js +const response = await fetch("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); +assert(response.ok); +const data = await response.arrayBuffer(); +// data contains binary data in the CAR format +``` + +> Note: At the moment, Zinnia does not provide any tools for interpreting the returned CAR data. We +> are discussing support for reading UnixFS data in +> [zinnia#245](https://github.com/filecoin-station/zinnia/issues/246). + +Under the hood, Zinnia handles `ipfs://bafy...` requests by calling Lassie's HTTP API. You can learn +more about supported parameters (request headers, query string arguments), response headers and +possible error status codes in +[Lassie's HTTP Specification](https://github.com/filecoin-project/lassie/blob/main/docs/HTTP_SPEC.md). +The format of CAR data returned by the retrieval client is described in +[Lassie's Returned CAR Specification](https://github.com/filecoin-project/lassie/blob/main/docs/CAR.md). + ## Testing Guide Zinnia provides lightweight tooling for writing and running automated tests. diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 62caa012..c0d15f2c 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,6 +21,8 @@ deno_fetch = "0.129.0" deno_url = "0.105.0" deno_web = "0.136.0" deno_webidl = "0.105.0" +lassie = "0.3.0" +# lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } log.workspace = true once_cell = "1.18.0" serde.workspace = true diff --git a/runtime/ext.rs b/runtime/ext.rs index 9bb23f51..146e69e1 100644 --- a/runtime/ext.rs +++ b/runtime/ext.rs @@ -52,6 +52,7 @@ deno_core::extension!( "90_zinnia_apis.js", "98_global_scope.js", "internals.js", + "fetch.js", "test.js", "vendored/asserts.bundle.js", "99_main.js", diff --git a/runtime/js/98_global_scope.js b/runtime/js/98_global_scope.js index 67c2f1a4..2fd432db 100644 --- a/runtime/js/98_global_scope.js +++ b/runtime/js/98_global_scope.js @@ -23,7 +23,7 @@ import * as fileReader from "ext:deno_web/10_filereader.js"; import * as formData from "ext:deno_fetch/21_formdata.js"; import * as request from "ext:deno_fetch/23_request.js"; import * as response from "ext:deno_fetch/23_response.js"; -import * as fetch from "ext:deno_fetch/26_fetch.js"; +import * as fetch from "ext:zinnia_runtime/fetch.js"; import * as messagePort from "ext:deno_web/13_message_port.js"; import * as webidl from "ext:deno_webidl/00_webidl.js"; import DOMException from "ext:deno_web/01_dom_exception.js"; diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 81856f6a..14c036e2 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -36,6 +36,7 @@ import { mainRuntimeGlobalProperties, windowOrWorkerGlobalScope, } from "ext:zinnia_runtime/98_global_scope.js"; +import { setLassieUrl } from "ext:zinnia_runtime/fetch.js"; function formatException(error) { if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) { @@ -66,6 +67,8 @@ function runtimeStart(runtimeOptions) { // deno-lint-ignore prefer-primordials Error.prepareStackTrace = core.prepareStackTrace; + + setLassieUrl(runtimeOptions.lassieUrl); } let hasBootstrapped = false; diff --git a/runtime/js/fetch.js b/runtime/js/fetch.js new file mode 100644 index 00000000..2ead4ae0 --- /dev/null +++ b/runtime/js/fetch.js @@ -0,0 +1,69 @@ +import { fetch as fetchImpl } from "ext:deno_fetch/26_fetch.js"; +import { fromInnerResponse, toInnerResponse } from "ext:deno_fetch/23_response.js"; +import { toInnerRequest, fromInnerRequest, Request } from "ext:deno_fetch/23_request.js"; +import { guardFromHeaders } from "ext:deno_fetch/20_headers.js"; + +const ipfsScheme = "ipfs://"; +let ipfsBaseUrl = undefined; + +export function setLassieUrl(/** @type {string} */ value) { + ipfsBaseUrl = value + "ipfs/"; +} + +export function fetch(resource, options) { + let request = new Request(resource, options); + // The `resource` arg can be a string or any other object with a stringifier - including a URL + // object - that provides the URL of the resource you want to fetch; or a Request object. + // See https://developer.mozilla.org/en-US/docs/Web/API/fetch#parameters + // Fortunately, Request's constructor handles the conversions, and Request#url is always a string. + // See https://developer.mozilla.org/en-US/docs/Web/API/Request/url + if (request.url.startsWith(ipfsScheme)) { + return fetchFromIpfs(request); + } else { + return fetchImpl(request); + } +} + +async function fetchFromIpfs(request) { + // Rewrite request URL to use Lassie + request = buildIpfsRequest(request); + + // Call Deno's `fetch` using the rewritten URL to make the actual HTTP request + const response = await fetchImpl(request); + + // Patch the response object to hide the fact that we are calling Lassie + // We don't want to leak Lassie's URL + return patchIpfsResponse(response); +} + +// Deno's Fetch Request is a thin immutable wrapper around InnerRequest. In order to modify the +// request URL, we must convert Request to InnerRequest first, make changes on the inner object, +// and finally convert the InnerRequest back to a new Request instance. +function buildIpfsRequest(request) { + const inner = toInnerRequest(request); + + inner.urlList = /** @type {(() => string)[]}*/ (inner.urlList).map((urlFn) => { + const url = urlFn(); + if (!url.startsWith(ipfsScheme)) return urlFn; + const newUrl = ipfsBaseUrl + url.slice(ipfsScheme.length); + return () => newUrl; + }); + inner.urlListProcessed = /** @type {string[]} */ (inner.urlListProcessed).map((url) => + url.startsWith(ipfsScheme) ? ipfsBaseUrl + url.slice(ipfsScheme.length) : url, + ); + + return fromInnerRequest(inner, request.signal, guardFromHeaders(request.headers)); +} + +// Deno's Fetch Response is a thin immutable wrapper around InnerResponse. In order to modify the +// response URL, we must convert Response to InnerResponse first, make changes on the inner object, +// and finally convert the InnerResponse back to a new Response instance. +function patchIpfsResponse(response) { + const inner = toInnerResponse(response); + + inner.urlList = /** @type {string[])} */ (inner.urlList).map((url) => + url.startsWith(ipfsBaseUrl) ? "ipfs://" + url.slice(ipfsBaseUrl.length) : url, + ); + + return fromInnerResponse(inner, guardFromHeaders(response.headers)); +} diff --git a/runtime/lib.rs b/runtime/lib.rs index f5d6fbd3..dc04255d 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -18,4 +18,6 @@ mod reporter; pub use console_reporter::*; pub use reporter::*; +pub use lassie; + mod ext; diff --git a/runtime/runtime.rs b/runtime/runtime.rs index 6dc7b43a..74eb8233 100644 --- a/runtime/runtime.rs +++ b/runtime/runtime.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; use std::rc::Rc; +use std::sync::Arc; use deno_core::{located_script_name, serde_json, JsRuntime, ModuleSpecifier, RuntimeOptions}; @@ -35,12 +36,17 @@ pub struct BootstrapOptions { /// Report activities pub reporter: Rc, + + /// Lassie daemon to use as the IPFS retrieval client. We must use Arc here to allow sharing of + /// the singleton Lassie instance between multiple threads spawned by Rust's test runner. + pub lassie_daemon: Arc, } impl BootstrapOptions { pub fn new( agent_version: String, reporter: Rc, + lassie_daemon: Arc, module_root: Option, ) -> Self { Self { @@ -52,6 +58,7 @@ impl BootstrapOptions { // See https://lotus.filecoin.io/lotus/manage/manage-fil/#public-key-address wallet_address: String::from("t1abjxfbp274xpdqcpuaykwkfb43omjotacm2p3za"), reporter, + lassie_daemon, } } @@ -60,6 +67,7 @@ impl BootstrapOptions { "noColor": self.no_color, "isTty": self.is_tty, "walletAddress": self.wallet_address, + "lassieUrl": format!("http://127.0.0.1:{}/", self.lassie_daemon.port()), }); serde_json::to_string_pretty(&payload).unwrap() } diff --git a/runtime/tests/fetch_api_tests.rs b/runtime/tests/fetch_api_tests.rs index 4362fc6a..9d548b2c 100644 --- a/runtime/tests/fetch_api_tests.rs +++ b/runtime/tests/fetch_api_tests.rs @@ -8,8 +8,12 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use zinnia_runtime::{anyhow, deno_core, run_js_module, BootstrapOptions, RecordingReporter}; +mod helpers; + #[tokio::test] async fn fetch_reports_user_agent() -> Result<()> { + let _ = env_logger::builder().is_test(true).try_init(); + let user_agent = "zinnia_fetch_api_tests agent/007"; let server_port = start_echo_server().await?; @@ -29,7 +33,12 @@ assertArrayIncludes(request_lines, ["user-agent: {user_agent}"]); &std::env::current_dir().context("unable to get current working directory")?, )?; let reporter = Rc::new(RecordingReporter::new()); - let config = BootstrapOptions::new(user_agent.into(), reporter.clone(), None); + let config = BootstrapOptions::new( + user_agent.into(), + reporter.clone(), + helpers::lassie_daemon(), + None, + ); run_js_module(&main_module, &config).await?; // the test passes when the JavaScript code does not throw Ok(()) diff --git a/runtime/tests/helpers/mod.rs b/runtime/tests/helpers/mod.rs new file mode 100644 index 00000000..ec9edc52 --- /dev/null +++ b/runtime/tests/helpers/mod.rs @@ -0,0 +1,14 @@ +use std::sync::{Arc, OnceLock}; + +pub fn lassie_daemon() -> Arc { + static LASSIE_DAEMON: OnceLock, lassie::StartError>> = + OnceLock::new(); + + let result = LASSIE_DAEMON + .get_or_init(|| lassie::Daemon::start(lassie::DaemonConfig::default()).map(Arc::new)); + + match result { + Ok(ptr) => Arc::clone(ptr), + Err(err) => panic!("could not start Lassie daemon: {err}"), + } +} diff --git a/runtime/tests/js/ipfs_retrieval_tests.js b/runtime/tests/js/ipfs_retrieval_tests.js new file mode 100644 index 00000000..08f6012b --- /dev/null +++ b/runtime/tests/js/ipfs_retrieval_tests.js @@ -0,0 +1,52 @@ +import { test } from "zinnia:test"; +import { assertEquals, AssertionError } from "zinnia:assert"; + +const EXPECTED_CAR_BASE64 = + "OqJlcm9vdHOB2CpYJQABcBIgO/KicpaH2Kj0sXyJNWLdY4kGpEe2mjY5zovBGRJ+6mpndmVyc2lvbgFrAXASIDvyonKWh9io9LF8iTVi3WOJBqRHtpo2Oc6LwRkSfupqCkUIAhI/TXkgbW9zdCBmYW1vdXMgZHJhd2luZywgYW5kIG9uZSBvZiB0aGUgZmlyc3QgSSBkaWQgZm9yIHRoZSBzaXRlGD8="; + +test("can retrieve CID content as a CAR file", async () => { + const requestUrl = "ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"; + const response = await fetch(requestUrl); + assertResponseIsOk(response); + + const payload = await response.arrayBuffer(); + assertEquals(payload.byteLength, 167, "CAR size in bytes"); + + const payload_encoded = btoa(String.fromCharCode(...new Uint8Array(payload))); + assertEquals(payload_encoded, EXPECTED_CAR_BASE64); + + assertEquals(response.url, requestUrl); +}); + +test("can retrieve IPFS content using URL", async () => { + const requestUrl = new URL("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); + const response = await fetch(requestUrl); + assertResponseIsOk(response); + + const payload = await response.arrayBuffer(); + assertEquals(payload.byteLength, 167, "CAR size in bytes"); + + assertEquals(response.url, requestUrl.toString()); +}); + +test("can retrieve IPFS content using Fetch Request object", async () => { + const request = new Request("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); + const response = await fetch(request); + assertResponseIsOk(response); + + const payload = await response.arrayBuffer(); + assertEquals(payload.byteLength, 167, "CAR size in bytes"); + + assertEquals(response.url, request.url); +}); + +/** + * @param {Response} response Fetch API response + */ +async function assertResponseIsOk(response) { + if (!response.ok) { + throw new AssertionError( + `Fetch request failed with status code ${response.status}: ${await response.text()}`, + ); + } +} diff --git a/runtime/tests/runtime_integration_tests.rs b/runtime/tests/runtime_integration_tests.rs index cc42c137..2b8839b1 100644 --- a/runtime/tests/runtime_integration_tests.rs +++ b/runtime/tests/runtime_integration_tests.rs @@ -14,6 +14,8 @@ use zinnia_runtime::{anyhow, deno_core, run_js_module, AnyError, BootstrapOption use pretty_assertions::assert_eq; +mod helpers; + macro_rules! js_tests( ( $name:ident ) => { #[tokio::test] @@ -87,12 +89,15 @@ js_tests!(libp2p_tests); js_tests!(station_apis_tests); js_tests!(station_reporting_tests check_activity); js_tests!(module_loader_tests); +js_tests!(ipfs_retrieval_tests); test_runner_tests!(passing_tests); test_runner_tests!(failing_tests expect_failure); // Run all tests in a single JS file async fn run_js_test_file(name: &str) -> Result<(Vec, Option), AnyError> { + let _ = env_logger::builder().is_test(true).try_init(); + let mut full_path = get_base_dir(); full_path.push(name); @@ -104,6 +109,7 @@ async fn run_js_test_file(name: &str) -> Result<(Vec, Option), let config = BootstrapOptions::new( format!("zinnia_runtime_tests/{}", env!("CARGO_PKG_VERSION")), reporter.clone(), + helpers::lassie_daemon(), None, ); let run_result = run_js_module(&main_module, &config).await;