From 8ebaf8dd833f64a1104156cea7ce5bd08e3af899 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 15 Dec 2023 17:37:17 +0100 Subject: [PATCH] `DataLoader`s 5: add support for external binary `DataLoader`s (PATH) (#4521) Turn any executable in your `PATH` into a `DataLoader` by logging data to stdout. Includes examples for all supported languages. ![image](https://github.com/rerun-io/rerun/assets/2910679/82391961-9ac1-4bd8-bdbe-a7378ed7a3fa) Checks: - [x] `rerun-loader-rust-file` just prints help message - [x] `rerun-loader-rust-file examples/rust/external_data_loader/src/main.rs` successfully does nothing - [x] `rerun-loader-rust-file examples/rust/external_data_loader/src/main.rs | rerun -` works - [x] `rerun-loader-python-file` just prints help message - [x] `rerun-loader-python-file examples/python/external_data_loader/main.py` successfully does nothing - [x] `rerun-loader-python-file examples/python/external_data_loader/main.py | rerun -` works - [x] `rerun-loader-cpp-file` just prints help message - [x] `rerun-loader-cpp-file examples/cpp/external_data_loader/main.cpp` successfully does nothing - [x] `rerun-loader-cpp-file examples/cpp/external_data_loader/main.cpp | rerun -` works - [x] `cargo r -p rerun-cli --no-default-features --features native_viewer -- examples/assets` - [x] Native: `File > Open > examples/assets/` - [x] Native: `Drag-n-drop > examples/assets/` --- Part of a series of PRs to make it possible to load _any_ file from the local filesystem, by any means, on web and native: - #4516 - #4517 - #4518 - #4519 - #4520 - #4521 - TODO: register custom loaders - TODO: high level docs and guides for everything related to loading files --- Cargo.lock | 8 + .../src/data_loader/loader_external.rs | 190 ++++++++++++++++++ crates/re_data_source/src/data_loader/mod.rs | 20 +- crates/re_data_source/src/lib.rs | 6 +- crates/re_data_source/src/load_file.rs | 9 +- crates/re_ui/src/command.rs | 2 +- crates/re_viewer/src/app.rs | 20 +- examples/assets/example.cpp | 4 + examples/assets/example.py | 4 + examples/assets/example.rs | 4 + examples/cpp/CMakeLists.txt | 2 + .../cpp/external_data_loader/CMakeLists.txt | 32 +++ examples/cpp/external_data_loader/README.md | 20 ++ examples/cpp/external_data_loader/main.cpp | 82 ++++++++ .../python/external_data_loader/README.md | 21 ++ examples/python/external_data_loader/main.py | 50 +++++ .../external_data_loader/requirements.txt | 1 + examples/python/requirements.txt | 1 + examples/rust/external_data_loader/Cargo.toml | 12 ++ examples/rust/external_data_loader/README.md | 20 ++ .../rust/external_data_loader/src/main.rs | 59 ++++++ 21 files changed, 557 insertions(+), 10 deletions(-) create mode 100644 crates/re_data_source/src/data_loader/loader_external.rs create mode 100644 examples/assets/example.cpp create mode 100644 examples/assets/example.py create mode 100644 examples/assets/example.rs create mode 100644 examples/cpp/external_data_loader/CMakeLists.txt create mode 100644 examples/cpp/external_data_loader/README.md create mode 100644 examples/cpp/external_data_loader/main.cpp create mode 100644 examples/python/external_data_loader/README.md create mode 100755 examples/python/external_data_loader/main.py create mode 100644 examples/python/external_data_loader/requirements.txt create mode 100644 examples/rust/external_data_loader/Cargo.toml create mode 100644 examples/rust/external_data_loader/README.md create mode 100644 examples/rust/external_data_loader/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 56a94c0db844..e186056d4287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5506,6 +5506,14 @@ dependencies = [ "tokio", ] +[[package]] +name = "rerun-loader-rust-file" +version = "0.12.0-alpha.1+dev" +dependencies = [ + "argh", + "rerun", +] + [[package]] name = "rerun_c" version = "0.12.0-alpha.1+dev" diff --git a/crates/re_data_source/src/data_loader/loader_external.rs b/crates/re_data_source/src/data_loader/loader_external.rs new file mode 100644 index 000000000000..715164b8de00 --- /dev/null +++ b/crates/re_data_source/src/data_loader/loader_external.rs @@ -0,0 +1,190 @@ +use std::io::Read; + +use once_cell::sync::Lazy; + +/// To register a new external data loader, simply add an executable in your $PATH whose name +/// starts with this prefix. +pub const EXTERNAL_DATA_LOADER_PREFIX: &str = "rerun-loader-"; + +/// Keeps track of the paths all external executable [`crate::DataLoader`]s. +/// +/// Lazy initialized the first time a file is opened by running a full scan of the `$PATH`. +/// +/// External loaders are _not_ registered on a per-extension basis: we want users to be able to +/// filter data on a much more fine-grained basis that just file extensions (e.g. checking the file +/// itself for magic bytes). +pub static EXTERNAL_LOADER_PATHS: Lazy> = Lazy::new(|| { + re_tracing::profile_function!(); + + use walkdir::WalkDir; + + let dirpaths = std::env::var("PATH") + .ok() + .into_iter() + .flat_map(|paths| paths.split(':').map(ToOwned::to_owned).collect::>()) + .map(std::path::PathBuf::from); + + let executables: ahash::HashSet<_> = dirpaths + .into_iter() + .flat_map(|dirpath| { + WalkDir::new(dirpath).into_iter().filter_map(|entry| { + let Ok(entry) = entry else { + return None; + }; + let filepath = entry.path(); + let is_rerun_loader = filepath.file_name().map_or(false, |filename| { + filename + .to_string_lossy() + .starts_with(EXTERNAL_DATA_LOADER_PREFIX) + }); + (filepath.is_file() && is_rerun_loader).then(|| filepath.to_owned()) + }) + }) + .collect(); + + // NOTE: We call all available loaders and do so in parallel: order is irrelevant here. + executables.into_iter().collect() +}); + +/// Iterator over all registered external [`crate::DataLoader`]s. +#[inline] +pub fn iter_external_loaders() -> impl ExactSizeIterator { + EXTERNAL_LOADER_PATHS.iter().cloned() +} + +// --- + +/// A [`crate::DataLoader`] that forwards the path to load to all executables present in +/// the user's `PATH` with a name that starts with `EXTERNAL_DATA_LOADER_PREFIX`. +/// +/// The external loaders are expected to log rrd data to their standard output. +/// +/// Refer to our `external_data_loader` example for more information. +pub struct ExternalLoader; + +impl crate::DataLoader for ExternalLoader { + #[inline] + fn name(&self) -> String { + "rerun.data_loaders.External".into() + } + + fn load_from_path( + &self, + store_id: re_log_types::StoreId, + filepath: std::path::PathBuf, + tx: std::sync::mpsc::Sender, + ) -> Result<(), crate::DataLoaderError> { + use std::process::{Command, Stdio}; + + re_tracing::profile_function!(filepath.display().to_string()); + + for exe in EXTERNAL_LOADER_PATHS.iter() { + let store_id = store_id.clone(); + let filepath = filepath.clone(); + let tx = tx.clone(); + + // NOTE: spawn is fine, the entire loader is native-only. + rayon::spawn(move || { + re_tracing::profile_function!(); + + let child = Command::new(exe) + .arg(filepath.clone()) + .args(["--recording-id".to_owned(), store_id.to_string()]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + + let mut child = match child { + Ok(child) => child, + Err(err) => { + re_log::error!(?filepath, loader = ?exe, %err, "Failed to execute external loader"); + return; + } + }; + + let Some(stdout) = child.stdout.take() else { + let reason = "stdout unreachable"; + re_log::error!(?filepath, loader = ?exe, %reason, "Failed to execute external loader"); + return; + }; + let Some(stderr) = child.stderr.take() else { + let reason = "stderr unreachable"; + re_log::error!(?filepath, loader = ?exe, %reason, "Failed to execute external loader"); + return; + }; + + re_log::debug!(?filepath, loader = ?exe, "Loading data from filesystem using external loader…",); + + let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let stdout = std::io::BufReader::new(stdout); + match re_log_encoding::decoder::Decoder::new(version_policy, stdout) { + Ok(decoder) => { + decode_and_stream(&filepath, &tx, decoder); + } + Err(re_log_encoding::decoder::DecodeError::Read(_)) => { + // The child was not interested in that file and left without logging + // anything. + // That's fine, we just need to make sure to check its exit status further + // down, still. + return; + } + Err(err) => { + re_log::error!(?filepath, loader = ?exe, %err, "Failed to decode external loader's output"); + return; + } + }; + + let status = match child.wait() { + Ok(output) => output, + Err(err) => { + re_log::error!(?filepath, loader = ?exe, %err, "Failed to execute external loader"); + return; + } + }; + + if !status.success() { + let mut stderr = std::io::BufReader::new(stderr); + let mut reason = String::new(); + stderr.read_to_string(&mut reason).ok(); + re_log::error!(?filepath, loader = ?exe, %reason, "Failed to execute external loader"); + } + }); + } + + Ok(()) + } + + #[inline] + fn load_from_file_contents( + &self, + _store_id: re_log_types::StoreId, + _path: std::path::PathBuf, + _contents: std::borrow::Cow<'_, [u8]>, + _tx: std::sync::mpsc::Sender, + ) -> Result<(), crate::DataLoaderError> { + // TODO(cmc): You could imagine a world where plugins can be streamed rrd data via their + // standard input… but today is not world. + Ok(()) // simply not interested + } +} + +fn decode_and_stream( + filepath: &std::path::Path, + tx: &std::sync::mpsc::Sender, + decoder: re_log_encoding::decoder::Decoder, +) { + re_tracing::profile_function!(filepath.display().to_string()); + + for msg in decoder { + let msg = match msg { + Ok(msg) => msg, + Err(err) => { + re_log::warn_once!("Failed to decode message in {filepath:?}: {err}"); + continue; + } + }; + if tx.send(msg.into()).is_err() { + break; // The other end has decided to hang up, not our problem. + } + } +} diff --git a/crates/re_data_source/src/data_loader/mod.rs b/crates/re_data_source/src/data_loader/mod.rs index 84eee2cc2676..5d7c239488fd 100644 --- a/crates/re_data_source/src/data_loader/mod.rs +++ b/crates/re_data_source/src/data_loader/mod.rs @@ -22,11 +22,14 @@ use re_log_types::{ArrowMsg, DataRow, LogMsg}; /// [There are plans to make this generic over any URI](https://github.com/rerun-io/rerun/issues/4525). /// /// Rerun comes with a few [`DataLoader`]s by default: -/// - [`RrdLoader`] for [Rerun files], +/// - [`RrdLoader`] for [Rerun files]. /// - [`ArchetypeLoader`] for: /// - [3D models] /// - [Images] +/// - [Point clouds] /// - [Text files] +/// - [`DirectoryLoader`] for recursively loading folders. +/// - [`ExternalLoader`], which looks for user-defined data loaders in $PATH. /// /// ## Execution /// @@ -36,9 +39,10 @@ use re_log_types::{ArrowMsg, DataRow, LogMsg}; /// /// On native, [`DataLoader`]s are executed in parallel. /// -/// [Rerun extensions]: crate::SUPPORTED_RERUN_EXTENSIONS +/// [Rerun files]: crate::SUPPORTED_RERUN_EXTENSIONS /// [3D models]: crate::SUPPORTED_MESH_EXTENSIONS /// [Images]: crate::SUPPORTED_IMAGE_EXTENSIONS +/// [Point clouds]: crate::SUPPORTED_POINT_CLOUD_EXTENSIONS /// [Text files]: crate::SUPPORTED_TEXT_EXTENSIONS // // TODO(#4525): `DataLoader`s should support arbitrary URIs @@ -206,6 +210,8 @@ static BUILTIN_LOADERS: Lazy>> = Lazy::new(|| { Arc::new(RrdLoader) as Arc, Arc::new(ArchetypeLoader), Arc::new(DirectoryLoader), + #[cfg(not(target_arch = "wasm32"))] + Arc::new(ExternalLoader), ] }); @@ -221,6 +227,16 @@ mod loader_archetype; mod loader_directory; mod loader_rrd; +#[cfg(not(target_arch = "wasm32"))] +mod loader_external; + pub use self::loader_archetype::ArchetypeLoader; pub use self::loader_directory::DirectoryLoader; pub use self::loader_rrd::RrdLoader; + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) use self::loader_external::EXTERNAL_LOADER_PATHS; +#[cfg(not(target_arch = "wasm32"))] +pub use self::loader_external::{ + iter_external_loaders, ExternalLoader, EXTERNAL_DATA_LOADER_PREFIX, +}; diff --git a/crates/re_data_source/src/lib.rs b/crates/re_data_source/src/lib.rs index 014f225d9906..3cfa30a8dcf8 100644 --- a/crates/re_data_source/src/lib.rs +++ b/crates/re_data_source/src/lib.rs @@ -15,12 +15,16 @@ mod web_sockets; mod load_stdin; pub use self::data_loader::{ - iter_loaders, ArchetypeLoader, DataLoader, DataLoaderError, LoadedData, RrdLoader, + iter_loaders, ArchetypeLoader, DataLoader, DataLoaderError, DirectoryLoader, LoadedData, + RrdLoader, }; pub use self::data_source::DataSource; pub use self::load_file::{extension, load_from_file_contents}; pub use self::web_sockets::connect_to_ws_url; +#[cfg(not(target_arch = "wasm32"))] +pub use self::data_loader::{iter_external_loaders, ExternalLoader}; + #[cfg(not(target_arch = "wasm32"))] pub use self::load_file::load_from_path; diff --git a/crates/re_data_source/src/load_file.rs b/crates/re_data_source/src/load_file.rs index 1603e6b7d0b4..1c0e7e2736bb 100644 --- a/crates/re_data_source/src/load_file.rs +++ b/crates/re_data_source/src/load_file.rs @@ -145,10 +145,17 @@ pub(crate) fn load( is_dir: bool, contents: Option>, ) -> Result, DataLoaderError> { + #[cfg(target_arch = "wasm32")] + let has_external_loaders = false; + #[cfg(not(target_arch = "wasm32"))] + let has_external_loaders = !crate::data_loader::EXTERNAL_LOADER_PATHS.is_empty(); + let extension = extension(path); let is_builtin = is_associated_with_builtin_loader(path, is_dir); - if !is_builtin { + // If there are no external loaders registered (which is always the case on wasm) and we don't + // have a builtin loader for it, then we know for a fact that we won't be able to load it. + if !is_builtin && !has_external_loaders { return if extension.is_empty() { Err(anyhow::anyhow!("files without extensions (file.XXX) are not supported").into()) } else { diff --git a/crates/re_ui/src/command.rs b/crates/re_ui/src/command.rs index 226d05156632..3b3710098c6f 100644 --- a/crates/re_ui/src/command.rs +++ b/crates/re_ui/src/command.rs @@ -86,7 +86,7 @@ impl UICommand { "Save data for the current loop selection to a Rerun data file (.rrd)", ), - UICommand::Open => ("Open…", "Open a Rerun Data File (.rrd)"), + UICommand::Open => ("Open…", "Open any supported files (.rrd, images, meshes, …)"), UICommand::CloseCurrentRecording => ( "Close current Recording", diff --git a/crates/re_viewer/src/app.rs b/crates/re_viewer/src/app.rs index 4e408ca2f1c5..cb7526bb035f 100644 --- a/crates/re_viewer/src/app.rs +++ b/crates/re_viewer/src/app.rs @@ -1297,11 +1297,21 @@ fn file_saver_progress_ui(egui_ctx: &egui::Context, background_tasks: &mut Backg #[cfg(not(target_arch = "wasm32"))] fn open_file_dialog_native() -> Vec { re_tracing::profile_function!(); - let supported: Vec<_> = re_data_source::supported_extensions().collect(); - rfd::FileDialog::new() - .add_filter("Supported files", &supported) - .pick_files() - .unwrap_or_default() + + let supported: Vec<_> = if re_data_source::iter_external_loaders().len() == 0 { + re_data_source::supported_extensions().collect() + } else { + vec![] + }; + + let mut dialog = rfd::FileDialog::new(); + + // If there's at least one external loader registered, then literally anything goes! + if !supported.is_empty() { + dialog = dialog.add_filter("Supported files", &supported); + } + + dialog.pick_files().unwrap_or_default() } #[cfg(target_arch = "wasm32")] diff --git a/examples/assets/example.cpp b/examples/assets/example.cpp new file mode 100644 index 000000000000..64963695450a --- /dev/null +++ b/examples/assets/example.cpp @@ -0,0 +1,4 @@ +int main() { + std::cout << "That will only work with the right plugin in your $PATH!" << std::endl; + std::cout << "Checkout the `external_data_loader` C++ example." << std::endl; +} diff --git a/examples/assets/example.py b/examples/assets/example.py new file mode 100644 index 000000000000..d73ba49536bb --- /dev/null +++ b/examples/assets/example.py @@ -0,0 +1,4 @@ +from __future__ import annotations + +print("That will only work with the right plugin in your $PATH!") +print("Checkout the `external_data_loader` Python example.") diff --git a/examples/assets/example.rs b/examples/assets/example.rs new file mode 100644 index 000000000000..435ad612cccb --- /dev/null +++ b/examples/assets/example.rs @@ -0,0 +1,4 @@ +fn main() { + println!("That will only work with the right plugin in your $PATH!"); + println!("Checkout the `external_data_loader` Rust example."); +} diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index c603e3c35dff..4d553e0f6063 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -1,6 +1,7 @@ add_subdirectory(clock) add_subdirectory(custom_collection_adapter) add_subdirectory(dna) +add_subdirectory(external_data_loader) add_subdirectory(minimal) add_subdirectory(shared_recording) add_subdirectory(spawn_viewer) @@ -15,3 +16,4 @@ add_dependencies(examples example_minimal) add_dependencies(examples example_shared_recording) add_dependencies(examples example_spawn_viewer) add_dependencies(examples example_stdio) +add_dependencies(examples rerun-loader-cpp-file) diff --git a/examples/cpp/external_data_loader/CMakeLists.txt b/examples/cpp/external_data_loader/CMakeLists.txt new file mode 100644 index 000000000000..3cf12ff2f2af --- /dev/null +++ b/examples/cpp/external_data_loader/CMakeLists.txt @@ -0,0 +1,32 @@ +cmake_minimum_required(VERSION 3.16...3.27) + +# If you use the example outside of the Rerun SDK you need to specify +# where the rerun_c build is to be found by setting the `RERUN_CPP_URL` variable. +# This can be done by passing `-DRERUN_CPP_URL=` to cmake. +if(DEFINED RERUN_REPOSITORY) + add_executable(rerun-loader-cpp-file main.cpp) + rerun_strict_warning_settings(rerun-loader-cpp-file) +else() + project(rerun-loader-cpp-file LANGUAGES CXX) + + add_executable(rerun-loader-cpp-file main.cpp) + + # Set the path to the rerun_c build. + set(RERUN_CPP_URL "https://github.com/rerun-io/rerun/releases/latest/download/rerun_cpp_sdk.zip" CACHE STRING "URL to the rerun_cpp zip.") + option(RERUN_FIND_PACKAGE "Whether to use find_package to find a preinstalled rerun package (instead of using FetchContent)." OFF) + + if(RERUN_FIND_PACKAGE) + find_package(rerun_sdk REQUIRED) + else() + # Download the rerun_sdk + include(FetchContent) + FetchContent_Declare(rerun_sdk URL ${RERUN_CPP_URL}) + FetchContent_MakeAvailable(rerun_sdk) + endif() + + # Rerun requires at least C++17, but it should be compatible with newer versions. + set_property(TARGET rerun-loader-cpp-file PROPERTY CXX_STANDARD 17) +endif() + +# Link against rerun_sdk. +target_link_libraries(rerun-loader-cpp-file PRIVATE rerun_sdk) diff --git a/examples/cpp/external_data_loader/README.md b/examples/cpp/external_data_loader/README.md new file mode 100644 index 000000000000..b03990f09d55 --- /dev/null +++ b/examples/cpp/external_data_loader/README.md @@ -0,0 +1,20 @@ +--- +title: External data-loader example +python: https://github.com/rerun-io/rerun/tree/latest/examples/python/external_data_loader/main.py?speculative-link +rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/external_data_loader/src/main.rs?speculative-link +cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/external_data_loader/main.cpp?speculative-link +thumbnail: https://static.rerun.io/external_data_loader_cpp/83cd3c2a322911cf597cf74aeda01c8fe83e275f/480w.png +--- + + + + + + + + + +This is an example executable data-loader plugin for the Rerun Viewer. + +It will log C++ source code files as markdown documents. +To try it out, compile it and place it in your $PATH, then open a C++ source file with Rerun (`rerun file.cpp`). diff --git a/examples/cpp/external_data_loader/main.cpp b/examples/cpp/external_data_loader/main.cpp new file mode 100644 index 000000000000..bffcf5f7af49 --- /dev/null +++ b/examples/cpp/external_data_loader/main.cpp @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +#include + +static const char* USAGE = R"( +This is an example executable data-loader plugin for the Rerun Viewer. + +It will log C++ source code files as markdown documents. +To try it out, compile it and place it in your $PATH, then open a C++ source file with Rerun (`rerun file.cpp`). + +USAGE: + rerun-loader-cpp-file [OPTIONS] FILEPATH + +FLAGS: + -h, --help Prints help information + +OPTIONS: + --recording-id RECORDING_ID ID of the shared recording + +ARGS: + +)"; + +int main(int argc, char* argv[]) { + // The Rerun Viewer will always pass these two pieces of information: + // 1. The path to be loaded, as a positional arg. + // 2. A shared recording ID, via the `--recording-id` flag. + // + // It is up to you whether you make use of that shared recording ID or not. + // If you use it, the data will end up in the same recording as all other plugins interested in + // that file, otherwise you can just create a dedicated recording for it. Or both. + std::string filepath; + std::string recording_id; + + for (int i = 1; i < argc; ++i) { + std::string arg(argv[i]); + + if (arg == "--recording-id") { + if (i + 1 < argc) { + recording_id = argv[i + 1]; + ++i; + } else { + std::cerr << USAGE << std::endl; + return 1; + } + } else { + filepath = arg; + } + } + + if (filepath.empty()) { + std::cerr << USAGE << std::endl; + return 1; + } + + bool is_file = std::filesystem::is_regular_file(filepath); + bool is_cpp_file = std::filesystem::path(filepath).extension().string() == ".cpp"; + + // We're not interested: just exit silently. + // Don't return an error, as that would show up to the end user in the Rerun Viewer! + if (!(is_file && is_cpp_file)) { + return 0; + } + + std::ifstream file(filepath); + std::stringstream body; + body << file.rdbuf(); + + std::string text = "## Some C++ code\n```cpp\n" + body.str() + "\n```\n"; + + const auto rec = rerun::RecordingStream("rerun_example_external_data_loader", recording_id); + // The most important part of this: log to standard output so the Rerun Viewer can ingest it! + rec.to_stdout().exit_on_failure(); + + rec.log_timeless( + filepath, + rerun::TextDocument(text).with_media_type(rerun::MediaType::markdown()) + ); +} diff --git a/examples/python/external_data_loader/README.md b/examples/python/external_data_loader/README.md new file mode 100644 index 000000000000..789262dae001 --- /dev/null +++ b/examples/python/external_data_loader/README.md @@ -0,0 +1,21 @@ +--- +title: External data-loader example +python: https://github.com/rerun-io/rerun/tree/latest/examples/python/external_data_loader/main.py?speculative-link +rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/external_data_loader/src/main.rs?speculative-link +cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/external_data_loader/main.cpp?speculative-link +thumbnail: https://static.rerun.io/external_data_loader_py/6c5609f5dd7d1de373c81babe19221b72d616da3/480w.png +thumbnail_dimensions: [480, 302] +--- + + + + + + + + + +This is an example executable data-loader plugin for the Rerun Viewer. + +It will log Python source code files as markdown documents. +To try it out, copy it in your $PATH as `rerun-loader-python-file`, then open a Python source file with Rerun (`rerun file.py`). diff --git a/examples/python/external_data_loader/main.py b/examples/python/external_data_loader/main.py new file mode 100755 index 000000000000..19363fe09529 --- /dev/null +++ b/examples/python/external_data_loader/main.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Example of an executable data-loader plugin for the Rerun Viewer.""" +from __future__ import annotations + +import argparse +import os + +import rerun as rr # pip install rerun-sdk + +# The Rerun Viewer will always pass these two pieces of information: +# 1. The path to be loaded, as a positional arg. +# 2. A shared recording ID, via the `--recording-id` flag. +# +# It is up to you whether you make use of that shared recording ID or not. +# If you use it, the data will end up in the same recording as all other plugins interested in +# that file, otherwise you can just create a dedicated recording for it. Or both. +parser = argparse.ArgumentParser( + description=""" +This is an example executable data-loader plugin for the Rerun Viewer. + +It will log Python source code files as markdown documents. +To try it out, copy it in your $PATH as `rerun-loader-python-file`, then open a Python source file with Rerun (`rerun file.py`). +""" +) +parser.add_argument("filepath", type=str) +parser.add_argument("--recording-id", type=str) +args = parser.parse_args() + + +def main() -> None: + is_file = os.path.isfile(args.filepath) + is_python_file = os.path.splitext(args.filepath)[1].lower() == ".py" + + # We're not interested: just exit silently. + # Don't return an error, as that would show up to the end user in the Rerun Viewer! + if not (is_file and is_python_file): + return + + rr.init("rerun_example_external_data_loader", recording_id=args.recording_id) + # The most important part of this: log to standard output so the Rerun Viewer can ingest it! + rr.stdout() + + with open(args.filepath) as file: + body = file.read() + text = f"""## Some Python code\n```python\n{body}\n```\n""" + rr.log(args.filepath, rr.TextDocument(text, media_type=rr.MediaType.MARKDOWN), timeless=True) + + +if __name__ == "__main__": + main() diff --git a/examples/python/external_data_loader/requirements.txt b/examples/python/external_data_loader/requirements.txt new file mode 100644 index 000000000000..ebb847ff0d2d --- /dev/null +++ b/examples/python/external_data_loader/requirements.txt @@ -0,0 +1 @@ +rerun-sdk diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt index 2cad455095bc..e5ef01dbcd75 100644 --- a/examples/python/requirements.txt +++ b/examples/python/requirements.txt @@ -7,6 +7,7 @@ -r detect_and_track_objects/requirements.txt -r dicom_mri/requirements.txt -r dna/requirements.txt +-r external_data_loader/requirements.txt -r face_tracking/requirements.txt -r human_pose_tracking/requirements.txt -r lidar/requirements.txt diff --git a/examples/rust/external_data_loader/Cargo.toml b/examples/rust/external_data_loader/Cargo.toml new file mode 100644 index 000000000000..a03f442605fc --- /dev/null +++ b/examples/rust/external_data_loader/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "rerun-loader-rust-file" +version = "0.12.0-alpha.1+dev" +edition = "2021" +rust-version = "1.72" +license = "MIT OR Apache-2.0" +publish = false + +[dependencies] +rerun = { path = "../../../crates/rerun" } + +argh = "0.1" diff --git a/examples/rust/external_data_loader/README.md b/examples/rust/external_data_loader/README.md new file mode 100644 index 000000000000..7b0c88460c24 --- /dev/null +++ b/examples/rust/external_data_loader/README.md @@ -0,0 +1,20 @@ +--- +title: External data-loader example +python: https://github.com/rerun-io/rerun/tree/latest/examples/python/external_data_loader/main.py?speculative-link +rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/external_data_loader/src/main.rs?speculative-link +cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/external_data_loader/main.cpp?speculative-link +thumbnail: https://static.rerun.io/external_data_loader_rs/74eecea3b16fee7fab01045e3bfdd90ba6c59bc9/480w.png +--- + + + + + + + + + +This is an example executable data-loader plugin for the Rerun Viewer. + +It will log Rust source code files as markdown documents. +To try it out, install it in your $PATH (`cargo install --path . -f`), then open a Rust source file with Rerun (`rerun file.rs`). diff --git a/examples/rust/external_data_loader/src/main.rs b/examples/rust/external_data_loader/src/main.rs new file mode 100644 index 000000000000..6f04184cd25a --- /dev/null +++ b/examples/rust/external_data_loader/src/main.rs @@ -0,0 +1,59 @@ +//! Example of an external data-loader executable plugin for the Rerun Viewer. + +use rerun::{external::re_data_source::extension, MediaType}; + +// The Rerun Viewer will always pass these two pieces of information: +// 1. The path to be loaded, as a positional arg. +// 2. A shared recording ID, via the `--recording-id` flag. +// +// It is up to you whether you make use of that shared recording ID or not. +// If you use it, the data will end up in the same recording as all other plugins interested in +// that file, otherwise you can just create a dedicated recording for it. Or both. + +/// This is an example executable data-loader plugin for the Rerun Viewer. +/// +/// It will log Rust source code files as markdown documents. +/// To try it out, install it in your $PATH (`cargo install --path . -f`), then open +/// Rust source file with Rerun (`rerun file.rs`). +#[derive(argh::FromArgs)] +struct Args { + #[argh(positional)] + filepath: std::path::PathBuf, + + /// optional ID of the shared recording + #[argh(option)] + recording_id: Option, +} + +fn main() -> Result<(), Box> { + let args: Args = argh::from_env(); + + let is_file = args.filepath.is_file(); + let is_rust_file = extension(&args.filepath) == "rs"; + + // We're not interested: just exit silently. + // Don't return an error, as that would show up to the end user in the Rerun Viewer! + if !(is_file && is_rust_file) { + return Ok(()); + } + + let rec = { + let mut rec = rerun::RecordingStreamBuilder::new("rerun_example_external_data_loader"); + if let Some(recording_id) = args.recording_id { + rec = rec.recording_id(recording_id); + }; + + // The most important part of this: log to standard output so the Rerun Viewer can ingest it! + rec.stdout()? + }; + + let body = std::fs::read_to_string(&args.filepath)?; + let text = format!("## Some Rust code\n```rust\n{body}\n```\n"); + + rec.log_timeless( + rerun::EntityPath::from_file_path(&args.filepath), + &rerun::TextDocument::new(text).with_media_type(MediaType::MARKDOWN), + )?; + + Ok(()) +}