Use TOML in examples (#4553)

### What

Closes #3355
Closes #2431

- [x] Convert all example `README.md` frontmatter to use TOML instead of
YAML
- [x] Convert `manifest.yml` to a `.toml` file
- [x] Support reading TOML frontmatter everywhere (linting,
`thumbnails.py`, `re_build_examples`)
- [x] Simplify both `README.md` frontmatter and the manifest file by
inferring more information from the filesystem
- [x] Use HTML comments instead of `---` for delimiters
- [x] Update `landing` to support TOML and new manifest/README formats
  - rerun-io/landing#594

After merging this PR, merge [the PR in
landing](rerun-io/landing#594) by following the
instructions in the PR body there.

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using newly built examples:
[app.rerun.io](https://app.rerun.io/pr/4553/index.html)
* Using examples from latest `main` build:
[app.rerun.io](https://app.rerun.io/pr/4553/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[app.rerun.io](https://app.rerun.io/pr/4553/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG

- [PR Build Summary](https://build.rerun.io/pr/4553)
- [Docs
preview](https://rerun.io/preview/7588e64e5e2f3676ebeb40fe75a3e8185da0b9e9/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/7588e64e5e2f3676ebeb40fe75a3e8185da0b9e9/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

---------

Co-authored-by: Emil Ernerfeldt <emil.ernerfeldt@gmail.com>

.github/workflows/contrib_checks.yml

@@ -230,7 +230,7 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: |
-          pip install gitignore_parser python-frontmatter
+          pip install gitignore_parser
           pip install -r ./scripts/ci/requirements.txt
 
       - name: Rerun lints

.github/workflows/documentation.yaml

@@ -35,7 +35,8 @@ jobs:
 
           # Avoid crates.io rate-limiting, skip changelog PR links (so many), and skip speculative links
           # TODO(#4085): https://rerun-io.github.io/rerun/dev/bench/ often 404:s for unknown reasons
-          linksToSkip: "https://crates.io/crates/.*, https://github.com/rerun-io/rerun/pull/.*, .*?speculative-link, https://rerun-io.github.io/rerun/dev/bench/"
+          # TODO(#4556): remove the `static.rerun.io` and `github.com` skips
+          linksToSkip: "https://static.rerun.io/.*, https://github.com/.*, https://crates.io/crates/.*, https://github.com/rerun-io/rerun/pull/.*, .*?speculative-link, https://rerun-io.github.io/rerun/dev/bench/"
           retry: true
           retryErrors: true
           retryErrorsCount: 5

.github/workflows/reusable_checks.yml

@@ -306,7 +306,7 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: |
-          pip install gitignore_parser python-frontmatter
+          pip install gitignore_parser
           pip install -r ./scripts/ci/requirements.txt
 
       - name: Rerun lints

Cargo.toml

@@ -192,7 +192,6 @@ serde = "1"
 serde_bytes = "0.11"
 serde_json = { version = "1", default-features = false, features = ["std"] }
 serde_test = "1"
-serde_yaml = { version = "0.9.21", default-features = false }
 sha2 = "0.10"
 similar-asserts = "1.4.2"
 slotmap = { version = "1.0.6", features = ["serde"] }
@@ -213,6 +212,7 @@ tinyvec = { version = "1.6", features = ["alloc", "rustc_1_55"] }
 tobj = "4.0"
 tokio = { version = "1.24", default-features = false }
 tokio-tungstenite = { version = "0.17.1", default-features = false }
+toml = { version = "0.7.8", default-features = false }
 tracing = { version = "0.1", default-features = false }
 tungstenite = { version = "0.17", default-features = false }
 type-map = "0.5"

crates/re_build_examples/Cargo.toml

@@ -26,5 +26,5 @@ argh.workspace = true
 indicatif.workspace = true
 rayon.workspace = true
 serde = { workspace = true, features = ["derive"] }
-serde_yaml.workspace = true
 serde_json.workspace = true
+toml = { workspace = true, features = ["parse", "preserve_order"] }

crates/re_build_examples/src/example.rs

@@ -154,22 +154,28 @@ struct Frontmatter {
 }
 
 fn parse_frontmatter<P: AsRef<Path>>(path: P) -> anyhow::Result<Option<Frontmatter>> {
+    const START: &str = "<!--[metadata]";
+    const END: &str = "-->";
+
     let path = path.as_ref();
     let content = std::fs::read_to_string(path)?;
-    let content = content.replace('\r', ""); // Windows, god damn you
-    re_build_tools::rerun_if_changed(path);
-    let Some(content) = content.strip_prefix("---\n") else {
+
+    let Some(start) = content.find(START) else {
         return Ok(None);
     };
-    let Some(end) = content.find("---") else {
-        anyhow::bail!("{:?} has invalid frontmatter: missing --- terminator", path);
+    let start = start + START.len();
+
+    let Some(end) = content[start..].find(END) else {
+        anyhow::bail!("{:?} has invalid frontmatter: missing --> terminator", path);
     };
-    Ok(Some(serde_yaml::from_str(&content[..end]).map_err(
-        |err| {
+    let end = start + end;
+
+    toml::from_str(content[start..end].trim())
+        .map(Some)
+        .map_err(|err| {
             anyhow::anyhow!(
-                "failed to read {:?}: {err}",
+                "Failed to parse TOML metadata of {:?}: {err}",
                 path.parent().unwrap().file_name().unwrap()
             )
-        },
-    )?))
+        })
 }

examples/README.md

@@ -41,12 +41,12 @@ tags: [2D, huggingface, object-detection, object-tracking, opencv]
 
 The contents of this `README.md` file and its frontmatter are used to render the examples in [the documentation](https://rerun.io/examples). Individual examples are currently "stitched together" to form one large markdown file for every category of examples (`artificial-data`, `real-data`).
 
-The `manifest.yml` file describes the structure of the examples contained in this repository. Only the examples which appear in the manifest are included in the [generated documentation](https://rerun.io/examples). The file contains a description of its own format.
+The `manifest.toml` file describes the structure of the examples contained in this repository. Only the examples which appear in the manifest are included in the [generated documentation](https://rerun.io/examples). The file contains a description of its own format.
 
 ## Adding a new example
 
 You can base your example off of `python/template` or `rust/template`.
-Once it's ready to be displayed in the docs, add it to the [manifest](./manifest.yml).
+Once it's ready to be displayed in the docs, add it to the [manifest](./manifest.toml).
 
 If you want to run the example on CI and include it in the in-viewer example page,
 add a `channel` entry to its README frontmatter. The available channels right now are:

examples/c/spawn_viewer/README.md

@@ -1,10 +1,8 @@
----
-title: Spawn Viewer
-tags: [spawn]
-c: https://github.com/rerun-io/rerun/tree/latest/examples/c/spawn_viewer/main.c
-cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/spawn_viewer/main.cpp
-rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/spawn_viewer/src/main.rs
----
+<!--[metadata]
+title = "Spawn Viewer"
+tags = ["spawn"]
+-->
+
 
 Shows how to spawn a new Rerun Viewer process ready to listen for TCP connections using an executable available in PATH.
 

examples/cpp/clock/README.md

@@ -1,10 +1,8 @@
----
-title: Clock
-python: https://github.com/rerun-io/rerun/tree/latest/examples/python/clock/main.py
-rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/clock/src/main.rs
-cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/clock/main.cpp
-thumbnail: https://static.rerun.io/clock/ae4b8970edba8480431cb71e57b8cddd9e1769c7/480w.png
----
+<!--[metadata]
+title = "Clock"
+thumbnail = "https://static.rerun.io/clock/ae4b8970edba8480431cb71e57b8cddd9e1769c7/480w.png"
+-->
+
 
 <picture>
   <source media="(max-width: 480px)" srcset="https://static.rerun.io/clock/05e69dc20c9a28005f1ffe7f0f2ac9eeaa95ba3b/480w.png">

examples/cpp/dna/README.md

@@ -1,14 +1,12 @@
----
-title: Helix
-python: https://github.com/rerun-io/rerun/tree/latest/examples/python/dna/main.py
-rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/dna/src/main.rs
-cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/dna/main.cpp
-tags: [3d, api-example]
-description: "Simple example of logging point and line primitives to draw a 3D helix."
-thumbnail: https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png
-thumbnail_dimensions: [480, 285]
-channel: main
----
+<!--[metadata]
+title = "Helix"
+tags = ["3d", "api-example"]
+description = "Simple example of logging point and line primitives to draw a 3D helix."
+thumbnail = "https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png"
+thumbnail_dimensions = [480, 285]
+channel = "main"
+-->
+
 
 <picture>
   <source media="(max-width: 480px)" srcset="https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png">

examples/cpp/eigen_opencv/README.md

@@ -1,10 +1,11 @@
----
-title: "Eigen and OpenCV C++ Integration"
-cpp: https://github.com/rerun-io/cpp-example-opencv-eigen
-tags: [2D, 3D, C++, Eigen, OpenCV]
-thumbnail: https://static.rerun.io/cpp-example-opencv-eigen/2fc6355fd87fbb4d07cda384ee8805edb68b5e01/480w.png
-thumbnail_dimensions: [480, 267]
----
+<!--[metadata]
+title = "Eigen and OpenCV C++ Integration"
+source = "https://github.com/rerun-io/cpp-example-opencv-eigen"
+tags = ["2D", "3D", "C++", "Eigen", "OpenCV"]
+thumbnail = "https://static.rerun.io/cpp-example-opencv-eigen/2fc6355fd87fbb4d07cda384ee8805edb68b5e01/480w.png"
+thumbnail_dimensions = [480, 267]
+-->
+
 
 This is a minimal CMake project that shows how to use Rerun in your code in conjunction with [Eigen](https://eigen.tuxfamily.org/) and [OpenCV](https://opencv.org/).
 

examples/cpp/minimal/README.md

@@ -1,10 +1,8 @@
----
-title: Minimal example
-python: https://github.com/rerun-io/rerun/tree/latest/examples/python/minimal/main.py
-rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/minimal/src/main.rs
-cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/minimal/main.cpp
-thumbnail: https://static.rerun.io/minimal/0e47ac513ab25d56cf2b493128097d499a07e5e8/480w.png
----
+<!--[metadata]
+title = "Minimal example"
+thumbnail = "https://static.rerun.io/minimal/0e47ac513ab25d56cf2b493128097d499a07e5e8/480w.png"
+-->
+
 
 <picture>
   <source media="(max-width: 480px)" srcset="https://static.rerun.io/minimal/0e47ac513ab25d56cf2b493128097d499a07e5e8/480w.png">

examples/cpp/shared_recording/README.md

@@ -1,8 +1,7 @@
----
-title: Shared Recording
-rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/shared_recording/src/main.rs?speculative-link
-cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/shared_recording/main.cpp?speculative-link
----
+<!--[metadata]
+title = "Shared Recording"
+-->
+
 
 <picture>
   <img src="https://static.rerun.io/shared_recording/c3da85f1d4c158b8c7afb6bd3278db000b58049d/full.png" alt="">

examples/cpp/spawn_viewer/README.md

@@ -1,10 +1,8 @@
----
-title: Spawn Viewer
-tags: [spawn]
-c: https://github.com/rerun-io/rerun/tree/latest/examples/c/spawn_viewer/main.c
-cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/spawn_viewer/main.cpp
-rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/spawn_viewer/src/main.rs
----
+<!--[metadata]
+title = "Spawn Viewer"
+tags = ["spawn"]
+-->
+
 
 Shows how to spawn a new Rerun Viewer process ready to listen for TCP connections using an executable available in PATH.
 

examples/cpp/stdio/README.md

@@ -1,10 +1,8 @@
----
-title: Standard Input/Output example
-python: https://github.com/rerun-io/rerun/tree/latest/examples/python/stdio/main.py?speculative-link
-rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/stdio/src/main.rs?speculative-link
-cpp: https://github.com/rerun-io/rerun/tree/latest/examples/cpp/stdio/main.cpp?speculative-link
-thumbnail: https://static.rerun.io/stdio/25c5aba992d4c8b3861386d8d9539a4823dca117/480w.png
----
+<!--[metadata]
+title = "Standard Input/Output example"
+thumbnail = "https://static.rerun.io/stdio/25c5aba992d4c8b3861386d8d9539a4823dca117/480w.png"
+-->
+
 
 <picture>
   <img src="https://static.rerun.io/stdio/25c5aba992d4c8b3861386d8d9539a4823dca117/full.png" alt="">

examples/cpp/vrs/README.md

@@ -1,10 +1,11 @@
----
-title: "VRS Viewer"
-cpp: https://github.com/rerun-io/cpp-example-vrs
-tags: [2D, 3D, vrs, viewer, C++]
-thumbnail: https://static.rerun.io/cpp-example-vrs/c765460d4448da27bb9ee2a2a15f092f82a402d2/480w.png
-thumbnail_dimensions: [480, 286]
----
+<!--[metadata]
+title = "VRS Viewer"
+source = "https://github.com/rerun-io/cpp-example-vrs"
+tags = ["2D", "3D", "vrs", "viewer", "C++"]
+thumbnail = "https://static.rerun.io/cpp-example-vrs/c765460d4448da27bb9ee2a2a15f092f82a402d2/480w.png"
+thumbnail_dimensions = [480, 286]
+-->
+
 
 This is an example that shows how to use [Rerun](https://github.com/rerun-io/rerun)'s C++ API to log and view [VRS](https://github.com/facebookresearch/vrs) files.