Test queue size latest latency

.github/workflows/ci.yml

@@ -316,6 +316,10 @@ jobs:
  dora stop --name ci-rust-dynamic --grace-duration 5s
  dora destroy
 
+ cargo build -p receive_data --release
+ # Run tests
+ dora start tests/queue_size_latest_data_rust/dataflow.yaml
+
  - name: "Test CLI (Python)"
  timeout-minutes: 30
  # fail-fast by using bash shell explictly
@@ -345,6 +349,9 @@ jobs:
  dora stop --name ci-python-dynamic --grace-duration 5s
  dora destroy
 
+ # Run tests
+ dora start ../tests/queue_size_latest_data_python/dataflow.yaml
+
  - name: "Test CLI (C)"
  timeout-minutes: 30
  # fail-fast by using bash shell explictly

Cargo.toml

@@ -23,6 +23,7 @@ members = [
  "examples/multiple-daemons/node",
  "examples/multiple-daemons/operator",
  "examples/multiple-daemons/sink",
+ "tests/queue_size_latest_data_rust/receive_data",
  "libraries/arrow-convert",
  "libraries/communication-layer/*",
  "libraries/core",

examples/camera/.gitignore

@@ -0,0 +1 @@
+*.pt

examples/camera/README.md

@@ -0,0 +1,9 @@
+# Quick example on how to use a camera
+
+Make sure to have, dora and pip installed.
+
+```bash
+dora up
+dora build dataflow.yml
+dora start dataflow.yml
+```

examples/camera/dataflow.yml

@@ -0,0 +1,20 @@
+nodes:
+ - id: camera
+ build: pip install ../../node-hub/opencv-video-capture
+ path: opencv-video-capture
+ inputs:
+ tick: dora/timer/millis/20
+ outputs:
+ - image
+ env:
+ CAPTURE_PATH: 0
+ IMAGE_WIDTH: 640
+ IMAGE_HEIGHT: 480
+
+ - id: plot
+ build: pip install ../../node-hub/opencv-plot
+ path: opencv-plot
+ inputs:
+ image:
+ source: camera/image
+ queue_size: 1

examples/camera/run.rs

@@ -0,0 +1,98 @@
+use dora_core::{get_pip_path, get_python_path, run};
+use dora_tracing::set_up_tracing;
+use eyre::{bail, ContextCompat, WrapErr};
+use std::path::Path;
+
+#[tokio::main]
+async fn main() -> eyre::Result<()> {
+ set_up_tracing("python-dataflow-runner")?;
+
+ let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+ std::env::set_current_dir(root.join(file!()).parent().unwrap())
+ .wrap_err("failed to set working dir")?;
+
+ run(
+ get_python_path().context("Could not get python binary")?,
+ &["-m", "venv", "../.env"],
+ None,
+ )
+ .await
+ .context("failed to create venv")?;
+ let venv = &root.join("examples").join(".env");
+ std::env::set_var(
+ "VIRTUAL_ENV",
+ venv.to_str().context("venv path not valid unicode")?,
+ );
+ let orig_path = std::env::var("PATH")?;
+ // bin folder is named Scripts on windows.
+ // 🤦‍♂️ See: https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1
+ let venv_bin = if cfg!(windows) {
+ venv.join("Scripts")
+ } else {
+ venv.join("bin")
+ };
+
+ if cfg!(windows) {
+ std::env::set_var(
+ "PATH",
+ format!(
+ "{};{orig_path}",
+ venv_bin.to_str().context("venv path not valid unicode")?
+ ),
+ );
+ } else {
+ std::env::set_var(
+ "PATH",
+ format!(
+ "{}:{orig_path}",
+ venv_bin.to_str().context("venv path not valid unicode")?
+ ),
+ );
+ }
+
+ run(
+ get_pip_path().context("Could not get pip binary")?,
+ &["install", "maturin"],
+ Some(venv),
+ )
+ .await
+ .context("pip install maturin failed")?;
+
+ run(
+ "maturin",
+ &["develop"],
+ Some(&root.join("apis").join("python").join("node")),
+ )
+ .await
+ .context("maturin develop failed")?;
+
+ let dataflow = Path::new("dataflow.yml");
+ run_dataflow(dataflow).await?;
+
+ Ok(())
+}
+
+async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
+ let cargo = std::env::var("CARGO").unwrap();
+
+ // First build the dataflow (install requirements)
+ let mut cmd = tokio::process::Command::new(&cargo);
+ cmd.arg("run");
+ cmd.arg("--package").arg("dora-cli");
+ cmd.arg("--").arg("build").arg(dataflow);
+ if !cmd.status().await?.success() {
+ bail!("failed to run dataflow");
+ };
+
+ let mut cmd = tokio::process::Command::new(&cargo);
+ cmd.arg("run");
+ cmd.arg("--package").arg("dora-cli");
+ cmd.arg("--")
+ .arg("daemon")
+ .arg("--run-dataflow")
+ .arg(dataflow);
+ if !cmd.status().await?.success() {
+ bail!("failed to run dataflow");
+ };
+ Ok(())
+}

examples/vlm/.gitignore

@@ -0,0 +1 @@
+*.pt

examples/vlm/README.md

@@ -0,0 +1 @@
+# Quick example on using a VLM with dora-rs

examples/vlm/dataflow.yml

@@ -0,0 +1,31 @@
+nodes:
+ - id: camera
+ build: pip install ../../node-hub/opencv-video-capture
+ path: opencv-video-capture
+ inputs:
+ tick: dora/timer/millis/20
+ outputs:
+ - image
+ env:
+ CAPTURE_PATH: 0
+ IMAGE_WIDTH: 640
+ IMAGE_HEIGHT: 480
+
+ - id: dora-qwenvl
+ build: pip install -e ../../node-hub/dora-qwenvl
+ path: dora-qwenvl
+ inputs:
+ image:
+ source: camera/image
+ queue_size: 1
+ outputs:
+ - text
+
+ - id: plot
+ build: pip install ../../node-hub/opencv-plot
+ path: opencv-plot
+ inputs:
+ image:
+ source: camera/image
+ queue_size: 1
+ text: dora-qwenvl/text

examples/vlm/run.rs

@@ -0,0 +1,98 @@
+use dora_core::{get_pip_path, get_python_path, run};
+use dora_tracing::set_up_tracing;
+use eyre::{bail, ContextCompat, WrapErr};
+use std::path::Path;
+
+#[tokio::main]
+async fn main() -> eyre::Result<()> {
+ set_up_tracing("python-dataflow-runner")?;
+
+ let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+ std::env::set_current_dir(root.join(file!()).parent().unwrap())
+ .wrap_err("failed to set working dir")?;
+
+ run(
+ get_python_path().context("Could not get python binary")?,
+ &["-m", "venv", "../.env"],
+ None,
+ )
+ .await
+ .context("failed to create venv")?;
+ let venv = &root.join("examples").join(".env");
+ std::env::set_var(
+ "VIRTUAL_ENV",
+ venv.to_str().context("venv path not valid unicode")?,
+ );
+ let orig_path = std::env::var("PATH")?;
+ // bin folder is named Scripts on windows.
+ // 🤦‍♂️ See: https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1
+ let venv_bin = if cfg!(windows) {
+ venv.join("Scripts")
+ } else {
+ venv.join("bin")
+ };
+
+ if cfg!(windows) {
+ std::env::set_var(
+ "PATH",
+ format!(
+ "{};{orig_path}",
+ venv_bin.to_str().context("venv path not valid unicode")?
+ ),
+ );
+ } else {
+ std::env::set_var(
+ "PATH",
+ format!(
+ "{}:{orig_path}",
+ venv_bin.to_str().context("venv path not valid unicode")?
+ ),
+ );
+ }
+
+ run(
+ get_pip_path().context("Could not get pip binary")?,
+ &["install", "maturin"],
+ Some(venv),
+ )
+ .await
+ .context("pip install maturin failed")?;
+
+ run(
+ "maturin",
+ &["develop"],
+ Some(&root.join("apis").join("python").join("node")),
+ )
+ .await
+ .context("maturin develop failed")?;
+
+ let dataflow = Path::new("dataflow.yml");
+ run_dataflow(dataflow).await?;
+
+ Ok(())
+}
+
+async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
+ let cargo = std::env::var("CARGO").unwrap();
+
+ // First build the dataflow (install requirements)
+ let mut cmd = tokio::process::Command::new(&cargo);
+ cmd.arg("run");
+ cmd.arg("--package").arg("dora-cli");
+ cmd.arg("--").arg("build").arg(dataflow);
+ if !cmd.status().await?.success() {
+ bail!("failed to run dataflow");
+ };
+
+ let mut cmd = tokio::process::Command::new(&cargo);
+ cmd.arg("run");
+ cmd.arg("--package").arg("dora-cli");
+ cmd.arg("--")
+ .arg("daemon")
+ .arg("--run-dataflow")
+ .arg(dataflow);
+ if !cmd.status().await?.success() {
+ bail!("failed to run dataflow");
+ };
+ Ok(())
+}

node-hub/dora-keyboard/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dora-keyboard"
-version = "0.3.5"
+version = "0.3.6"
 authors = [
  "Haixuan Xavier Tao <tao.xavier@outlook.com>",
  "Enzo Le Van <dev@enzo-le-van.fr>",
@@ -13,7 +13,7 @@ readme = "README.md"
 packages = [{ include = "dora_keyboard" }]
 
 [tool.poetry.dependencies]
-dora-rs = "0.3.5"
+dora-rs = "^0.3.6"
 numpy = "< 2.0.0"
 pyarrow = ">= 5.0.0"
 pynput = "^1.7.6"

node-hub/dora-qwenvl/README.md

@@ -0,0 +1,3 @@
+# Dora QwenVL2 node
+
+Experimental node for using a VLM within dora.

node-hub/dora-qwenvl/dora_qwenvl/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+# Define the path to the README file relative to the package directory
+readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md")
+
+# Read the content of the README file
+try:
+ with open(readme_path, "r", encoding="utf-8") as f:
+ __doc__ = f.read()
+except FileNotFoundError:
+ __doc__ = "README file not found."