Skip to content

Commit

Permalink
feat(engine): add Wasm action processor and integrate into workflow (#…
Browse files Browse the repository at this point in the history
miseyu authored Nov 25, 2024
1 parent 817ea4e commit d4463e7
Showing 18 changed files with 91 additions and 20 deletions.
31 changes: 28 additions & 3 deletions engine/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions engine/Cargo.toml
Original file line number Diff line number Diff line change
@@ -50,6 +50,7 @@ reearth-flow-action-plateau-processor = { path = "runtime/action-plateau-process
reearth-flow-action-processor = { path = "runtime/action-processor" }
reearth-flow-action-sink = { path = "runtime/action-sink" }
reearth-flow-action-source = { path = "runtime/action-source" }
reearth-flow-action-wasm-processor = { path = "runtime/action-wasm-processor" }
reearth-flow-common = { path = "runtime/common" }
reearth-flow-eval-expr = { path = "runtime/eval-expr" }
reearth-flow-geometry = { path = "runtime/geometry" }
1 change: 1 addition & 0 deletions engine/cli/Cargo.toml
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@ reearth-flow-action-plateau-processor.workspace = true
reearth-flow-action-processor.workspace = true
reearth-flow-action-sink.workspace = true
reearth-flow-action-source.workspace = true
reearth-flow-action-wasm-processor.workspace = true
reearth-flow-common.workspace = true
reearth-flow-runner.workspace = true
reearth-flow-runtime.workspace = true
8 changes: 7 additions & 1 deletion engine/cli/src/doc_action.rs
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ use indoc::indoc;
use reearth_flow_runtime::node::{NodeKind, RouterFactory};

use crate::{
factory::{BUILTIN_ACTION_FACTORIES, PLATEAU_ACTION_FACTORIES},
factory::{BUILTIN_ACTION_FACTORIES, PLATEAU_ACTION_FACTORIES, WASM_ACTION_FACTORIES},
utils::create_action_schema,
};

@@ -37,6 +37,12 @@ impl DocActionCliCommand {
.map(|kind| create_action_schema(kind, false))
.collect::<Vec<_>>();
actions.extend(plateau_actions);
let wasm_actions = WASM_ACTION_FACTORIES
.clone()
.values()
.map(|kind| create_action_schema(kind, false))
.collect::<Vec<_>>();
actions.extend(wasm_actions);
actions.sort_by(|a, b| a.name.cmp(&b.name));
println!("# Actions");
actions.iter().for_each(|action| {
4 changes: 4 additions & 0 deletions engine/cli/src/factory.rs
Original file line number Diff line number Diff line change
@@ -19,10 +19,14 @@ pub(crate) static BUILTIN_ACTION_FACTORIES: Lazy<HashMap<String, NodeKind>> = La
pub(crate) static PLATEAU_ACTION_FACTORIES: Lazy<HashMap<String, NodeKind>> =
Lazy::new(|| PLATEAU_MAPPINGS.clone());

pub(crate) static WASM_ACTION_FACTORIES: Lazy<HashMap<String, NodeKind>> =
Lazy::new(|| reearth_flow_action_wasm_processor::mapping::ACTION_FACTORY_MAPPINGS.clone());

pub(crate) static ALL_ACTION_FACTORIES: Lazy<HashMap<String, NodeKind>> = Lazy::new(|| {
BUILTIN_ACTION_FACTORIES
.iter()
.chain(PLATEAU_ACTION_FACTORIES.iter())
.chain(WASM_ACTION_FACTORIES.iter())
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
});
8 changes: 7 additions & 1 deletion engine/cli/src/schema_action.rs
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ use reearth_flow_runtime::node::{NodeKind, RouterFactory};
use serde::{Deserialize, Serialize};

use crate::{
factory::{BUILTIN_ACTION_FACTORIES, PLATEAU_ACTION_FACTORIES},
factory::{BUILTIN_ACTION_FACTORIES, PLATEAU_ACTION_FACTORIES, WASM_ACTION_FACTORIES},
utils::{create_action_schema, ActionSchema},
};

@@ -43,6 +43,12 @@ impl SchemaActionCliCommand {
.map(|kind| create_action_schema(kind, false))
.collect::<Vec<_>>();
actions.extend(plateau_actions);
let wasm_actions = WASM_ACTION_FACTORIES
.clone()
.values()
.map(|kind| create_action_schema(kind, false))
.collect::<Vec<_>>();
actions.extend(wasm_actions);
actions.sort_by(|a, b| a.name.cmp(&b.name));
let root = RootActionSchema { actions };
println!("{}", serde_json::to_string_pretty(&root).unwrap());
7 changes: 1 addition & 6 deletions engine/containers/worker/Dockerfile
Original file line number Diff line number Diff line change
@@ -49,14 +49,9 @@ RUN \
apt-get update && \
apt-get install -y \
ca-certificates \
libxml2 \
python3.11 \
python3.11-distutils \
python3-pip && \
libxml2 && \
rm -rf /var/lib/apt/lists/*

RUN python3.11 -m pip install --upgrade pip && pip install py2wasm

COPY --chmod=0755 --from=builder /tmp/reearth-flow-worker /bin

CMD [ "/bin/reearth-flow-worker" ]
3 changes: 0 additions & 3 deletions engine/runtime/action-processor/Cargo.toml
Original file line number Diff line number Diff line change
@@ -42,14 +42,11 @@ rstar.workspace = true
schemars.workspace = true
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
thiserror.workspace = true
tokio.workspace = true
tracing.workspace = true
url.workspace = true
uuid.workspace = true
wasmer.workspace = true
wasmer-wasix.workspace = true

[dev-dependencies]
bytes.workspace = true
1 change: 0 additions & 1 deletion engine/runtime/action-processor/src/lib.rs
Original file line number Diff line number Diff line change
@@ -5,7 +5,6 @@ pub(crate) mod file;
pub(crate) mod geometry;
pub mod mapping;
pub(crate) mod noop;
pub(crate) mod wasm;
pub(crate) mod xml;

#[cfg(test)]
4 changes: 1 addition & 3 deletions engine/runtime/action-processor/src/mapping.rs
Original file line number Diff line number Diff line change
@@ -4,8 +4,7 @@ use once_cell::sync::Lazy;
use reearth_flow_runtime::node::{NodeKind, ProcessorFactory};

use crate::{
attribute, echo::EchoProcessorFactory, feature, file, geometry, noop::NoopProcessorFactory,
wasm, xml,
attribute, echo::EchoProcessorFactory, feature, file, geometry, noop::NoopProcessorFactory, xml,
};

pub static ACTION_FACTORY_MAPPINGS: Lazy<HashMap<String, NodeKind>> = Lazy::new(|| {
@@ -23,7 +22,6 @@ pub static ACTION_FACTORY_MAPPINGS: Lazy<HashMap<String, NodeKind>> = Lazy::new(
mapping.extend(attribute::mapping::ACTION_FACTORY_MAPPINGS.clone());
mapping.extend(feature::mapping::ACTION_FACTORY_MAPPINGS.clone());
mapping.extend(geometry::mapping::ACTION_FACTORY_MAPPINGS.clone());
mapping.extend(wasm::mapping::ACTION_FACTORY_MAPPINGS.clone());
mapping.extend(xml::mapping::ACTION_FACTORY_MAPPINGS.clone());
mapping.extend(file::mapping::ACTION_FACTORY_MAPPINGS.clone());
mapping
36 changes: 36 additions & 0 deletions engine/runtime/action-wasm-processor/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
[package]
description = "Re:Earth Flow Action Wasm Processor"
name = "reearth-flow-action-wasm-processor"

authors.workspace = true
edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true

[dependencies]
reearth-flow-common.workspace = true
reearth-flow-eval-expr.workspace = true
reearth-flow-geometry.workspace = true
reearth-flow-runtime.workspace = true
reearth-flow-storage.workspace = true
reearth-flow-types.workspace = true

chrono.workspace = true
itertools.workspace = true
once_cell.workspace = true
regex.workspace = true
schemars.workspace = true
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
thiserror.workspace = true
tokio.workspace = true
tracing.workspace = true
wasmer.workspace = true
wasmer-wasix.workspace = true

[dev-dependencies]
bytes.workspace = true
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
pub(crate) mod errors;
pub(crate) mod mapping;
pub mod mapping;
pub(crate) mod runtime_executor;
1 change: 1 addition & 0 deletions engine/runtime/tests/Cargo.toml
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@ reearth-flow-action-log.workspace = true
reearth-flow-action-processor.workspace = true
reearth-flow-action-sink.workspace = true
reearth-flow-action-source.workspace = true
reearth-flow-action-wasm-processor.workspace = true
reearth-flow-common.workspace = true
reearth-flow-eval-expr.workspace = true
reearth-flow-runner.workspace = true
2 changes: 2 additions & 0 deletions engine/runtime/tests/helper.rs
Original file line number Diff line number Diff line change
@@ -21,9 +21,11 @@ pub(crate) static BUILTIN_ACTION_FACTORIES: Lazy<HashMap<String, NodeKind>> = La
let sink = SINK_MAPPINGS.clone();
let source = SOURCE_MAPPINGS.clone();
let processor = PROCESSOR_MAPPINGS.clone();
let wasm = reearth_flow_action_wasm_processor::mapping::ACTION_FACTORY_MAPPINGS.clone();
common.extend(sink);
common.extend(source);
common.extend(processor);
common.extend(wasm);
common
});

2 changes: 1 addition & 1 deletion engine/schema/actions.json
Original file line number Diff line number Diff line change
@@ -2709,7 +2709,7 @@
}
}
},
"builtin": true,
"builtin": false,
"inputPorts": [
"default"
],

0 comments on commit d4463e7

Please sign in to comment.