From 088838e030829f703db9b0598491abc9ad569174 Mon Sep 17 00:00:00 2001 From: Alan Chen Date: Mon, 6 May 2024 08:21:26 -0700 Subject: [PATCH] fix: explicitly use wasi on smartengine uses remove feature flag, wasi is always on if the engine is on --- Cargo.toml | 22 +++++++++--------- crates/fluvio-connector-common/Cargo.toml | 5 ++-- .../fluvio-connector-common/src/consumer.rs | 8 +++++-- crates/fluvio-smartengine/Cargo.toml | 5 ++-- .../src/engine/wasmtime/state.rs | 23 ------------------- crates/fluvio-spu/Cargo.toml | 2 +- crates/fluvio/src/consumer/mod.rs | 5 +++- crates/smartmodule-development-kit/Cargo.toml | 2 +- 8 files changed, 27 insertions(+), 45 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 362d2f48ee5..890892bff09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,28 +165,28 @@ k8-diff = { version = "0.1.2" } trybuild = { branch = "check_option", git = "https://github.com/infinyon/trybuild" } # Internal fluvio dependencies -fluvio = { version = "0.22.0", path = "crates/fluvio" } +fluvio = { path = "crates/fluvio" } fluvio-auth = { path = "crates/fluvio-auth" } fluvio-channel = { path = "crates/fluvio-channel" } fluvio-cli-common = { path = "crates/fluvio-cli-common"} -fluvio-compression = { version = "0.3", path = "crates/fluvio-compression" } +fluvio-compression = { path = "crates/fluvio-compression" } fluvio-connector-package = { path = "crates/fluvio-connector-package/" } fluvio-controlplane = { path = "crates/fluvio-controlplane" } fluvio-controlplane-metadata = { version = "0.28.0", default-features = false, path = "crates/fluvio-controlplane-metadata" } fluvio-extension-common = { path = "crates/fluvio-extension-common", default-features = false } fluvio-hub-util = { path = "crates/fluvio-hub-util" } fluvio-package-index = { version = "0.7.0", path = "crates/fluvio-package-index", default-features = false } -fluvio-protocol = { version = "0.10.12", path = "crates/fluvio-protocol" } -fluvio-sc-schema = { version = "0.24.0", path = "crates/fluvio-sc-schema", default-features = false } +fluvio-protocol = { path = "crates/fluvio-protocol" } +fluvio-sc-schema = { path = "crates/fluvio-sc-schema", default-features = false } fluvio-service = { path = "crates/fluvio-service" } -fluvio-smartengine = { version = "0.7.0", path = "crates/fluvio-smartengine", default-features = false } -fluvio-smartmodule = { version = "0.7.0", path = "crates/fluvio-smartmodule", default-features = false } -fluvio-socket = { version = "0.14.3", path = "crates/fluvio-socket", default-features = false } -fluvio-spu-schema = { version = "0.14.5", path = "crates/fluvio-spu-schema", default-features = false } +fluvio-smartengine = { path = "crates/fluvio-smartengine", default-features = false } +fluvio-smartmodule = { path = "crates/fluvio-smartmodule", default-features = false } +fluvio-socket = { path = "crates/fluvio-socket", default-features = false } +fluvio-spu-schema = { path = "crates/fluvio-spu-schema", default-features = false } fluvio-storage = { path = "crates/fluvio-storage" } -fluvio-stream-dispatcher = { version = "0.13.0", path = "crates/fluvio-stream-dispatcher" } -fluvio-stream-model = { version = "0.11.0", path = "crates/fluvio-stream-model", default-features = false } -fluvio-types = { version = "0.4.4", path = "crates/fluvio-types", default-features = false } +fluvio-stream-dispatcher = { path = "crates/fluvio-stream-dispatcher" } +fluvio-stream-model = { path = "crates/fluvio-stream-model", default-features = false } +fluvio-types = { path = "crates/fluvio-types", default-features = false } fluvio-kv-storage = { path = "crates/fluvio-kv-storage", default-features = false } # Used to make eyre faster on debug builds diff --git a/crates/fluvio-connector-common/Cargo.toml b/crates/fluvio-connector-common/Cargo.toml index 53624b8e346..51c56409c83 100644 --- a/crates/fluvio-connector-common/Cargo.toml +++ b/crates/fluvio-connector-common/Cargo.toml @@ -10,7 +10,6 @@ description = "Fluvio Connector Common SDK" [features] default = [] derive = ["fluvio-connector-derive"] -wasi = ["fluvio-smartengine/wasi"] [[test]] name = "derive-test" @@ -33,10 +32,10 @@ fluvio-future = { workspace = true, features = ["subscriber"] } fluvio-connector-package = { workspace = true } fluvio-connector-derive = { path = "../fluvio-connector-derive/", optional = true } fluvio-sc-schema = { workspace = true } -fluvio-smartengine = { workspace = true , features = [ "transformation"] } +fluvio-smartengine = { workspace = true , features = [ "transformation", "engine"] } [dev-dependencies] -trybuild = { version = "1.0" } # default workspace dep is forked and fails for this crate +trybuild = { version = "1.0" } # default workspace dep is forked and fails for this crate serde = { workspace = true, features = ["derive"]} fluvio = { workspace = true } diff --git a/crates/fluvio-connector-common/src/consumer.rs b/crates/fluvio-connector-common/src/consumer.rs index 992a2077fa9..875629482b0 100644 --- a/crates/fluvio-connector-common/src/consumer.rs +++ b/crates/fluvio-connector-common/src/consumer.rs @@ -66,8 +66,12 @@ pub async fn consumer_stream_from_config( if let Some(smartmodules) = smartmodule_vec_from_config(config) { builder.smartmodule(smartmodules); } - - let stream = fluvio.consumer_with_config(builder.build()?).await?; + tracing::info!("Building config"); + let cfg = builder.build().map_err(|e| { + tracing::error!("Config build error: {e}"); + e + })?; + let stream = fluvio.consumer_with_config(cfg).await?; Ok((fluvio, stream)) } diff --git a/crates/fluvio-smartengine/Cargo.toml b/crates/fluvio-smartengine/Cargo.toml index fe3aae79630..bd9177160b0 100644 --- a/crates/fluvio-smartengine/Cargo.toml +++ b/crates/fluvio-smartengine/Cargo.toml @@ -12,10 +12,9 @@ description = "The official Fluvio SmartEngine" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -engine = ["wasmtime"] -wasi = ["wasmtime-wasi", "engine"] +engine = ["wasmtime", "wasmtime-wasi"] transformation = ["serde_json", "serde_yaml", "humantime-serde"] -default = ["engine", "wasi"] +default = ["engine"] [dependencies] diff --git a/crates/fluvio-smartengine/src/engine/wasmtime/state.rs b/crates/fluvio-smartengine/src/engine/wasmtime/state.rs index 73fb8276c51..ed37e510ea1 100644 --- a/crates/fluvio-smartengine/src/engine/wasmtime/state.rs +++ b/crates/fluvio-smartengine/src/engine/wasmtime/state.rs @@ -18,7 +18,6 @@ pub struct WasmState(Store); pub struct Context { limiter: StoreResourceLimiter, - #[cfg(feature = "wasi")] wasi_ctx: wasmtime_wasi::WasiCtx, } @@ -54,28 +53,6 @@ impl WasmState { } } -#[cfg(not(feature = "wasi"))] -impl WasmState { - pub(crate) fn new(engine: &Engine, limiter: StoreResourceLimiter) -> Self { - let mut s = Self(Store::new(engine, Context { limiter })); - s.0.limiter(|inner| &mut inner.limiter); - s.top_up_fuel(); - s - } - - pub(crate) fn instantiate( - &mut self, - module: &Module, - host_fn: impl IntoFunc<::Data, Params, Args>, - ) -> Result { - use wasmtime::Func; - - let func = Func::wrap(&mut *self, host_fn); - Instance::new(self, module, &[func.into()]) - } -} - -#[cfg(feature = "wasi")] impl WasmState { pub(crate) fn new(engine: &Engine, limiter: StoreResourceLimiter) -> Self { let wasi_ctx = wasmtime_wasi::WasiCtxBuilder::new() diff --git a/crates/fluvio-spu/Cargo.toml b/crates/fluvio-spu/Cargo.toml index b7749d8eb01..3fcfe570352 100644 --- a/crates/fluvio-spu/Cargo.toml +++ b/crates/fluvio-spu/Cargo.toml @@ -66,7 +66,7 @@ fluvio-future = { workspace = true,features = [ "openssl_tls", "zero_copy", ] } -fluvio-smartengine = { workspace = true, optional = true } +fluvio-smartengine = { workspace = true, optional = true, features = ["engine"] } fluvio-smartmodule = { workspace = true} fluvio-kv-storage = { workspace = true} diff --git a/crates/fluvio/src/consumer/mod.rs b/crates/fluvio/src/consumer/mod.rs index 7f00e24077c..650e74dfa4d 100644 --- a/crates/fluvio/src/consumer/mod.rs +++ b/crates/fluvio/src/consumer/mod.rs @@ -316,7 +316,10 @@ where let batch: Result = raw_batch.try_into(); match batch { Ok(batch) => Ok(batch), - Err(err) => Err(ErrorCode::Other(err.to_string())), + Err(err) => { + tracing::error!("{err:?}"); + Err(ErrorCode::Other(err.to_string())) + } } }); let error = { diff --git a/crates/smartmodule-development-kit/Cargo.toml b/crates/smartmodule-development-kit/Cargo.toml index 32a4cd12ecb..0fc25259f7f 100644 --- a/crates/smartmodule-development-kit/Cargo.toml +++ b/crates/smartmodule-development-kit/Cargo.toml @@ -32,7 +32,7 @@ fluvio = { path = "../fluvio", default-features = false } fluvio-hub-util = { path = "../fluvio-hub-util" } fluvio-protocol = { path = "../fluvio-protocol", features=["record","api"] } fluvio-future = { workspace = true, features = ["subscriber"]} -fluvio-smartengine = { path = "../fluvio-smartengine", features = ["transformation", "wasi"] } +fluvio-smartengine = { path = "../fluvio-smartengine", features = ["transformation"] } fluvio-extension-common = { path = "../fluvio-extension-common", features = ["target"] } fluvio-controlplane-metadata = { path = "../fluvio-controlplane-metadata", features = ["smartmodule"] } fluvio-sc-schema = { path = "../fluvio-sc-schema" }