PVF: Move PVF workers into separate crate (#7101)

* Move PVF workers into separate crate * Fix indentation * Fix compilation errors * Fix more compilation errors * Rename `worker.rs` files, make host interface to worker more clear * Fix more compilation errors * Fix more compilation errors * Add link to issue * Address review comments * Update comment
paritytech · Apr 21, 2023 · 81da250 · 81da250
1 parent fc95bc6
commit 81da250
Show file tree

Hide file tree

Showing 42 changed files with 878 additions and 627 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -24,7 +24,7 @@ tikv-jemallocator = "0.5.0"
 
 # Crates in our workspace, defined as dependencies so we can pass them feature flags.
 polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ]  }
-polkadot-node-core-pvf = { path = "node/core/pvf" }
+polkadot-node-core-pvf-worker = { path = "node/core/pvf/worker" }
 polkadot-overseer = { path = "node/overseer" }
 
 [dev-dependencies]
@@ -80,6 +80,7 @@ members = [
 	"node/core/parachains-inherent",
 	"node/core/provisioner",
 	"node/core/pvf",
+	"node/core/pvf/worker",
 	"node/core/pvf-checker",
 	"node/core/runtime-api",
 	"node/network/approval-distribution",
@@ -206,7 +207,7 @@ try-runtime = [ "polkadot-cli/try-runtime" ]
 fast-runtime = [ "polkadot-cli/fast-runtime" ]
 runtime-metrics = [ "polkadot-cli/runtime-metrics" ]
 pyroscope = ["polkadot-cli/pyroscope"]
-jemalloc-allocator = ["polkadot-node-core-pvf/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator"]
+jemalloc-allocator = ["polkadot-node-core-pvf-worker/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator"]
 
 # Configuration for building a .deb package - for use with `cargo-deb`
 [package.metadata.deb]

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -22,12 +22,13 @@ pyro = { package = "pyroscope", version = "0.3.1", optional = true }
 
 service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true }
 polkadot-client = { path = "../node/client", optional = true }
-polkadot-node-core-pvf = { path = "../node/core/pvf", optional = true }
+polkadot-node-core-pvf-worker = { path = "../node/core/pvf/worker", optional = true }
 polkadot-performance-test = { path = "../node/test/performance-test", optional = true }
 
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" }
 frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
@@ -52,7 +53,7 @@ cli = [
 	"frame-benchmarking-cli",
 	"try-runtime-cli",
 	"polkadot-client",
-	"polkadot-node-core-pvf",
+	"polkadot-node-core-pvf-worker",
 ]
 runtime-benchmarks = [
 	"service/runtime-benchmarks",

diff --git a/cli/src/command.rs b/cli/src/command.rs
@@ -494,7 +494,7 @@ pub fn run() -> Result<()> {
 
 			#[cfg(not(target_os = "android"))]
 			{
-				polkadot_node_core_pvf::prepare_worker_entrypoint(
+				polkadot_node_core_pvf_worker::prepare_worker_entrypoint(
 					&cmd.socket_path,
 					Some(&cmd.node_impl_version),
 				);
@@ -516,7 +516,7 @@ pub fn run() -> Result<()> {
 
 			#[cfg(not(target_os = "android"))]
 			{
-				polkadot_node_core_pvf::execute_worker_entrypoint(
+				polkadot_node_core_pvf_worker::execute_worker_entrypoint(
 					&cmd.socket_path,
 					Some(&cmd.node_impl_version),
 				);

diff --git a/cli/src/host_perf_check.rs b/cli/src/host_perf_check.rs
@@ -15,7 +15,6 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use log::info;
-use polkadot_node_core_pvf::sp_maybe_compressed_blob;
 use polkadot_performance_test::{
 	measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS,
 	ERASURE_CODING_TIME_LIMIT, PVF_PREPARE_TIME_LIMIT, VALIDATION_CODE_BOMB_LIMIT,

diff --git a/node/core/pvf/Cargo.toml b/node/core/pvf/Cargo.toml
@@ -4,24 +4,15 @@ version.workspace = true
 authors.workspace = true
 edition.workspace = true
 
-[[bin]]
-name = "puppet_worker"
-path = "bin/puppet_worker.rs"
-
 [dependencies]
 always-assert = "0.1"
-assert_matches = "1.4.0"
-cpu-time = "1.0.0"
 futures = "0.3.21"
 futures-timer = "3.0.2"
 gum = { package = "tracing-gum", path = "../../gum" }
 libc = "0.2.139"
 pin-project = "1.0.9"
 rand = "0.8.5"
-rayon = "1.5.1"
 slotmap = "1.0"
-tempfile = "3.3.0"
-tikv-jemalloc-ctl = { version = "0.5.0", optional = true }
 tokio = { version = "1.24.2", features = ["fs", "process"] }
 
 parity-scale-codec = { version = "3.4.0", default-features = false, features = ["derive"] }
@@ -30,13 +21,8 @@ polkadot-parachain = { path = "../../../parachain" }
 polkadot-core-primitives = { path = "../../../core-primitives" }
 polkadot-node-metrics = { path = "../../metrics" }
 polkadot-node-primitives = { path = "../../primitives" }
-
 polkadot-primitives = { path = "../../../primitives" }
-sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sc-executor-wasmtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
+
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -45,14 +31,7 @@ sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master
 [build-dependencies]
 substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-[target.'cfg(target_os = "linux")'.dependencies]
-tikv-jemalloc-ctl = "0.5.0"
-
 [dev-dependencies]
-adder = { package = "test-parachain-adder", path = "../../../parachain/test-parachains/adder" }
-halt = { package = "test-parachain-halt", path = "../../../parachain/test-parachains/halt" }
-hex-literal = "0.4.1"
+assert_matches = "1.4.0"
+hex-literal = "0.3.4"
 tempfile = "3.3.0"
-
-[features]
-jemalloc-allocator = ["dep:tikv-jemalloc-ctl"]
diff --git a/node/core/pvf/src/artifacts.rs b/node/core/pvf/src/artifacts.rs
@@ -65,9 +65,11 @@ use std::{
 	time::{Duration, SystemTime},
 };
 
+/// Contains the bytes for a successfully compiled artifact.
 pub struct CompiledArtifact(Vec<u8>);
 
 impl CompiledArtifact {
+	/// Creates a `CompiledArtifact`.
 	pub fn new(code: Vec<u8>) -> Self {
 		Self(code)
 	}