From dc62114ce4c3c46c2b31df756d96c8571e10b1af Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Fri, 12 Nov 2021 15:55:24 +0000 Subject: [PATCH 1/2] Limit the number of PVF workers In particular, limit the number of preparation workers to 1 (soft & hard) and limit the number of execution workers to 2. The reason why we are doing this is that it seems many workers launched at the same time can cause problems. I.e. if there are more than 2 preparation workers, the time for preparation rises significantly to the point of reaching the timeout. This was mostly observed with parallel_compilation=true, so each worker used `numcpu` threads and now we are looking to flip that parameter to `false`. That said, we want to err on the safe side here and gradually enable it later if our measurements show that we can do that safely. --- node/core/pvf/src/host.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index 292e37cdc30d..a50f7f27702e 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -128,11 +128,11 @@ impl Config { cache_path, prepare_worker_program_path: program_path.clone(), prepare_worker_spawn_timeout: Duration::from_secs(3), - prepare_workers_soft_max_num: 8, - prepare_workers_hard_max_num: 5, + prepare_workers_soft_max_num: 1, + prepare_workers_hard_max_num: 1, execute_worker_program_path: program_path, execute_worker_spawn_timeout: Duration::from_secs(3), - execute_workers_max_num: 5, + execute_workers_max_num: 2, } } } From 20516d955aa757687ac34d6617f368b6f58d8696 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Fri, 12 Nov 2021 16:49:25 +0000 Subject: [PATCH 2/2] Adjust the test to accomodate the changed config value --- node/core/pvf/tests/it/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/core/pvf/tests/it/main.rs b/node/core/pvf/tests/it/main.rs index e8fd7b665aa3..bf0983d50874 100644 --- a/node/core/pvf/tests/it/main.rs +++ b/node/core/pvf/tests/it/main.rs @@ -135,7 +135,7 @@ async fn parallel_execution() { #[async_std::test] async fn execute_queue_doesnt_stall_if_workers_died() { let host = TestHost::new_with_config(|cfg| { - assert_eq!(cfg.execute_workers_max_num, 5); + cfg.execute_workers_max_num = 5; }); // Here we spawn 8 validation jobs for the `halt` PVF and share those between 5 workers. The