From 012bd0638ac5b0148610788660d65ad5f9b42266 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Wed, 4 Dec 2024 10:59:46 +0100 Subject: [PATCH 01/14] refactor: remove unused `with` statement in flake.nix --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 09d8df2..d51329f 100644 --- a/flake.nix +++ b/flake.nix @@ -69,7 +69,7 @@ ]; devShells.default = pkgs.mkShell { - packages = with pkgs; [ + packages = [ pkgs.influxdb2-cli pkgs.influxdb2-server # TODO https://docs.influxdata.com/telegraf/v1/install/#ntp From efbd0e76addd3652a53438faf6415c580a683238 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Wed, 4 Dec 2024 11:07:32 +0100 Subject: [PATCH 02/14] chore(nix): add Nomad package to the default devShell --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index d51329f..98d422d 100644 --- a/flake.nix +++ b/flake.nix @@ -54,6 +54,7 @@ systems = builtins.attrNames inputs.holonix.devShells; perSystem = { inputs', pkgs, system, config, ... }: let + unfreePkgs = import nixpkgs { inherit system; config.allowUnfree = true; }; rustMod = flake-parts-lib.importApply ./nix/modules/rust.nix { inherit crane rust-overlay nixpkgs; }; in { @@ -68,6 +69,7 @@ ./nix/modules/zomes.nix ]; + devShells.default = pkgs.mkShell { packages = [ pkgs.influxdb2-cli @@ -81,6 +83,7 @@ pkgs.taplo pkgs.yamlfmt pkgs.perl + unfreePkgs.nomad config.rustHelper.rust inputs'.holonix.packages.holochain inputs'.holonix.packages.lair-keystore From b62427689c1b4b561e0f1fd19bf500d1f550c209 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Wed, 4 Dec 2024 15:29:59 +0100 Subject: [PATCH 03/14] chore: add result to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 22e0f69..c6f2db8 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ logs/**/*.log scenarios/*/logs run_summary.jsonl summariser-report-*.json + +# A symlink to the output directory of nix builds +result From 412fb29dd19822346562fcfa24a571f2c5542033 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Wed, 4 Dec 2024 15:30:56 +0100 Subject: [PATCH 04/14] feat: add Nomad job to run app_install scenario locally with Nomad --- README.md | 35 +++++++++++++++++++++++++++ nomad/run-app_install-local.nomad.hcl | 35 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 nomad/run-app_install-local.nomad.hcl diff --git a/README.md b/README.md index 3a9fa97..e8af944 100644 --- a/README.md +++ b/README.md @@ -354,6 +354,41 @@ You can then start a second terminal and run one of the scenarios in the `scenar ```bash RUST_LOG=info CONDUCTOR_CONFIG="CI" TRYCP_RUST_LOG="info" MIN_PEERS=2 cargo run --package trycp_write_validated -- --targets targets-ci.yaml --instances-per-target 2 --duration 60 ``` +#### Running Wind Tunnel Scenarios with Nomad + +> [!Warning] +> This is a work in progress and currently only works with the `app_install` scenario. + +##### Running Locally + +You can easily test the Wind Tunnel scenarios with [Nomad](https://www.nomadproject.io) by running them locally, this requires running a Nomad agent locally +as both a client and a server. + +First, enter the Nix `devShell` with `nix develop` to make sure you have all the packages install. +Alternatively, [install Nomad](https://developer.hashicorp.com/nomad/install) and Holochain locally so that both `nomad` and `hc` are in your `PATH`. + +Once Nomad is installed, run the agent in `dev` mode to spin up both a server and client, do this with: +```shell +nomad agent -dev +``` + +Now navigate to to view the Nomad dashboard. + +Next, in a new terminal window, build the `app_install` scenario with: +```shell +nix build .#app_install +``` + +Once the scenario is built you can run the Nomad job with: +```shell +nomad job run nomad/run-app_install-local.nomad.hcl +``` + +Then, navigate to where you should see one allocation (the Nomad name for an instance of the job) +this allocation should have two tasks: the `start-holochain` task and the `run_scenario` task. You can view the logs of these tasks to see the results. +The allocation should be marked as "Completed" after a few seconds. + +Once you've finished testing you can kill the Nomad agent with `^C` in the first terminal running the agent. ### Published crates diff --git a/nomad/run-app_install-local.nomad.hcl b/nomad/run-app_install-local.nomad.hcl new file mode 100644 index 0000000..214d52f --- /dev/null +++ b/nomad/run-app_install-local.nomad.hcl @@ -0,0 +1,35 @@ +job "app_install_scenario" { + type = "batch" + group "app_install" { + task "start_holochain" { + lifecycle { + hook = "prestart" + sidecar = true + } + + driver = "raw_exec" + config { + command = "bash" + args = ["-c", "hc s clean && echo 1234 | hc s --piped create && echo 1234 | hc s --piped -f 8888 run"] + } + } + + task "run_scenario" { + driver = "raw_exec" + env { + RUST_LOG = "info" + } + config { + command = abspath("result/bin/app_install") + args = [ + "--connection-string", "ws://localhost:8888", + "--agents", "2", + "--behaviour", "minimal:1", + "--behaviour", "large:1", + "--duration", "5", + "--no-progress" + ] + } + } + } +} From 90d2bc2a4ccff572cc54c8f7ea361bf9b5dee292 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Fri, 6 Dec 2024 15:29:48 +0100 Subject: [PATCH 05/14] refactor: use a single job spec with scenario name variable This allows us to run all scenarios as Nomad jobs using the same job spec. --- ...l-local.nomad.hcl => run_scenario_local.nomad.hcl} | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) rename nomad/{run-app_install-local.nomad.hcl => run_scenario_local.nomad.hcl} (81%) diff --git a/nomad/run-app_install-local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl similarity index 81% rename from nomad/run-app_install-local.nomad.hcl rename to nomad/run_scenario_local.nomad.hcl index 214d52f..992d16c 100644 --- a/nomad/run-app_install-local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -1,6 +1,11 @@ -job "app_install_scenario" { +variable "scenario_name" { + type = string +} + +job "run_scenario" { type = "batch" - group "app_install" { + + group "scenario_runnner" { task "start_holochain" { lifecycle { hook = "prestart" @@ -20,7 +25,7 @@ job "app_install_scenario" { RUST_LOG = "info" } config { - command = abspath("result/bin/app_install") + command = abspath("result/bin/${var.scenario_name}") args = [ "--connection-string", "ws://localhost:8888", "--agents", "2", From c868a15d7e8ebd6cdda054fc3417f1b0af58f6f3 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 14:13:08 +0100 Subject: [PATCH 06/14] refactor: use kebab-case for variables --- nomad/run_scenario_local.nomad.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomad/run_scenario_local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl index 992d16c..86c63f2 100644 --- a/nomad/run_scenario_local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -1,4 +1,4 @@ -variable "scenario_name" { +variable "scenario-name" { type = string } @@ -25,7 +25,7 @@ job "run_scenario" { RUST_LOG = "info" } config { - command = abspath("result/bin/${var.scenario_name}") + command = abspath("result/bin/${var.scenario-name}") args = [ "--connection-string", "ws://localhost:8888", "--agents", "2", From fa99d07cee1a10e059695f111f2a05f7925d09bf Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 15:30:07 +0100 Subject: [PATCH 07/14] feat(nomad): add connection-string variable with localhost default --- nomad/run_scenario_local.nomad.hcl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nomad/run_scenario_local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl index 86c63f2..db33420 100644 --- a/nomad/run_scenario_local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -2,6 +2,11 @@ variable "scenario-name" { type = string } +variable "connection-string" { + type = string + default = "ws://localhost:8888" +} + job "run_scenario" { type = "batch" @@ -27,7 +32,7 @@ job "run_scenario" { config { command = abspath("result/bin/${var.scenario-name}") args = [ - "--connection-string", "ws://localhost:8888", + "--connection-string=${var.connection-string}", "--agents", "2", "--behaviour", "minimal:1", "--behaviour", "large:1", From 7b21dc113077895820db3b55f7d26068b0ef2851 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 15:36:30 +0100 Subject: [PATCH 08/14] feat(nomad): add agents variable to set the number of agents required --- nomad/run_scenario_local.nomad.hcl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nomad/run_scenario_local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl index db33420..b7fc4d7 100644 --- a/nomad/run_scenario_local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -7,6 +7,11 @@ variable "connection-string" { default = "ws://localhost:8888" } +variable "agents" { + type = number + default = null +} + job "run_scenario" { type = "batch" @@ -31,14 +36,15 @@ job "run_scenario" { } config { command = abspath("result/bin/${var.scenario-name}") - args = [ + // The `compact` function removes empty strings and `null` items from the list. + args = compact([ "--connection-string=${var.connection-string}", - "--agents", "2", + var.agents != null ? "--agents=${var.agents}" : null, "--behaviour", "minimal:1", "--behaviour", "large:1", "--duration", "5", "--no-progress" - ] + ]) } } } From f5432e037f89ee0fac0390e7f27683f953dfca36 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 15:44:02 +0100 Subject: [PATCH 09/14] feat(nomad): add duration variable to set the max duration of the run --- nomad/run_scenario_local.nomad.hcl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nomad/run_scenario_local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl index b7fc4d7..67bd25b 100644 --- a/nomad/run_scenario_local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -12,6 +12,11 @@ variable "agents" { default = null } +variable "duration" { + type = number + default = null +} + job "run_scenario" { type = "batch" @@ -40,9 +45,9 @@ job "run_scenario" { args = compact([ "--connection-string=${var.connection-string}", var.agents != null ? "--agents=${var.agents}" : null, + var.duration != null ? "--duration=${var.duration}" : null, "--behaviour", "minimal:1", "--behaviour", "large:1", - "--duration", "5", "--no-progress" ]) } From 5e2a2051741e5cb629f19ba68cb31af25ea20bd4 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 15:46:49 +0100 Subject: [PATCH 10/14] feat(nomad): add behaviours variable to set the scenario behaviours --- nomad/run_scenario_local.nomad.hcl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nomad/run_scenario_local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl index 67bd25b..2f2342c 100644 --- a/nomad/run_scenario_local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -17,6 +17,11 @@ variable "duration" { default = null } +variable "behaviours" { + type = map(string) + default = {} +} + job "run_scenario" { type = "batch" @@ -42,13 +47,13 @@ job "run_scenario" { config { command = abspath("result/bin/${var.scenario-name}") // The `compact` function removes empty strings and `null` items from the list. - args = compact([ + args = concat(compact([ "--connection-string=${var.connection-string}", var.agents != null ? "--agents=${var.agents}" : null, var.duration != null ? "--duration=${var.duration}" : null, - "--behaviour", "minimal:1", - "--behaviour", "large:1", "--no-progress" + ]), [ + for k, v in var.behaviours : "--behaviour=${k}:${v}" ]) } } From 31d37dae111544df6e4c74d15eb4854cb6432670 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 16:04:01 +0100 Subject: [PATCH 11/14] feat(nomad): add reporter variable that defaults to influx-file --- nomad/run_scenario_local.nomad.hcl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nomad/run_scenario_local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl index 2f2342c..c1263d1 100644 --- a/nomad/run_scenario_local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -17,6 +17,11 @@ variable "duration" { default = null } +variable "reporter" { + type = string + default = "influx-file" +} + variable "behaviours" { type = map(string) default = {} @@ -51,6 +56,7 @@ job "run_scenario" { "--connection-string=${var.connection-string}", var.agents != null ? "--agents=${var.agents}" : null, var.duration != null ? "--duration=${var.duration}" : null, + var.reporter != null ? "--reporter=${var.reporter}" : null, "--no-progress" ]), [ for k, v in var.behaviours : "--behaviour=${k}:${v}" From 932f15499082cd99a4e26a8f547d95732dc416ef Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 16:14:30 +0100 Subject: [PATCH 12/14] docs(nomad): add descriptions to the job variables --- nomad/run_scenario_local.nomad.hcl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nomad/run_scenario_local.nomad.hcl b/nomad/run_scenario_local.nomad.hcl index c1263d1..8a67583 100644 --- a/nomad/run_scenario_local.nomad.hcl +++ b/nomad/run_scenario_local.nomad.hcl @@ -1,29 +1,35 @@ variable "scenario-name" { type = string + description = "The name of the scenario to run" } variable "connection-string" { type = string + description = "The URL to be used to connect to the service being tested" default = "ws://localhost:8888" } variable "agents" { type = number + description = "The number of agents to create" default = null } variable "duration" { type = number + description = "The maximum duration of the scenario run" default = null } variable "reporter" { type = string + description = "The method used to report the logs" default = "influx-file" } variable "behaviours" { type = map(string) + description = "Custom behaviours defined and used by the scenarios" default = {} } From 1babd161e0de27bb9a77ad774352c028f5b934fc Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 16:46:03 +0100 Subject: [PATCH 13/14] feat(nomad): add var-files for each local scenario run --- nomad/var_files/app_install_large.vars | 2 ++ nomad/var_files/app_install_minimal.vars | 2 ++ nomad/var_files/dht_sync_lag.vars | 6 ++++++ nomad/var_files/first_call.vars | 2 ++ nomad/var_files/local_signals.vars | 1 + nomad/var_files/single_write_many_read.vars | 1 + nomad/var_files/write_query.vars | 1 + nomad/var_files/write_read.vars | 1 + nomad/var_files/write_validated.vars | 1 + nomad/var_files/zome_call_single_value.vars | 1 + 10 files changed, 18 insertions(+) create mode 100644 nomad/var_files/app_install_large.vars create mode 100644 nomad/var_files/app_install_minimal.vars create mode 100644 nomad/var_files/dht_sync_lag.vars create mode 100644 nomad/var_files/first_call.vars create mode 100644 nomad/var_files/local_signals.vars create mode 100644 nomad/var_files/single_write_many_read.vars create mode 100644 nomad/var_files/write_query.vars create mode 100644 nomad/var_files/write_read.vars create mode 100644 nomad/var_files/write_validated.vars create mode 100644 nomad/var_files/zome_call_single_value.vars diff --git a/nomad/var_files/app_install_large.vars b/nomad/var_files/app_install_large.vars new file mode 100644 index 0000000..3a57523 --- /dev/null +++ b/nomad/var_files/app_install_large.vars @@ -0,0 +1,2 @@ +scenario-name = "app_install" +behaviours = { large = 1 } diff --git a/nomad/var_files/app_install_minimal.vars b/nomad/var_files/app_install_minimal.vars new file mode 100644 index 0000000..f712705 --- /dev/null +++ b/nomad/var_files/app_install_minimal.vars @@ -0,0 +1,2 @@ +scenario-name = "app_install" +behaviours = { minimal = 1 } diff --git a/nomad/var_files/dht_sync_lag.vars b/nomad/var_files/dht_sync_lag.vars new file mode 100644 index 0000000..4ffb0ff --- /dev/null +++ b/nomad/var_files/dht_sync_lag.vars @@ -0,0 +1,6 @@ +scenario-name = "dht_sync_lag" +agents = 2 +behaviours = { + write = 1, + record_lag = 1, +} diff --git a/nomad/var_files/first_call.vars b/nomad/var_files/first_call.vars new file mode 100644 index 0000000..bceb3cb --- /dev/null +++ b/nomad/var_files/first_call.vars @@ -0,0 +1,2 @@ +scenario-name = "first_call" +behaviours = { local = 1 } diff --git a/nomad/var_files/local_signals.vars b/nomad/var_files/local_signals.vars new file mode 100644 index 0000000..c635b2f --- /dev/null +++ b/nomad/var_files/local_signals.vars @@ -0,0 +1 @@ +scenario-name = "local_signals" diff --git a/nomad/var_files/single_write_many_read.vars b/nomad/var_files/single_write_many_read.vars new file mode 100644 index 0000000..236bf2d --- /dev/null +++ b/nomad/var_files/single_write_many_read.vars @@ -0,0 +1 @@ +scenario-name = "single_write_many_read" diff --git a/nomad/var_files/write_query.vars b/nomad/var_files/write_query.vars new file mode 100644 index 0000000..4b8035c --- /dev/null +++ b/nomad/var_files/write_query.vars @@ -0,0 +1 @@ +scenario-name = "write_query" diff --git a/nomad/var_files/write_read.vars b/nomad/var_files/write_read.vars new file mode 100644 index 0000000..a64e993 --- /dev/null +++ b/nomad/var_files/write_read.vars @@ -0,0 +1 @@ +scenario-name = "write_read" diff --git a/nomad/var_files/write_validated.vars b/nomad/var_files/write_validated.vars new file mode 100644 index 0000000..e026c27 --- /dev/null +++ b/nomad/var_files/write_validated.vars @@ -0,0 +1 @@ +scenario-name = "write_validated" diff --git a/nomad/var_files/zome_call_single_value.vars b/nomad/var_files/zome_call_single_value.vars new file mode 100644 index 0000000..188bb42 --- /dev/null +++ b/nomad/var_files/zome_call_single_value.vars @@ -0,0 +1 @@ +scenario-name = "zome_call_single_value" From 69dc49153c5c8ceb8571496ffbd7e6fa21901eb5 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 9 Dec 2024 16:53:52 +0100 Subject: [PATCH 14/14] docs: update the Nomad section in the readme --- README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e8af944..81f367a 100644 --- a/README.md +++ b/README.md @@ -357,7 +357,7 @@ RUST_LOG=info CONDUCTOR_CONFIG="CI" TRYCP_RUST_LOG="info" MIN_PEERS=2 cargo run #### Running Wind Tunnel Scenarios with Nomad > [!Warning] -> This is a work in progress and currently only works with the `app_install` scenario. +> This is a work in progress and currently only works running the scenarios locally. ##### Running Locally @@ -374,19 +374,29 @@ nomad agent -dev Now navigate to to view the Nomad dashboard. -Next, in a new terminal window, build the `app_install` scenario with: +Next, in a new terminal window, build the scenario you want to run with: ```shell nix build .#app_install ``` +where `app_install` is the name of the scenario. Once the scenario is built you can run the Nomad job with: ```shell -nomad job run nomad/run-app_install-local.nomad.hcl +nomad job run -var-file=nomad/var_files/app_install_minimal.vars nomad/run_scenario_local.nomad.hcl ``` +where the `-var-file` path should point to the var file in `nomad/var_files` with the matching scenario name. + +You can also override existing and omitted variables with the `-var` flag. For example, to set the duration (omitted) and reporter (override) use: + +```shell +nomad job run -var-file=nomad/var_files/app_install_minimal.vars -var duration=5 -var reporter=in-memory nomad/run_scenario_local.nomad.hcl +``` +> [!Note] +> Make sure the `var` options are after the `var-file` option otherwise the values in the file will take precedence. Then, navigate to where you should see one allocation (the Nomad name for an instance of the job) -this allocation should have two tasks: the `start-holochain` task and the `run_scenario` task. You can view the logs of these tasks to see the results. -The allocation should be marked as "Completed" after a few seconds. +this allocation should have two tasks: the `start_holochain` task and the `run_scenario` task. You can view the logs of these tasks to see the results. +The allocation should be marked as "Completed" after the duration specified. Once you've finished testing you can kill the Nomad agent with `^C` in the first terminal running the agent.