refactor: prepare and dump state inside params-estimator

.buildkite/pipeline.yml

@@ -96,8 +96,7 @@ steps:
       ./build.sh
       cd ..
       RUSTFLAGS='-D warnings' cargo run --release --package neard --bin neard -- --home /tmp/data init --test-seed=alice.near --account-id=test.near --fast
-      RUSTFLAGS='-D warnings' cargo run --release --package genesis-populate --bin genesis-populate -- --additional-accounts-num=200000 --home /tmp/data
-      RUSTFLAGS='-D warnings' cargo run --release --package runtime-params-estimator --bin runtime-params-estimator -- --home /tmp/data --accounts-num 20000 --iters 1 --warmup-iters 1 --metric time
+      RUSTFLAGS='-D warnings' cargo run --release --package runtime-params-estimator --bin runtime-params-estimator -- --home /tmp/data --accounts-num 20000 --additional-accounts-num 200000 --iters 1 --warmup-iters 1 --metric time
 
     branches: "!master !beta !stable"
     timeout: 60

runtime/runtime-params-estimator/Cargo.toml

@@ -30,6 +30,8 @@ near-vm-runner = {path = "../../runtime/near-vm-runner" }
 node-runtime = { path = "../../runtime/runtime" }
 near-store = { path = "../../core/store" }
 near-primitives = { path = "../../core/primitives" }
+near-test-contracts = { path = "../../runtime/near-test-contracts" }
+
 testlib = { path = "../../test-utils/testlib" }
 state-viewer = { path = "../../test-utils/state-viewer" }
 nearcore = { path = "../../nearcore" }

runtime/runtime-params-estimator/README.md

@@ -2,23 +2,22 @@
 
 Use this tool to measure the running time of elementary runtime operations that have associated fees.
 
-1. Create genesis config and generate state dump
+1. Initialize home folder for the estimator
     ```bash
     cargo run --release --package neard --bin neard -- --home /tmp/data init --test-seed=alice.near --account-id=test.near --fast
-    cargo run --release --package genesis-populate --bin genesis-populate -- --additional-accounts-num=200000 --home /tmp/data
     ```
 
 2. Run the estimator
     ```bash
-    cargo run --release --package runtime-params-estimator --features required --bin runtime-params-estimator -- --home /tmp/data --accounts-num 20000 --iters 1 --warmup-iters 1 --metric time
+    cargo run --release --package runtime-params-estimator --features required --bin runtime-params-estimator -- --home /tmp/data --accounts-num 20000 --additional-accounts-num 200000 --iters 1 --warmup-iters 1 --metric time
     ```
 
     With the given parameters above estimator will run relatively fast.
     Note the `--metric time` flag: it instructs the estimator to use wall-clock time for estimation, which is quick, but highly variable between runs and physical machines.
     To get more robust estimates, use these arguments:
 
     ```bash
-    --docker --home /tmp/data --accounts-num 20000 --iters 1 --warmup-iters 1 --metric icount
+    --docker --home /tmp/data --accounts-num 20000 --additional-accounts-num 200000 --iters 1 --warmup-iters 1 --metric icount
     ```
 
     This will run and build the estimator inside a docker container, using QEMU to precisely count the number of executed instructions.

runtime/runtime-params-estimator/emu-cost/README.md

@@ -32,15 +32,14 @@ Start container and build estimator with:
     host> ./run.sh
     docker> cd /host/nearcore
     docker> cargo run -j2 --release --package neard --bin neard -- --home /tmp/data init --test-seed=alice.near --account-id=test.near --fast
-    docker> cargo run -j2 --release --package genesis-populate --bin genesis-populate -- --additional-accounts-num=200000 --home /tmp/data
     docker> cd /host/nearcore/runtime/runtime-params-estimator
     docker> pushd ./test-contract && ./build.sh && popd
     docker> cargo build --release --package runtime-params-estimator --features required
 
 Now start the estimator under QEMU with the counter plugin enabled (note, that Rust compiler produces SSE4, so specify recent CPU):
 
     docker> ./emu-cost/counter_plugin/qemu-x86_64 -cpu Westmere-v1 -plugin file=./emu-cost/counter_plugin/libcounter.so \
-         ../../target/release/runtime-params-estimator --home /tmp/data --accounts-num 20000 --iters 1 --warmup-iters 1
+         ../../target/release/runtime-params-estimator --home /tmp/data --accounts-num 20000 --additional-accounts-num 200000 --iters 1 --warmup-iters 1
 
 ### Notes
 

runtime/runtime-params-estimator/src/main.rs

@@ -1,7 +1,9 @@
 use anyhow::Context;
 use clap::Clap;
+use genesis_populate::GenesisBuilder;
+use near_store::create_store;
 use near_vm_runner::VMKind;
-use nearcore::get_default_home;
+use nearcore::{get_default_home, get_store_path, load_config};
 use runtime_params_estimator::cases::run;
 use runtime_params_estimator::costs_to_runtime_config;
 use runtime_params_estimator::testbed_runners::Config;
@@ -13,6 +15,7 @@ use std::fs;
 use std::path::Path;
 use std::path::PathBuf;
 use std::process::Command;
+use std::sync::Arc;
 use std::time;
 
 #[derive(Clap)]
@@ -26,9 +29,12 @@ struct CliArgs {
     /// How many iterations per block are we going to try.
     #[clap(long, default_value = "10")]
     iters: usize,
-    /// How many accounts were generated with `genesis-populate`.
-    #[clap(long, default_value = "10000")]
+    /// Number of active accounts in the state (accounts used for estimation).
+    #[clap(long, default_value = "20000")]
     accounts_num: usize,
+    /// Number of additional accounts to add to the state, among which active accounts are selected.
+    #[clap(long, default_value = "200000")]
+    additional_accounts_num: usize,
     /// What metric to use.
     #[clap(long, default_value = "icount", possible_values = &["icount", "time"])]
     metric: String,
@@ -53,6 +59,24 @@ fn main() -> anyhow::Result<()> {
 
     let state_dump_path = cli_args.home.unwrap_or_else(|| get_default_home().into());
 
+    let additional_accounts_num = cli_args.additional_accounts_num as u64;
+    if additional_accounts_num > 0 {
+        let near_config = load_config(&state_dump_path);
+        let store = create_store(&get_store_path(&state_dump_path));
+        GenesisBuilder::from_config_and_store(
+            &state_dump_path,
+            Arc::new(near_config.genesis),
+            store,
+        )
+        .add_additional_accounts(additional_accounts_num)
+        .add_additional_accounts_contract(near_test_contracts::tiny_contract().to_vec())
+        .print_progress()
+        .build()
+        .unwrap()
+        .dump_state()
+        .unwrap();
+    }
+
     if cli_args.docker {
         return main_docker(&state_dump_path);
     }
@@ -146,6 +170,7 @@ fn main_docker(state_dump_path: &Path) -> anyhow::Result<()> {
 
         let mut buf = String::new();
         buf.push_str("set -ex;\n");
+        buf.push_str("cd /host/nearcore;\n");
         buf.push_str(
             "\
 cargo build --manifest-path /host/nearcore/Cargo.toml \
@@ -167,6 +192,11 @@ cargo build --manifest-path /host/nearcore/Cargo.toml \
         while let Some(arg) = args.next() {
             match arg.as_str() {
                 "--docker" => continue,
+                "--additional-accounts-num" => {
+                    args.next();
+                    write!(buf, " {:?} 0", arg).unwrap();
+                    continue;
+                }
                 "--home" => {
                     args.next();
                     continue;