Add slam tool to consensus server repo (mobilecoinfoundation#733)

* Add slam tool to consensus server repo

The slam tool is a load testing tool for the consensus servers.
Generally, test code should be versioned with the code under test.

This moves the slam tool to the same repo as the consensus servers,
and will make it easier to run this test in CD and more easily use
it to validate changes to the consensus servers.

* fix build, docu, review comments, copyright

* slam -> mc-slam and update docu

* lockfiles and fix build

* fixups to readme

Cargo.toml

@@ -69,6 +69,7 @@ members = [
     "sgx/report-cache/untrusted",
     "sgx/slog-edl",
     "sgx/urts-sys",
+    "slam",
     "test-vectors/account-keys",
     "test-vectors/b58-encodings",
     "transaction/core",

slam/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "mc-slam"
+version = "1.0.0"
+authors = ["MobileCoin"]
+edition = "2018"
+
+[dependencies]
+mc-account-keys = { path = "../account-keys" }
+mc-attest-core = { path = "../attest/core" }
+mc-common = { path = "../common", features = ["log"] }
+mc-connection = { path = "../connection" }
+mc-consensus-enclave-measurement = { path = "../consensus/enclave/measurement" }
+mc-consensus-scp = { path = "../consensus/scp" }
+mc-crypto-keys = { path = "../crypto/keys" }
+mc-fog-report-validation = { path = "../fog/report/validation" }
+mc-ledger-db = { path = "../ledger/db" }
+mc-ledger-sync = { path = "../ledger/sync" }
+mc-mobilecoind = { path = "../mobilecoind" }
+mc-transaction-core = { path = "../transaction/core" }
+mc-transaction-std = { path = "../transaction/std" }
+mc-util-keyfile = { path = "../util/keyfile" }
+mc-util-uri = { path = "../util/uri" }
+
+crossbeam-channel = "0.5"
+grpcio = "0.6.0"
+lazy_static = "1.4"
+rand = "0.7"
+structopt = "0.3"
+tempdir = "0.3"
+
+[target.'cfg(any(target_feature = "avx2", target_feature = "avx"))'.dependencies]
+curve25519-dalek = { version = "3.0", default-features = false, features = ["simd_backend", "nightly"] }
+
+[target.'cfg(not(any(target_feature = "avx2", target_feature = "avx")))'.dependencies]
+curve25519-dalek = { version = "3.0", default-features = false, features = ["nightly", "u64_backend"] }
+
+[dev-dependencies]
+mc-common = { path = "../common", features = ["loggers"] }

slam/README.md

@@ -0,0 +1,68 @@
+# Slam Load Testing
+
+Slam is a load testing tool that rapidly submits transactions to a consensus network.
+
+Slam testing a consensus network usually involves:
+
+1. Obtaining a copy of the initial ledger content and corresponding private keys.
+1. Running Slam.
+1. Monitoring the network's performance.
+
+# Obtaining an initial ledger and keys
+
+In order to create transactions, Slam requires a copy of the ledger and the set of private keys that own the contents of the ledger. The ledger and keys must **exactly** match those used by the consensus network; if not, Slam's transactions will likely be rejected with `InvalidTxOutMembershipProof` or `LedgerDbError`.
+
+## Generating a local ledger and keys
+If you know how the consensus network's ledger was initialized, you can initialize the same ledger locally with:
+
+```
+    mobilecoin $ mkdir -p target/sample_data
+    mobilecoin $ cd sample_data
+    mobilecoin/target/sample_data $ cargo run -p mc-util-keyfile --bin sample-keys --release -- --num 1000
+
+    mobilecoin/target/sample_data $ cargo run -p mc-util-generate-sample-ledger --bin generate-sample-ledger --release -- --num 100
+```
+
+
+## Using a deployed ledger
+Alternatively, Slam can use the ledger from a deployed network instead of a locally-generated one:
+
+```
+    docker pull mobilecoin/node_hw:master-latest
+    docker run -it --detach=true --entrypoint="/bin/bash" --name=extract_ledger mobilecoin/node_hw:master-latest
+    docker cp extract_ledger:/var/lib/mobilecoin/ledger/ /tmp/ledger
+```
+
+# Running Slam
+
+To Run Slam against a deployed network (e.g. "other"), set one of the following environment variables. If you get them wrong, you'll probably see "Attestation failure" messages.
+
+```
+# aws s3 cp s3://enclave-distribution.other.mobilecoin.com/consensus/consensus-enclave.css ./s
+export CONSENSUS_ENCLAVE_CSS=/home/you/consensus-enclave.css
+
+# Local development
+export CONSENSUS_ENCLAVE_PRIVKEY=/home/you/Enclave_private.pem
+```
+
+Then, run slam in `release` mode:
+
+```
+    cargo run -p mc-slam --release -- --sample-data-dir target/sample_data/ \
+        --peer mc://node1.demo.mobilecoin.com \
+        --peer mc://node2.demo.mobilecoin.com \
+        --peer mc://node3.demo.mobilecoin.com \
+        --peer mc://node4.demo.mobilecoin.com \
+        --peer mc://node5.demo.mobilecoin.com \
+        --add-tx-delay-ms 500 \
+        --tombstone-block 100 \
+        --with-ledger-sync \
+        --tx-source-url https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node1.demo.mobilecoin.com/
+```
+
+## Running Slam with a local consensus network
+
+If you are running a consensus network locally, you will replace the peer URIs above with either:
+
+* `insecure-mc://localhost:3223` if running outside Docker, with the port matching the local ports corresponding to the consensus nodes.
+* `insecure-mc://<container_name>:3223` if running inside Docker, making sure that the ports are published on the docker container

slam/src/config.rs

@@ -0,0 +1,124 @@
+// Copyright (c) 2018-2021 The MobileCoin Foundation
+
+//! Configuration parameters for the slam script
+
+use grpcio::EnvBuilder;
+use mc_attest_core::{MrSignerVerifier, Verifier, DEBUG_ENCLAVE};
+use mc_common::logger::{o, Logger};
+use mc_connection::{
+    HardcodedCredentialsProvider, Result as ConnectionResult, SyncConnection, ThickClient,
+};
+use mc_mobilecoind::config::PeersConfig;
+use mc_util_uri::ConnectionUri;
+use std::{fs, path::PathBuf, str::FromStr, sync::Arc};
+use structopt::StructOpt;
+
+#[derive(Clone, Debug, StructOpt)]
+#[structopt(name = "slam", about = "Generate valid txs.")]
+pub struct SlamConfig {
+    /// Path to sample data for keys/ and ledger/
+    #[structopt(long, parse(from_os_str))]
+    pub sample_data_dir: PathBuf,
+
+    /// Number of transactions to send per account
+    #[structopt(long, default_value = "-1")]
+    pub num_tx_to_send: isize,
+
+    /// Number of inputs in the ring
+    #[structopt(long, default_value = "11")]
+    pub ring_size: usize,
+
+    /// Block after which to tombstone
+    #[structopt(long, default_value = "50")]
+    pub tombstone_block: u64,
+
+    #[structopt(long, default_value = "1")]
+    pub num_inputs: usize,
+
+    /// Ask consensus for the current block to set tombstone appropriately
+    #[structopt(long)]
+    pub query_consensus_for_cur_block: bool,
+
+    /// Offset into transactions to start
+    #[structopt(long, default_value = "0")]
+    pub start_offset: usize,
+
+    /// Num transactions per account - must set this if using start_offset
+    #[structopt(long, default_value = "0")]
+    pub num_transactions_per_account: usize,
+
+    /// Offset into accounts
+    #[structopt(long, default_value = "0")]
+    pub account_offset: usize,
+
+    /// Number of threads with which to submit transactions (threadpool uses min with cpu)
+    #[structopt(long, default_value = "32")]
+    pub max_threads: usize,
+
+    /// Delay (in milliseconds) before each add_transaction call
+    #[structopt(long, default_value = "0")]
+    pub add_tx_delay_ms: u64,
+
+    /// Enable ledger sync, which allows slam to run indefinitely (until it runs out of money).
+    #[structopt(long)]
+    pub with_ledger_sync: bool,
+
+    /// URLs to use for transaction data.
+    ///
+    /// For example: https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node1.master.mobilecoin.com/
+    #[structopt(long = "tx-source-url")]
+    pub tx_source_urls: Vec<String>,
+
+    #[structopt(flatten)]
+    pub peers_config: PeersConfig,
+}
+
+impl SlamConfig {
+    pub fn get_connections(
+        &self,
+        logger: &Logger,
+    ) -> ConnectionResult<Vec<SyncConnection<ThickClient<HardcodedCredentialsProvider>>>> {
+        let mut mr_signer_verifier =
+            MrSignerVerifier::from(mc_consensus_enclave_measurement::sigstruct());
+        mr_signer_verifier.allow_hardening_advisory("INTEL-SA-00334");
+
+        let mut verifier = Verifier::default();
+        verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE);
+
+        self.peers_config
+            .peers
+            .clone()
+            .unwrap()
+            .iter()
+            .map(|uri| {
+                // We create a new environment for each peer to maintain current behavior
+                let env = Arc::new(
+                    EnvBuilder::new()
+                        .name_prefix(format!("slam-{}", uri.addr()))
+                        .build(),
+                );
+                let logger = logger.new(o!("mc.cxn" => uri.addr()));
+                ThickClient::new(
+                    uri.clone(),
+                    verifier.clone(),
+                    env,
+                    HardcodedCredentialsProvider::from(uri),
+                    logger.clone(),
+                )
+                .map(|inner| SyncConnection::new(inner, logger))
+            })
+            .collect()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct FileData(pub Vec<u8>);
+impl FromStr for FileData {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(fs::read(s).map_err(|e| {
+            format!("Failed reading \"{}\": {:?}", s, e)
+        })?))
+    }
+}

slam/src/lib.rs

@@ -0,0 +1,5 @@
+// Copyright (c) 2018-2021 The MobileCoin Foundation
+
+pub mod config;
+
+pub use crate::config::SlamConfig;