From c32160e32e835db7c7a64a097ffd38fbbb65f1c5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:26:31 +0300 Subject: [PATCH] Add support for memory-profiling on subsystem-bench (#5522) Add support in subsystem-benchmarks to profile memory usage using the jemalloc builting profiler, this allows us to run each benchmark with profiling enabled and determine if the memory usage patters are in conformance with our expectations. --------- Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 46 +++++++++++++++++++ Cargo.toml | 1 + polkadot/node/subsystem-bench/Cargo.toml | 8 ++++ polkadot/node/subsystem-bench/README.md | 35 ++++++++++++++ .../src/cli/subsystem-bench.rs | 11 +++++ 5 files changed, 101 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bb0f01542d31..8eb4fa5b4e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7350,6 +7350,23 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jemalloc_pprof" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96368c0fc161a0a1a20b3952b6fd31ee342fffc87ed9e48ac1ed49fb25686655" +dependencies = [ + "anyhow", + "libc", + "mappings", + "once_cell", + "pprof_util", + "tempfile", + "tikv-jemalloc-ctl", + "tokio", + "tracing", +] + [[package]] name = "jni" version = "0.19.0" @@ -8550,6 +8567,19 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "mappings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa2605f461115ef6336342b12f0d8cabdfd7b258fed86f5f98c725535843601" +dependencies = [ + "anyhow", + "libc", + "once_cell", + "pprof_util", + "tracing", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -14927,6 +14957,7 @@ dependencies = [ "futures-timer", "hex", "itertools 0.11.0", + "jemalloc_pprof", "kvdb-memorydb", "log", "orchestra", @@ -14950,6 +14981,7 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", + "polkadot-service", "polkadot-statement-distribution", "prometheus", "pyroscope", @@ -14978,6 +15010,7 @@ dependencies = [ "sp-tracing 16.0.0", "strum 0.26.2", "substrate-prometheus-endpoint", + "tikv-jemallocator", "tokio", "tracing-gum", ] @@ -15426,6 +15459,19 @@ dependencies = [ "thiserror", ] +[[package]] +name = "pprof_util" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c620a1858d6ebf10d7c60256629078b2d106968d0e6ff63b850d9ecd84008fbe" +dependencies = [ + "anyhow", + "flate2", + "num", + "paste", + "prost 0.11.9", +] + [[package]] name = "ppv-lite86" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index f26a894960a7..7cd18bc8a590 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -810,6 +810,7 @@ is-terminal = { version = "0.4.9" } is_executable = { version = "1.0.1" } isahc = { version = "1.2" } itertools = { version = "0.11" } +jemalloc_pprof = { version = "0.4" } jobserver = { version = "0.1.26" } jsonpath_lib = { version = "0.3" } jsonrpsee = { version = "0.24.3" } diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 9384f8142a97..ae798cf2640a 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -19,7 +19,11 @@ path = "src/cli/subsystem-bench.rs" # Prevent rustdoc error. Already documented from top-level Cargo.toml. doc = false + [dependencies] +tikv-jemallocator = { features = ["profiling", "unprefixed_malloc_on_supported_platforms"], workspace = true, optional = true } +jemalloc_pprof = { workspace = true, optional = true } +polkadot-service = { workspace = true, default-features = true } polkadot-node-subsystem = { workspace = true, default-features = true } polkadot-node-subsystem-util = { workspace = true, default-features = true } polkadot-node-subsystem-types = { workspace = true, default-features = true } @@ -93,3 +97,7 @@ strum = { features = ["derive"], workspace = true, default-features = true } [features] default = [] +memprofile = [ + "dep:jemalloc_pprof", + "dep:tikv-jemallocator", +] diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 228fba41c46c..8d20f1f49c0e 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -260,6 +260,41 @@ This file is best interpreted with `cg_annotate --auto=yes cachegrind.out.` For finer profiling of cache misses, better use `perf` on a bare-metal machine. +### Profile memory usage using jemalloc + +Bellow you can find instructions how to setup and run profiling with jemalloc, this is complementary +with using other memory profiling tools like: . + +#### Prerequisites + +Install tooling with: + +``` +sudo apt install libjemalloc-dev graphviz +``` + +#### Generate memory usage snapshots + +Memory usage can be profiled by running any subsystem benchmark with `--features memprofile`, e.g: + +``` +RUSTFLAGS=-g cargo run -p polkadot-subsystem-bench --release --features memprofile -- polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +``` + +#### Interpret the results + +After the benchmark ran the memory usage snapshots can be found in `/tmp/subsystem-bench*`, to extract the information +from a snapshot you can use `jeprof` like this: + +``` +jeprof --text PATH_TO_EXECUTABLE_WITH_DEBUG_SYMBOLS /tmp/subsystem-bench.1222895.199.i199.heap > statistics.txt +``` + +Useful links: + +- Tutorial: +- Jemalloc configuration options: + ## Create new test objectives This tool is intended to make it easy to write new test objectives that focus individual subsystems, diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 153efdda4056..0f68b905b4ca 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -182,6 +182,17 @@ impl BenchCli { } } +#[cfg(feature = "memprofile")] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[cfg(feature = "memprofile")] +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +// See https://jemalloc.net/jemalloc.3.html for more information on the configuration options. +pub static malloc_conf: &[u8] = + b"prof:true,prof_active:true,lg_prof_interval:30,lg_prof_sample:21,prof_prefix:/tmp/subsystem-bench\0"; + fn main() -> eyre::Result<()> { color_eyre::install()?; sp_tracing::try_init_simple();