[Merged by Bors] - Tune GNU malloc

Cargo.toml

@@ -32,6 +32,7 @@ members = [
  "common/lockfile",
  "common/logging",
  "common/lru_cache",
+ "common/malloc_utils",
  "common/remote_signer_consumer",
  "common/sensitive_url",
  "common/slot_clock",

beacon_node/http_metrics/Cargo.toml

@@ -20,6 +20,7 @@ lazy_static = "1.4.0"
 eth2 = { path = "../../common/eth2" }
 lighthouse_version = { path = "../../common/lighthouse_version" }
 warp_utils = { path = "../../common/warp_utils" }
+malloc_utils = { path = "../../common/malloc_utils" }
 
 [dev-dependencies]
 tokio = { version = "1.1.0", features = ["sync"] }

beacon_node/http_metrics/src/lib.rs

@@ -49,6 +49,7 @@ pub struct Config {
  pub listen_addr: Ipv4Addr,
  pub listen_port: u16,
  pub allow_origin: Option<String>,
+ pub allocator_metrics_enabled: bool,
 }
 
 impl Default for Config {
@@ -58,6 +59,7 @@ impl Default for Config {
  listen_addr: Ipv4Addr::new(127, 0, 0, 1),
  listen_port: 5054,
  allow_origin: None,
+ allocator_metrics_enabled: true,
  }
  }
 }

beacon_node/http_metrics/src/metrics.rs

@@ -1,6 +1,7 @@
 use crate::Context;
 use beacon_chain::BeaconChainTypes;
 use lighthouse_metrics::{Encoder, TextEncoder};
+use malloc_utils::scrape_allocator_metrics;
 
 pub use lighthouse_metrics::*;
 
@@ -41,6 +42,12 @@ pub fn gather_prometheus_metrics<T: BeaconChainTypes>(
 
  warp_utils::metrics::scrape_health_metrics();
 
+ // It's important to ensure these metrics are explicitly enabled in the case that users aren't
+ // using glibc and this function causes panics.
+ if ctx.config.allocator_metrics_enabled {
+ scrape_allocator_metrics();
+ }
+
  encoder
  .encode(&lighthouse_metrics::gather(), &mut buffer)
  .unwrap();

beacon_node/http_metrics/tests/tests.rs

@@ -20,6 +20,7 @@ async fn returns_200_ok() {
  listen_addr: Ipv4Addr::new(127, 0, 0, 1),
  listen_port: 0,
  allow_origin: None,
+ allocator_metrics_enabled: true,
  },
  chain: None,
  db_path: None,

beacon_node/src/config.rs

@@ -1,5 +1,5 @@
 use clap::ArgMatches;
-use clap_utils::BAD_TESTNET_DIR_MESSAGE;
+use clap_utils::{flags::DISABLE_MALLOC_TUNING_FLAG, BAD_TESTNET_DIR_MESSAGE};
 use client::{ClientConfig, ClientGenesis};
 use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR};
 use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
@@ -145,6 +145,11 @@ pub fn get_config<E: EthSpec>(
  );
  }
 
+ // Do not scrape for malloc metrics if we've disabled tuning malloc as it may cause panics.
+ if cli_args.is_present(DISABLE_MALLOC_TUNING_FLAG) {
+ client_config.http_metrics.allocator_metrics_enabled = false;
+ }
+
  /*
  * Eth1
  */

common/clap_utils/src/flags.rs

@@ -0,0 +1,3 @@
+//! CLI flags used across the Lighthouse code base can be located here.
+
+pub const DISABLE_MALLOC_TUNING_FLAG: &str = "disable-malloc-tuning";

common/clap_utils/src/lib.rs

@@ -6,6 +6,8 @@ use ssz::Decode;
 use std::path::PathBuf;
 use std::str::FromStr;
 
+pub mod flags;
+
 pub const BAD_TESTNET_DIR_MESSAGE: &str = "The hard-coded testnet directory was invalid. \
  This happens when Lighthouse is migrating between spec versions \
  or when there is no default public network to connect to. \

common/malloc_utils/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "malloc_utils"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+lighthouse_metrics = { path = "../lighthouse_metrics" }
+lazy_static = "1.4.0"
+libc = "0.2.79"
+parking_lot = "0.11.0"
+num_cpus = "1.13.0"

common/malloc_utils/src/glibc.rs

@@ -0,0 +1,210 @@
+//! Contains functions for tuning and controlling "The GNU Allocator", included in the `glibc`
+//! library.
+//!
+//! https://www.gnu.org/software/libc/manual/html_node/The-GNU-Allocator.html
+//!
+//! These functions are generally only suitable for Linux systems.
+use lazy_static::lazy_static;
+use lighthouse_metrics::*;
+use parking_lot::Mutex;
+use std::env;
+use std::os::raw::c_int;
+use std::result::Result;
+
+/// The value to be provided to `malloc_mmap_threshold`.
+///
+/// Value chosen so that values of the validators tree hash cache will *not* be allocated via
+/// `mmap`.
+///
+/// The size of a single chunk is:
+///
+/// NODES_PER_VALIDATOR * VALIDATORS_PER_ARENA * 32 = 15 * 4096 * 32 = 1.875 MiB
+const OPTIMAL_MMAP_THRESHOLD: c_int = 2 * 1_024 * 1_024;
+
+/// The maximum number of arenas allowed to be created by malloc.
+///
+/// See `ArenaMaxSetting` docs for details.
+const OPTIMAL_ARENA_MAX: ArenaMaxSetting = ArenaMaxSetting::NumCpus;
+
+/// Constants used to configure malloc internals.
+///
+/// Source:
+///
+/// https://github.com/lattera/glibc/blob/895ef79e04a953cac1493863bcae29ad85657ee1/malloc/malloc.h#L115-L123
+const M_MMAP_THRESHOLD: c_int = -4;
+const M_ARENA_MAX: c_int = -8;
+
+/// Environment variables used to configure malloc.
+///
+/// Source:
+///
+/// https://man7.org/linux/man-pages/man3/mallopt.3.html
+const ENV_VAR_ARENA_MAX: &str = "MALLOC_ARENA_MAX";
+const ENV_VAR_MMAP_THRESHOLD: &str = "MALLOC_MMAP_THRESHOLD_";
+
+#[allow(dead_code)]
+enum ArenaMaxSetting {
+ /// Do not set any value for MALLOC_ARENA_MAX, leave it as default.
+ DoNotSet,
+ /// Set a fixed value.
+ Fixed(c_int),
+ /// Read the number of CPUs at runtime and use that value.
+ NumCpus,
+}
+
+lazy_static! {
+ pub static ref GLOBAL_LOCK: Mutex<()> = <_>::default();
+}
+
+// Metrics for the malloc. For more information, see:
+//
+// https://man7.org/linux/man-pages/man3/mallinfo.3.html
+lazy_static! {
+ pub static ref MALLINFO_ARENA: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_arena",
+ "The total amount of memory allocated by means other than mmap(2). \
+ This figure includes both in-use blocks and blocks on the free list.",
+ );
+ pub static ref MALLINFO_ORDBLKS: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_ordblks",
+ "The number of ordinary (i.e., non-fastbin) free blocks.",
+ );
+ pub static ref MALLINFO_SMBLKS: lighthouse_metrics::Result<IntGauge> =
+ try_create_int_gauge("mallinfo_smblks", "The number of fastbin free blocks.",);
+ pub static ref MALLINFO_HBLKS: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_hblks",
+ "The number of blocks currently allocated using mmap.",
+ );
+ pub static ref MALLINFO_HBLKHD: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_hblkhd",
+ "The number of bytes in blocks currently allocated using mmap.",
+ );
+ pub static ref MALLINFO_FSMBLKS: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_fsmblks",
+ "The total number of bytes in fastbin free blocks.",
+ );
+ pub static ref MALLINFO_UORDBLKS: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_uordblks",
+ "The total number of bytes used by in-use allocations.",
+ );
+ pub static ref MALLINFO_FORDBLKS: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_fordblks",
+ "The total number of bytes in free blocks.",
+ );
+ pub static ref MALLINFO_KEEPCOST: lighthouse_metrics::Result<IntGauge> = try_create_int_gauge(
+ "mallinfo_keepcost",
+ "The total amount of releasable free space at the top of the heap..",
+ );
+}
+
+/// Calls `mallinfo` and updates Prometheus metrics with the results.
+pub fn scrape_mallinfo_metrics() {
+ // The docs for this function say it is thread-unsafe since it may return inconsistent results.
+ // Since these are just metrics it's not a concern to us if they're sometimes inconsistent.
+ //
+ // There exists a `malloc2` function, however it was release in February 2021 and this seems too
+ // recent to rely on.
+ //
+ // Docs:
+ //
+ // https://man7.org/linux/man-pages/man3/mallinfo.3.html
+ let mallinfo = mallinfo();
+
+ set_gauge(&MALLINFO_ARENA, mallinfo.arena as i64);
+ set_gauge(&MALLINFO_ORDBLKS, mallinfo.ordblks as i64);
+ set_gauge(&MALLINFO_SMBLKS, mallinfo.smblks as i64);
+ set_gauge(&MALLINFO_HBLKS, mallinfo.hblks as i64);
+ set_gauge(&MALLINFO_HBLKHD, mallinfo.hblkhd as i64);
+ set_gauge(&MALLINFO_FSMBLKS, mallinfo.fsmblks as i64);
+ set_gauge(&MALLINFO_UORDBLKS, mallinfo.uordblks as i64);
+ set_gauge(&MALLINFO_FORDBLKS, mallinfo.fordblks as i64);
+ set_gauge(&MALLINFO_KEEPCOST, mallinfo.keepcost as i64);
+}
+
+/// Perform all configuration routines.
+pub fn configure_glibc_malloc() -> Result<(), String> {
+ if !env_var_present(ENV_VAR_ARENA_MAX) {
+ let arena_max = match OPTIMAL_ARENA_MAX {
+ ArenaMaxSetting::DoNotSet => None,
+ ArenaMaxSetting::Fixed(n) => Some(n),
+ ArenaMaxSetting::NumCpus => Some(num_cpus::get() as c_int),
+ };
+
+ if let Some(max) = arena_max {
+ if let Err(e) = malloc_arena_max(max) {
+ return Err(format!("failed (code {}) to set malloc max arena count", e));
+ }
+ }
+ }
+
+ if !env_var_present(ENV_VAR_MMAP_THRESHOLD) {
+ if let Err(e) = malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD) {
+ return Err(format!("failed (code {}) to set malloc mmap threshold", e));
+ }
+ }
+
+ Ok(())
+}
+
+/// Returns `true` if an environment variable is present.
+fn env_var_present(name: &str) -> bool {
+ env::var(name) != Err(env::VarError::NotPresent)
+}
+
+/// Uses `mallopt` to set the `M_ARENA_MAX` value, specifying the number of memory arenas to be
+/// created by malloc.
+///
+/// Generally speaking, a smaller arena count reduces memory fragmentation at the cost of memory contention
+/// between threads.
+///
+/// ## Resources
+///
+/// - https://man7.org/linux/man-pages/man3/mallopt.3.html
+fn malloc_arena_max(num_arenas: c_int) -> Result<(), c_int> {
+ into_result(mallopt(M_ARENA_MAX, num_arenas))
+}
+
+/// Uses `mallopt` to set the `M_MMAP_THRESHOLD` value, specifying the threshold where objects of this
+/// size or larger are allocated via an `mmap`.
+///
+/// ## Resources
+///
+/// - https://man7.org/linux/man-pages/man3/mallopt.3.html
+fn malloc_mmap_threshold(num_arenas: c_int) -> Result<(), c_int> {
+ into_result(mallopt(M_MMAP_THRESHOLD, num_arenas))
+}
+
+fn mallopt(param: c_int, val: c_int) -> c_int {
+ // Prevent this function from being called in parallel with any other non-thread-safe function.
+ let _lock = GLOBAL_LOCK.lock();
+ unsafe { libc::mallopt(param, val) }
+}
+
+fn mallinfo() -> libc::mallinfo {
+ // Prevent this function from being called in parallel with any other non-thread-safe function.
+ let _lock = GLOBAL_LOCK.lock();
+ unsafe { libc::mallinfo() }
+}
+
+fn into_result(result: c_int) -> Result<(), c_int> {
+ if result == 1 {
+ Ok(())
+ } else {
+ Err(result)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn malloc_arena_max_does_not_panic() {
+ malloc_arena_max(2).unwrap();
+ }
+
+ #[test]
+ fn malloc_mmap_threshold_does_not_panic() {
+ malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD).unwrap();
+ }
+}