Switch allocator to jemalloc (#3697)

Squashed commit of the following:

commit 974b335
Merge: ac205b7 480309f
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Wed Jan 18 10:01:26 2023 +1100

    Merge remote-tracking branch 'origin/unstable' into jemalloc

commit 480309f
Author: aliask <aliask@gmail.com>
Date:   Tue Jan 17 05:13:49 2023 +0000

    Fix some dead links in markdown files (#3885)

    ## Issue Addressed

    No issue has been raised for these broken links.

    ## Proposed Changes

    Update links with the new URLs for the same document.

    ## Additional Info

    ~The link for the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb/) mailing list is also broken, but I can't find the correct link.~

    Co-authored-by: Paul Hauner <paul@paulhauner.com>

commit b4d9fc0
Author: GeemoCandama <geemo@tutanota.com>
Date:   Tue Jan 17 05:13:48 2023 +0000

    add logging for starting request  and receiving block (#3858)

    ## Issue Addressed

    #3853

    ## Proposed Changes

    Added `INFO` level logs for requesting and receiving the unsigned block.

    ## Additional Info

    Logging for successfully publishing the signed block is already there. And seemingly there is a log for when "We realize we are going to produce a block" in the `start_update_service`: `info!(log, "Block production service started");
    `.  Is there anywhere else you'd like to see logging around this event?

    Co-authored-by: GeemoCandama <104614073+GeemoCandama@users.noreply.github.com>

commit 9a970ce
Author: David Theodore <prodigalsonsolutions@gmail.com>
Date:   Tue Jan 17 05:13:47 2023 +0000

    add better err reporting UnableToOpenVotingKeystore (#3781)

    ## Issue Addressed

    #3780

    ## Proposed Changes

    Add error reporting that notifies the node operator that the `voting_keystore_path` in their `validator_definitions.yml` file may be incorrect.

    ## Additional Info

    There is more info in issue #3780

    Co-authored-by: Paul Hauner <paul@paulhauner.com>

commit ac205b7
Merge: 93457d8 bf533c8
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Fri Nov 25 16:32:33 2022 +1100

    Merge remote-tracking branch 'origin/unstable' into jemalloc

commit 93457d8
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Wed Nov 9 11:53:59 2022 +1100

    Fix cargo-udeps

commit 6c42aef
Author: Michael Sproul <micsproul@gmail.com>
Date:   Tue Nov 8 19:12:19 2022 +1100

    Fixups

commit f14b87b
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Tue Nov 8 16:28:16 2022 +1100

    Update docs

commit 5005dc3
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Tue Nov 8 16:22:42 2022 +1100

    Fix lcli

commit a082ba5
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Tue Nov 8 16:17:10 2022 +1100

    Remove check-consensus

commit 81441e9
Author: Michael Sproul <micsproul@gmail.com>
Date:   Tue Nov 8 15:28:11 2022 +1100

    Disable jemalloc on Windows

commit 41eac5d
Author: Michael Sproul <micsproul@gmail.com>
Date:   Tue Nov 8 13:46:17 2022 +1100

    Compatibility with macOS

commit 69ecba7
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Mon Nov 7 18:48:31 2022 +1100

    Add jemalloc support

.github/workflows/test-suite.yml

@@ -306,16 +306,6 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Typecheck benchmark code without running it
       run: make check-benches
-  check-consensus:
-    name: check-consensus
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v3
-    - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Typecheck consensus code in strict mode
-      run: make check-consensus
   clippy:
     name: clippy
     runs-on: ubuntu-latest
@@ -382,14 +372,12 @@ jobs:
     - uses: actions/checkout@v3
     - name: Install Rust (${{ env.PINNED_NIGHTLY }})
       run: rustup toolchain install $PINNED_NIGHTLY
-    # NOTE: cargo-udeps version is pinned until this issue is resolved:
-    # https://github.com/est31/cargo-udeps/issues/135
     - name: Install Protoc
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install cargo-udeps
-      run: cargo install cargo-udeps --locked --force --version 0.1.30
+      run: cargo install cargo-udeps --locked --force --version $CARGO_UDEPS_VERSION
     - name: Create Cargo config dir
       run: mkdir -p .cargo
     - name: Install custom Cargo config

Cargo.toml

@@ -81,6 +81,7 @@ members = [
     "validator_client",
     "validator_client/slashing_protection",
 ]
+resolver = "2"
 
 [patch]
 [patch.crates-io]

Makefile

@@ -14,8 +14,16 @@ BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 PINNED_NIGHTLY ?= nightly
 CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19
 
+# List of features to use when building natively. Can be overriden via the environment.
+# No jemalloc on Windows
+ifeq ($(OS),Windows_NT)
+    FEATURES?=
+else
+    FEATURES?=jemalloc
+endif
+
 # List of features to use when cross-compiling. Can be overridden via the environment.
-CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx
+CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,jemalloc
 
 # Cargo profile for Cross builds. Default is for local builds, CI uses an override.
 CROSS_PROFILE ?= release
@@ -101,10 +109,6 @@ cargo-fmt:
 check-benches:
 	cargo check --workspace --benches
 
-# Typechecks consensus code *without* allowing deprecated legacy arithmetic or metrics.
-check-consensus:
-	cargo check -p state_processing --no-default-features
-
 # Runs only the ef-test vectors.
 run-ef-tests:
 	rm -rf $(EF_TESTS)/.accessed_file_log.txt

README.md

@@ -66,7 +66,7 @@ of the Lighthouse book.
 The best place for discussion is the [Lighthouse Discord
 server](https://discord.gg/cyAszAh).
 
-Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb/) mailing list for email
+Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb) mailing list for email
 notifications about releases, network status and other important information.
 
 Encrypt sensitive messages using our [PGP

book/src/installation-source.md

@@ -64,6 +64,7 @@ choco install protoc
 These dependencies are for compiling Lighthouse natively on Windows. Lighthouse can also run
 successfully under the [Windows Subsystem for Linux (WSL)][WSL]. If using Ubuntu under WSL, you
 should follow the instructions for Ubuntu listed in the [Dependencies (Ubuntu)](#ubuntu) section.
+
 [WSL]: https://docs.microsoft.com/en-us/windows/wsl/about
 
 ## Build Lighthouse
@@ -128,8 +129,12 @@ Commonly used features include:
 * `gnosis`: support for the Gnosis Beacon Chain.
 * `portable`: support for legacy hardware.
 * `modern`: support for exclusively modern hardware.
-* `slasher-mdbx`: support for the MDBX slasher backend (enabled by default).
+* `slasher-mdbx`: support for the MDBX slasher backend. Enabled by default.
 * `slasher-lmdb`: support for the LMDB slasher backend.
+* `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS.
+  Not supported on Windows.
+
+[jemalloc]: https://jemalloc.net/
 
 ## Compilation Profiles
 

book/src/merge-migration.md

@@ -58,7 +58,7 @@ supported.
 Each execution engine has its own flags for configuring the engine API and JWT. Please consult
 the relevant page for your execution engine for the required flags:
 
-- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/interface/consensus-clients)
+- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/getting-started/consensus-clients)
 - [Nethermind: Running Nethermind Post Merge](https://docs.nethermind.io/nethermind/first-steps-with-nethermind/running-nethermind-post-merge)
 - [Besu: Prepare For The Merge](https://besu.hyperledger.org/en/stable/HowTo/Upgrade/Prepare-for-The-Merge/)
 - [Erigon: Beacon Chain (Consensus Layer)](https://github.com/ledgerwatch/erigon#beacon-chain-consensus-layer)
@@ -203,5 +203,5 @@ guidance for specific setups.
 - [Ethereum.org: The Merge](https://ethereum.org/en/upgrades/merge/)
 - [Ethereum Staking Launchpad: Merge Readiness](https://launchpad.ethereum.org/en/merge-readiness).
 - [CoinCashew: Ethereum Merge Upgrade Checklist](https://www.coincashew.com/coins/overview-eth/ethereum-merge-upgrade-checklist-for-home-stakers-and-validators)
-- [EthDocker: Merge Preparation](https://eth-docker.net/docs/About/MergePrep/)
+- [EthDocker: Merge Preparation](https://eth-docker.net/About/MergePrep/)
 - [Remy Roy: How to join the Goerli/Prater merge testnet](https://github.com/remyroy/ethstaker/blob/main/merge-goerli-prater.md)

book/src/run_a_node.md

@@ -26,7 +26,7 @@ has authority to control the execution engine.
 Each execution engine has its own flags for configuring the engine API and JWT.
 Please consult the relevant page of your execution engine for the required flags:
 
-- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/interface/consensus-clients)
+- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/getting-started/consensus-clients)
 - [Nethermind: Running Nethermind & CL](https://docs.nethermind.io/nethermind/first-steps-with-nethermind/running-nethermind-post-merge)
 - [Besu: Connect to Mainnet](https://besu.hyperledger.org/en/stable/public-networks/get-started/connect/mainnet/)
 - [Erigon: Beacon Chain (Consensus Layer)](https://github.com/ledgerwatch/erigon#beacon-chain-consensus-layer)

bors.toml

@@ -10,7 +10,6 @@ status = [
     "merge-transition-ubuntu",
     "no-eth1-simulator-ubuntu",
     "check-benchmarks",
-    "check-consensus",
     "clippy",
     "arbitrary-check",
     "cargo-audit",

common/malloc_utils/Cargo.toml

@@ -4,18 +4,21 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 lighthouse_metrics = { path = "../lighthouse_metrics" }
 lazy_static = "1.4.0"
 libc = "0.2.79"
 parking_lot = "0.12.0"
-jemallocator = { version = "0.3.0", optional = true, features = ["background_threads"] }
-jemalloc-sys = { version = "0.3.0", optional = true }
+jemalloc-ctl = { version = "0.5.0", optional = true }
+
+# Jemalloc's background_threads feature requires Linux (pthreads).
+[target.'cfg(target_os = "linux")'.dependencies]
+jemallocator = { version = "0.5.0", optional = true, features = ["stats", "background_threads"] }
+
+[target.'cfg(not(target_os = "linux"))'.dependencies]
+jemallocator = { version = "0.5.0", optional = true, features = ["stats"] }
 
 [features]
 mallinfo2 = []
-jemalloc = ["jemallocator", "jemalloc-sys"]
-jemalloc-stats = ["jemallocator/stats"]
+jemalloc = ["jemallocator", "jemalloc-ctl"]
 jemalloc-profiling = ["jemallocator/profiling"]

common/malloc_utils/src/jemalloc.rs

@@ -7,6 +7,46 @@
 //!
 //! A) `JEMALLOC_SYS_WITH_MALLOC_CONF` at compile-time.
 //! B) `_RJEM_MALLOC_CONF` at runtime.
+use jemalloc_ctl::{arenas, epoch, stats, Error};
+use lazy_static::lazy_static;
+use lighthouse_metrics::{set_gauge, try_create_int_gauge, IntGauge};
 
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
+// Metrics for jemalloc.
+lazy_static! {
+    pub static ref NUM_ARENAS: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_num_arenas", "The number of arenas in use");
+    pub static ref BYTES_ALLOCATED: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_allocated", "Equivalent to stats.allocated");
+    pub static ref BYTES_ACTIVE: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_active", "Equivalent to stats.active");
+    pub static ref BYTES_MAPPED: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_mapped", "Equivalent to stats.mapped");
+    pub static ref BYTES_METADATA: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_metadata", "Equivalent to stats.metadata");
+    pub static ref BYTES_RESIDENT: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_resident", "Equivalent to stats.resident");
+    pub static ref BYTES_RETAINED: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_retained", "Equivalent to stats.retained");
+}
+
+pub fn scrape_jemalloc_metrics() {
+    scrape_jemalloc_metrics_fallible().unwrap()
+}
+
+pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> {
+    // Advance the epoch so that the underlying statistics are updated.
+    epoch::advance()?;
+
+    set_gauge(&NUM_ARENAS, arenas::narenas::read()? as i64);
+    set_gauge(&BYTES_ALLOCATED, stats::allocated::read()? as i64);
+    set_gauge(&BYTES_ACTIVE, stats::active::read()? as i64);
+    set_gauge(&BYTES_MAPPED, stats::mapped::read()? as i64);
+    set_gauge(&BYTES_METADATA, stats::metadata::read()? as i64);
+    set_gauge(&BYTES_RESIDENT, stats::resident::read()? as i64);
+    set_gauge(&BYTES_RETAINED, stats::retained::read()? as i64);
+
+    Ok(())
+}

common/malloc_utils/src/lib.rs

@@ -26,7 +26,8 @@
 
 #[cfg(all(
     target_os = "linux",
-    not(any(target_env = "musl", feature = "jemalloc"))
+    not(target_env = "musl"),
+    not(feature = "jemalloc")
 ))]
 mod glibc;
 
@@ -37,14 +38,28 @@ pub use interface::*;
 
 #[cfg(all(
     target_os = "linux",
-    not(any(target_env = "musl", feature = "jemalloc"))
+    not(target_env = "musl"),
+    not(feature = "jemalloc")
 ))]
 mod interface {
     pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator;
     pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics;
 }
 
-#[cfg(any(not(target_os = "linux"), target_env = "musl", feature = "jemalloc"))]
+#[cfg(feature = "jemalloc")]
+mod interface {
+    #[allow(dead_code)]
+    pub fn configure_memory_allocator() -> Result<(), String> {
+        Ok(())
+    }
+
+    pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics;
+}
+
+#[cfg(all(
+    any(not(target_os = "linux"), target_env = "musl"),
+    not(feature = "jemalloc")
+))]
 mod interface {
     #[allow(dead_code, clippy::unnecessary_wraps)]
     pub fn configure_memory_allocator() -> Result<(), String> {

lcli/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2021"
 [features]
 portable = ["bls/supranational-portable"]
 fake_crypto = ['bls/fake_crypto']
+jemalloc = ["malloc_utils/jemalloc"]
 
 [dependencies]
 bls = { path = "../crypto/bls" }
@@ -40,3 +41,7 @@ eth2 = { path = "../common/eth2" }
 snap = "1.0.1"
 beacon_chain = { path = "../beacon_node/beacon_chain" }
 store = { path = "../beacon_node/store" }
+malloc_utils = { path = "../common/malloc_utils" }
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["malloc_utils"]

lighthouse/Cargo.toml

@@ -24,6 +24,8 @@ gnosis = []
 slasher-mdbx = ["slasher/mdbx"]
 # Support slasher LMDB backend.
 slasher-lmdb = ["slasher/lmdb"]
+# Use jemalloc.
+jemalloc = ["malloc_utils/jemalloc"]
 
 [dependencies]
 beacon_node = { "path" = "../beacon_node" }

lighthouse/src/main.rs

@@ -31,6 +31,14 @@ fn bls_library_name() -> &'static str {
     }
 }
 
+fn allocator_name() -> &'static str {
+    if cfg!(feature = "jemalloc") {
+        "jemalloc"
+    } else {
+        "system"
+    }
+}
+
 fn main() {
     // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
     if std::env::var("RUST_BACKTRACE").is_err() {
@@ -51,10 +59,12 @@ fn main() {
                 "{}\n\
                  BLS library: {}\n\
                  SHA256 hardware acceleration: {}\n\
+                 Allocator: {}\n\
                  Specs: mainnet (true), minimal ({}), gnosis ({})",
                  VERSION.replace("Lighthouse/", ""),
                  bls_library_name(),
                  have_sha_extensions(),
+                 allocator_name(),
                  cfg!(feature = "spec-minimal"),
                  cfg!(feature = "gnosis"),
             ).as_str()

validator_client/src/block_service.rs

@@ -333,6 +333,11 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
         let proposer_index = self.validator_store.validator_index(&validator_pubkey);
         let validator_pubkey_ref = &validator_pubkey;
 
+        info!(
+            log,
+            "Requesting unsigned block";
+            "slot" => slot.as_u64(),
+        );
         // Request block from first responsive beacon node.
         let block = self
             .beacon_nodes
@@ -383,6 +388,11 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
                         }
                     };
 
+                    info!(
+                        log,
+                        "Received unsigned block";
+                        "slot" => slot.as_u64(),
+                    );
                     if proposer_index != Some(block.proposer_index()) {
                         return Err(BlockError::Recoverable(
                             "Proposer index does not match block proposer. Beacon chain re-orged"
@@ -401,6 +411,11 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
             .await
             .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?;
 
+        info!(
+            log,
+            "Publishing signed block";
+            "slot" => slot.as_u64(),
+        );
         // Publish block with first available beacon node.
         self.beacon_nodes
             .first_success(