From ca241e22c40245c261c2a98a4223c6eac3a7e56d Mon Sep 17 00:00:00 2001 From: Brandon Kite Date: Fri, 8 Sep 2023 10:37:12 -0700 Subject: [PATCH] Importer Metrics (#1355) closes: #1163 Adds the following metrics to prometheus: - `importer_tx_count` - `importer_block_height` - `importer_latest_block_commit_timestamp_s` - `importer_execute_and_commit_duration_s` Additional related changes: - New field in the metadata table to keep track of the total number of transactions so that the metric is accurate across restarts - Removed the compiler feature flag for "metrics" since it was sneaked into all builds anyways with the addition of futures tracking. This allows us to _finally_ start measuring the following: - Transactions per second - Delays in synchronization (ie. comparing the block heights between nodes, and comparing the committed timestamps) - Various measures of time related to block importing/execution on synchronizing nodes (i.e. validation mode) such as avg, max, min etc --------- Co-authored-by: Brandon Vrooman --- CHANGELOG.md | 2 + Cargo.lock | 2 +- benches/Cargo.toml | 2 +- bin/e2e-test-client/Cargo.toml | 2 +- bin/fuel-core/Cargo.toml | 5 +- crates/fuel-core/Cargo.toml | 5 +- crates/fuel-core/src/database/metadata.rs | 18 +++++ crates/fuel-core/src/graphql_api.rs | 1 - .../src/graphql_api/metrics_extension.rs | 6 +- crates/fuel-core/src/graphql_api/service.rs | 20 +++-- .../src/service/adapters/block_importer.rs | 8 +- crates/fuel-core/src/service/metrics.rs | 11 +-- crates/fuel-core/src/state/rocks_db.rs | 81 ++++++++----------- crates/metrics/Cargo.toml | 1 - crates/metrics/src/core_metrics.rs | 11 +-- crates/metrics/src/graphql_metrics.rs | 13 ++- crates/metrics/src/importer.rs | 72 +++++++++++++++++ crates/metrics/src/lib.rs | 13 +++ crates/metrics/src/p2p_metrics.rs | 8 +- crates/metrics/src/response.rs | 25 +++--- crates/metrics/src/services.rs | 16 ++-- crates/metrics/src/txpool_metrics.rs | 11 ++- crates/services/importer/Cargo.toml | 1 + crates/services/importer/src/importer.rs | 66 ++++++++++++++- crates/services/importer/src/importer/test.rs | 4 +- crates/services/importer/src/ports.rs | 3 + crates/services/p2p/src/gossipsub/config.rs | 4 +- crates/services/p2p/src/p2p_service.rs | 4 +- crates/services/src/service.rs | 4 +- crates/services/txpool/src/txpool.rs | 9 +-- deployment/e2e-client.Dockerfile | 2 +- tests/Cargo.toml | 11 +-- tests/tests/lib.rs | 1 + 33 files changed, 296 insertions(+), 146 deletions(-) create mode 100644 crates/metrics/src/importer.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bdbd931655..2024710d67a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Description of the upcoming release here. ### Added +- [#1355](https://github.com/FuelLabs/fuel-core/pull/1355): Added new metrics related to block importing, such as tps, sync delays etc - [#1324](https://github.com/FuelLabs/fuel-core/pull/1324): Added pyroscope profiling to fuel-core, intended to be used by a secondary docker image that has debug symbols enabled. - [#1309](https://github.com/FuelLabs/fuel-core/pull/1309): Add documentation for running debug builds with CLion and Visual Studio Code. - [#1308](https://github.com/FuelLabs/fuel-core/pull/1308): Add support for loading .env files when compiling with the `env` feature. This allows users to conveniently supply CLI arguments in a secure and IDE-agnostic way. @@ -30,6 +31,7 @@ Description of the upcoming release here. - [#1342](https://github.com/FuelLabs/fuel-core/pull/1342): Add error handling for P2P requests to return `None` to requester and log error ### Breaking +- [#1355](https://github.com/FuelLabs/fuel-core/pull/1355): Removed the `metrics` feature flag from the fuel-core crate, and metrics are now included by default. - [#1318](https://github.com/FuelLabs/fuel-core/pull/1318): Removed the `--sync-max-header-batch-requests` CLI argument, and renamed `--sync-max-get-txns` to `--sync-block-stream-buffer-size` to better represent the current behavior in the import. - [#1290](https://github.com/FuelLabs/fuel-core/pull/1290): Standardize CLI args to use `-` instead of `_`. - [#1279](https://github.com/FuelLabs/fuel-core/pull/1279): Added a new CLI flag to enable the Relayer service `--enable-relayer`, and disabled the Relayer service by default. When supplying the `--enable-relayer` flag, the `--relayer` argument becomes mandatory, and omitting it is an error. Similarly, providing a `--relayer` argument without the `--enable-relayer` flag is an error. Lastly, providing the `--keypair` or `--network` arguments will also produce an error if the `--enable-p2p` flag is not set. diff --git a/Cargo.lock b/Cargo.lock index 4fcc8de86da..7f420468739 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2944,6 +2944,7 @@ name = "fuel-core-importer" version = "0.20.4" dependencies = [ "anyhow", + "fuel-core-metrics", "fuel-core-storage", "fuel-core-trace", "fuel-core-types", @@ -2970,7 +2971,6 @@ name = "fuel-core-metrics" version = "0.20.4" dependencies = [ "axum", - "lazy_static", "once_cell", "pin-project-lite 0.2.12", "prometheus-client 0.18.1", diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 89e2c62e91b..bc95bd636d5 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -14,7 +14,7 @@ ctrlc = "3.2.3" ed25519-dalek = "1.0" # TODO: upgrade to 2.0 when it's released, and remove rand below ed25519-dalek_old_rand = { package = "rand", version = "0.7.3" } ethnum = "1.3" -fuel-core = { path = "../crates/fuel-core", default-features = false, features = ["metrics", "rocksdb-production"] } +fuel-core = { path = "../crates/fuel-core", default-features = false, features = ["rocksdb-production"] } fuel-core-services = { path = "./../crates/services" } fuel-core-storage = { path = "./../crates/storage" } fuel-core-sync = { path = "./../crates/services/sync", features = ["benchmarking"] } diff --git a/bin/e2e-test-client/Cargo.toml b/bin/e2e-test-client/Cargo.toml index 2561920d27b..b9587955f4c 100644 --- a/bin/e2e-test-client/Cargo.toml +++ b/bin/e2e-test-client/Cargo.toml @@ -36,6 +36,6 @@ insta = { workspace = true } tempfile = { workspace = true } [features] -default = ["fuel-core?/default", "fuel-core?/metrics"] +default = ["fuel-core?/default"] p2p = ["fuel-core?/p2p"] dev-deps = ["fuel-core/test-helpers"] diff --git a/bin/fuel-core/Cargo.toml b/bin/fuel-core/Cargo.toml index d26a55adfc0..dbbd6cf9ed7 100644 --- a/bin/fuel-core/Cargo.toml +++ b/bin/fuel-core/Cargo.toml @@ -41,12 +41,11 @@ url = { version = "2.2", optional = true } test-case = { workspace = true } [features] -default = ["env", "metrics", "relayer", "rocksdb"] +default = ["env", "relayer", "rocksdb"] env = ["dep:dotenvy"] -metrics = ["fuel-core/metrics"] p2p = ["fuel-core/p2p", "const_format"] relayer = ["fuel-core/relayer", "dep:url", "dep:serde_json"] rocksdb = ["fuel-core/rocksdb"] rocksdb-production = ["fuel-core/rocksdb-production"] # features to enable in production, but increase build times -production = ["env", "metrics", "relayer", "rocksdb-production", "p2p"] +production = ["env", "relayer", "rocksdb-production", "p2p"] diff --git a/crates/fuel-core/Cargo.toml b/crates/fuel-core/Cargo.toml index 47ffbd8dfa2..10aab388a4c 100644 --- a/crates/fuel-core/Cargo.toml +++ b/crates/fuel-core/Cargo.toml @@ -25,7 +25,7 @@ fuel-core-consensus-module = { workspace = true } fuel-core-database = { workspace = true } fuel-core-executor = { workspace = true } fuel-core-importer = { workspace = true } -fuel-core-metrics = { workspace = true, optional = true } +fuel-core-metrics = { workspace = true } fuel-core-p2p = { workspace = true, optional = true } fuel-core-poa = { workspace = true } fuel-core-producer = { workspace = true } @@ -73,8 +73,7 @@ test-case = { workspace = true } test-strategy = { workspace = true } [features] -default = ["metrics", "rocksdb"] -metrics = ["dep:fuel-core-metrics"] +default = ["rocksdb"] p2p = ["dep:fuel-core-p2p", "dep:fuel-core-sync"] relayer = ["dep:fuel-core-relayer"] rocksdb = ["dep:rocksdb", "dep:tempfile"] diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index af74538e2e6..88ae9391ba1 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -8,6 +8,9 @@ use fuel_core_chain_config::ChainConfig; pub(crate) const DB_VERSION_KEY: &[u8] = b"version"; pub(crate) const CHAIN_NAME_KEY: &[u8] = b"chain_name"; +/// Tracks the total number of transactions written to the chain +/// It's useful for analyzing TPS or other metrics. +pub(crate) const TX_COUNT: &[u8] = b"total_tx_count"; /// Can be used to perform migrations in the future. pub(crate) const DB_VERSION: u32 = 0x00; @@ -45,4 +48,19 @@ impl Database { pub fn get_chain_name(&self) -> DatabaseResult> { self.get(CHAIN_NAME_KEY, Column::Metadata) } + + pub fn increase_tx_count(&self, new_txs: u64) -> DatabaseResult { + // TODO: how should tx count be initialized after regenesis? + let current_tx_count: u64 = + self.get(TX_COUNT, Column::Metadata)?.unwrap_or_default(); + // Using saturating_add because this value doesn't significantly impact the correctness of execution. + let new_tx_count = current_tx_count.saturating_add(new_txs); + self.insert::<_, _, u64>(TX_COUNT, Column::Metadata, &new_tx_count)?; + Ok(new_tx_count) + } + + pub fn get_tx_count(&self) -> DatabaseResult { + self.get(TX_COUNT, Column::Metadata) + .map(|v| v.unwrap_or_default()) + } } diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index e64c68c1405..3fd27a3c19b 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -9,7 +9,6 @@ use fuel_core_types::{ }; use std::net::SocketAddr; -#[cfg(feature = "metrics")] pub(crate) mod metrics_extension; pub mod ports; pub mod service; diff --git a/crates/fuel-core/src/graphql_api/metrics_extension.rs b/crates/fuel-core/src/graphql_api/metrics_extension.rs index 7d88d4e9f06..a178fa136c7 100644 --- a/crates/fuel-core/src/graphql_api/metrics_extension.rs +++ b/crates/fuel-core/src/graphql_api/metrics_extension.rs @@ -15,7 +15,7 @@ use async_graphql::{ Value, Variables, }; -use fuel_core_metrics::graphql_metrics::GRAPHQL_METRICS; +use fuel_core_metrics::graphql_metrics::graphql_metrics; use std::{ sync::{ Arc, @@ -59,7 +59,7 @@ impl Extension for MetricsExtInner { let start_time = Instant::now(); let result = next.run(ctx).await; let seconds = start_time.elapsed().as_secs_f64(); - GRAPHQL_METRICS.graphql_observe("request", seconds); + graphql_metrics().graphql_observe("request", seconds); result } @@ -95,7 +95,7 @@ impl Extension for MetricsExtInner { let elapsed = start_time.elapsed(); if let Some(field_name) = field_name { - GRAPHQL_METRICS.graphql_observe(field_name, elapsed.as_secs_f64()); + graphql_metrics().graphql_observe(field_name, elapsed.as_secs_f64()); } if elapsed > self.log_threshold_ms { diff --git a/crates/fuel-core/src/graphql_api/service.rs b/crates/fuel-core/src/graphql_api/service.rs index a24e5634827..536e7510c34 100644 --- a/crates/fuel-core/src/graphql_api/service.rs +++ b/crates/fuel-core/src/graphql_api/service.rs @@ -1,5 +1,3 @@ -#[cfg(feature = "metrics")] -use crate::graphql_api::metrics_extension::MetricsExtension; use crate::{ fuel_core_graphql_api::ports::{ BlockProducerPort, @@ -7,7 +5,10 @@ use crate::{ DatabasePort, TxPoolPort, }, - graphql_api::Config, + graphql_api::{ + metrics_extension::MetricsExtension, + Config, + }, schema::{ CoreSchema, CoreSchemaBuilder, @@ -166,18 +167,15 @@ pub fn new_service( ) -> anyhow::Result { let network_addr = config.addr; - let builder = schema + let schema = schema .data(config) .data(database) .data(txpool) .data(producer) - .data(consensus_module); - let builder = builder.extension(async_graphql::extensions::Tracing); - - #[cfg(feature = "metrics")] - let builder = builder.extension(MetricsExtension::new(_log_threshold_ms)); - - let schema = builder.finish(); + .data(consensus_module) + .extension(async_graphql::extensions::Tracing) + .extension(MetricsExtension::new(_log_threshold_ms)) + .finish(); let router = Router::new() .route("/playground", get(graphql_playground)) diff --git a/crates/fuel-core/src/service/adapters/block_importer.rs b/crates/fuel-core/src/service/adapters/block_importer.rs index 53347208aea..3fc939a0f7b 100644 --- a/crates/fuel-core/src/service/adapters/block_importer.rs +++ b/crates/fuel-core/src/service/adapters/block_importer.rs @@ -51,8 +51,10 @@ impl BlockImporterAdapter { executor: ExecutorAdapter, verifier: VerifierAdapter, ) -> Self { + let importer = Importer::new(config, database, executor, verifier); + importer.init_metrics(); Self { - block_importer: Arc::new(Importer::new(config, database, executor, verifier)), + block_importer: Arc::new(importer), } } @@ -113,6 +115,10 @@ impl ImporterDatabase for Database { fn latest_block_height(&self) -> StorageResult { self.latest_height() } + + fn increase_tx_count(&self, new_txs_count: u64) -> StorageResult { + self.increase_tx_count(new_txs_count).map_err(Into::into) + } } impl ExecutorDatabase for Database { diff --git a/crates/fuel-core/src/service/metrics.rs b/crates/fuel-core/src/service/metrics.rs index e04ffe5c771..1acd7e48661 100644 --- a/crates/fuel-core/src/service/metrics.rs +++ b/crates/fuel-core/src/service/metrics.rs @@ -3,17 +3,8 @@ use axum::{ http::Request, response::IntoResponse, }; -#[cfg(feature = "metrics")] use fuel_core_metrics::response::encode_metrics_response; pub async fn metrics(_req: Request) -> impl IntoResponse { - #[cfg(feature = "metrics")] - { - encode_metrics_response() - } - #[cfg(not(feature = "metrics"))] - { - use axum::http::StatusCode; - (StatusCode::NOT_FOUND, "Metrics collection disabled") - } + encode_metrics_response() } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 204fece9bd8..889b6aa63ed 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -15,8 +15,7 @@ use crate::{ WriteOperation, }, }; -#[cfg(feature = "metrics")] -use fuel_core_metrics::core_metrics::DATABASE_METRICS; +use fuel_core_metrics::core_metrics::database_metrics; use fuel_core_storage::iter::{ BoxedIter, IntoBoxedIter, @@ -142,13 +141,12 @@ impl RocksDb { item.map(|(key, value)| { let value_as_vec = Vec::from(value); let key_as_vec = Vec::from(key); - #[cfg(feature = "metrics")] - { - DATABASE_METRICS.read_meter.inc(); - DATABASE_METRICS - .bytes_read - .observe((key_as_vec.len() + value_as_vec.len()) as f64); - } + + database_metrics().read_meter.inc(); + database_metrics() + .bytes_read + .observe((key_as_vec.len() + value_as_vec.len()) as f64); + (key_as_vec, Arc::new(value_as_vec)) }) .map_err(|e| DatabaseError::Other(e.into())) @@ -158,18 +156,16 @@ impl RocksDb { impl KeyValueStore for RocksDb { fn get(&self, key: &[u8], column: Column) -> DatabaseResult> { - #[cfg(feature = "metrics")] - DATABASE_METRICS.read_meter.inc(); + database_metrics().read_meter.inc(); let value = self .db .get_cf(&self.cf(column), key) .map_err(|e| DatabaseError::Other(e.into())); - #[cfg(feature = "metrics")] - { - if let Ok(Some(value)) = &value { - DATABASE_METRICS.bytes_read.observe(value.len() as f64); - } + + if let Ok(Some(value)) = &value { + database_metrics().bytes_read.observe(value.len() as f64); } + value.map(|value| value.map(Arc::new)) } @@ -179,11 +175,9 @@ impl KeyValueStore for RocksDb { column: Column, value: Value, ) -> DatabaseResult> { - #[cfg(feature = "metrics")] - { - DATABASE_METRICS.write_meter.inc(); - DATABASE_METRICS.bytes_written.observe(value.len() as f64); - } + database_metrics().write_meter.inc(); + database_metrics().bytes_written.observe(value.len() as f64); + // FIXME: This is a race condition. We should use a transaction. let prev = self.get(key, column)?; // FIXME: This is a race condition. We should use a transaction. @@ -273,8 +267,7 @@ impl KeyValueStore for RocksDb { } fn size_of_value(&self, key: &[u8], column: Column) -> DatabaseResult> { - #[cfg(feature = "metrics")] - DATABASE_METRICS.read_meter.inc(); + database_metrics().read_meter.inc(); Ok(self .db @@ -289,8 +282,7 @@ impl KeyValueStore for RocksDb { column: Column, mut buf: &mut [u8], ) -> DatabaseResult> { - #[cfg(feature = "metrics")] - DATABASE_METRICS.read_meter.inc(); + database_metrics().read_meter.inc(); let r = self .db @@ -304,21 +296,16 @@ impl KeyValueStore for RocksDb { }) .transpose()?; - #[cfg(feature = "metrics")] - { - if let Some(r) = &r { - DATABASE_METRICS.bytes_read.observe(*r as f64); - } + if let Some(r) = &r { + database_metrics().bytes_read.observe(*r as f64); } + Ok(r) } fn write(&self, key: &[u8], column: Column, buf: &[u8]) -> DatabaseResult { - #[cfg(feature = "metrics")] - { - DATABASE_METRICS.write_meter.inc(); - DATABASE_METRICS.bytes_written.observe(buf.len() as f64); - } + database_metrics().write_meter.inc(); + database_metrics().bytes_written.observe(buf.len() as f64); let r = buf.len(); self.db @@ -329,8 +316,7 @@ impl KeyValueStore for RocksDb { } fn read_alloc(&self, key: &[u8], column: Column) -> DatabaseResult> { - #[cfg(feature = "metrics")] - DATABASE_METRICS.read_meter.inc(); + database_metrics().read_meter.inc(); let r = self .db @@ -338,12 +324,10 @@ impl KeyValueStore for RocksDb { .map_err(|e| DatabaseError::Other(e.into()))? .map(|value| value.to_vec()); - #[cfg(feature = "metrics")] - { - if let Some(r) = &r { - DATABASE_METRICS.bytes_read.observe(r.len() as f64); - } + if let Some(r) = &r { + database_metrics().bytes_read.observe(r.len() as f64); } + Ok(r.map(Arc::new)) } @@ -389,13 +373,12 @@ impl BatchOperations for RocksDb { } } } - #[cfg(feature = "metrics")] - { - DATABASE_METRICS.write_meter.inc(); - DATABASE_METRICS - .bytes_written - .observe(batch.size_in_bytes() as f64); - } + + database_metrics().write_meter.inc(); + database_metrics() + .bytes_written + .observe(batch.size_in_bytes() as f64); + self.db .write(batch) .map_err(|e| DatabaseError::Other(e.into())) diff --git a/crates/metrics/Cargo.toml b/crates/metrics/Cargo.toml index 1726aade240..fe14ec609fe 100644 --- a/crates/metrics/Cargo.toml +++ b/crates/metrics/Cargo.toml @@ -12,7 +12,6 @@ description = "Fuel metrics" [dependencies] axum = { workspace = true } -lazy_static = { workspace = true } libp2p-prom-client = { workspace = true } once_cell = { workspace = true } pin-project-lite = { workspace = true } diff --git a/crates/metrics/src/core_metrics.rs b/crates/metrics/src/core_metrics.rs index 0b037f3cb89..0b691c6fe0c 100644 --- a/crates/metrics/src/core_metrics.rs +++ b/crates/metrics/src/core_metrics.rs @@ -1,4 +1,3 @@ -use lazy_static::lazy_static; use prometheus_client::{ metrics::{ counter::Counter, @@ -6,6 +5,7 @@ use prometheus_client::{ }, registry::Registry, }; +use std::sync::OnceLock; pub struct DatabaseMetrics { pub registry: Registry, @@ -64,10 +64,11 @@ pub fn init(mut metrics: DatabaseMetrics) -> DatabaseMetrics { metrics } -lazy_static! { - pub static ref DATABASE_METRICS: DatabaseMetrics = { - let registry = DatabaseMetrics::new(); +static DATABASE_METRICS: OnceLock = OnceLock::new(); +pub fn database_metrics() -> &'static DatabaseMetrics { + DATABASE_METRICS.get_or_init(|| { + let registry = DatabaseMetrics::new(); init(registry) - }; + }) } diff --git a/crates/metrics/src/graphql_metrics.rs b/crates/metrics/src/graphql_metrics.rs index 0a2d6641db9..508c18dc1cf 100644 --- a/crates/metrics/src/graphql_metrics.rs +++ b/crates/metrics/src/graphql_metrics.rs @@ -1,4 +1,4 @@ -use lazy_static::lazy_static; +use crate::timing_buckets; use prometheus_client::{ encoding::EncodeLabelSet, metrics::{ @@ -7,6 +7,7 @@ use prometheus_client::{ }, registry::Registry, }; +use std::sync::OnceLock; #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] pub struct Label { @@ -23,7 +24,7 @@ impl GraphqlMetrics { fn new() -> Self { let mut registry = Registry::default(); let requests = Family::::new_with_constructor(|| { - Histogram::new(BUCKETS.iter().cloned()) + Histogram::new(timing_buckets().iter().cloned()) }); registry.register("graphql_request_duration_seconds", "", requests.clone()); Self { registry, requests } @@ -37,9 +38,7 @@ impl GraphqlMetrics { } } -lazy_static! { - pub static ref GRAPHQL_METRICS: GraphqlMetrics = GraphqlMetrics::new(); - // recommended bucket defaults for API response times - static ref BUCKETS: Vec = - vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]; +static GRAPHQL_METRICS: OnceLock = OnceLock::new(); +pub fn graphql_metrics() -> &'static GraphqlMetrics { + GRAPHQL_METRICS.get_or_init(GraphqlMetrics::new) } diff --git a/crates/metrics/src/importer.rs b/crates/metrics/src/importer.rs new file mode 100644 index 00000000000..41e75d6d222 --- /dev/null +++ b/crates/metrics/src/importer.rs @@ -0,0 +1,72 @@ +use crate::timing_buckets; +use prometheus_client::{ + metrics::{ + gauge::Gauge, + histogram::Histogram, + }, + registry::Registry, +}; +use std::sync::{ + atomic::AtomicU64, + OnceLock, +}; + +pub struct ImporterMetrics { + pub registry: Registry, + // using gauges in case blocks are rolled back for any reason + pub total_txs_count: Gauge, + pub block_height: Gauge, + pub latest_block_import_timestamp: Gauge, + pub execute_and_commit_duration: Histogram, +} + +impl Default for ImporterMetrics { + fn default() -> Self { + let mut registry = Registry::default(); + + let tx_count_gauge = Gauge::default(); + let block_height_gauge = Gauge::default(); + let latest_block_import_ms = Gauge::default(); + let execute_and_commit_duration = + Histogram::new(timing_buckets().iter().cloned()); + + registry.register( + "importer_tx_count", + "the total amount of transactions that have been imported on chain", + tx_count_gauge.clone(), + ); + + registry.register( + "importer_block_height", + "the current height of the chain", + block_height_gauge.clone(), + ); + + registry.register( + "importer_latest_block_commit_timestamp_s", + "A timestamp of when the current block was imported", + latest_block_import_ms.clone(), + ); + + registry.register( + "importer_execute_and_commit_duration_s", + "Records the duration time of executing and committing a block", + execute_and_commit_duration.clone(), + ); + + Self { + registry, + total_txs_count: tx_count_gauge, + block_height: block_height_gauge, + latest_block_import_timestamp: latest_block_import_ms, + execute_and_commit_duration, + } + } +} + +// Setup a global static for accessing importer metrics +static IMPORTER_METRICS: OnceLock = OnceLock::new(); + +pub fn importer_metrics() -> &'static ImporterMetrics { + IMPORTER_METRICS.get_or_init(ImporterMetrics::default) +} diff --git a/crates/metrics/src/lib.rs b/crates/metrics/src/lib.rs index 2fe6ac19306..1e1489fa61c 100644 --- a/crates/metrics/src/lib.rs +++ b/crates/metrics/src/lib.rs @@ -1,10 +1,23 @@ #![deny(unused_crate_dependencies)] #![deny(warnings)] +use std::sync::OnceLock; + pub mod core_metrics; pub mod future_tracker; pub mod graphql_metrics; +pub mod importer; pub mod p2p_metrics; pub mod response; pub mod services; pub mod txpool_metrics; + +// recommended bucket defaults for logging response times +static BUCKETS: OnceLock> = OnceLock::new(); +pub fn timing_buckets() -> &'static Vec { + BUCKETS.get_or_init(|| { + vec![ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ] + }) +} diff --git a/crates/metrics/src/p2p_metrics.rs b/crates/metrics/src/p2p_metrics.rs index b93375073ff..bedf4841c19 100644 --- a/crates/metrics/src/p2p_metrics.rs +++ b/crates/metrics/src/p2p_metrics.rs @@ -1,9 +1,9 @@ -use lazy_static::lazy_static; use libp2p_prom_client::{ metrics::counter::Counter, registry::Registry, }; use once_cell::race::OnceBox; +use std::sync::OnceLock; pub struct P2PMetrics { pub gossip_sub_registry: OnceBox, @@ -34,6 +34,8 @@ impl P2PMetrics { } } -lazy_static! { - pub static ref P2P_METRICS: P2PMetrics = P2PMetrics::new(); +static P2P_METRICS: OnceLock = OnceLock::new(); + +pub fn p2p_metrics() -> &'static P2PMetrics { + P2P_METRICS.get_or_init(P2PMetrics::new) } diff --git a/crates/metrics/src/response.rs b/crates/metrics/src/response.rs index aaac7008b0d..9fd8d0f0dc0 100644 --- a/crates/metrics/src/response.rs +++ b/crates/metrics/src/response.rs @@ -1,8 +1,9 @@ use crate::{ - graphql_metrics::GRAPHQL_METRICS, - p2p_metrics::P2P_METRICS, - services::SERVICES_METRICS, - txpool_metrics::TXPOOL_METRICS, + graphql_metrics::graphql_metrics, + importer::importer_metrics, + p2p_metrics::p2p_metrics, + services::services_metrics, + txpool_metrics::txpool_metrics, }; use axum::{ body::Body, @@ -18,19 +19,20 @@ use prometheus_client::encoding::text::encode; pub fn encode_metrics_response() -> impl IntoResponse { // encode libp2p metrics using older prometheus let mut libp2p_bytes = Vec::::new(); - if let Some(value) = P2P_METRICS.gossip_sub_registry.get() { + if let Some(value) = p2p_metrics().gossip_sub_registry.get() { if libp2p_encode(&mut libp2p_bytes, value).is_err() { return error_body() } } - if libp2p_encode(&mut libp2p_bytes, &P2P_METRICS.peer_metrics).is_err() { + if libp2p_encode(&mut libp2p_bytes, &p2p_metrics().peer_metrics).is_err() { return error_body() } let mut encoded = String::from_utf8_lossy(&libp2p_bytes).into_owned(); + // encode the rest of the fuel-core metrics using latest prometheus { - let lock = SERVICES_METRICS + let lock = services_metrics() .registry .lock() .expect("The service metrics lock is poisoned"); @@ -39,12 +41,15 @@ pub fn encode_metrics_response() -> impl IntoResponse { } } - // encode the rest of the fuel-core metrics using latest prometheus - if encode(&mut encoded, &TXPOOL_METRICS.registry).is_err() { + if encode(&mut encoded, &txpool_metrics().registry).is_err() { + return error_body() + } + + if encode(&mut encoded, &graphql_metrics().registry).is_err() { return error_body() } - if encode(&mut encoded, &GRAPHQL_METRICS.registry).is_err() { + if encode(&mut encoded, &importer_metrics().registry).is_err() { return error_body() } diff --git a/crates/metrics/src/services.rs b/crates/metrics/src/services.rs index 6b3c5fc49b6..cbbcf49b168 100644 --- a/crates/metrics/src/services.rs +++ b/crates/metrics/src/services.rs @@ -1,4 +1,3 @@ -use lazy_static::lazy_static; use prometheus_client::{ encoding::text::encode, metrics::counter::Counter, @@ -6,7 +5,10 @@ use prometheus_client::{ }; use std::{ ops::Deref, - sync::Mutex, + sync::{ + Mutex, + OnceLock, + }, }; /// The statistic of the service life cycle. @@ -62,12 +64,14 @@ impl ServicesMetrics { } } -lazy_static! { - pub static ref SERVICES_METRICS: ServicesMetrics = ServicesMetrics::default(); +static SERVICES_METRICS: OnceLock = OnceLock::new(); + +pub fn services_metrics() -> &'static ServicesMetrics { + SERVICES_METRICS.get_or_init(ServicesMetrics::default) } #[test] fn register_success() { - SERVICES_METRICS.register_service("Foo"); - SERVICES_METRICS.register_service("Bar"); + services_metrics().register_service("Foo"); + services_metrics().register_service("Bar"); } diff --git a/crates/metrics/src/txpool_metrics.rs b/crates/metrics/src/txpool_metrics.rs index b06ebd2dd38..14fc1d0b1b8 100644 --- a/crates/metrics/src/txpool_metrics.rs +++ b/crates/metrics/src/txpool_metrics.rs @@ -1,9 +1,11 @@ -use lazy_static::lazy_static; use prometheus_client::{ metrics::histogram::Histogram, registry::Registry, }; -use std::default::Default; +use std::{ + default::Default, + sync::OnceLock, +}; pub struct TxPoolMetrics { // Attaches each Metric to the Registry @@ -46,6 +48,7 @@ impl Default for TxPoolMetrics { } } -lazy_static! { - pub static ref TXPOOL_METRICS: TxPoolMetrics = TxPoolMetrics::default(); +static TXPOOL_METRICS: OnceLock = OnceLock::new(); +pub fn txpool_metrics() -> &'static TxPoolMetrics { + TXPOOL_METRICS.get_or_init(TxPoolMetrics::default) } diff --git a/crates/services/importer/Cargo.toml b/crates/services/importer/Cargo.toml index f361a0530e5..4423268ace6 100644 --- a/crates/services/importer/Cargo.toml +++ b/crates/services/importer/Cargo.toml @@ -11,6 +11,7 @@ description = "Fuel Block Importer" [dependencies] anyhow = { workspace = true } +fuel-core-metrics = { workspace = true } fuel-core-storage = { workspace = true } fuel-core-types = { workspace = true } thiserror = { workspace = true } diff --git a/crates/services/importer/src/importer.rs b/crates/services/importer/src/importer.rs index 228877e10e6..433bf0e7cba 100644 --- a/crates/services/importer/src/importer.rs +++ b/crates/services/importer/src/importer.rs @@ -7,6 +7,7 @@ use crate::{ }, Config, }; +use fuel_core_metrics::importer::importer_metrics; use fuel_core_storage::{ transactional::StorageTransaction, Error as StorageError, @@ -32,7 +33,14 @@ use fuel_core_types::{ Uncommitted, }, }; -use std::sync::Arc; +use std::{ + sync::Arc, + time::{ + Instant, + SystemTime, + UNIX_EPOCH, + }, +}; use tokio::sync::{ broadcast, TryAcquireError, @@ -94,6 +102,7 @@ pub struct Importer { impl Importer { pub fn new(config: Config, database: D, executor: E, verifier: V) -> Self { let (broadcast, _) = broadcast::channel(config.max_block_notify_buffer); + Self { database, executor, @@ -223,12 +232,57 @@ where .seal_block(&block_id, &result.sealed_block.consensus)? .should_be_unique(&expected_next_height)?; + // Update the total tx count in chain metadata + let total_txs = db_after_execution + // Safety: casting len to u64 since it's impossible to execute a block with more than 2^64 txs + .increase_tx_count(result.sealed_block.entity.transactions().len() as u64)?; + db_tx.commit()?; - tracing::info!("Committed block"); + // update the importer metrics after the block is successfully committed + importer_metrics().total_txs_count.set(total_txs as i64); + importer_metrics() + .block_height + .set(actual_height.as_usize() as i64); + let current_time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + importer_metrics() + .latest_block_import_timestamp + .set(current_time); + + tracing::info!("Committed block {:#x}", result.sealed_block.entity.id()); let _ = self.broadcast.send(Arc::new(result)); Ok(()) } + + /// Should only be called once after startup to set importer metrics to their initial values + pub fn init_metrics(&self) { + // load starting values from database + + // Errors are optimistically handled via fallback to default values since the metrics + // should get updated regularly anyways and these errors will be discovered and handled + // correctly in more mission critical areas (such as _commit_result) + let current_block_height = + self.database.latest_block_height().unwrap_or_default(); + let total_tx_count = self.database.increase_tx_count(0).unwrap_or_default(); + + importer_metrics() + .total_txs_count + .set(total_tx_count as i64); + importer_metrics() + .block_height + .set(current_block_height.as_usize() as i64); + // on init just set to current time since it's not worth tracking in the db + let current_time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + importer_metrics() + .latest_block_import_timestamp + .set(current_time); + } } impl Importer @@ -314,8 +368,14 @@ where /// It is a combination of the [`Importer::verify_and_execute_block`] and [`Importer::commit_result`]. pub fn execute_and_commit(&self, sealed_block: SealedBlock) -> Result<(), Error> { let _guard = self.lock()?; + let start = Instant::now(); let result = self.verify_and_execute_block(sealed_block)?; - self._commit_result(result) + let commit_result = self._commit_result(result); + // record the execution time to prometheus + let time = start.elapsed().as_secs_f64(); + importer_metrics().execute_and_commit_duration.observe(time); + // return execution result + commit_result } } diff --git a/crates/services/importer/src/importer/test.rs b/crates/services/importer/src/importer/test.rs index 6dee917d3fe..24db5d043c3 100644 --- a/crates/services/importer/src/importer/test.rs +++ b/crates/services/importer/src/importer/test.rs @@ -51,6 +51,7 @@ mockall::mock! { impl ImporterDatabase for Database { fn latest_block_height(&self) -> StorageResult; + fn increase_tx_count(&self, new_txs_count: u64) -> StorageResult; } impl ExecutorDatabase for Database { @@ -115,6 +116,7 @@ where let mut db = MockDatabase::default(); db.expect_latest_block_height() .returning(move || result().map(Into::into)); + db.expect_increase_tx_count().returning(Ok); db } } @@ -132,7 +134,7 @@ where .returning(move || height().map(Into::into)); db.expect_seal_block().returning(move |_, _| seal()); db.expect_commit().times(commits).returning(|| Ok(())); - + db.expect_increase_tx_count().returning(Ok); db } } diff --git a/crates/services/importer/src/ports.rs b/crates/services/importer/src/ports.rs index 02ff02504a8..ce0449a8743 100644 --- a/crates/services/importer/src/ports.rs +++ b/crates/services/importer/src/ports.rs @@ -33,6 +33,9 @@ pub trait Executor: Send + Sync { pub trait ImporterDatabase { /// Returns the latest block height. fn latest_block_height(&self) -> StorageResult; + /// Update metadata about the total number of transactions on the chain. + /// Returns the total count after the update. + fn increase_tx_count(&self, new_txs_count: u64) -> StorageResult; } /// The port for returned database from the executor. diff --git a/crates/services/p2p/src/gossipsub/config.rs b/crates/services/p2p/src/gossipsub/config.rs index c78ce7db52c..6901a8277b1 100644 --- a/crates/services/p2p/src/gossipsub/config.rs +++ b/crates/services/p2p/src/gossipsub/config.rs @@ -2,7 +2,7 @@ use crate::config::{ Config, MAX_RESPONSE_SIZE, }; -use fuel_core_metrics::p2p_metrics::P2P_METRICS; +use fuel_core_metrics::p2p_metrics::p2p_metrics; use libp2p::gossipsub::{ metrics::Config as MetricsConfig, FastMessageId, @@ -190,7 +190,7 @@ pub(crate) fn build_gossipsub_behaviour(p2p_config: &Config) -> Gossipsub { .expect("gossipsub initialized"); // This couldn't be set unless multiple p2p services are running? So it's ok to unwrap - P2P_METRICS + p2p_metrics() .gossip_sub_registry .set(Box::new(p2p_registry)) .unwrap_or(()); diff --git a/crates/services/p2p/src/p2p_service.rs b/crates/services/p2p/src/p2p_service.rs index 499f2ed0d5e..967cd09f1c1 100644 --- a/crates/services/p2p/src/p2p_service.rs +++ b/crates/services/p2p/src/p2p_service.rs @@ -30,7 +30,7 @@ use crate::{ ResponseMessage, }, }; -use fuel_core_metrics::p2p_metrics::P2P_METRICS; +use fuel_core_metrics::p2p_metrics::p2p_metrics; use fuel_core_types::{ fuel_types::BlockHeight, services::p2p::peer_reputation::AppScore, @@ -501,7 +501,7 @@ impl FuelP2PService { agent_version, } => { if self.metrics { - P2P_METRICS.unique_peers.inc(); + p2p_metrics().unique_peers.inc(); } self.peer_manager.handle_peer_identified( diff --git a/crates/services/src/service.rs b/crates/services/src/service.rs index c5e5b93f58b..ab78de7519c 100644 --- a/crates/services/src/service.rs +++ b/crates/services/src/service.rs @@ -6,8 +6,8 @@ use anyhow::anyhow; use fuel_core_metrics::{ future_tracker::FutureTracker, services::{ + services_metrics, ServiceLifecycle, - SERVICES_METRICS, }, }; use futures::FutureExt; @@ -153,7 +153,7 @@ where /// Initializes a new `ServiceRunner` containing a `RunnableService` with parameters for underlying `Task` pub fn new_with_params(service: S, params: S::TaskParams) -> Self { let shared = service.shared_data(); - let metric = SERVICES_METRICS.register_service(S::NAME); + let metric = services_metrics().register_service(S::NAME); let state = initialize_loop(service, params, metric); Self { shared, state } } diff --git a/crates/services/txpool/src/txpool.rs b/crates/services/txpool/src/txpool.rs index e196addaed2..411de068402 100644 --- a/crates/services/txpool/src/txpool.rs +++ b/crates/services/txpool/src/txpool.rs @@ -11,8 +11,6 @@ use crate::{ Error, TxInfo, }; - -use fuel_core_metrics::txpool_metrics::TXPOOL_METRICS; use fuel_core_types::{ fuel_tx::{ Chargeable, @@ -37,6 +35,7 @@ use fuel_core_types::{ tai64::Tai64, }; +use fuel_core_metrics::txpool_metrics::txpool_metrics; use fuel_core_types::fuel_vm::checked_transaction::CheckPredicateParams; use std::{ cmp::Reverse, @@ -130,11 +129,11 @@ where } } if self.config.metrics { - TXPOOL_METRICS + txpool_metrics() .gas_price_histogram .observe(tx.price() as f64); - TXPOOL_METRICS + txpool_metrics() .tx_size_histogram .observe(tx.metered_bytes_size() as f64); } @@ -422,7 +421,7 @@ fn verify_tx_min_gas_price(tx: &Transaction, config: &Config) -> Result<(), Erro // Gas Price metrics are recorded here to avoid double matching for // every single transaction, but also means metrics aren't collected on gas // price if there is no minimum gas price - TXPOOL_METRICS.gas_price_histogram.observe(price as f64); + txpool_metrics().gas_price_histogram.observe(price as f64); } if price < config.min_gas_price { return Err(Error::NotInsertedGasPriceTooLow) diff --git a/deployment/e2e-client.Dockerfile b/deployment/e2e-client.Dockerfile index 7fbf1934cf7..ba1dd1d2214 100644 --- a/deployment/e2e-client.Dockerfile +++ b/deployment/e2e-client.Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Build -FROM rust:1.68.1 AS chef +FROM rust:1.71.0 AS chef RUN cargo install cargo-chef WORKDIR /build/ # hadolint ignore=DL3008 diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 4f043d20854..eea22e44248 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -16,14 +16,6 @@ harness = true name = "integration_tests" path = "tests/lib.rs" -# Metrics tests need to query /metrics without and explicit endpoint, and without -# a seperate endpoint, the noise can cause issues -[[test]] -harness = true -name = "metrics_tests" -path = "tests/metrics.rs" -required-features = ["metrics"] - [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } @@ -57,7 +49,6 @@ tokio = { workspace = true, features = [ ] } [features] -default = ["fuel-core/default", "metrics", "relayer"] -metrics = ["fuel-core/metrics", "fuel-core/rocksdb"] +default = ["fuel-core/default", "relayer"] p2p = ["fuel-core/p2p", "fuel-core-p2p"] relayer = ["fuel-core/relayer", "fuel-core-relayer"] diff --git a/tests/tests/lib.rs b/tests/tests/lib.rs index 717bc96cac2..821395859c3 100644 --- a/tests/tests/lib.rs +++ b/tests/tests/lib.rs @@ -12,6 +12,7 @@ mod deployment; mod health; mod helpers; mod messages; +mod metrics; mod node_info; mod poa; #[cfg(feature = "relayer")]