From 4786d9fed7c5f6404a25458eef5f4dc621471770 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Tue, 15 Aug 2023 14:35:12 +1000 Subject: [PATCH 1/4] add opentelemetry --- Cargo.lock | 289 ++++++++++++++++++++++++++-- integration-tests/src/main.rs | 2 +- mpc-recovery/Cargo.toml | 6 + mpc-recovery/src/leader_node/mod.rs | 10 + mpc-recovery/src/main.rs | 51 ++++- 5 files changed, 332 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0dee1efc4..b145d8059 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -445,6 +445,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-tracing-opentelemetry" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8ad3d73e77ce712d52d620de73d2cbef4686328742e82bac94d5a616d05ccb" +dependencies = [ + "axum", + "futures-core", + "futures-util", + "http", + "opentelemetry 0.20.0", + "pin-project-lite", + "tower", + "tracing", + "tracing-opentelemetry 0.20.0", + "tracing-opentelemetry-instrumentation-sdk", +] + [[package]] name = "backtrace" version = "0.3.68" @@ -2423,6 +2441,22 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "init-tracing-opentelemetry" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ffe76c930735369e9c4a1f7d24a16c5018590d371e72948580eb9487411365" +dependencies = [ + "opentelemetry 0.20.0", + "opentelemetry-otlp 0.13.0", + "opentelemetry-semantic-conventions 0.12.0", + "opentelemetry-stdout", + "thiserror", + "tracing", + "tracing-opentelemetry 0.20.0", + "tracing-subscriber", +] + [[package]] name = "inout" version = "0.1.3" @@ -2734,6 +2768,7 @@ dependencies = [ "async-trait", "axum", "axum-extra", + "axum-tracing-opentelemetry", "borsh 0.10.3", "chrono", "clap 4.3.19", @@ -2745,6 +2780,7 @@ dependencies = [ "hex 0.4.3", "hyper", "hyper-rustls", + "init-tracing-opentelemetry", "jsonwebtoken", "lazy_static", "multi-party-eddsa", @@ -2752,6 +2788,7 @@ dependencies = [ "near-jsonrpc-client", "near-jsonrpc-primitives", "near-primitives 0.17.0", + "opentelemetry 0.20.0", "prometheus", "rand 0.7.3", "rand 0.8.5", @@ -3018,9 +3055,9 @@ dependencies = [ "near-crypto 0.17.0", "near-primitives-core 0.17.0", "once_cell", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", + "opentelemetry 0.17.0", + "opentelemetry-otlp 0.10.0", + "opentelemetry-semantic-conventions 0.9.0", "prometheus", "serde", "strum", @@ -3028,7 +3065,7 @@ dependencies = [ "tokio", "tracing", "tracing-appender", - "tracing-opentelemetry", + "tracing-opentelemetry 0.17.4", "tracing-subscriber", ] @@ -3319,7 +3356,7 @@ dependencies = [ "near-vm-errors 0.14.0", "ripemd", "serde", - "sha2 0.8.2", + "sha2 0.10.7", "sha3", "zeropool-bn", ] @@ -3578,6 +3615,28 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-http" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry_api", +] + [[package]] name = "opentelemetry-otlp" version = "0.10.0" @@ -3588,21 +3647,124 @@ dependencies = [ "futures", "futures-util", "http", - "opentelemetry", - "prost", + "opentelemetry 0.17.0", + "prost 0.9.0", "thiserror", "tokio", - "tonic", + "tonic 0.6.2", "tonic-build", ] +[[package]] +name = "opentelemetry-otlp" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry-semantic-conventions 0.12.0", + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", + "thiserror", + "tokio", + "tonic 0.9.2", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", + "tonic 0.9.2", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" dependencies = [ - "opentelemetry", + "opentelemetry 0.17.0", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +dependencies = [ + "opentelemetry 0.20.0", +] + +[[package]] +name = "opentelemetry-stdout" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bd550321bc0f9d3f6dcbfe5c75262789de5b3e2776da2cbcfd2392aa05db0c6" +dependencies = [ + "futures-util", + "opentelemetry_api", + "opentelemetry_sdk", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "opentelemetry_api" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "ordered-float", + "percent-encoding 2.3.0", + "rand 0.8.5", + "regex", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "ordered-float" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" +dependencies = [ + "num-traits", ] [[package]] @@ -3986,7 +4148,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.9.0", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", ] [[package]] @@ -4002,7 +4174,7 @@ dependencies = [ "log", "multimap", "petgraph", - "prost", + "prost 0.9.0", "prost-types", "regex", "tempfile", @@ -4022,6 +4194,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-types" version = "0.9.0" @@ -4029,7 +4214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" dependencies = [ "bytes", - "prost", + "prost 0.9.0", ] [[package]] @@ -5548,8 +5733,8 @@ dependencies = [ "hyper-timeout", "percent-encoding 2.3.0", "pin-project", - "prost", - "prost-derive", + "prost 0.9.0", + "prost-derive 0.9.0", "tokio", "tokio-stream", "tokio-util 0.6.10", @@ -5560,6 +5745,34 @@ dependencies = [ "tracing-futures", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64 0.21.2", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding 2.3.0", + "pin-project", + "prost 0.11.9", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" version = "0.6.2" @@ -5695,13 +5908,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" dependencies = [ "once_cell", - "opentelemetry", + "opentelemetry 0.17.0", "tracing", "tracing-core", "tracing-log", "tracing-subscriber", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc09e402904a5261e42cf27aea09ccb7d5318c6717a9eec3d8e2e65c56b18f19" +dependencies = [ + "once_cell", + "opentelemetry 0.20.0", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + +[[package]] +name = "tracing-opentelemetry-instrumentation-sdk" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad80bb643d9a70b13d10eb5d3241e2955afc880d3a7a15a40d6cc6f620091a02" +dependencies = [ + "http", + "opentelemetry-http", + "opentelemetry_api", + "tracing", + "tracing-opentelemetry 0.20.0", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.17" @@ -5712,12 +5962,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -5831,6 +6084,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8parse" version = "0.2.1" diff --git a/integration-tests/src/main.rs b/integration-tests/src/main.rs index 22934b56c..522688bae 100644 --- a/integration-tests/src/main.rs +++ b/integration-tests/src/main.rs @@ -109,7 +109,7 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Please run the command below to start a leader node:"); tracing::info!( - "RUST_LOG=mpc_recovery=debug cargo run --bin mpc-recovery -- {}", + "RUST_BACKTRACE=1 RUST_LOG=mpc_recovery=debug cargo run --bin mpc-recovery -- {}", cmd.join(" ") ); tracing::info!("===================================="); diff --git a/mpc-recovery/Cargo.toml b/mpc-recovery/Cargo.toml index 24d5e75b8..7cc074b0e 100644 --- a/mpc-recovery/Cargo.toml +++ b/mpc-recovery/Cargo.toml @@ -14,6 +14,7 @@ anyhow = "1" async-trait = "0.1" axum = "0.6.19" axum-extra = "0.7" +axum-tracing-opentelemetry = "0.13.1" borsh = "0.10.3" chrono = "0.4.24" clap = { version = "4.2", features = ["derive", "env"] } @@ -23,8 +24,13 @@ google-secretmanager1 = "5" hex = "0.4.3" hyper = { version = "0.14", features = ["full"] } hyper-rustls = { version = "=0.23", features = ["http2"] } +init-tracing-opentelemetry = { version = "0.13.1", features = [ + "tracing_subscriber_ext", + "stdout", +] } jsonwebtoken = "8.3.0" lazy_static = "1.4.0" +opentelemetry = "0.20.0" prometheus = { version = "0.13.3", features = ["process"] } rand = "0.7" rand8 = { package = "rand", version = "0.8" } diff --git a/mpc-recovery/src/leader_node/mod.rs b/mpc-recovery/src/leader_node/mod.rs index b8f1ffb08..8ff7027ba 100644 --- a/mpc-recovery/src/leader_node/mod.rs +++ b/mpc-recovery/src/leader_node/mod.rs @@ -24,6 +24,7 @@ use axum::{ Extension, Json, Router, }; use axum_extra::extract::WithRejection; +use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; use curv::elliptic::curves::{Ed25519, Point}; use near_crypto::SecretKey; use near_primitives::delegate_action::NonDelegateAction; @@ -133,16 +134,25 @@ pub async fn run(config: Config) { .route("/metrics", get(metrics)) .route_layer(middleware::from_fn(track_metrics)) .layer(Extension(state)) + // Include trace context as header into the response + .layer(OtelInResponseLayer::default()) + // Start OpenTelemetry trace on incoming request + .layer(OtelAxumLayer::default()) .layer(cors_layer); let addr = SocketAddr::from(([0, 0, 0, 0], port)); tracing::debug!(?addr, "starting http server"); axum::Server::bind(&addr) .serve(app.into_make_service()) + // .with_graceful_shutdown(shutdown_signal()) .await .unwrap(); } +// async fn shutdown_signal() { +// opentelemetry::global::shutdown_tracer_provider(); +// } + async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { let timer = Instant::now(); let path = if let Some(matched_path) = req.extensions().get::() { diff --git a/mpc-recovery/src/main.rs b/mpc-recovery/src/main.rs index 0cfebedc6..78906ea81 100644 --- a/mpc-recovery/src/main.rs +++ b/mpc-recovery/src/main.rs @@ -3,6 +3,9 @@ use aes_gcm::{ Aes256Gcm, }; use clap::Parser; +use init_tracing_opentelemetry::tracing_subscriber_ext::{ + build_loglevel_filter_layer, build_otel_layer, +}; use mpc_recovery::{ gcp::GcpService, oauth::{PagodaFirebaseTokenVerifier, UniversalTokenVerifier}, @@ -11,7 +14,10 @@ use mpc_recovery::{ }; use multi_party_eddsa::protocols::ExpandedKeyPair; use near_primitives::types::AccountId; -use tracing_subscriber::EnvFilter; +use tracing::Subscriber; +use tracing_subscriber::{ + fmt::format::FmtSpan, prelude::__tracing_subscriber_SubscriberExt, registry::LookupSpan, Layer, +}; #[derive(Parser, Debug)] enum Cli { @@ -165,20 +171,45 @@ async fn load_account_creator_sk( } } -#[tokio::main] -async fn main() -> anyhow::Result<()> { - // Install global collector configured based on RUST_LOG env var. - let mut subscriber = tracing_subscriber::fmt() - .with_thread_ids(true) - .with_env_filter(EnvFilter::from_default_env()); +fn build_tracing_layer() -> Box + Send + Sync + 'static> +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + let mut layer = tracing_subscriber::fmt::layer() + .pretty() + .with_line_number(true) + .with_thread_names(true) + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .with_timer(tracing_subscriber::fmt::time::uptime()); // Check if running in Google Cloud Run: https://cloud.google.com/run/docs/container-contract#services-env-vars if std::env::var("K_SERVICE").is_ok() { // Disable colored logging as it messes up Google's log formatting - subscriber = subscriber.with_ansi(false); + layer = layer.with_ansi(false); } - subscriber.init(); - let _span = tracing::trace_span!("cli").entered(); + Box::new(layer) +} + +fn init_subscribers() -> anyhow::Result<()> { + // Setup a temporary subscriber to log output during setup + let subscriber = tracing_subscriber::registry() + .with(build_loglevel_filter_layer()) + .with(build_tracing_layer()); + let _guard = tracing::subscriber::set_default(subscriber); + tracing::info!("init logging & tracing"); + let subscriber = tracing_subscriber::registry() + .with(build_otel_layer()?) + .with(build_loglevel_filter_layer()) + .with(build_tracing_layer()); + tracing::subscriber::set_global_default(subscriber)?; + Ok(()) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + init_subscribers()?; + + let _span = tracing::trace_span!("cli").entered(); match Cli::parse() { Cli::Generate { n } => { let GenerateResult { pk_set, secrets } = mpc_recovery::generate(n); From 6bce7bcf6ef265be0a1df0cd0b43ac24bd583e96 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Fri, 13 Oct 2023 12:29:57 +1100 Subject: [PATCH 2/4] refactor logging --- Cargo.lock | 67 ++----- integration-tests/src/containers.rs | 4 +- integration-tests/tests/lib.rs | 6 +- integration-tests/tests/mpc/mod.rs | 2 +- integration-tests/tests/mpc/negative.rs | 2 +- integration-tests/tests/mpc/positive.rs | 2 +- mpc-recovery/Cargo.toml | 10 +- mpc-recovery/src/lib.rs | 1 + mpc-recovery/src/logging.rs | 249 ++++++++++++++++++++++++ mpc-recovery/src/main.rs | 90 ++++----- 10 files changed, 319 insertions(+), 114 deletions(-) create mode 100644 mpc-recovery/src/logging.rs diff --git a/Cargo.lock b/Cargo.lock index 18dbccbb2..a310e4d17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "axum-tracing-opentelemetry" -version = "0.13.1" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c8ad3d73e77ce712d52d620de73d2cbef4686328742e82bac94d5a616d05ccb" +checksum = "06985105829f176e9a3f113b1c71cc24e08f600ef0df4e70cd90d144f889e19f" dependencies = [ "axum", "futures-core", @@ -401,7 +401,7 @@ dependencies = [ "pin-project-lite", "tower", "tracing", - "tracing-opentelemetry 0.20.0", + "tracing-opentelemetry 0.21.0", "tracing-opentelemetry-instrumentation-sdk", ] @@ -2430,22 +2430,6 @@ dependencies = [ "serde", ] -[[package]] -name = "init-tracing-opentelemetry" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ffe76c930735369e9c4a1f7d24a16c5018590d371e72948580eb9487411365" -dependencies = [ - "opentelemetry 0.20.0", - "opentelemetry-otlp 0.13.0", - "opentelemetry-semantic-conventions 0.12.0", - "opentelemetry-stdout", - "thiserror", - "tracing", - "tracing-opentelemetry 0.20.0", - "tracing-subscriber", -] - [[package]] name = "inout" version = "0.1.3" @@ -2747,6 +2731,7 @@ dependencies = [ "aes-gcm", "anyhow", "async-trait", + "atty", "axum", "axum-extra", "axum-tracing-opentelemetry", @@ -2762,7 +2747,6 @@ dependencies = [ "hex 0.4.3", "hyper", "hyper-rustls 0.23.2", - "init-tracing-opentelemetry", "jsonwebtoken", "lazy_static", "multi-party-eddsa", @@ -2771,6 +2755,7 @@ dependencies = [ "near-jsonrpc-primitives", "near-primitives 0.17.0", "opentelemetry 0.20.0", + "opentelemetry-otlp 0.13.0", "prometheus", "rand 0.7.3", "rand 0.8.5", @@ -2785,6 +2770,8 @@ dependencies = [ "tokio-retry", "tower-http", "tracing", + "tracing-appender", + "tracing-opentelemetry 0.21.0", "tracing-subscriber", "yup-oauth2", ] @@ -3713,7 +3700,6 @@ dependencies = [ "async-trait", "futures-core", "http", - "opentelemetry-http", "opentelemetry-proto", "opentelemetry-semantic-conventions 0.12.0", "opentelemetry_api", @@ -3754,20 +3740,6 @@ dependencies = [ "opentelemetry 0.20.0", ] -[[package]] -name = "opentelemetry-stdout" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd550321bc0f9d3f6dcbfe5c75262789de5b3e2776da2cbcfd2392aa05db0c6" -dependencies = [ - "futures-util", - "opentelemetry_api", - "opentelemetry_sdk", - "ordered-float", - "serde", - "serde_json", -] - [[package]] name = "opentelemetry_api" version = "0.20.0" @@ -6084,12 +6056,14 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc09e402904a5261e42cf27aea09ccb7d5318c6717a9eec3d8e2e65c56b18f19" +checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" dependencies = [ "once_cell", "opentelemetry 0.20.0", + "opentelemetry_sdk", + "smallvec", "tracing", "tracing-core", "tracing-log", @@ -6098,25 +6072,15 @@ dependencies = [ [[package]] name = "tracing-opentelemetry-instrumentation-sdk" -version = "0.13.1" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad80bb643d9a70b13d10eb5d3241e2955afc880d3a7a15a40d6cc6f620091a02" +checksum = "752ddd669b14a08036a89045e4ac4497ff7ce254e11386647751f36d7c7849ea" dependencies = [ "http", "opentelemetry-http", "opentelemetry_api", "tracing", - "tracing-opentelemetry 0.20.0", -] - -[[package]] -name = "tracing-serde" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" -dependencies = [ - "serde", - "tracing-core", + "tracing-opentelemetry 0.21.0", ] [[package]] @@ -6129,15 +6093,12 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", - "serde", - "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", - "tracing-serde", ] [[package]] diff --git a/integration-tests/src/containers.rs b/integration-tests/src/containers.rs index 00c88e043..2f0c3e92f 100644 --- a/integration-tests/src/containers.rs +++ b/integration-tests/src/containers.rs @@ -512,7 +512,7 @@ impl<'a> SignerNode<'a> { container.exec(ExecCommand { cmd: format!("bash -c 'while [[ \"$(curl -s -o /dev/null -w ''%{{http_code}}'' localhost:{})\" != \"200\" ]]; do sleep 1; done'", Self::CONTAINER_PORT), - ready_conditions: vec![WaitFor::message_on_stdout("node is ready to accept connections")] + ready_conditions: vec![WaitFor::message_on_stderr("node is ready to accept connections")] }); let full_address = format!("http://{ip_address}:{}", Self::CONTAINER_PORT); @@ -662,7 +662,7 @@ impl<'a> LeaderNode<'a> { container.exec(ExecCommand { cmd: format!("bash -c 'while [[ \"$(curl -s -o /dev/null -w ''%{{http_code}}'' localhost:{})\" != \"200\" ]]; do sleep 1; done'", Self::CONTAINER_PORT), - ready_conditions: vec![WaitFor::message_on_stdout("node is ready to accept connections")] + ready_conditions: vec![WaitFor::message_on_stderr("node is ready to accept connections")] }); let full_address = format!("http://{ip_address}:{}", Self::CONTAINER_PORT); diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index b6e31ad21..6903f06bc 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -12,7 +12,7 @@ use mpc_recovery::{ }; use mpc_recovery_integration_tests::containers; use near_primitives::utils::generate_random_string; -use workspaces::{network::Sandbox, Worker}; +use near_workspaces::{network::Sandbox, Worker}; const NETWORK: &str = "mpc_it_network"; const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; @@ -115,8 +115,8 @@ where } mod account { + use near_workspaces::{network::Sandbox, AccountId, Worker}; use rand::{distributions::Alphanumeric, Rng}; - use workspaces::{network::Sandbox, AccountId, Worker}; pub fn random(worker: &Worker) -> anyhow::Result { let account_id_rand: String = rand::thread_rng() @@ -174,7 +174,7 @@ mod key { mod check { use crate::TestContext; use near_crypto::PublicKey; - use workspaces::AccountId; + use near_workspaces::AccountId; pub async fn access_key_exists( ctx: &TestContext, diff --git a/integration-tests/tests/mpc/mod.rs b/integration-tests/tests/mpc/mod.rs index e198ff3ab..d2ffddc32 100644 --- a/integration-tests/tests/mpc/mod.rs +++ b/integration-tests/tests/mpc/mod.rs @@ -4,7 +4,7 @@ use mpc_recovery::msg::{NewAccountResponse, UserCredentialsResponse}; use mpc_recovery::sign_node::oidc::OidcToken; use mpc_recovery::transaction::LimitedAccessKey; use near_crypto::{PublicKey, SecretKey}; -use workspaces::AccountId; +use near_workspaces::AccountId; use crate::{account, check, key, MpcCheck, TestContext}; diff --git a/integration-tests/tests/mpc/negative.rs b/integration-tests/tests/mpc/negative.rs index abd2f50bc..ba44b3f0b 100644 --- a/integration-tests/tests/mpc/negative.rs +++ b/integration-tests/tests/mpc/negative.rs @@ -19,9 +19,9 @@ use near_primitives::{ DeployContractAction, FunctionCallAction, StakeAction, TransferAction, }, }; +use near_workspaces::AccountId; use std::{str::FromStr, time::Duration}; use test_log::test; -use workspaces::AccountId; #[test(tokio::test)] async fn whitlisted_actions_test() -> anyhow::Result<()> { diff --git a/integration-tests/tests/mpc/positive.rs b/integration-tests/tests/mpc/positive.rs index ddf3d8bf0..a77d83cc2 100644 --- a/integration-tests/tests/mpc/positive.rs +++ b/integration-tests/tests/mpc/positive.rs @@ -4,7 +4,7 @@ use crate::mpc::{add_pk_and_check_validity, fetch_recovery_pk, new_random_accoun use crate::{account, key, with_nodes, MpcCheck}; use hyper::StatusCode; -use workspaces::types::AccessKeyPermission; +use near_workspaces::types::AccessKeyPermission; use mpc_recovery::{ gcp::value::{FromValue, IntoValue}, diff --git a/mpc-recovery/Cargo.toml b/mpc-recovery/Cargo.toml index 575c78290..dd8c3acc0 100644 --- a/mpc-recovery/Cargo.toml +++ b/mpc-recovery/Cargo.toml @@ -12,9 +12,10 @@ aes-gcm = "0.10" actix-rt = "2.8" anyhow = "1" async-trait = "0.1" +atty = "0.2" axum = "0.6.19" axum-extra = "0.7" -axum-tracing-opentelemetry = "0.13.1" +axum-tracing-opentelemetry = "0.14.1" base64 = "0.21" borsh = "0.10.3" chrono = "0.4.24" @@ -25,13 +26,10 @@ google-secretmanager1 = "5" hex = "0.4.3" hyper = { version = "0.14", features = ["full"] } hyper-rustls = { version = "=0.23", features = ["http2"] } -init-tracing-opentelemetry = { version = "0.13.1", features = [ - "tracing_subscriber_ext", - "stdout", -] } jsonwebtoken = "8.3.0" lazy_static = "1.4.0" opentelemetry = "0.20.0" +opentelemetry-otlp = "0.13.0" prometheus = { version = "0.13.3", features = ["process"] } rand = "0.7" rand8 = { package = "rand", version = "0.8" } @@ -43,7 +41,9 @@ thiserror = "1" tokio = { version = "1.28", features = ["full"] } tokio-retry = "0.3" tracing = "0.1" +tracing-appender = "0.2.2" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-opentelemetry = "0.21.0" near-jsonrpc-client = "0.6" near-jsonrpc-primitives = "0.17" near-primitives = "0.17" diff --git a/mpc-recovery/src/lib.rs b/mpc-recovery/src/lib.rs index 5aa11f347..6b53a1837 100644 --- a/mpc-recovery/src/lib.rs +++ b/mpc-recovery/src/lib.rs @@ -15,6 +15,7 @@ pub mod firewall; pub mod gcp; pub mod key_recovery; pub mod leader_node; +pub mod logging; pub mod metrics; pub mod msg; pub mod nar; diff --git a/mpc-recovery/src/logging.rs b/mpc-recovery/src/logging.rs new file mode 100644 index 000000000..4ee600947 --- /dev/null +++ b/mpc-recovery/src/logging.rs @@ -0,0 +1,249 @@ +use opentelemetry::sdk::trace::{self, RandomIdGenerator, Sampler, Tracer}; +use opentelemetry::sdk::Resource; +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; +use std::sync::OnceLock; +use tracing::subscriber::DefaultGuard; +use tracing_appender::non_blocking::NonBlocking; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::filter::{Filtered, LevelFilter}; +use tracing_subscriber::layer::{Layered, SubscriberExt}; +use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::{fmt, reload, EnvFilter, Layer, Registry}; + +static LOG_LAYER_RELOAD_HANDLE: OnceLock> = OnceLock::new(); +static OTLP_LAYER_RELOAD_HANDLE: OnceLock>> = + OnceLock::new(); + +type LogLayer = Layered< + Filtered< + fmt::Layer, + reload::Layer, + Inner, + >, + Inner, +>; + +type TracingLayer = Layered< + Filtered, reload::Layer, Inner>, + Inner, +>; + +// Records the level of opentelemetry tracing verbosity configured via command-line flags at the startup. +static DEFAULT_OTLP_LEVEL: OnceLock = OnceLock::new(); + +// Doesn't define WARN and ERROR, because the highest verbosity of spans is INFO. +#[derive(Copy, Clone, Debug, Default, clap::ValueEnum, serde::Serialize, serde::Deserialize)] +pub enum OpenTelemetryLevel { + #[default] + OFF, + INFO, + DEBUG, + TRACE, +} + +/// Whether to use colored log format. +/// Option `Auto` enables color output only if the logging is done to a terminal and +/// `NO_COLOR` environment variable is not set. +#[derive(clap::ValueEnum, Debug, Clone, Default)] +pub enum ColorOutput { + #[default] + Auto, + Always, + Never, +} + +/// Configures exporter of span and trace data. +#[derive(Debug, Default, clap::Parser)] +pub struct Options { + /// Enables export of span data using opentelemetry exporters. + #[clap(long, value_enum, default_value = "off")] + opentelemetry: OpenTelemetryLevel, + + /// Whether the log needs to be colored. + #[clap(long, value_enum, default_value = "auto")] + color: ColorOutput, + + /// Enable logging of spans. For instance, this prints timestamps of entering and exiting a span, + /// together with the span duration and used/idle CPU time. + #[clap(long)] + log_span_events: bool, +} + +fn use_color_output(options: &Options) -> bool { + fn use_color_auto() -> bool { + std::env::var_os("NO_COLOR").is_none() && is_terminal() + } + + fn is_terminal() -> bool { + // Crate `atty` provides a platform-independent way of checking whether the output is a tty. + atty::is(atty::Stream::Stderr) + } + + match options.color { + ColorOutput::Auto => use_color_auto(), + ColorOutput::Always => true, + ColorOutput::Never => false, + } +} + +fn get_fmt_span(with_span_events: bool) -> fmt::format::FmtSpan { + if with_span_events { + fmt::format::FmtSpan::ENTER | fmt::format::FmtSpan::CLOSE + } else { + fmt::format::FmtSpan::NONE + } +} + +fn add_non_blocking_log_layer( + filter: EnvFilter, + writer: NonBlocking, + ansi: bool, + with_span_events: bool, + subscriber: S, +) -> (LogLayer, reload::Handle) +where + S: tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync, +{ + let (filter, handle) = reload::Layer::::new(filter); + + let layer = fmt::layer() + .with_ansi(ansi) + .with_span_events(get_fmt_span(with_span_events)) + .with_writer(writer) + .with_line_number(true) + .with_thread_names(true) + .with_filter(filter); + + (subscriber.with(layer), handle) +} + +/// Constructs an OpenTelemetryConfig which sends span data to an external collector. +async fn add_opentelemetry_layer( + opentelemetry_level: OpenTelemetryLevel, + env: String, + node_id: String, + subscriber: S, +) -> (TracingLayer, reload::Handle) +where + S: tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync, +{ + let filter = match opentelemetry_level { + OpenTelemetryLevel::OFF => LevelFilter::OFF, + OpenTelemetryLevel::INFO => LevelFilter::INFO, + OpenTelemetryLevel::DEBUG => LevelFilter::DEBUG, + OpenTelemetryLevel::TRACE => LevelFilter::TRACE, + }; + let (filter, handle) = reload::Layer::::new(filter); + + let resource = vec![KeyValue::new("env", env), KeyValue::new("node_id", node_id)]; + + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint("http://localhost:4317"), + ) + .with_trace_config( + trace::config() + .with_sampler(Sampler::AlwaysOn) + .with_id_generator(RandomIdGenerator::default()) + .with_resource(Resource::new(resource)), + ) + .install_batch(opentelemetry::runtime::Tokio) + .unwrap(); + let layer = tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(filter); + (subscriber.with(layer), handle) +} + +fn set_default_otlp_level(options: &Options) { + // Record the initial tracing level specified as a command-line flag. Use this recorded value to + // reset opentelemetry filter when the LogConfig file gets deleted. + DEFAULT_OTLP_LEVEL.set(options.opentelemetry).unwrap(); +} + +/// The resource representing a registered subscriber. +/// +/// Once dropped, the subscriber is unregistered, and the output is flushed. Any messages output +/// after this value is dropped will be delivered to a previously active subscriber, if any. +pub struct DefaultSubscriberGuard { + // NB: the field order matters here. We must first drop the `local_subscriber_guard` so that + // no new messages are delivered to this subscriber while we take care of flushing the + // messages already in queue. If dropped the other way around, the events/spans generated + // while the subscriber drop guard runs would be lost. + subscriber: Option, + local_subscriber_guard: Option, + #[allow(dead_code)] // This field is never read, but has semantic purpose as a drop guard. + writer_guard: Option, +} + +impl DefaultSubscriberGuard { + /// Register this default subscriber globally , for all threads. + /// + /// Must not be called more than once. Mutually exclusive with `Self::local`. + pub fn global(mut self) -> Self { + if let Some(subscriber) = self.subscriber.take() { + tracing::subscriber::set_global_default(subscriber) + .expect("could not set a global subscriber"); + } else { + panic!("trying to set a default subscriber that has been already taken") + } + self + } + + /// Register this default subscriber for the current thread. + /// + /// Must not be called more than once. Mutually exclusive with `Self::global`. + pub fn local(mut self) -> Self { + if let Some(subscriber) = self.subscriber.take() { + self.local_subscriber_guard = Some(tracing::subscriber::set_default(subscriber)); + } else { + panic!("trying to set a default subscriber that has been already taken") + } + self + } +} + +pub async fn default_subscriber_with_opentelemetry( + env_filter: EnvFilter, + options: &Options, + env: String, + node_id: String, +) -> DefaultSubscriberGuard { + let color_output = use_color_output(options); + + // Do not lock the `stderr` here to allow for things like `dbg!()` work during development. + let stderr = std::io::stderr(); + let lined_stderr = std::io::LineWriter::new(stderr); + let (writer, writer_guard) = tracing_appender::non_blocking(lined_stderr); + + let subscriber = tracing_subscriber::registry(); + + set_default_otlp_level(options); + + let (subscriber, handle) = add_non_blocking_log_layer( + env_filter, + writer, + color_output, + options.log_span_events, + subscriber, + ); + LOG_LAYER_RELOAD_HANDLE + .set(handle) + .unwrap_or_else(|_| panic!("Failed to set Log Layer Filter")); + + let (subscriber, handle) = + add_opentelemetry_layer(options.opentelemetry, env, node_id, subscriber).await; + OTLP_LAYER_RELOAD_HANDLE + .set(handle) + .unwrap_or_else(|_| panic!("Failed to set OTLP Layer Filter")); + + DefaultSubscriberGuard { + subscriber: Some(subscriber), + local_subscriber_guard: None, + writer_guard: Some(writer_guard), + } +} diff --git a/mpc-recovery/src/main.rs b/mpc-recovery/src/main.rs index 3fdbaf19c..83e55afe2 100644 --- a/mpc-recovery/src/main.rs +++ b/mpc-recovery/src/main.rs @@ -1,13 +1,9 @@ -use std::path::PathBuf; - use aes_gcm::{ aead::{consts::U32, generic_array::GenericArray, KeyInit}, Aes256Gcm, }; use clap::Parser; -use init_tracing_opentelemetry::tracing_subscriber_ext::{ - build_loglevel_filter_layer, build_otel_layer, -}; +use mpc_recovery::logging; use mpc_recovery::{ firewall::allowed::{OidcProviderList, PartnerList}, gcp::GcpService, @@ -18,10 +14,8 @@ use mpc_recovery::{ use multi_party_eddsa::protocols::ExpandedKeyPair; use near_primitives::types::AccountId; use serde::de::DeserializeOwned; -use tracing::Subscriber; -use tracing_subscriber::{ - fmt::format::FmtSpan, prelude::__tracing_subscriber_SubscriberExt, registry::LookupSpan, Layer, -}; +use std::path::PathBuf; +use tracing_subscriber::EnvFilter; #[derive(Parser, Debug)] enum Cli { @@ -69,6 +63,9 @@ enum Cli { /// Whether to accept test tokens #[arg(long, env("MPC_RECOVERY_TEST"), default_value("false"))] test: bool, + /// Enables export of span data using opentelemetry protocol. + #[clap(flatten)] + logging_options: logging::Options, }, StartSign { /// Environment to run in (`dev` or `prod`) @@ -101,6 +98,9 @@ enum Cli { /// Whether to accept test tokens #[arg(long, env("MPC_RECOVERY_TEST"), default_value("false"))] test: bool, + /// Enables export of span data using opentelemetry protocol. + #[clap(flatten)] + logging_options: logging::Options, }, RotateSignNodeCipher { /// Environment to run in (`dev` or `prod`) @@ -124,6 +124,9 @@ enum Cli { /// GCP datastore URL #[arg(long, env("MPC_RECOVERY_GCP_DATASTORE_URL"))] gcp_datastore_url: Option, + /// Enables export of span data using opentelemetry protocol. + #[clap(flatten)] + logging_options: logging::Options, }, } @@ -171,24 +174,6 @@ async fn load_account_creator_sk( } } -fn build_tracing_layer() -> Box + Send + Sync + 'static> -where - S: Subscriber + for<'a> LookupSpan<'a>, -{ - let mut layer = tracing_subscriber::fmt::layer() - .pretty() - .with_line_number(true) - .with_thread_names(true) - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .with_timer(tracing_subscriber::fmt::time::uptime()); - // Check if running in Google Cloud Run: https://cloud.google.com/run/docs/container-contract#services-env-vars - if std::env::var("K_SERVICE").is_ok() { - // Disable colored logging as it messes up Google's log formatting - layer = layer.with_ansi(false); - } - Box::new(layer) -} - async fn load_entries( gcp_service: &GcpService, env: &str, @@ -218,38 +203,20 @@ where Ok(entries) } -fn init_subscribers() -> anyhow::Result<()> { - // Setup a temporary subscriber to log output during setup - let subscriber = tracing_subscriber::registry() - .with(build_loglevel_filter_layer()) - .with(build_tracing_layer()); - let _guard = tracing::subscriber::set_default(subscriber); - tracing::info!("init logging & tracing"); - - let subscriber = tracing_subscriber::registry() - .with(build_otel_layer()?) - .with(build_loglevel_filter_layer()) - .with(build_tracing_layer()); - tracing::subscriber::set_global_default(subscriber)?; - Ok(()) -} - #[tokio::main] async fn main() -> anyhow::Result<()> { - init_subscribers()?; - let _span = tracing::trace_span!("cli").entered(); match Cli::parse() { Cli::Generate { n } => { let GenerateResult { pk_set, secrets } = mpc_recovery::generate(n); - tracing::info!("Public key set: {}", serde_json::to_string(&pk_set)?); + println!("Public key set: {}", serde_json::to_string(&pk_set)?); for (i, (sk_share, cipher_key)) in secrets.iter().enumerate() { - tracing::info!( + println!( "Secret key share {}: {}", i, serde_json::to_string(sk_share)? ); - tracing::info!("Cipher {}: {}", i, hex::encode(cipher_key)); + println!("Cipher {}: {}", i, hex::encode(cipher_key)); } } Cli::StartLeader { @@ -265,7 +232,16 @@ async fn main() -> anyhow::Result<()> { gcp_project_id, gcp_datastore_url, test, + logging_options, } => { + let _subscriber_guard = logging::default_subscriber_with_opentelemetry( + EnvFilter::from_default_env(), + &logging_options, + env.clone(), + "leader".to_string(), + ) + .await + .global(); let gcp_service = GcpService::new(env.clone(), gcp_project_id, gcp_datastore_url).await?; let account_creator_sk = @@ -306,7 +282,16 @@ async fn main() -> anyhow::Result<()> { gcp_project_id, gcp_datastore_url, test, + logging_options, } => { + let _subscriber_guard = logging::default_subscriber_with_opentelemetry( + EnvFilter::from_default_env(), + &logging_options, + env.clone(), + "leader".to_string(), + ) + .await + .global(); let gcp_service = GcpService::new(env.clone(), gcp_project_id, gcp_datastore_url).await?; let oidc_providers = OidcProviderList { @@ -351,7 +336,16 @@ async fn main() -> anyhow::Result<()> { new_cipher_key, gcp_project_id, gcp_datastore_url, + logging_options, } => { + let _subscriber_guard = logging::default_subscriber_with_opentelemetry( + EnvFilter::from_default_env(), + &logging_options, + env.clone(), + "leader".to_string(), + ) + .await + .global(); let gcp_service = GcpService::new( env.clone(), gcp_project_id.clone(), From e652b33085ee16cad2d41d8058343460a49e7af6 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Fri, 13 Oct 2023 14:05:11 +1100 Subject: [PATCH 3/4] configurable endpoint --- Cargo.lock | 1 + mpc-recovery/Cargo.toml | 1 + mpc-recovery/src/logging.rs | 24 ++++++++++++++++++++---- mpc-recovery/src/main.rs | 4 ++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a310e4d17..0da08a82e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2756,6 +2756,7 @@ dependencies = [ "near-primitives 0.17.0", "opentelemetry 0.20.0", "opentelemetry-otlp 0.13.0", + "opentelemetry-semantic-conventions 0.12.0", "prometheus", "rand 0.7.3", "rand 0.8.5", diff --git a/mpc-recovery/Cargo.toml b/mpc-recovery/Cargo.toml index dd8c3acc0..9e082dc14 100644 --- a/mpc-recovery/Cargo.toml +++ b/mpc-recovery/Cargo.toml @@ -30,6 +30,7 @@ jsonwebtoken = "8.3.0" lazy_static = "1.4.0" opentelemetry = "0.20.0" opentelemetry-otlp = "0.13.0" +opentelemetry-semantic-conventions = "0.12.0" prometheus = { version = "0.13.3", features = ["process"] } rand = "0.7" rand8 = { package = "rand", version = "0.8" } diff --git a/mpc-recovery/src/logging.rs b/mpc-recovery/src/logging.rs index 4ee600947..6d483efa3 100644 --- a/mpc-recovery/src/logging.rs +++ b/mpc-recovery/src/logging.rs @@ -2,6 +2,7 @@ use opentelemetry::sdk::trace::{self, RandomIdGenerator, Sampler, Tracer}; use opentelemetry::sdk::Resource; use opentelemetry::KeyValue; use opentelemetry_otlp::WithExportConfig; +use opentelemetry_semantic_conventions::resource::SERVICE_NAME; use std::sync::OnceLock; use tracing::subscriber::DefaultGuard; use tracing_appender::non_blocking::NonBlocking; @@ -60,6 +61,10 @@ pub struct Options { #[clap(long, value_enum, default_value = "off")] opentelemetry: OpenTelemetryLevel, + /// Opentelemetry gRPC collector endpoint. + #[clap(long, default_value = "http://localhost:4317")] + otlp_endpoint: String, + /// Whether the log needs to be colored. #[clap(long, value_enum, default_value = "auto")] color: ColorOutput, @@ -121,6 +126,7 @@ where /// Constructs an OpenTelemetryConfig which sends span data to an external collector. async fn add_opentelemetry_layer( opentelemetry_level: OpenTelemetryLevel, + otlp_endpoint: &str, env: String, node_id: String, subscriber: S, @@ -136,14 +142,18 @@ where }; let (filter, handle) = reload::Layer::::new(filter); - let resource = vec![KeyValue::new("env", env), KeyValue::new("node_id", node_id)]; + let resource = vec![ + KeyValue::new(SERVICE_NAME, format!("mpc:{}", node_id)), + KeyValue::new("env", env), + KeyValue::new("node_id", node_id), + ]; let tracer = opentelemetry_otlp::new_pipeline() .tracing() .with_exporter( opentelemetry_otlp::new_exporter() .tonic() - .with_endpoint("http://localhost:4317"), + .with_endpoint(otlp_endpoint), ) .with_trace_config( trace::config() @@ -235,8 +245,14 @@ pub async fn default_subscriber_with_opentelemetry( .set(handle) .unwrap_or_else(|_| panic!("Failed to set Log Layer Filter")); - let (subscriber, handle) = - add_opentelemetry_layer(options.opentelemetry, env, node_id, subscriber).await; + let (subscriber, handle) = add_opentelemetry_layer( + options.opentelemetry, + &options.otlp_endpoint, + env, + node_id, + subscriber, + ) + .await; OTLP_LAYER_RELOAD_HANDLE .set(handle) .unwrap_or_else(|_| panic!("Failed to set OTLP Layer Filter")); diff --git a/mpc-recovery/src/main.rs b/mpc-recovery/src/main.rs index f2b924d12..4b8222ac5 100644 --- a/mpc-recovery/src/main.rs +++ b/mpc-recovery/src/main.rs @@ -294,7 +294,7 @@ async fn main() -> anyhow::Result<()> { EnvFilter::from_default_env(), &logging_options, env.clone(), - "leader".to_string(), + node_id.to_string(), ) .await .global(); @@ -348,7 +348,7 @@ async fn main() -> anyhow::Result<()> { EnvFilter::from_default_env(), &logging_options, env.clone(), - "leader".to_string(), + node_id.to_string(), ) .await .global(); From b4a169bb46dd13cc2f973741882fb6fcac3ee42d Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Fri, 13 Oct 2023 20:44:37 +1100 Subject: [PATCH 4/4] remove axum telemetry for now --- Cargo.lock | 46 ----------------------------- mpc-recovery/Cargo.toml | 1 - mpc-recovery/src/leader_node/mod.rs | 12 -------- 3 files changed, 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0da08a82e..cd80d3620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,24 +387,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "axum-tracing-opentelemetry" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06985105829f176e9a3f113b1c71cc24e08f600ef0df4e70cd90d144f889e19f" -dependencies = [ - "axum", - "futures-core", - "futures-util", - "http", - "opentelemetry 0.20.0", - "pin-project-lite", - "tower", - "tracing", - "tracing-opentelemetry 0.21.0", - "tracing-opentelemetry-instrumentation-sdk", -] - [[package]] name = "backtrace" version = "0.3.69" @@ -2734,7 +2716,6 @@ dependencies = [ "atty", "axum", "axum-extra", - "axum-tracing-opentelemetry", "base64 0.21.4", "borsh 0.10.3", "chrono", @@ -3662,18 +3643,6 @@ dependencies = [ "opentelemetry_sdk", ] -[[package]] -name = "opentelemetry-http" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" -dependencies = [ - "async-trait", - "bytes", - "http", - "opentelemetry_api", -] - [[package]] name = "opentelemetry-otlp" version = "0.10.0" @@ -3776,8 +3745,6 @@ dependencies = [ "regex", "serde_json", "thiserror", - "tokio", - "tokio-stream", ] [[package]] @@ -6071,19 +6038,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "tracing-opentelemetry-instrumentation-sdk" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "752ddd669b14a08036a89045e4ac4497ff7ce254e11386647751f36d7c7849ea" -dependencies = [ - "http", - "opentelemetry-http", - "opentelemetry_api", - "tracing", - "tracing-opentelemetry 0.21.0", -] - [[package]] name = "tracing-subscriber" version = "0.3.17" diff --git a/mpc-recovery/Cargo.toml b/mpc-recovery/Cargo.toml index 9e082dc14..745d67a6d 100644 --- a/mpc-recovery/Cargo.toml +++ b/mpc-recovery/Cargo.toml @@ -15,7 +15,6 @@ async-trait = "0.1" atty = "0.2" axum = "0.6.19" axum-extra = "0.7" -axum-tracing-opentelemetry = "0.14.1" base64 = "0.21" borsh = "0.10.3" chrono = "0.4.24" diff --git a/mpc-recovery/src/leader_node/mod.rs b/mpc-recovery/src/leader_node/mod.rs index bac64ccc1..4a9d08a17 100644 --- a/mpc-recovery/src/leader_node/mod.rs +++ b/mpc-recovery/src/leader_node/mod.rs @@ -15,7 +15,6 @@ use crate::transaction::{ }; use crate::utils::{check_digest_signature, user_credentials_request_digest}; use crate::{metrics, nar}; - use anyhow::Context; use axum::extract::MatchedPath; use axum::middleware::{self, Next}; @@ -27,7 +26,6 @@ use axum::{ Extension, Json, Router, }; use axum_extra::extract::WithRejection; -use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; use borsh::BorshDeserialize; use curv::elliptic::curves::{Ed25519, Point}; use near_crypto::SecretKey; @@ -36,7 +34,6 @@ use near_primitives::transaction::{Action, DeleteKeyAction}; use near_primitives::types::AccountId; use prometheus::{Encoder, TextEncoder}; use rand::{distributions::Alphanumeric, Rng}; - use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; @@ -142,25 +139,16 @@ pub async fn run(config: Config) { .route("/metrics", get(metrics)) .route_layer(middleware::from_fn(track_metrics)) .layer(Extension(state)) - // Include trace context as header into the response - .layer(OtelInResponseLayer::default()) - // Start OpenTelemetry trace on incoming request - .layer(OtelAxumLayer::default()) .layer(cors_layer); let addr = SocketAddr::from(([0, 0, 0, 0], port)); tracing::debug!(?addr, "starting http server"); axum::Server::bind(&addr) .serve(app.into_make_service()) - // .with_graceful_shutdown(shutdown_signal()) .await .unwrap(); } -// async fn shutdown_signal() { -// opentelemetry::global::shutdown_tracer_provider(); -// } - async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { let timer = Instant::now(); let path = if let Some(matched_path) = req.extensions().get::() {