feat: use Grafana OTLP collector (#345)

near · Nov 3, 2023 · 6e8e60d · 6e8e60d
1 parent 9b34247
commit 6e8e60d
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 12 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/infra/main.tf b/infra/main.tf
@@ -148,6 +148,9 @@ module "leader" {
 
   jwt_signature_pk_url = var.jwt_signature_pk_url
 
+  otlp_endpoint       = var.otlp_endpoint
+  opentelemetry_level = var.opentelemetry_level
+
   depends_on = [
     google_secret_manager_secret_iam_member.account_creator_secret_access,
     google_secret_manager_secret_iam_member.fast_auth_partners_secret_access,

diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf
@@ -61,11 +61,18 @@ resource "google_cloud_run_v2_service" "leader" {
           }
         }
       }
-
       env {
         name  = "MPC_RECOVERY_JWT_SIGNATURE_PK_URL"
         value = var.jwt_signature_pk_url
       }
+      env {
+        name  = "MPC_RECOVERY_OTLP_ENDPOINT"
+        value = var.otlp_endpoint
+      }
+      env {
+        name  = "MPC_RECOVERY_OPENTELEMETRY_LEVEL"
+        value = var.opentelemetry_level
+      }
 
       env {
         name  = "RUST_LOG"

diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf
@@ -51,3 +51,11 @@ variable "fast_auth_partners_secret_id" {
 variable "jwt_signature_pk_url" {
   type = string
 }
+
+variable "otlp_endpoint" {
+  type = string
+}
+
+variable "opentelemetry_level" {
+  type = string
+}
diff --git a/infra/terraform-dev.tfvars b/infra/terraform-dev.tfvars
@@ -20,3 +20,5 @@ signer_configs = [
   }
 ]
 jwt_signature_pk_url = "https://www.googleapis.com/robot/v1/metadata/x509/securetoken@system.gserviceaccount.com"
+otlp_endpoint        = "https://otel.dev.api.pagoda.co:443/v1/traces"
+opentelemetry_level  = "debug"
diff --git a/infra/variables.tf b/infra/variables.tf
@@ -53,3 +53,13 @@ variable "signer_configs" {
 variable "jwt_signature_pk_url" {
   type = string
 }
+
+variable "otlp_endpoint" {
+  type    = string
+  default = "http://localhost:4317"
+}
+
+variable "opentelemetry_level" {
+  type    = string
+  default = "off"
+}
diff --git a/mpc-recovery/Cargo.toml b/mpc-recovery/Cargo.toml
@@ -28,7 +28,10 @@ hyper-rustls = { version = "=0.23", features = ["http2"] }
 jsonwebtoken = "8.3.0"
 lazy_static = "1.4.0"
 opentelemetry = { version = "0.20.0", features = ["rt-tokio", "trace"] }
-opentelemetry-otlp = "0.13.0"
+opentelemetry-otlp = { version = "0.13.0", features = [
+    "http-proto",
+    "reqwest-client",
+] }
 opentelemetry-semantic-conventions = "0.12.0"
 prometheus = { version = "0.13.3", features = ["process"] }
 rand = "0.7"

diff --git a/mpc-recovery/src/logging.rs b/mpc-recovery/src/logging.rs
@@ -79,14 +79,23 @@ impl Display for ColorOutput {
 }
 
 /// Configures exporter of span and trace data.
-#[derive(Debug, Default, clap::Parser)]
+#[derive(Debug, clap::Parser)]
 pub struct Options {
     /// Enables export of span data using opentelemetry exporters.
-    #[clap(long, value_enum, default_value = "off")]
-    opentelemetry: OpenTelemetryLevel,
+    #[clap(
+        long,
+        env("MPC_RECOVERY_OPENTELEMETRY_LEVEL"),
+        value_enum,
+        default_value = "off"
+    )]
+    opentelemetry_level: OpenTelemetryLevel,
 
     /// Opentelemetry gRPC collector endpoint.
-    #[clap(long, default_value = "http://localhost:4317")]
+    #[clap(
+        long,
+        env("MPC_RECOVERY_OTLP_ENDPOINT"),
+        default_value = "http://localhost:4317"
+    )]
     otlp_endpoint: String,
 
     /// Whether the log needs to be colored.
@@ -99,11 +108,22 @@ pub struct Options {
     log_span_events: bool,
 }
 
+impl Default for Options {
+    fn default() -> Self {
+        Self {
+            opentelemetry_level: OpenTelemetryLevel::OFF,
+            otlp_endpoint: "http://localhost:4317".to_string(),
+            color: ColorOutput::Auto,
+            log_span_events: false,
+        }
+    }
+}
+
 impl Options {
     pub fn into_str_args(self) -> Vec<String> {
         let mut buf = vec![
-            "--opentelemetry".to_string(),
-            self.opentelemetry.to_string(),
+            "--opentelemetry-level".to_string(),
+            self.opentelemetry_level.to_string(),
             "--otlp-endpoint".to_string(),
             self.otlp_endpoint,
             "--color".to_string(),
@@ -184,7 +204,7 @@ where
     let (filter, handle) = reload::Layer::<LevelFilter, S>::new(filter);
 
     let resource = vec![
-        KeyValue::new(SERVICE_NAME, format!("mpc:{}", node_id)),
+        KeyValue::new(SERVICE_NAME, format!("mpc:{}:{}", env, node_id)),
         KeyValue::new("env", env),
         KeyValue::new("node_id", node_id),
     ];
@@ -193,7 +213,7 @@ where
         .tracing()
         .with_exporter(
             opentelemetry_otlp::new_exporter()
-                .tonic()
+                .http()
                 .with_endpoint(otlp_endpoint),
         )
         .with_trace_config(
@@ -213,7 +233,7 @@ where
 fn set_default_otlp_level(options: &Options) {
     // Record the initial tracing level specified as a command-line flag. Use this recorded value to
     // reset opentelemetry filter when the LogConfig file gets deleted.
-    DEFAULT_OTLP_LEVEL.set(options.opentelemetry).unwrap();
+    DEFAULT_OTLP_LEVEL.set(options.opentelemetry_level).unwrap();
 }
 
 /// The resource representing a registered subscriber.
@@ -287,7 +307,7 @@ pub async fn default_subscriber_with_opentelemetry(
         .unwrap_or_else(|_| panic!("Failed to set Log Layer Filter"));
 
     let (subscriber, handle) = add_opentelemetry_layer(
-        options.opentelemetry,
+        options.opentelemetry_level,
         &options.otlp_endpoint,
         env,
         node_id,