Add broadcast_errors metric

.changelog/unreleased/features/ibc-telemetry/3708-add-broadcast-errors.md

@@ -0,0 +1,3 @@
+- Add a new metric `broadcast_errors`` which
+ records the errors observed when broadcasting Txs
+ ([\#3708](https://github.com/informalsystems/hermes/issues/3708))

crates/relayer/src/chain/cosmos/retry.rs

@@ -107,6 +107,13 @@ async fn do_send_tx_with_account_sequence_retry(
  refreshing account sequence number and retrying once"
  );
 
+ telemetry!(
+ broadcast_errors,
+ &account.address.to_string(),
+ response.code.into(),
+ &response.log,
+ );
+
  refresh_account_and_retry_send_tx_with_account_sequence(
  rpc_client, config, key_pair, account, tx_memo, messages,
  )
@@ -147,6 +154,13 @@ async fn do_send_tx_with_account_sequence_retry(
  "failed to broadcast tx with unrecoverable error"
  );
 
+ telemetry!(
+ broadcast_errors,
+ &account.address.to_string(),
+ code.into(),
+ &response.log
+ );
+
  Ok(response)
  }
  }

crates/telemetry/src/state.rs

@@ -197,6 +197,9 @@ pub struct TelemetryState {
 
  /// Sum of rewarded fees over the past FEE_LIFETIME seconds
  period_fees: ObservableGauge<u64>,
+
+ /// Number of errors observed by Hermes when broadcasting a Tx
+ broadcast_errors: Counter<u64>,
 }
 
 impl TelemetryState {
@@ -371,6 +374,13 @@ impl TelemetryState {
  .u64_observable_gauge("ics29_period_fees")
  .with_description("Amount of ICS29 fees rewarded over the past 7 days")
  .init(),
+
+ broadcast_errors: meter
+ .u64_counter("broadcast_errors")
+ .with_description(
+ "Number of errors observed by Hermes when broadcasting a Tx",
+ )
+ .init(),
  }
  }
 
@@ -1069,6 +1079,20 @@ impl TelemetryState {
  pub fn add_visible_fee_address(&self, address: String) {
  self.visible_fee_addresses.insert(address);
  }
+
+ /// Add an error and its description to the list of errors observed after broadcasting
+ /// a Tx with a specific account.
+ pub fn broadcast_errors(&self, address: &String, error_code: u32, error_description: &String) {
+ let cx = Context::current();
+
+ let labels = &[
+ KeyValue::new("account", address.to_string()),
+ KeyValue::new("error_code", error_code.to_string()),
+ KeyValue::new("error_description", error_description.to_string()),
+ ];
+
+ self.broadcast_errors.add(&cx, 1, labels);
+ }
 }
 
 use std::sync::Arc;

guide/src/documentation/telemetry/operators.md

@@ -142,6 +142,7 @@ the `backlog_oldest_sequence` that is blocked.
 | `tx_latency_submitted` | Latency for all transactions submitted to a chain (i.e., difference between the moment when Hermes received an event until the corresponding transaction(s) were submitted), per chain, counterparty chain, channel and port | `u64` ValueRecorder | None |
 | `cleared_send_packet_count_total`  | Number of SendPacket events received during the initial and periodic clearing, per chain, counterparty chain, channel and port | `u64` Counter | Packet workers enabled, and periodic packet clearing or clear on start enabled |
 | `cleared_acknowledgment_count_total` | Number of WriteAcknowledgement events received during the initial and periodic clearing, per chain, counterparty chain, channel and port | `u64` Counter | Packet workers enabled, and periodic packet clearing or clear on start enabled |
+| `broadcast_errors_total` | Number of errors observed by Hermes when broadcasting a Tx, per error type and account | `u64` Counter | Packet workers enabled |
 
 Notes:
 - The two metrics `cleared_send_packet_count_total` and `cleared_acknowledgment_count_total` are only populated if `tx_confirmation = true`.