Rework metrics to use metrics crate

Signed-off-by: Ludvig Liljenberg <lliljenberg@microsoft.com>

Author: ludfjig

Cargo.lock

@@ -34,7 +34,6 @@ lazy_static = "1.4.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 log = "0.4.26"
-once_cell = { version = "1.21.1" }
 tracing = { version = "0.1.41", features = ["log"] }
 tracing-log = "0.2.0"
 tracing-core = "0.1.33"
@@ -43,11 +42,10 @@ vmm-sys-util = "0.12.1"
 crossbeam = "0.8.0"
 crossbeam-channel = "0.5.14"
 thiserror = "2.0.12"
-prometheus = "0.13.3"
-strum = { version = "0.27", features = ["derive"] }
 tempfile = { version = "3.18", optional = true }
 serde_yaml = "0.9"
 anyhow = "1.0"
+metrics = "0.24.1"
 
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.59", features = [
@@ -104,6 +102,8 @@ opentelemetry_sdk = { version = "0.28", features = ["rt-tokio"] }
 tokio = { version = "1.43.0", features = ["full"] }
 criterion = "0.5.1"
 tracing-chrome = "0.7.2"
+metrics-util = "0.19.0"
+metrics-exporter-prometheus = "0.16.2"
 
 [target.'cfg(windows)'.dev-dependencies]
 windows = { version = "0.59", features = [
@@ -119,7 +119,7 @@ cfg_aliases = "0.2.1"
 built = { version = "0.7.7", features = ["chrono", "git2"] }
 
 [features]
-default = ["kvm", "mshv2", "seccomp"]
+default = ["kvm", "mshv2", "seccomp", "function_call_metrics"]
 seccomp = ["dep:seccompiler"]
 function_call_metrics = []
 executable_heap = []

src/hyperlight_host/Cargo.toml

@@ -22,22 +22,27 @@ use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, Ret
 use hyperlight_host::sandbox::uninitialized::UninitializedSandbox;
 use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox;
 use hyperlight_host::sandbox_state::transition::Noop;
-use hyperlight_host::{set_metrics_registry, GuestBinary, MultiUseSandbox, Result};
+use hyperlight_host::{GuestBinary, MultiUseSandbox, Result};
 use hyperlight_testing::simple_guest_as_string;
-use lazy_static::lazy_static;
-use prometheus::Registry;
 
-lazy_static! {
-    static ref HOST_REGISTRY: Registry = Registry::new();
-}
-fn fn_writer(_msg: String) -> Result<i32> {
-    Ok(0)
+fn main() {
+    // Install prometheus metrics exporter.
+    // We only install the metrics recorder here, but you can also use the
+    // `metrics_exporter_prometheus::PrometheusBuilder::new().install()` method
+    // to install a HTTP listener that serves the metrics.
+    let prometheus_handle = metrics_exporter_prometheus::PrometheusBuilder::new()
+        .install_recorder()
+        .expect("Failed to install Prometheus exporter");
+
+    // Do some hyperlight stuff to generate metrics.
+    do_hyperlight_stuff();
+
+    // Get the metrics and print them in prometheus exposition format.
+    let payload = prometheus_handle.render();
+    println!("Prometheus metrics:\n{}", payload);
 }
 
-fn main() -> Result<()> {
-    // If this is not called then the default registry `prometheus::default_registry` will be used.
-    set_metrics_registry(&HOST_REGISTRY)?;
-
+fn do_hyperlight_stuff() {
     // Get the path to a simple guest binary.
     let hyperlight_guest_path =
         simple_guest_as_string().expect("Cannot find the guest binary at the expected location.");
@@ -60,7 +65,7 @@ fn main() -> Result<()> {
 
             let no_op = Noop::<UninitializedSandbox, MultiUseSandbox>::default();
 
-            let mut multiuse_sandbox = usandbox.evolve(no_op)?;
+            let mut multiuse_sandbox = usandbox.evolve(no_op).expect("Failed to evolve sandbox");
 
             // Call a guest function 5 times to generate some metrics.
             for _ in 0..5 {
@@ -97,13 +102,14 @@ fn main() -> Result<()> {
         None,
         None,
         None,
-    )?;
+    )
+    .expect("Failed to create UninitializedSandbox");
 
     // Initialize the sandbox.
 
     let no_op = Noop::<UninitializedSandbox, MultiUseSandbox>::default();
 
-    let mut multiuse_sandbox = usandbox.evolve(no_op)?;
+    let mut multiuse_sandbox = usandbox.evolve(no_op).expect("Failed to evolve sandbox");
 
     // Call a function that gets cancelled by the host function 5 times to generate some metrics.
 
@@ -121,73 +127,8 @@ fn main() -> Result<()> {
         let result = join_handle.join();
         assert!(result.is_ok());
     }
-
-    get_metrics();
-
-    Ok(())
 }
 
-fn get_metrics() {
-    // Get the metrics from the registry.
-
-    let metrics = HOST_REGISTRY.gather();
-
-    // Print the metrics.
-
-    print!("\nMETRICS:\n");
-
-    for metric in metrics.iter() {
-        match metric.get_field_type() {
-            prometheus::proto::MetricType::COUNTER => {
-                println!("Counter: {:?}", metric.get_help());
-                metric.get_metric().iter().for_each(|metric| {
-                    let pair = metric.get_label();
-                    for pair in pair.iter() {
-                        println!("Label: {:?} Name: {:?}", pair.get_name(), pair.get_value());
-                    }
-                    println!("Value: {:?}", metric.get_counter().get_value());
-                });
-            }
-            prometheus::proto::MetricType::GAUGE => {
-                println!("Gauge: {:?}", metric.get_help());
-                metric.get_metric().iter().for_each(|metric| {
-                    let pair = metric.get_label();
-                    for pair in pair.iter() {
-                        println!("Label: {:?} Name: {:?}", pair.get_name(), pair.get_value());
-                    }
-                    println!("Value: {:?}", metric.get_gauge().get_value());
-                });
-            }
-            prometheus::proto::MetricType::UNTYPED => {
-                println!("Metric: {:?}", metric.get_help());
-            }
-            prometheus::proto::MetricType::HISTOGRAM => {
-                println!("Histogram: {:?}", metric.get_help());
-                for metric in metric.get_metric() {
-                    let pair = metric.get_label();
-                    for pair in pair.iter() {
-                        println!("Label: {:?} Name: {:?}", pair.get_name(), pair.get_value());
-                    }
-                    let count = metric.get_histogram().get_sample_count();
-                    println!("Number of observations: {:?}", count);
-                    let sm = metric.get_histogram().get_sample_sum();
-                    println!("Sum of observations: {:?}", sm);
-                    metric
-                        .get_histogram()
-                        .get_bucket()
-                        .iter()
-                        .for_each(|bucket| {
-                            println!(
-                                "Bucket: {:?} Count: {:?}",
-                                bucket.get_upper_bound(),
-                                bucket.get_cumulative_count()
-                            )
-                        });
-                }
-            }
-            prometheus::proto::MetricType::SUMMARY => {
-                println!("Summary: {:?}", metric.get_help());
-            }
-        }
-    }
+fn fn_writer(_msg: String) -> Result<i32> {
+    Ok(0)
 }

src/hyperlight_host/examples/metrics/main.rs

@@ -237,10 +237,6 @@ pub enum HyperlightError {
     #[error("Failure processing PE File {0:?}")]
     PEFileProcessingFailure(#[from] goblin::error::Error),
 
-    /// a Prometheus error occurred
-    #[error("Prometheus Error {0:?}")]
-    Prometheus(#[from] prometheus::Error),
-
     /// Raw pointer is less than base address
     #[error("Raw pointer ({0:?}) was less than the base address ({1})")]
     RawPointerLessThanBaseAddress(RawPtr, u64),

src/hyperlight_host/src/error.rs

@@ -14,15 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-use hyperlight_common::flatbuffer_wrappers::guest_error::{
-    ErrorCode, GuestError as GuestErrorStruct,
-};
+use hyperlight_common::flatbuffer_wrappers::guest_error::ErrorCode;
 
 use crate::error::HyperlightError::{GuestError, OutBHandlingError, StackOverflow};
 use crate::mem::shared_mem::HostSharedMemory;
+use crate::metrics::{self, CounterMetric};
 use crate::sandbox::mem_mgr::MemMgrWrapper;
-use crate::sandbox::metrics::SandboxMetric::GuestErrorCount;
-use crate::{int_counter_vec_inc, log_then_return, Result};
+use crate::{log_then_return, Result};
 /// Check for a guest error and return an `Err` if one was found,
 /// and `Ok` if one was not found.
 pub(crate) fn check_for_guest_error(mgr: &MemMgrWrapper<HostSharedMemory>) -> Result<()> {
@@ -31,7 +29,7 @@ pub(crate) fn check_for_guest_error(mgr: &MemMgrWrapper<HostSharedMemory>) -> Re
         ErrorCode::NoError => Ok(()),
         ErrorCode::OutbError => match mgr.as_ref().get_host_error()? {
             Some(host_err) => {
-                increment_guest_error_count(&guest_err);
+                metrics::increase_counter(CounterMetric::NumGuestErrors);
                 log_then_return!(OutBHandlingError(
                     host_err.source.clone(),
                     guest_err.message.clone()
@@ -41,23 +39,15 @@ pub(crate) fn check_for_guest_error(mgr: &MemMgrWrapper<HostSharedMemory>) -> Re
             None => Ok(()),
         },
         ErrorCode::StackOverflow => {
-            increment_guest_error_count(&guest_err.clone());
+            metrics::increase_counter(CounterMetric::NumGuestErrors);
             log_then_return!(StackOverflow());
         }
         _ => {
-            increment_guest_error_count(&guest_err.clone());
+            metrics::increase_counter(CounterMetric::NumGuestErrors);
             log_then_return!(GuestError(
                 guest_err.code.clone(),
                 guest_err.message.clone()
             ));
         }
     }
 }
-
-fn increment_guest_error_count(guest_err: &GuestErrorStruct) {
-    let guest_err_code_string: String = guest_err.code.clone().into();
-    int_counter_vec_inc!(
-        &GuestErrorCount,
-        &[&guest_err_code_string, guest_err.message.clone().as_str()]
-    );
-}

src/hyperlight_host/src/func/guest_err.rs

@@ -37,8 +37,6 @@ use windows::Win32::System::Hypervisor::{WHvCancelRunVirtualProcessor, WHV_PARTI
 
 #[cfg(gdb)]
 use super::gdb::create_gdb_thread;
-#[cfg(feature = "function_call_metrics")]
-use crate::histogram_vec_observe;
 #[cfg(gdb)]
 use crate::hypervisor::handlers::DbgMemAccessHandlerWrapper;
 use crate::hypervisor::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
@@ -53,8 +51,6 @@ use crate::mem::shared_mem::{GuestSharedMemory, HostSharedMemory, SharedMemory};
 #[cfg(gdb)]
 use crate::sandbox::config::DebugInfo;
 use crate::sandbox::hypervisor::{get_available_hypervisor, HypervisorType};
-#[cfg(feature = "function_call_metrics")]
-use crate::sandbox::metrics::SandboxMetric::GuestFunctionCallDurationMicroseconds;
 #[cfg(target_os = "linux")]
 use crate::signal_handlers::setup_signal_handlers;
 use crate::HyperlightError::{
@@ -438,6 +434,8 @@ impl HypervisorHandler {
                                 let res = {
                                     #[cfg(feature = "function_call_metrics")]
                                     {
+                                        use crate::metrics::{self, HistogramMetric};
+
                                         let start = std::time::Instant::now();
                                         let result = hv.dispatch_call_from_host(
                                             dispatch_function_addr,
@@ -447,11 +445,8 @@ impl HypervisorHandler {
                                             #[cfg(gdb)]
                                             configuration.dbg_mem_access_handler.clone(),
                                         );
-                                        histogram_vec_observe!(
-                                            &GuestFunctionCallDurationMicroseconds,
-                                            &[function_name.as_str()],
-                                            start.elapsed().as_micros() as f64
-                                        );
+                                        let elapsed = start.elapsed();
+                                        metrics::record_histogram_duration(HistogramMetric::GuestCallDuration, elapsed);
                                         result
                                     }