From 5ab15adb1034999bafdaa25c08633d4eed4472e3 Mon Sep 17 00:00:00 2001 From: Jay Allen Date: Tue, 4 Jun 2024 14:18:21 +0900 Subject: [PATCH] Integrating recommended changes into telemetry doc --- docs/how_to_guides/telemetry.mdx | 40 ++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/how_to_guides/telemetry.mdx b/docs/how_to_guides/telemetry.mdx index 50b21ce63..100670aac 100644 --- a/docs/how_to_guides/telemetry.mdx +++ b/docs/how_to_guides/telemetry.mdx @@ -2,7 +2,11 @@ import Dashboard from './assets/dashboard.png'; import ViewTraces from './assets/view_traces.mp4'; import GrafanaOtelConfig from './assets/grafana_otel_config.mp4'; -# Capture Metrics for your Guards +# Overview + +In this document, we explain how to set up Guardrails with [OpenTelemetry (OTEL)](https://opentelemetry.io/) using either Grafana or a self-hosted OTEL collector. With this functionality enabled, you can measure latency of Guards, Large Language Models (LLMs), scuccess rates, and other metrics for your Guardrails-protected LLM calls. + +## Metrics you can capture using OTEL This package is instrumented using the OpenTelemetry Python SDK. By viewing the captured traces and derived metrics, we're able to get useful insights into how our Guards, and our LLM apps in general perform. Among other things, we're able to find: @@ -12,12 +16,11 @@ This package is instrumented using the OpenTelemetry Python SDK. By viewing the 4. The rate at which validators Pass and Fail, within a Guard and across Guards 5. Deep dives into singular guard and validator calls -Since we are using OpenTelemetry, traces and metrics can be written to any OpenTelemetry enabled service or OTLP endpoint. This includes all major metrics providers like Grafana, New Relic, Prometheus, and Splunk. +Since we are using OpenTelemetry, traces and metrics can be written to any OpenTelemetry-enabled service or OTLP endpoint. This includes all major metrics providers like Grafana, New Relic, Prometheus, and Splunk. -This guide will show how to set up your python project to log traces to Grafana and an OTEL collector. +This guide will show how to set up your Python project to log traces to Grafana and to a self-hosted OTEL collector. For other OTEL endpoints, consult your metrics provider's documentation on OTEL support. - -## Setup with Grafana +## Configure OTEL for Grafana Grafana Cloud is a free offering by Grafana that's easy to setup and is our preferred location for storing metrics. @@ -41,11 +44,11 @@ OTEL_EXPORTER_OTLP_HEADERS ### Setup a Guard with Telemetry -We first have to install the ```ValidLength``` guardrail from Guardrails Hub. +1. First, install the ```ValidLength``` guardrail from Guardrails Hub. ```guardrails hub install hub://guardrails/valid_length``` -Then, set up your Guard the default tracer provided in the guardrails library. You can still use your desired validators +2. Next, set up your Guard the default tracer provided in the guardrails library. You can still use your desired validators:
main.py
```python @@ -69,7 +72,7 @@ guard( ) ``` -Before running the file, make sure to set the environment variables you got from Grafana +3. Before running the file, make sure to set the environment variables you got from Grafana ```bash export GRAFANA_INSTANCE_ID= @@ -82,7 +85,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT= export OTEL_EXPORTER_OTLP_HEADERS= ``` -Finally, run the python script +4. Finally, run the python script ```bash @@ -90,26 +93,33 @@ python main.py ``` -### Viewing traces +### View traces + +There are two ways to view traces: using the Explore tab or using the Guardrails Grafana dashboard template. -The simplest way to do this is to go to your grafana stack and click on the "Explore" tab. You should see a list of traces that you can filter by service name, operation name, and more. +#### Use the Explore tab + +The simplest way to do this is to go to your grafana stack and click on the "**Explore** tab. You should see a list of traces that you can filter by service name, operation name, and more. +#### Use the Guardrails Grafana dashboard template + While this is easy to do, it's not the best way to get a big-picture view of how your guards are doing. For that, we should use the Guardrails Grafana dashboard template. -The template can be found here https://grafana.com/grafana/dashboards/20600-standard-guardrails-dash/ -and instructions to use the template are found here https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/ +**[Use the template](https://grafana.com/grafana/dashboards/20600-standard-guardrails-dash/)** + +**[Template instructions](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/)** -## OTEL Collector +## Configure OTEL for a self-hosted OpenTelemetry Collector For advanced use cases (like if you have a metrics provider in a VPC), you can use a self-hosted OpenTelemetry Collector to receive traces and metrics from your Guard. -Standard [open telemetry environment variables](https://opentelemetry-python.readthedocs.io/en/stable/getting-started.html#configure-the-exporter) are used to configure the collector. Use the default_otel_collector_tracer when configuring your guard. +Standard [open telemetry environment variables](https://opentelemetry.io/docs/languages/python/exporters/) are used to configure the collector. Use the `default_otel_collector_tracer` when configuring your guard. ```python from guardrails import Guard, OnFailAction