Add Grafana Tempo

Makefile

@@ -47,10 +47,10 @@ install-tools-and-app: install-tools install-app
 install-tools-and-app-local: install-tools-local install-app-local
 
 .PHONY: install-tools-local
-install-tools-local: create-namespace-observability install-cert-manager install-jaeger install-kube-prometheus-stack install-opentelemetry-operator install-opentelemetry-collector-local
+install-tools-local: create-namespace-observability install-cert-manager install-jaeger install-tempo install-kube-prometheus-stack install-opentelemetry-operator install-opentelemetry-collector-local
 
 .PHONY: install-tools
-install-tools: create-namespace-observability install-cert-manager install-jaeger install-kube-prometheus-stack install-opentelemetry-operator install-opentelemetry-collector
+install-tools: create-namespace-observability install-cert-manager install-jaeger install-tempo install-kube-prometheus-stack install-opentelemetry-operator install-opentelemetry-collector
 
 .PHONY: create-namespace-observability
 create-namespace-observability:
@@ -68,6 +68,10 @@ install-jaeger:
 install-kube-prometheus-stack:
 	./scripts/kube-prometheus-stack_install.sh
 
+.PHONY: install-tempo
+install-tempo:
+	./scripts/tempo_install.sh
+
 .PHONY: install-opentelemetry-operator
 install-opentelemetry-operator:
 	./scripts/opentelemetry_operator_install.sh
@@ -135,17 +139,14 @@ push-images:
 .PHONY: port-forward-jaeger
 port-forward-jaeger:
 	kubectl port-forward -n observability svc/jaeger-query 16686
-	@echo "Navigate to http://localhost:16686"
 
 .PHONY: port-forward-grafana
 port-forward-grafana:
 	kubectl port-forward -n observability svc/prometheus-grafana 8080:80
-	@echo "Navigate to http://localhost:8080"
 
 .PHONY: port-forward-prometheus
 port-forward-prometheus:
 	kubectl port-forward -n observability svc/prometheus-kube-prometheus-prometheus 9090
-	@echo "Navigate to http://localhost:9090"
 
 .PHONY: e2e
 e2e:

cmd/cart/main.go

@@ -16,6 +16,7 @@ import (
 	"go.opentelemetry.io/otel/baggage"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/propagation"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/trstringer/otel-shopping-cart/pkg/cart"
@@ -41,7 +42,17 @@ var rootCmd = &cobra.Command{
 	Long:  `Shopping cart application for OpenTelemetry example.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		validateParams()
-		setupObservability()
+		tp, err := setupObservability()
+		if err != nil {
+			fmt.Printf("Error setting up observability: %v\n", err)
+			os.Exit(1)
+		}
+		defer func() {
+			if err := tp.Shutdown(context.Background()); err != nil {
+				fmt.Printf("Error shutting down tracer provider: %v", err)
+				os.Exit(1)
+			}
+		}()
 		runServer()
 	},
 }
@@ -68,25 +79,19 @@ func main() {
 	Execute()
 }
 
-func setupObservability() {
+func setupObservability() (*sdktrace.TracerProvider, error) {
 	fmt.Printf("otelReceiver is %s\n", otelReceiver)
 	tp, err := telemetry.OTLPTracerProvider(otelReceiver, "cart", "v1.0.0")
 	if err != nil {
-		fmt.Printf("Error setting tracer provider: %v\n", err)
-		os.Exit(1)
+		return nil, fmt.Errorf("error setting tracer provider: %w", err)
 	}
 	otel.SetTracerProvider(tp)
 	otel.SetTextMapPropagator(
 		propagation.NewCompositeTextMapPropagator(
 			propagation.TraceContext{},
 			propagation.Baggage{}),
 	)
-	defer func() {
-		if err := tp.Shutdown(context.Background()); err != nil {
-			fmt.Printf("Error shutting down tracer provider: %v", err)
-			os.Exit(1)
-		}
-	}()
+	return tp, nil
 }
 
 func validateParams() {

cmd/users/main.go

@@ -14,6 +14,7 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/baggage"
 	"go.opentelemetry.io/otel/propagation"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 
 	"github.com/trstringer/otel-shopping-cart/pkg/telemetry"
 	"github.com/trstringer/otel-shopping-cart/pkg/users"
@@ -35,7 +36,17 @@ var rootCmd = &cobra.Command{
 	Long:  `Users application for OpenTelemetry example.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		validateParams()
-		setupObservability()
+		tp, err := setupObservability()
+		if err != nil {
+			fmt.Printf("Error setting up observability: %v\n", err)
+			os.Exit(1)
+		}
+		defer func() {
+			if err := tp.Shutdown(context.Background()); err != nil {
+				fmt.Printf("Error shutting down tracer provider: %v", err)
+				os.Exit(1)
+			}
+		}()
 		runServer()
 	},
 }
@@ -60,24 +71,18 @@ func main() {
 	Execute()
 }
 
-func setupObservability() {
+func setupObservability() (*sdktrace.TracerProvider, error) {
 	tp, err := telemetry.OTLPTracerProvider(otelReceiver, "users", "v1.0.0")
 	if err != nil {
-		fmt.Printf("Error setting tracer provider: %v\n", err)
-		os.Exit(1)
+		return nil, fmt.Errorf("error setting tracer provider: %w", err)
 	}
 	otel.SetTracerProvider(tp)
 	otel.SetTextMapPropagator(
 		propagation.NewCompositeTextMapPropagator(
 			propagation.TraceContext{},
 			propagation.Baggage{}),
 	)
-	defer func() {
-		if err := tp.Shutdown(context.Background()); err != nil {
-			fmt.Printf("Error shutting down tracer provider: %v", err)
-			os.Exit(1)
-		}
-	}()
+	return tp, nil
 }
 
 func validateParams() {

collector/opentelemetry/templates/opentelemetry-collector.yaml

@@ -40,10 +40,14 @@ spec:
     exporters:
       logging:
         loglevel: debug
-      otlp:
+      otlp/jaeger:
         endpoint: jaeger-collector:4317
         tls:
           insecure: true
+      otlp/tempo:
+        endpoint: tempo:4317
+        tls:
+          insecure: true
       prometheus:
         endpoint: "0.0.0.0:9090"
 
@@ -52,7 +56,7 @@ spec:
         traces:
           receivers: [otlp]
           processors: [batch]
-          exporters: [otlp]
+          exporters: [otlp/jaeger,otlp/tempo]
         metrics:
           receivers: [prometheus]
           processors: []

scripts/kube-prometheus-stack_install.sh

@@ -5,15 +5,10 @@ if ! helm repo list | grep promethues-community; then
     helm repo update
 fi
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 helm upgrade \
     -n observability \
     --install \
-    --set "grafana.grafana\.ini.auth\.anonymous.enabled=true" \
-    --set "grafana.grafana\.ini.auth\.anonymous.org_role=Editor" \
-    --set "grafana.grafana\.ini.auth.disable_login_form=true" \
-    --set "grafana.grafana\.ini.auth.disable_signout_menu=true" \
-    --set "grafana.grafana\.ini.users.disable_signout_menu=true" \
-    --set "prometheus.prometheusSpec.serviceMonitorSelector.matchLabels.release=otel" \
-    --set "prometheus.prometheusSpec.podMonitorSelector.matchLabels.release=otel" \
+    -f "${SCRIPT_DIR}/kube-prometheus-stack_values.yaml" \
     prometheus \
     prometheus-community/kube-prometheus-stack

scripts/kube-prometheus-stack_values.yaml

@@ -0,0 +1,30 @@
+grafana:
+  grafana.ini:
+    auth:
+      disable_login_form: true
+      disable_signout_menu: true
+    users:
+      disable_signout_menu: true
+    auth.anonymous:
+      enabled: true
+      org_role: Editor
+  additionalDataSources:
+    - name: Tempo
+      url: http://tempo:3100
+      access: proxy
+      basicAuth: false
+      type: tempo
+      orgId: 1
+      version: 1
+      uid: tempo
+      editable: false
+      apiVersion: 1
+
+prometheus:
+  prometheusSpec:
+    serviceMonitorSelector:
+      matchLabels:
+        release: otel
+    podMonitorSelector:
+      matchLabels:
+        release: otel

scripts/tempo_install.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if ! helm repo list | grep grafana; then
+    helm repo add grafana https://grafana.github.io/helm-charts
+fi
+helm repo update
+
+helm upgrade \
+    -n observability \
+    --install \
+    tempo \
+    grafana/tempo

version

@@ -1 +1 @@
-1.0
+1.1