Prometheus metrics (#133)

Closes #113 and #112.
trstringer · Mar 24, 2024 · bd830f8 · bd830f8
1 parent 4f878a5
commit bd830f8
Show file tree

Hide file tree

Showing 16 changed files with 423 additions and 314 deletions.
diff --git a/Makefile b/Makefile
@@ -125,7 +125,7 @@ install-app-local: create-namespace-app build-images push-images
 	helm upgrade -n app --install otel-shopping-cart ./charts/otel-shopping-cart
 
 .PHONY: build-images
-build-images: build-collector
+build-images:
 	docker build -t $(CART_IMAGE_REPO):$(IMAGE_TAG) -f ./dockerfiles/Dockerfile.cart .
 	docker build -t $(DATASEED_IMAGE_REPO):$(IMAGE_TAG) -f ./dockerfiles/Dockerfile.dataseed .
 	docker build -t $(INTERRUPTER_IMAGE_REPO):$(IMAGE_TAG) -f ./dockerfiles/Dockerfile.interrupter .

diff --git a/charts/otel-shopping-cart/templates/cart-service.yaml b/charts/otel-shopping-cart/templates/cart-service.yaml
@@ -2,9 +2,12 @@ kind: Service
 apiVersion: v1
 metadata:
   name: {{ .Values.cart.serviceName }}
+  labels:
+    app: cart
 spec:
   selector:
     app: {{ .Values.cart.serviceName }}
   ports:
     - port: {{ .Values.cart.port }}
+      name: http
       targetPort: {{ .Values.cart.port }}
diff --git a/charts/otel-shopping-cart/templates/cart-servicemonitor.yaml b/charts/otel-shopping-cart/templates/cart-servicemonitor.yaml
@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Values.cart.serviceName }}
+  labels:
+    release: prometheus
+spec:
+  endpoints:
+    - port: http
+  selector:
+    matchLabels:
+      app: cart
diff --git a/charts/otel-shopping-cart/templates/user-service.yaml b/charts/otel-shopping-cart/templates/user-service.yaml
@@ -2,9 +2,12 @@ kind: Service
 apiVersion: v1
 metadata:
   name: {{ .Values.user.serviceName }}
+  labels:
+    app: user
 spec:
   selector:
     app: {{ .Values.user.serviceName }}
   ports:
     - port: {{ .Values.user.port }}
+      name: http
       targetPort: {{ .Values.user.port }}
diff --git a/charts/otel-shopping-cart/templates/user-servicemonitor.yaml b/charts/otel-shopping-cart/templates/user-servicemonitor.yaml
@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Values.user.serviceName }}
+  labels:
+    release: prometheus
+spec:
+  endpoints:
+    - port: http
+  selector:
+    matchLabels:
+      app: user
diff --git a/cmd/cart/main.go b/cmd/cart/main.go
@@ -7,8 +7,10 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel"
@@ -20,6 +22,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/trstringer/otel-shopping-cart/pkg/cart"
+	"github.com/trstringer/otel-shopping-cart/pkg/dbmanager"
 	"github.com/trstringer/otel-shopping-cart/pkg/telemetry"
 	"github.com/trstringer/otel-shopping-cart/pkg/users"
 )
@@ -127,12 +130,14 @@ func validateParams() {
 }
 
 func userCart(w http.ResponseWriter, r *http.Request) {
+	httpRequest.Inc()
 	ctx, span := otel.Tracer(telemetry.TelemetryLibrary).Start(r.Context(), "get_user_cart")
 	defer span.End()
 
 	if r.Method != http.MethodGet && r.Method != http.MethodPost {
 		err := fmt.Errorf("unsupported request method: %s", r.Method)
 		userRequestError(ctx, w, err, http.StatusBadRequest, true)
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusBadRequest)).Inc()
 		return
 	}
 
@@ -145,6 +150,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 			http.StatusInternalServerError,
 			true,
 		)
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 		fmt.Printf("error creating baggage member: %v\n", err)
 		return
 	}
@@ -158,6 +164,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 			http.StatusInternalServerError,
 			true,
 		)
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 		fmt.Printf("error creating baggage: %v\n", err)
 		return
 	}
@@ -167,7 +174,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 	fmt.Printf("Received cart request for %s\n", userName)
 	span.SetAttributes(attribute.String("user.name", userName))
 
-	cartManager := cart.NewDBManager(
+	cartManager := dbmanager.NewDBManager(
 		dbSQLAddress,
 		"otel_shopping_cart",
 		dbSQLUser,
@@ -183,6 +190,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 			http.StatusInternalServerError,
 			true,
 		)
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 		fmt.Printf("error getting user: %v\n", err)
 		return
 	}
@@ -195,6 +203,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 			http.StatusInternalServerError,
 			true,
 		)
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 		fmt.Printf("error getting user cart: %v\n", err)
 		return
 	}
@@ -209,6 +218,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 				http.StatusInternalServerError,
 				true,
 			)
+			httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 			fmt.Printf("error reading body data: %v\n", err)
 			return
 		}
@@ -221,6 +231,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 				http.StatusInternalServerError,
 				true,
 			)
+			httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 			fmt.Printf("error unmarshalling data: %v\n", err)
 			return
 		}
@@ -232,6 +243,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 				http.StatusInternalServerError,
 				true,
 			)
+			httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 			fmt.Printf("error adding item to cart: %v\n", err)
 			return
 		}
@@ -245,6 +257,7 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 				http.StatusInternalServerError,
 				true,
 			)
+			httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 			fmt.Printf("error getting user cart: %v\n", err)
 			return
 		}
@@ -259,10 +272,12 @@ func userCart(w http.ResponseWriter, r *http.Request) {
 			http.StatusInternalServerError,
 			true,
 		)
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 		fmt.Printf("error marshalling cart: %v\n", err)
 		return
 	}
 
+	httpResponses.WithLabelValues(strconv.Itoa(http.StatusOK)).Inc()
 	w.Write([]byte(jsonCart))
 }
 
@@ -357,14 +372,16 @@ func userRequestError(ctx context.Context, w http.ResponseWriter, err error, htt
 }
 
 func runServer() {
+	http.Handle("/metrics", promhttp.Handler())
 	http.Handle(
 		fmt.Sprintf("/%s/", rootPath),
 		otelhttp.NewHandler(
 			http.HandlerFunc(userCart),
 			"http_user_cart",
 			otelhttp.WithTracerProvider(otel.GetTracerProvider()),
 			otelhttp.WithPropagators(otel.GetTextMapPropagator()),
-		))
+		),
+	)
 
 	addr := fmt.Sprintf(":%d", port)
 	fmt.Printf("Running server on %s\n", addr)

diff --git a/cmd/cart/metrics.go b/cmd/cart/metrics.go
@@ -0,0 +1,20 @@
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	httpRequest = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "cart_http_request",
+		Help: "HTTP request",
+	})
+	httpResponses = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "cart_http_response",
+			Help: "HTTP response",
+		},
+		[]string{"status"},
+	)
+)
diff --git a/cmd/interrupter/main.go b/cmd/interrupter/main.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/trstringer/otel-shopping-cart/pkg/users"
+	"github.com/trstringer/otel-shopping-cart/pkg/dbmanager"
 )
 
 var (
@@ -23,7 +23,7 @@ var rootCmd = &cobra.Command{
 	Short: "Service interrupter",
 	Long:  `Interrupt service and cause quality issues.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		dbm := users.NewDBManager(dbSQLAddress, "otel_shopping_cart", dbSQLUser, os.Getenv("DB_PASSWORD"))
+		dbm := dbmanager.NewDBManager(dbSQLAddress, "otel_shopping_cart", dbSQLUser, os.Getenv("DB_PASSWORD"))
 		users, err := dbm.GetAllUsers()
 		if err != nil {
 			fmt.Printf("Error getting users: %v\n", err)

diff --git a/cmd/users/main.go b/cmd/users/main.go
@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel"
@@ -16,6 +18,7 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 
+	"github.com/trstringer/otel-shopping-cart/pkg/dbmanager"
 	"github.com/trstringer/otel-shopping-cart/pkg/telemetry"
 	"github.com/trstringer/otel-shopping-cart/pkg/users"
 )
@@ -108,7 +111,8 @@ func validateParams() {
 }
 
 func allUsers(w http.ResponseWriter, r *http.Request) {
-	userManager := users.NewDBManager(
+	httpRequest.Inc()
+	userManager := dbmanager.NewDBManager(
 		dbSQLAddress,
 		"otel_shopping_cart",
 		dbSQLUser,
@@ -119,20 +123,23 @@ func allUsers(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusBadRequest)
 		fmt.Printf("error retrieving all users: %v\n", err)
 		w.Write([]byte(fmt.Sprintf("error retrieving all users: %v", err)))
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusBadRequest)).Inc()
 		return
 	}
 	userData, err := json.Marshal(allUsers)
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		fmt.Printf("error marshalling all users: %v\n", err)
 		w.Write([]byte(fmt.Sprintf("error marshalling all users: %v", err)))
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusBadRequest)).Inc()
 		return
 	}
 
 	w.Write([]byte(userData))
 }
 
 func user(w http.ResponseWriter, r *http.Request) {
+	httpRequest.Inc()
 	ctx := r.Context()
 	ctx, span := otel.Tracer(telemetry.TelemetryLibrary).Start(ctx, "get_user")
 	defer span.End()
@@ -146,7 +153,7 @@ func user(w http.ResponseWriter, r *http.Request) {
 	userName := strings.TrimPrefix(r.URL.Path, fmt.Sprintf("/%s/", rootPath))
 	fmt.Printf("Received user request for %q\n", userName)
 
-	userManager := users.NewDBManager(
+	userManager := dbmanager.NewDBManager(
 		dbSQLAddress,
 		"otel_shopping_cart",
 		dbSQLUser,
@@ -158,6 +165,7 @@ func user(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusBadRequest)
 		fmt.Printf("error retrieving user: %v\n", err)
 		w.Write([]byte(fmt.Sprintf("error retrieving user: %v", err)))
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusBadRequest)).Inc()
 		return
 	}
 
@@ -167,9 +175,11 @@ func user(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusInternalServerError)
 		fmt.Printf("error retrieving user: %v\n", err)
 		w.Write([]byte(fmt.Sprintf("error marshalling user data: %v", err)))
+		httpResponses.WithLabelValues(strconv.Itoa(http.StatusInternalServerError)).Inc()
 		return
 	}
 
+	httpResponses.WithLabelValues(strconv.Itoa(http.StatusOK)).Inc()
 	w.Write([]byte(userData))
 }
 
@@ -178,8 +188,8 @@ func getUser(ctx context.Context, userManager users.Manager, userName string) (*
 }
 
 func runServer() {
+	http.Handle("/metrics", promhttp.Handler())
 	http.HandleFunc(fmt.Sprintf("/%s", rootPath), allUsers)
-
 	http.Handle(
 		fmt.Sprintf("/%s/", rootPath),
 		otelhttp.NewHandler(

diff --git a/cmd/users/metrics.go b/cmd/users/metrics.go
@@ -0,0 +1,20 @@
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	httpRequest = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "users_http_request",
+		Help: "HTTP request",
+	})
+	httpResponses = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "users_http_response",
+			Help: "HTTP response",
+		},
+		[]string{"status"},
+	)
+)
diff --git a/go.mod b/go.mod
@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/lib/pq v1.10.9
+	github.com/prometheus/client_golang v1.19.0
 	github.com/spf13/cobra v1.8.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0
 	go.opentelemetry.io/otel v1.24.0
@@ -15,13 +16,18 @@ require (
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect