Actually hook up to loki_write metrics

grafana · Dec 18, 2024 · 0064049 · 0064049
1 parent f9367e0
commit 0064049
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 27 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -58,6 +58,8 @@ Main (unreleased)
 
 - Update `prometheus.write.queue` library for performance increases in cpu. (@mattdurham)
 
+- Add `tenant` label to remaining `loki_write_.+` metrics
+
 ### Bugfixes
 
 - Fixed issue with automemlimit logging bad messages and trying to access cgroup on non-linux builds (@dehaansa)

diff --git a/internal/component/common/loki/client/client.go b/internal/component/common/loki/client/client.go
@@ -60,7 +60,6 @@ type Metrics struct {
 	mutatedBytes                 *prometheus.CounterVec
 	requestDuration              *prometheus.HistogramVec
 	batchRetries                 *prometheus.CounterVec
-	countersWithHost             []*prometheus.CounterVec
 	countersWithHostTenant       []*prometheus.CounterVec
 	countersWithHostTenantReason []*prometheus.CounterVec
 }
@@ -105,12 +104,8 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
 		Help: "Number of times batches has had to be retried.",
 	}, []string{HostLabel, TenantLabel})
 
-	m.countersWithHost = []*prometheus.CounterVec{
-		m.encodedBytes, m.sentBytes, m.sentEntries,
-	}
-
 	m.countersWithHostTenant = []*prometheus.CounterVec{
-		m.batchRetries,
+		m.batchRetries, m.encodedBytes, m.sentBytes, m.sentEntries,
 	}
 
 	m.countersWithHostTenantReason = []*prometheus.CounterVec{
@@ -210,12 +205,6 @@ func newClient(metrics *Metrics, cfg Config, maxStreams, maxLineSize int, maxLin
 
 	c.client.Timeout = cfg.Timeout
 
-	// Initialize counters to 0 so the metrics are exported before the first
-	// occurrence of incrementing to avoid missing metrics.
-	for _, counter := range c.metrics.countersWithHost {
-		counter.WithLabelValues(c.cfg.URL.Host).Add(0)
-	}
-
 	c.wg.Add(1)
 	go c.run()
 	return c, nil

diff --git a/internal/component/common/loki/client/client_test.go b/internal/component/common/loki/client/client_test.go
@@ -83,7 +83,7 @@ func TestClient_Handle(t *testing.T) {
 			expectedMetrics: `
                                # HELP loki_write_sent_entries_total Number of log entries sent to the ingester.
                                # TYPE loki_write_sent_entries_total counter
-                               loki_write_sent_entries_total{host="__HOST__"} 3.0
+                               loki_write_sent_entries_total{host="__HOST__",tenant=""} 3.0
                                # HELP loki_write_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
                                # TYPE loki_write_dropped_entries_total counter
                                loki_write_dropped_entries_total{host="__HOST__",reason="ingester_error",tenant=""} 0
@@ -121,7 +121,7 @@ func TestClient_Handle(t *testing.T) {
 			expectedMetrics: `
                                # HELP loki_write_sent_entries_total Number of log entries sent to the ingester.
                                # TYPE loki_write_sent_entries_total counter
-                               loki_write_sent_entries_total{host="__HOST__"} 2.0
+                               loki_write_sent_entries_total{host="__HOST__",tenant=""} 2.0
                                # HELP loki_write_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
                                # TYPE loki_write_dropped_entries_total counter
                                loki_write_dropped_entries_total{host="__HOST__",reason="ingester_error",tenant=""} 0
@@ -166,7 +166,7 @@ func TestClient_Handle(t *testing.T) {
 			expectedMetrics: `
                                # HELP loki_write_sent_entries_total Number of log entries sent to the ingester.
                                # TYPE loki_write_sent_entries_total counter
-                               loki_write_sent_entries_total{host="__HOST__"} 3.0
+                               loki_write_sent_entries_total{host="__HOST__",tenant=""} 3.0
                                # HELP loki_write_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
                                # TYPE loki_write_dropped_entries_total counter
                                loki_write_dropped_entries_total{host="__HOST__",reason="ingester_error",tenant=""} 0
@@ -208,7 +208,7 @@ func TestClient_Handle(t *testing.T) {
 			expectedMetrics: `
                               # HELP loki_write_sent_entries_total Number of log entries sent to the ingester.
                               # TYPE loki_write_sent_entries_total counter
-                              loki_write_sent_entries_total{host="__HOST__"} 2.0
+                              loki_write_sent_entries_total{host="__HOST__",tenant=""} 2.0
                               # HELP loki_write_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
                               # TYPE loki_write_dropped_entries_total counter
                               loki_write_dropped_entries_total{host="__HOST__",reason="ingester_error",tenant=""} 0
@@ -270,7 +270,7 @@ func TestClient_Handle(t *testing.T) {
                               loki_write_mutated_bytes_total{host="__HOST__",reason="stream_limited",tenant=""} 0
                               # HELP loki_write_sent_entries_total Number of log entries sent to the ingester.
                               # TYPE loki_write_sent_entries_total counter
-                              loki_write_sent_entries_total{host="__HOST__"} 0
+                              loki_write_sent_entries_total{host="__HOST__",tenant=""} 0
                        `,
 		},
 		"do not retry send a batch in case the server responds with a 4xx": {
@@ -306,7 +306,7 @@ func TestClient_Handle(t *testing.T) {
                               loki_write_mutated_bytes_total{host="__HOST__",reason="stream_limited",tenant=""} 0
                               # HELP loki_write_sent_entries_total Number of log entries sent to the ingester.
                               # TYPE loki_write_sent_entries_total counter
-                              loki_write_sent_entries_total{host="__HOST__"} 0
+                              loki_write_sent_entries_total{host="__HOST__",tenant=""} 0
                        `,
 		},
 		"do retry sending a batch in case the server responds with a 429": {

diff --git a/internal/component/common/loki/client/queue_client.go b/internal/component/common/loki/client/queue_client.go
@@ -233,12 +233,6 @@ func newQueueClient(metrics *Metrics, qcMetrics *QueueClientMetrics, cfg Config,
 
 	c.client.Timeout = cfg.Timeout
 
-	// Initialize counters to 0 so the metrics are exported before the first
-	// occurrence of incrementing to avoid missing metrics.
-	for _, counter := range c.metrics.countersWithHost {
-		counter.WithLabelValues(c.cfg.URL.Host).Add(0)
-	}
-
 	c.wg.Add(1)
 	go c.runSendOldBatches()
 	return c, nil
@@ -456,7 +450,7 @@ func (c *queueClient) sendBatch(ctx context.Context, tenantID string, batch *bat
 		return
 	}
 	bufBytes := float64(len(buf))
-	c.metrics.encodedBytes.WithLabelValues(c.cfg.URL.Host).Add(bufBytes)
+	c.metrics.encodedBytes.WithLabelValues(c.cfg.URL.Host, tenantID).Add(bufBytes)
 
 	backoff := backoff.New(c.ctx, c.cfg.BackoffConfig)
 	var status int
@@ -476,8 +470,8 @@ func (c *queueClient) sendBatch(ctx context.Context, tenantID string, batch *bat
 		}
 
 		if err == nil {
-			c.metrics.sentBytes.WithLabelValues(c.cfg.URL.Host).Add(bufBytes)
-			c.metrics.sentEntries.WithLabelValues(c.cfg.URL.Host).Add(float64(entriesCount))
+			c.metrics.sentBytes.WithLabelValues(c.cfg.URL.Host, tenantID).Add(bufBytes)
+			c.metrics.sentEntries.WithLabelValues(c.cfg.URL.Host, tenantID).Add(float64(entriesCount))
 
 			return
 		}