Merge branch 'main' into fix-cluster-label

.drone/drone.jsonnet

@@ -450,23 +450,6 @@ local build_image_tag = '0.33.1';
       },
     ],
   },
-  pipeline('mixins') {
-    workspace: {
-      base: '/src',
-      path: 'loki',
-    },
-    steps: [
-      make('lint-jsonnet', container=false) {
-        // Docker image defined at https://github.com/grafana/jsonnet-libs/tree/master/build
-        image: 'grafana/jsonnet-build:c8b75df',
-        depends_on: ['clone'],
-      },
-      make('loki-mixin-check', container=false) {
-        depends_on: ['clone'],
-        when: onPRs + onPath('production/loki-mixin/**'),
-      },
-    ],
-  },
   pipeline('documentation-checks') {
     workspace: {
       base: '/src',

.drone/drone.yml

@@ -132,38 +132,6 @@ workspace:
   path: loki
 ---
 kind: pipeline
-name: mixins
-steps:
-- commands:
-  - make BUILD_IN_CONTAINER=false lint-jsonnet
-  depends_on:
-  - clone
-  environment: {}
-  image: grafana/jsonnet-build:c8b75df
-  name: lint-jsonnet
-- commands:
-  - make BUILD_IN_CONTAINER=false loki-mixin-check
-  depends_on:
-  - clone
-  environment: {}
-  image: grafana/loki-build-image:0.33.1
-  name: loki-mixin-check
-  when:
-    event:
-    - pull_request
-    paths:
-    - production/loki-mixin/**
-trigger:
-  ref:
-  - refs/heads/main
-  - refs/heads/k???
-  - refs/tags/v*
-  - refs/pull/*/head
-workspace:
-  base: /src
-  path: loki
----
-kind: pipeline
 name: documentation-checks
 steps:
 - commands:
@@ -1340,6 +1308,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: dbc4d2b5c84e0464f24846abc8e7e73a5a937df289a0ecdb501f3bca28ebb8e3
+hmac: e0940674c7a2b5ae47c6509b0bc97dc594a054e5b881fd1962b81837d6b1dee6
 
 ...

.github/workflows/lint-jsonnet.yml

@@ -0,0 +1,32 @@
+---
+name: lint-jsonnet
+on: [pull_request]
+  # pull_request:
+  #   paths: "production/**"
+
+jobs:
+  check-mixin:
+    name: Check mixin jsonnet files
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22.2'
+      - name: setup jsonnet
+        run: |
+          go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
+          go install github.com/google/go-jsonnet/cmd/jsonnetfmt@v0.20.0
+          go install github.com/google/go-jsonnet/cmd/jsonnet-lint@v0.20.0
+          go install github.com/monitoring-mixins/mixtool/cmd/mixtool@16dc166166d91e93475b86b9355a4faed2400c18
+          go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1
+      - name: run linting
+        run: make BUILD_IN_CONTAINER=false lint-jsonnet
+      - name: check compiled mixin has been updated
+        run: |
+          make BUILD_IN_CONTAINER=false loki-mixin-check
+      
+
+

.github/workflows/promtail-windows-test.yml

@@ -1,8 +1,10 @@
 name: Promtail Windows Test
 on:
   pull_request:
-  tags: ['v[0-9].[0-9]+.[0-9]+']
-  branches: [main, k*]
+    branches: ["main", "k*", "release-[0-9]+.[0-9]+.x"]
+  push:
+    tags: ['v[0-9].[0-9]+.[0-9]+']
+    branches: ["main", "k*", "release-[0-9]+.[0-9]+.x"]
 jobs:
   build:
     runs-on: windows-latest

.gitignore

@@ -12,6 +12,7 @@ cmd/loki/loki
 cmd/logcli/logcli
 clients/cmd/promtail/promtail
 cmd/loki/loki-debug
+cmd/lokitool/lokitool
 clients/cmd/promtail/promtail-debug
 clients/cmd/docker-driver/docker-driver
 cmd/loki-canary/loki-canary

Makefile

@@ -204,6 +204,15 @@ loki-querytee: cmd/querytee/querytee ## build loki-querytee executable
 cmd/querytee/querytee:
 	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
 
+############
+# lokitool #
+############
+.PHONY: cmd/lokitool/lokitool
+lokitool: cmd/lokitool/lokitool ## build lokitool executable
+
+cmd/lokitool/lokitool:
+	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./cmd/lokitool
+
 ############
 # Promtail #
 ############

clients/cmd/promtail/Dockerfile

@@ -1,21 +1,15 @@
-FROM golang:1.21.9-bullseye as build
+FROM golang:1.21.9-bookworm as build
 
 COPY . /src/loki
 WORKDIR /src/loki
-# Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression
-RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list
-RUN apt-get update && apt-get install -t bullseye-backports -qy libsystemd-dev
+RUN apt-get update && apt-get install -qy libsystemd-dev
 RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail
 
 # Promtail requires debian as the base image to support systemd journal reading
-FROM debian:bullseye-slim
+FROM debian:12.5-slim
 # tzdata required for the timestamp stage to work
-# Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression
-RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list
 RUN apt-get update && \
-  apt-get install -qy \
-  tzdata ca-certificates
-RUN apt-get install -t bullseye-backports -qy libsystemd-dev && \
+  apt-get install -qy tzdata ca-certificates libsystemd-dev && \
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 COPY --from=build /src/loki/clients/cmd/promtail/promtail /usr/bin/promtail
 COPY clients/cmd/promtail/promtail-docker-config.yaml /etc/promtail/config.yml

clients/cmd/promtail/Dockerfile.arm32

@@ -1,21 +1,15 @@
-FROM golang:1.21.9-bullseye as build
+FROM golang:1.21.9-bookworm as build
 
 COPY . /src/loki
 WORKDIR /src/loki
-# Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression
-RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list
-RUN apt-get update && apt-get install -t bullseye-backports -qy libsystemd-dev
+RUN apt-get update && apt-get install -qy libsystemd-dev
 RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail
 
 # Promtail requires debian as the base image to support systemd journal reading
-FROM debian:bullseye-slim
+FROM debian:12.5-slim
 # tzdata required for the timestamp stage to work
-# Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression
-RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list
 RUN apt-get update && \
-  apt-get install -qy \
-  tzdata ca-certificates
-RUN apt-get install -t bullseye-backports -qy libsystemd-dev && \
+  apt-get install -qy tzdata ca-certificates libsystemd-dev && \
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 COPY --from=build /src/loki/clients/cmd/promtail/promtail /usr/bin/promtail
 COPY clients/cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml

clients/cmd/promtail/Dockerfile.cross

@@ -13,11 +13,10 @@ WORKDIR /src/loki
 RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail
 
 # Promtail requires debian as the base image to support systemd journal reading
-FROM debian:stretch-slim
+FROM debian:12.5-slim
 # tzdata required for the timestamp stage to work
 RUN apt-get update && \
-  apt-get install -qy \
-  tzdata ca-certificates libsystemd-dev && \
+  apt-get install -qy tzdata ca-certificates libsystemd-dev && \
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 COPY --from=build /src/loki/clients/cmd/promtail/promtail /usr/bin/promtail
 COPY clients/cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml

clients/pkg/promtail/client/client_writeto_test.go

@@ -29,11 +29,14 @@ func TestClientWriter_LogEntriesAreReconstructedAndForwardedCorrectly(t *testing
 	ch := make(chan api.Entry)
 	defer close(ch)
 
+	var mu sync.Mutex
 	var receivedEntries []api.Entry
 
 	go func() {
 		for e := range ch {
+			mu.Lock()
 			receivedEntries = append(receivedEntries, e)
+			mu.Unlock()
 		}
 	}()
 
@@ -72,12 +75,16 @@ func TestClientWriter_LogEntriesAreReconstructedAndForwardedCorrectly(t *testing
 	}
 
 	require.Eventually(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
 		return len(receivedEntries) == len(lines)
 	}, time.Second*10, time.Second)
+	mu.Lock()
 	for _, receivedEntry := range receivedEntries {
 		require.Contains(t, lines, receivedEntry.Line, "entry line was not expected")
 		require.Equal(t, model.LabelValue("test"), receivedEntry.Labels["app"])
 	}
+	mu.Unlock()
 }
 
 func TestClientWriter_LogEntriesWithoutMatchingSeriesAreIgnored(t *testing.T) {

clients/pkg/promtail/client/manager_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"sync"
 	"testing"
 	"time"
 
@@ -127,10 +128,13 @@ func TestManager_WALEnabled(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "wal:test-client", manager.Name())
 
+	var mu sync.Mutex
 	receivedRequests := []utils.RemoteWriteRequest{}
 	go func() {
 		for req := range rwReceivedReqs {
+			mu.Lock()
 			receivedRequests = append(receivedRequests, req)
+			mu.Unlock()
 		}
 	}()
 
@@ -155,17 +159,21 @@ func TestManager_WALEnabled(t *testing.T) {
 	}
 
 	require.Eventually(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
 		return len(receivedRequests) == totalLines
 	}, 5*time.Second, time.Second, "timed out waiting for requests to be received")
 
 	var seenEntries = map[string]struct{}{}
 	// assert over rw client received entries
+	mu.Lock()
 	for _, req := range receivedRequests {
 		require.Len(t, req.Request.Streams, 1, "expected 1 stream requests to be received")
 		require.Len(t, req.Request.Streams[0].Entries, 1, "expected 1 entry in the only stream received per request")
 		require.Equal(t, `{wal_enabled="true"}`, req.Request.Streams[0].Labels)
 		seenEntries[req.Request.Streams[0].Entries[0].Line] = struct{}{}
 	}
+	mu.Unlock()
 	require.Len(t, seenEntries, totalLines)
 }
 
@@ -182,10 +190,13 @@ func TestManager_WALDisabled(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "multi:test-client", manager.Name())
 
+	var mu sync.Mutex
 	receivedRequests := []utils.RemoteWriteRequest{}
 	go func() {
 		for req := range rwReceivedReqs {
+			mu.Lock()
 			receivedRequests = append(receivedRequests, req)
+			mu.Unlock()
 		}
 	}()
 
@@ -209,17 +220,21 @@ func TestManager_WALDisabled(t *testing.T) {
 	}
 
 	require.Eventually(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
 		return len(receivedRequests) == totalLines
 	}, 5*time.Second, time.Second, "timed out waiting for requests to be received")
 
 	var seenEntries = map[string]struct{}{}
 	// assert over rw client received entries
+	mu.Lock()
 	for _, req := range receivedRequests {
 		require.Len(t, req.Request.Streams, 1, "expected 1 stream requests to be received")
 		require.Len(t, req.Request.Streams[0].Entries, 1, "expected 1 entry in the only stream received per request")
 		require.Equal(t, `{pizza-flavour="fugazzeta"}`, req.Request.Streams[0].Labels)
 		seenEntries[req.Request.Streams[0].Entries[0].Line] = struct{}{}
 	}
+	mu.Unlock()
 	require.Len(t, seenEntries, totalLines)
 }
 
@@ -250,15 +265,20 @@ func TestManager_WALDisabled_MultipleConfigs(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "multi:test-client,test-client-2", manager.Name())
 
+	var mu sync.Mutex
 	receivedRequests := []utils.RemoteWriteRequest{}
 	ctx, cancel := context.WithCancel(context.Background())
 	go func(ctx context.Context) {
 		for {
 			select {
 			case req := <-rwReceivedReqs:
+				mu.Lock()
 				receivedRequests = append(receivedRequests, req)
+				mu.Unlock()
 			case req := <-rwReceivedReqs2:
+				mu.Lock()
 				receivedRequests = append(receivedRequests, req)
+				mu.Unlock()
 			case <-ctx.Done():
 				return
 			}
@@ -289,16 +309,20 @@ func TestManager_WALDisabled_MultipleConfigs(t *testing.T) {
 	// times 2 due to clients being run
 	expectedTotalLines := totalLines * 2
 	require.Eventually(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
 		return len(receivedRequests) == expectedTotalLines
 	}, 5*time.Second, time.Second, "timed out waiting for requests to be received")
 
 	var seenEntries = map[string]struct{}{}
 	// assert over rw client received entries
+	mu.Lock()
 	for _, req := range receivedRequests {
 		require.Len(t, req.Request.Streams, 1, "expected 1 stream requests to be received")
 		require.Len(t, req.Request.Streams[0].Entries, 1, "expected 1 entry in the only stream received per request")
 		seenEntries[fmt.Sprintf("%s-%s", req.Request.Streams[0].Labels, req.Request.Streams[0].Entries[0].Line)] = struct{}{}
 	}
+	mu.Unlock()
 	require.Len(t, seenEntries, expectedTotalLines)
 }