Merge branch 'main' into fix_create_rbac_consul

.github/jsonnetfile.json

@@ -8,7 +8,7 @@
           "subdir": "workflows"
         }
       },
-      "version": "main"
+      "version": "124c4d996f9625478a79f1884465e29ea082d224"
     }
   ],
   "legacyImports": true

.github/jsonnetfile.lock.json

@@ -8,8 +8,8 @@
           "subdir": "workflows"
         }
       },
-      "version": "634945b73e8eed4f5161ec08810178ddeca7505b",
-      "sum": "BOnwSjzyOjWwv9ikwJSAgPBNnYHTU2PEDJ0PWY6nr7I="
+      "version": "124c4d996f9625478a79f1884465e29ea082d224",
+      "sum": "8wrJURq48ZBAtZcReO1W7AiXmvUyLqb932Q9sXyfFVo="
     }
   ],
   "legacyImports": false

.github/release-workflows.jsonnet

@@ -1,10 +1,7 @@
 local lokiRelease = import 'workflows/main.jsonnet';
 local build = lokiRelease.build;
 
-local releaseLibRef = std.filter(
-  function(dep) dep.source.git.remote == 'https://github.com/grafana/loki-release.git',
-  (import 'jsonnetfile.json').dependencies
-)[0].version;
+local releaseLibRef = 'main';
 
 local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef;
 

.github/workflows/check.yml

@@ -12,4 +12,4 @@
   "pull_request": {}
   "push":
     "branches":
-    - "main"
+    - "main"

.github/workflows/metrics-collector.yml

@@ -5,6 +5,7 @@ on:
 
 jobs:
   main:
+    if: github.owner == "grafana"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions

.github/workflows/minor-release-pr.yml

@@ -58,7 +58,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -753,7 +753,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -826,4 +826,4 @@ name: "Prepare Minor Release PR from Weekly"
 permissions:
   contents: "write"
   id-token: "write"
-  pull-requests: "write"
+  pull-requests: "write"

.github/workflows/patch-release-pr.yml

@@ -58,7 +58,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -753,7 +753,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -826,4 +826,4 @@ name: "Prepare Patch Release PR"
 permissions:
   contents: "write"
   id-token: "write"
-  pull-requests: "write"
+  pull-requests: "write"

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -158,7 +158,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -219,4 +219,4 @@ name: "create release"
 permissions:
   contents: "write"
   id-token: "write"
-  pull-requests: "write"
+  pull-requests: "write"

.github/workflows/verify-release-workflow.yaml

@@ -0,0 +1,18 @@
+name: Verify release workflow updates
+on: [pull_request]
+jobs:
+  check-release-changes:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: setup go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22.2'
+    - name: setup jsonnet
+      run: |
+        go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
+        go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1
+    - name: Check that the release workflows have been updated properly
+      run: |
+          make BUILD_IN_CONTAINER=false release-workflows-check

Makefile

@@ -900,3 +900,9 @@ scan-vulnerabilities: trivy snyk
 release-workflows:
 	pushd $(CURDIR)/.github && jb update && popd
 	jsonnet -SJ .github/vendor -m .github/workflows .github/release-workflows.jsonnet
+
+.PHONY: release-workflows-check
+release-workflows-check:
+	@$(MAKE) release-workflows
+	@echo "Checking diff"
+	@git diff --exit-code -- ".github/workflows/*release*" || (echo "Please build release workflows by running 'make release-workflows'" && false)

cmd/logcli/main.go

@@ -16,6 +16,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/grafana/loki/v3/pkg/logcli/client"
+	"github.com/grafana/loki/v3/pkg/logcli/detected"
 	"github.com/grafana/loki/v3/pkg/logcli/index"
 	"github.com/grafana/loki/v3/pkg/logcli/labelquery"
 	"github.com/grafana/loki/v3/pkg/logcli/output"
@@ -253,6 +254,39 @@ Example:
 	   'my-query'
   `)
 	volumeRangeQuery = newVolumeQuery(true, volumeRangeCmd)
+
+	detectedFieldsCmd = app.Command("detected-fields", `Run a query for detected fields..
+
+The "detected-fields" command will return information about fields detected using either
+the "logfmt" or "json" parser against the log lines returned by the provided query for the
+provided time range.
+
+The "detected-fields" command will output extra information about the query
+and its results, such as the API URL, set of common labels, and set
+of excluded labels. This extra information can be suppressed with the
+--quiet flag.
+
+By default we look over the last hour of data; use --since to modify
+or provide specific start and end times with --from and --to respectively.
+
+Notice that when using --from and --to then ensure to use RFC3339Nano
+time format, but without timezone at the end. The local timezone will be added
+automatically or if using  --timezone flag.
+
+Example:
+
+	logcli detected-fields
+	   --timezone=UTC
+	   --from="2021-01-19T10:00:00Z"
+	   --to="2021-01-19T20:00:00Z"
+	   --output=jsonl
+	   'my-query'
+
+The output is limited to 100 fields by default; use --field-limit to increase.
+The query is limited to processing 1000 lines per subquery; use --line-limit to increase.
+`)
+
+	detectedFieldsQuery = newDetectedFieldsQuery(detectedFieldsCmd)
 )
 
 func main() {
@@ -388,6 +422,8 @@ func main() {
 		} else {
 			index.GetVolume(volumeQuery, queryClient, out, *statistics)
 		}
+	case detectedFieldsCmd.FullCommand():
+		detectedFieldsQuery.Do(queryClient, *outputMode)
 	}
 }
 
@@ -652,3 +688,41 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query {
 
 	return q
 }
+
+func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery {
+	// calculate query range from cli params
+	var from, to string
+	var since time.Duration
+
+	q := &detected.FieldsQuery{}
+
+	// executed after all command flags are parsed
+	cmd.Action(func(c *kingpin.ParseContext) error {
+		defaultEnd := time.Now()
+		defaultStart := defaultEnd.Add(-since)
+
+		q.Start = mustParse(from, defaultStart)
+		q.End = mustParse(to, defaultEnd)
+
+		q.Quiet = *quiet
+
+		return nil
+	})
+
+	cmd.Flag("field-limit", "Limit on number of fields to return.").
+		Default("100").
+		IntVar(&q.FieldLimit)
+	cmd.Flag("line-limit", "Limit the number of lines each subquery is allowed to process.").
+		Default("1000").
+		IntVar(&q.LineLimit)
+	cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").
+		Required().
+		StringVar(&q.QueryString)
+	cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
+	cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
+	cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
+	cmd.Flag("step", "Query resolution step width, for metric queries. Evaluate the query at the specified step over the time range.").
+		DurationVar(&q.Step)
+
+	return q
+}

docs/sources/get-started/quick-start.md

@@ -12,14 +12,15 @@ If you want to experiment with Loki, you can run Loki locally using the Docker C
 The Docker Compose configuration instantiates the following components, each in its own container:
 
 - **flog** a sample application which generates log lines.  [flog](https://github.com/mingrammer/flog) is a log generator for common log formats.
-- **Promtail** which scrapes the log lines from flog, and pushes them to Loki through the gateway.
+- **Grafana Alloy** which scrapes the log lines from flog, and pushes them to Loki through the gateway.
 - **Gateway** (NGINX) which receives requests and redirects them to the appropriate container based on the request's URL.
-- One Loki **read** component.
-- One Loki **write** component.
+- One Loki **read** component (Query Frontend, Querier).
+- One Loki **write** component (Distributor, Ingester).
+- One Loki **backend** component (Index Gateway, Compactor, Ruler, Bloom Compactor (Experimental), Bloom Gateway (Experimental)).
 - **Minio** an S3-compatible object store which Loki uses to store its index and chunks.
 - **Grafana** which provides visualization of the log lines captured within Loki.
 
-{{< figure max-width="75%" src="/media/docs/loki/get-started-flog-v2.png" caption="Getting started sample application" alt="Getting started sample application">}}
+{{< figure max-width="75%" src="/media/docs/loki/get-started-flog-v3.png" caption="Getting started sample application" alt="Getting started sample application">}}
 
 ## Installing Loki and collecting sample logs
 
@@ -41,11 +42,11 @@ This quickstart assumes you are running Linux.
     cd evaluate-loki
     ```
 
-1. Download `loki-config.yaml`, `promtail-local-config.yaml`, and `docker-compose.yaml`:
+1. Download `loki-config.yaml`, `alloy-local-config.yaml`, and `docker-compose.yaml`:
 
     ```bash
     wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/loki-config.yaml -O loki-config.yaml
-    wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/promtail-local-config.yaml -O promtail-local-config.yaml
+    wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/alloy-local-config.yaml -O alloy-local-config.yaml
     wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/docker-compose.yaml -O docker-compose.yaml
     ```
 
@@ -63,16 +64,20 @@ This quickstart assumes you are running Linux.
     ✔ Network evaluate-loki_loki          Created      0.1s 
     ✔ Container evaluate-loki-minio-1     Started      0.6s 
     ✔ Container evaluate-loki-flog-1      Started      0.6s 
+    ✔ Container evaluate-loki-backend-1   Started      0.8s 
     ✔ Container evaluate-loki-write-1     Started      0.8s 
     ✔ Container evaluate-loki-read-1      Started      0.8s 
     ✔ Container evaluate-loki-gateway-1   Started      1.1s 
     ✔ Container evaluate-loki-grafana-1   Started      1.4s 
-    ✔ Container evaluate-loki-promtail-1  Started      1.4s 
+    ✔ Container evaluate-loki-alloy-1     Started      1.4s
     ```
 
 1. (Optional) Verify that the Loki cluster is up and running.
     - The read component returns `ready` when you point a web browser at [http://localhost:3101/ready](http://localhost:3101/ready). The message `Query Frontend not ready: not ready: number of schedulers this worker is connected to is 0` will show prior to the read component being ready.
     - The write component returns `ready` when you point a web browser at [http://localhost:3102/ready](http://localhost:3102/ready). The message `Ingester not ready: waiting for 15s after being ready` will show prior to the write component being ready.
+
+1. (Optional) Verify that Grafana Alloy is running.
+    - Grafana Alloy's UI can be accessed at [http://localhost:12345](http://localhost:12345).  
 
 ## Viewing your logs in Grafana
 

docs/sources/operations/storage/_index.md

@@ -9,53 +9,53 @@ You can read a high level overview of Loki storage [here](https://grafana.com/do
 
 Grafana Loki needs to store two different types of data: **chunks** and **indexes**.
 
+When using Accelerated Search (experimental), then a third data type is used: **bloom blocks**.
+
 Loki receives logs in separate streams, where each stream is uniquely identified
 by its tenant ID and its set of labels. As log entries from a stream arrive,
-they are compressed as "chunks" and saved in the chunks store. See [chunk
+they are compressed as **chunks** and saved in the chunks store. See [chunk
 format](#chunk-format) for how chunks are stored internally.
 
 The **index** stores each stream's label set and links them to the individual
-chunks.
-
-Refer to Loki's [configuration](https://grafana.com/docs/loki/<LOKI_VERSION>/configure/) for details on
-how to configure the storage and the index.
+chunks. Refer to Loki's [configuration](https://grafana.com/docs/loki/<LOKI_VERSION>/configure/) for
+details on how to configure the storage and the index.
 
 For more information:
 
 - [Table Manager](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/storage/table-manager/)
 - [Retention](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/storage/retention/)
 - [Logs Deletion](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/storage/logs-deletion/)
 
-## Supported Stores
+## Store Types
 
-The following are supported for the index:
+### ✅ Supported index stores
 
-- [TSDB](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/storage/tsdb/) index store which stores TSDB index files in the object store. This is the recommended index store for Loki 2.8 and newer.
-- [Single Store (boltdb-shipper)](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/storage/boltdb-shipper/) index store which stores boltdb index files in the object store. 
-- [Amazon DynamoDB](https://aws.amazon.com/dynamodb)
-- [Google Bigtable](https://cloud.google.com/bigtable)
-- [Apache Cassandra](https://cassandra.apache.org)
-- [BoltDB](https://github.com/boltdb/bolt) (doesn't work when clustering Loki)
+- [Single Store TSDB](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/storage/tsdb/) index store which stores TSDB index files in the object store.
+  **This is the recommended index store for Loki 2.8 and newer.**
+- [Single Store BoltDB (boltdb-shipper)](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/storage/boltdb-shipper/) index store which stores boltdb index files in the object store.
 
-The following are deprecated for the index and will be removed in a future release:
+### ❌ Deprecated index stores
 
 - [Amazon DynamoDB](https://aws.amazon.com/dynamodb). Support for this is deprecated and will be removed in a future release.
 - [Google Bigtable](https://cloud.google.com/bigtable). Support for this is deprecated and will be removed in a future release.
 - [Apache Cassandra](https://cassandra.apache.org). Support for this is deprecated and will be removed in a future release.
+- [BoltDB](https://github.com/boltdb/bolt) (doesn't work when clustering Loki)
 
-The following are supported and recommended for the chunks:
+### ✅ Supported and recommended chunks stores
 
-- [Amazon S3](https://aws.amazon.com/s3)
-- [Google Cloud Storage](https://cloud.google.com/storage/)
-- [Filesystem]({{< relref "./filesystem" >}}) (please read more about the filesystem to understand the pros/cons before using with production data)
-- [Baidu Object Storage](https://cloud.baidu.com/product/bos.html)
-- [IBM Cloud Object Storage](https://www.ibm.com/cloud/object-storage)
+- [Amazon Simple Storage Storage (S3)](https://aws.amazon.com/s3)
+- [Google Cloud Storage (GCS)](https://cloud.google.com/storage/)
+- [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs)
+- [IBM Cloud Object Storage (COS)](https://www.ibm.com/cloud/object-storage)
+- [Baidu Object Storage (BOS)](https://cloud.baidu.com/product/bos.html)
+- [Alibaba Object Storage Service (OSS)](https://www.alibabacloud.com/product/object-storage-service)
 
-The following are supported for the chunks, but not typically recommended for production use:
+### ⚠️ Supported chunks stores, not typically recommended for production use
 
 - [Filesystem]({{< relref "./filesystem" >}}) (please read more about the filesystem to understand the pros/cons before using with production data)
+- S3 API compatible storage, such as [MinIO](https://min.io/)
 
-The following are deprecated for the chunks and will be removed in a future release:
+### ❌ Deprecated chunks stores
 
 - [Amazon DynamoDB](https://aws.amazon.com/dynamodb). Support for this is deprecated and will be removed in a future release.
 - [Google Bigtable](https://cloud.google.com/bigtable). Support for this is deprecated and will be removed in a future release.