diff --git a/cmd/docker-driver/README.md b/cmd/docker-driver/README.md index 229158871e35..166a010dcda3 100644 --- a/cmd/docker-driver/README.md +++ b/cmd/docker-driver/README.md @@ -159,6 +159,31 @@ This is a bit more difficult as you need to properly escape bash special charact Providing both `loki-pipeline-stage-file` and `loki-pipeline-stages` will cause an error. +## Relabeling + +You can use [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) configuration to modify labels discovered by the driver. The configuration must be passed as a YAML string like the [pipeline stages](#pipeline-stages). + +For example the configuration below will rename the label `swarm_stack` and `swarm_service` to respectively `namespace` and `service`. + +```yaml +version: "3" +services: + nginx: + image: grafana/grafana + logging: + driver: loki + options: + loki-url: http://host.docker.internal:3100/loki/api/v1/push + loki-relabel-config: | + - action: labelmap + regex: swarm_stack + replacement: namespace + - action: labelmap + regex: swarm_(service) + ports: + - "3000:3000" +``` + ## log-opt options To specify additional logging driver options, you can use the --log-opt NAME=VALUE flag. @@ -174,7 +199,8 @@ To specify additional logging driver options, you can use the --log-opt NAME=VAL | `loki-max-backoff` | No | `10s` | The maximum amount of time to wait before retrying a batch. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". | | `loki-retries` | No | `10` | The maximum amount of retries for a log batch. | | `loki-pipeline-stage-file` | No | | The location of a pipeline stage configuration file ([example](./pipeline-example.yaml)). Pipeline stages allows to parse log lines to extract more labels. [see documentation](../../docs/logentry/processing-log-lines.md) | -| `loki-pipeline-stages` | No | | The pipeline stage configuration provided as a string [see](#pipeline-stages) and [see documentation](../../docs/clients/promtail/pipelines.md) | +| `loki-pipeline-stages` | No | | The pipeline stage configuration provided as a string [see](#pipeline-stages) and [see documentation](../../docs/clients/promtail/pipelines.md) | +| `loki-relabel-config` | No | | A [Prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) allowing you to rename labels [see](#relabeling) | | `loki-tenant-id` | No | | Set the tenant id (http header`X-Scope-OrgID`) when sending logs to Loki. It can be overrides by a pipeline stage. | | `loki-tls-ca-file` | No | | Set the path to a custom certificate authority. | | `loki-tls-cert-file` | No | | Set the path to a client certificate file. | @@ -190,6 +216,7 @@ To specify additional logging driver options, you can use the --log-opt NAME=VAL | `env` | No | | Comma-separated list of keys of environment variables to be included in message if they specified for a container. | | `env-regex` | No | | A regular expression to match logging-related environment variables. Used for advanced log label options. If there is collision between the label and env keys, the value of the env takes precedence. Both options add additional fields to the labels of a logging message. | + ## Uninstall the plugin To cleanly disable and remove the plugin, run: diff --git a/cmd/docker-driver/config.go b/cmd/docker-driver/config.go index 1fcf584d3fe5..488eec3d88a3 100644 --- a/cmd/docker-driver/config.go +++ b/cmd/docker-driver/config.go @@ -9,17 +9,20 @@ import ( "strings" "time" - "github.com/cortexproject/cortex/pkg/util" + cortex_util "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/templates" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/relabel" yaml "gopkg.in/yaml.v2" "github.com/grafana/loki/pkg/helpers" "github.com/grafana/loki/pkg/logentry/stages" "github.com/grafana/loki/pkg/promtail/client" "github.com/grafana/loki/pkg/promtail/targets" + "github.com/grafana/loki/pkg/util" ) const ( @@ -44,6 +47,7 @@ const ( cfgTenantIDKey = "loki-tenant-id" cfgNofile = "no-file" cfgKeepFile = "keep-file" + cfgRelabelKey = "loki-relabel-config" swarmServiceLabelKey = "com.docker.swarm.service.name" swarmStackLabelKey = "com.docker.stack.namespace" @@ -65,7 +69,7 @@ var ( defaultClientConfig = client.Config{ BatchWait: 1 * time.Second, BatchSize: 100 * 1024, - BackoffConfig: util.BackoffConfig{ + BackoffConfig: cortex_util.BackoffConfig{ MinBackoff: 100 * time.Millisecond, MaxBackoff: 10 * time.Second, MaxRetries: 10, @@ -106,6 +110,7 @@ func validateDriverOpt(loggerInfo logger.Info) error { case cfgPipelineStagesKey: case cfgPipelineStagesFileKey: case cfgTenantIDKey: + case cfgRelabelKey: case cfgNofile: case cfgKeepFile: case "labels": @@ -280,6 +285,15 @@ func parseConfig(logCtx logger.Info) (*config, error) { } labels[targets.FilenameLabel] = model.LabelValue(logCtx.LogPath) + // Process relabel configs. + if relabelString, ok := logCtx.Config[cfgRelabelKey]; ok && relabelString != "" { + relabeled, err := relabelConfig(relabelString, labels) + if err != nil { + return nil, fmt.Errorf("error applying relabel config: %s err:%s", relabelString, err) + } + labels = relabeled + } + // parse pipeline stages pipeline, err := parsePipeline(logCtx) if err != nil { @@ -347,6 +361,15 @@ func parseInt(key string, logCtx logger.Info, set func(i int)) error { return nil } +func relabelConfig(config string, lbs model.LabelSet) (model.LabelSet, error) { + relabelConfig := make([]*relabel.Config, 0) + if err := yaml.UnmarshalStrict([]byte(config), &relabelConfig); err != nil { + return nil, err + } + relabed := relabel.Process(labels.FromMap(util.ModelLabelSetToMap(lbs)), relabelConfig...) + return model.LabelSet(util.LabelsToMetric(relabed)), nil +} + func parseBoolean(key string, logCtx logger.Info, defaultValue bool) (bool, error) { value, ok := logCtx.Config[key] if !ok || value == "" { diff --git a/cmd/docker-driver/config_test.go b/cmd/docker-driver/config_test.go index 8baffeee2e61..0dd1dd8fe1a0 100644 --- a/cmd/docker-driver/config_test.go +++ b/cmd/docker-driver/config_test.go @@ -10,10 +10,55 @@ import ( "github.com/cortexproject/cortex/pkg/util" "github.com/docker/docker/daemon/logger" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logentry/stages" ) +var jobRename = ` +- regex: (.*) + source_labels: [swarm_stack] + target_label: job +- regex: ^swarm_stack$ + action: labeldrop` + +func Test_relabelConfig(t *testing.T) { + + tests := []struct { + name string + config string + in model.LabelSet + out model.LabelSet + wantErr bool + }{ + { + "config err", + "foo", + nil, + nil, + true, + }, + { + "config err", + jobRename, + model.LabelSet{"swarm_stack": "foo", "bar": "buzz"}, + model.LabelSet{"job": "foo", "bar": "buzz"}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := relabelConfig(tt.config, tt.in) + if (err != nil) != tt.wantErr { + t.Errorf("relabelConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + require.True(t, got.Equal(tt.out)) + }) + } +} + var pipelineString = ` - regex: expression: '(level|lvl|severity)=(?P\w+)'