diff --git a/clients/pkg/promtail/client/client.go b/clients/pkg/promtail/client/client.go index 03b5665d8d3ac..6e25e532a57eb 100644 --- a/clients/pkg/promtail/client/client.go +++ b/clients/pkg/promtail/client/client.go @@ -320,14 +320,21 @@ func (c *client) sendBatch(tenantID string, batch *batch) { } var lblSet model.LabelSet for i := range lbls { - if lbls[i].Name == LatencyLabel { - lblSet = model.LabelSet{ - model.LabelName(HostLabel): model.LabelValue(c.cfg.URL.Host), - model.LabelName(LatencyLabel): model.LabelValue(lbls[i].Value), + for _, lbl := range c.cfg.StreamLagLabels { + if lbls[i].Name == lbl { + if lblSet == nil { + lblSet = model.LabelSet{} + } + + lblSet = lblSet.Merge(model.LabelSet{ + model.LabelName(lbl): model.LabelValue(lbls[i].Value), + }) } } } if lblSet != nil { + // always set host + lblSet = lblSet.Merge(model.LabelSet{model.LabelName(HostLabel): model.LabelValue(c.cfg.URL.Host)}) c.metrics.streamLag.With(lblSet).Set(time.Since(s.Entries[len(s.Entries)-1].Timestamp).Seconds()) } } diff --git a/clients/pkg/promtail/client/config.go b/clients/pkg/promtail/client/config.go index 1507e04eb10fd..bd38aa71ccc54 100644 --- a/clients/pkg/promtail/client/config.go +++ b/clients/pkg/promtail/client/config.go @@ -37,6 +37,8 @@ type Config struct { // The tenant ID to use when pushing logs to Loki (empty string means // single tenant mode) TenantID string `yaml:"tenant_id"` + + StreamLagLabels flagext.StringSliceCSV `yaml:"stream_lag_labels"` } // RegisterFlags with prefix registers flags where every name is prefixed by @@ -53,6 +55,9 @@ func (c *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.Var(&c.ExternalLabels, prefix+"client.external-labels", "list of external labels to add to each log (e.g: --client.external-labels=lb1=v1,lb2=v2)") f.StringVar(&c.TenantID, prefix+"client.tenant-id", "", "Tenant ID to use when pushing logs to Loki.") + + c.StreamLagLabels = []string{"filename"} + f.Var(&c.StreamLagLabels, prefix+"client.stream-lag-labels", "Comma-separated list of labels to use when calculating stream lag") } // RegisterFlags registers flags. @@ -75,9 +80,10 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { MaxRetries: MaxRetries, MinBackoff: MinBackoff, }, - BatchSize: BatchSize, - BatchWait: BatchWait, - Timeout: Timeout, + BatchSize: BatchSize, + BatchWait: BatchWait, + Timeout: Timeout, + StreamLagLabels: []string{"filename"}, } } diff --git a/clients/pkg/promtail/client/config_test.go b/clients/pkg/promtail/client/config_test.go index f5ee8aad3c057..b237fdd9c39ce 100644 --- a/clients/pkg/promtail/client/config_test.go +++ b/clients/pkg/promtail/client/config_test.go @@ -48,9 +48,10 @@ func Test_Config(t *testing.T) { MaxRetries: MaxRetries, MinBackoff: MinBackoff, }, - BatchSize: BatchSize, - BatchWait: BatchWait, - Timeout: Timeout, + BatchSize: BatchSize, + BatchWait: BatchWait, + Timeout: Timeout, + StreamLagLabels: []string{"filename"}, }, }, { @@ -64,9 +65,10 @@ func Test_Config(t *testing.T) { MaxRetries: 20, MinBackoff: 5 * time.Second, }, - BatchSize: 100 * 2048, - BatchWait: 5 * time.Second, - Timeout: 5 * time.Second, + BatchSize: 100 * 2048, + BatchWait: 5 * time.Second, + Timeout: 5 * time.Second, + StreamLagLabels: []string{"filename"}, }, }, } diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md index 4cb4bee7a9649..b1e0050bf1d93 100644 --- a/docs/sources/clients/promtail/configuration.md +++ b/docs/sources/clients/promtail/configuration.md @@ -273,6 +273,12 @@ external_labels: # Maximum time to wait for a server to respond to a request [timeout: | default = 10s] + +# A comma-separated list of labels to include in the stream lag metric `promtail_stream_lag_seconds`. +# The default value is "filename". A "host" label is always included. +# The stream lag metric indicates which streams are falling behind on writes to Loki; +# be mindful about not using too many labels here as it can explode cardinality. +[stream_lag_labels: | default = "filename"] ``` ## positions