From ba30afa94e7f0177fb61004eebd13c76efc439ff Mon Sep 17 00:00:00 2001 From: Karen Miller Date: Thu, 31 Mar 2022 10:37:51 -0700 Subject: [PATCH 1/4] Docs: Document Promtail global rate limiting --- CHANGELOG.md | 1 + .../sources/clients/promtail/configuration.md | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 004d72c0f2e37..1f183921af11e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ * [5163](https://github.com/grafana/loki/pull/5163) **chaudum** Fix regression in fluentd plugin introduced with #5107 that caused `NoMethodError` when parsing non-string values of log lines. * [5144](https://github.com/grafana/loki/pull/5144) **dannykopping** Ruler: fix remote write basic auth credentials. * [5091](https://github.com/grafana/loki/pull/5091) **owen-d**: Changes `ingester.concurrent-flushes` default to 32 +* [5031](https://github.com/grafana/loki/pull/5031) **liguozhong**: Promtail: Add global read rate limiting. * [4879](https://github.com/grafana/loki/pull/4879) **cyriltovena**: LogQL: add __line__ function to | line_format template. * [5081](https://github.com/grafana/loki/pull/5081) **SasSwart**: Add the option to configure memory ballast for Loki * [5085](https://github.com/grafana/loki/pull/5085) **aknuds1**: Upgrade Cortex to [e0807c4eb487](https://github.com/cortexproject/cortex/compare/4e9fc3a2b5ab..e0807c4eb487) and Prometheus to [692a54649ed7](https://github.com/prometheus/prometheus/compare/2a3d62ac8456..692a54649ed7) diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md index 3078fa3d04e58..27d3b629c2984 100644 --- a/docs/sources/clients/promtail/configuration.md +++ b/docs/sources/clients/promtail/configuration.md @@ -95,6 +95,9 @@ clients: scrape_configs: - [] +# Configures global limits for this instance of Promtail +[limit_config: ] + # Configures how tailed targets will be watched. [target_config: ] @@ -1756,6 +1759,26 @@ scrape_configs: target_label: 'container' ``` +## limit_config + +The optional `limit_config` block configures global limits for this instance of Promtail. + +```yaml +# When true, enforces rate limiting on this instance of Promtail. +[readline_rate_enabled: | default = false] + +# The rate limit in log lines per second that this instance of Promtail may push to Loki. +[readline_rate: | default = 10000] + +# The cap in the quantity of burst lines that this instance of Promtail may push +# to Loki. +[readline_burst: | default = 10000] + +# When true, exceeding the rate limit causes this instance of Promtail to discard +# log lines, rather than sending them to Loki. When false, exceeding the rate limit +# causes this instance of Promtail to hold off on sending the log lines. +[readline_rate_drop: | default = true] +``` ## target_config From b1e34031598ade2b8b5b88d586ecdaa324a8d3d3 Mon Sep 17 00:00:00 2001 From: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com> Date: Thu, 31 Mar 2022 14:00:16 -0700 Subject: [PATCH 2/4] Update docs/sources/clients/promtail/configuration.md Co-authored-by: Jennifer Villa --- docs/sources/clients/promtail/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md index 27d3b629c2984..e59a589bee25d 100644 --- a/docs/sources/clients/promtail/configuration.md +++ b/docs/sources/clients/promtail/configuration.md @@ -1776,7 +1776,7 @@ The optional `limit_config` block configures global limits for this instance of # When true, exceeding the rate limit causes this instance of Promtail to discard # log lines, rather than sending them to Loki. When false, exceeding the rate limit -# causes this instance of Promtail to hold off on sending the log lines. +# causes this instance of Promtail to temporarily hold off on sending the log lines and retry later. [readline_rate_drop: | default = true] ``` From e92c5cab2c823288529a12f7d547c5efd2c99af0 Mon Sep 17 00:00:00 2001 From: Karen Miller Date: Fri, 1 Apr 2022 11:15:32 -0700 Subject: [PATCH 3/4] Revise strings that explain the new configuration parameters --- clients/pkg/promtail/limit/config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/pkg/promtail/limit/config.go b/clients/pkg/promtail/limit/config.go index 6ecef6fdd523e..4584502233b18 100644 --- a/clients/pkg/promtail/limit/config.go +++ b/clients/pkg/promtail/limit/config.go @@ -12,8 +12,8 @@ type Config struct { } func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.Float64Var(&cfg.ReadlineRate, prefix+"limit.readline-rate", 10000, "promtail readline Rate.") - f.IntVar(&cfg.ReadlineBurst, prefix+"limit.readline-burst", 10000, "promtail readline Burst.") - f.BoolVar(&cfg.ReadlineRateEnabled, prefix+"limit.readline-rate-enabled", false, "Set to false to disable readline rate limit.") - f.BoolVar(&cfg.ReadlineRateDrop, prefix+"limit.readline-rate-drop", true, "Set to true to drop log when rate limit.") + f.Float64Var(&cfg.ReadlineRate, prefix+"limit.readline-rate", 10000, "The rate limit in log lines per second that this instance of Promtail may push to Loki.") + f.IntVar(&cfg.ReadlineBurst, prefix+"limit.readline-burst", 10000, "The cap in the quantity of burst lines that this instance of Promtail may push to Loki.") + f.BoolVar(&cfg.ReadlineRateEnabled, prefix+"limit.readline-rate-enabled", false, "When true, enforces rate limiting on this instance of Promtail.") + f.BoolVar(&cfg.ReadlineRateDrop, prefix+"limit.readline-rate-drop", true, "When true, exceeding the rate limit causes this instance of Promtail to discard log lines, rather than sending them to Loki.") } From 618bdd86579ee1fa60e2096aacd15dcb8aaf2843 Mon Sep 17 00:00:00 2001 From: Karen Miller Date: Mon, 4 Apr 2022 13:18:24 -0700 Subject: [PATCH 4/4] For rate limiting, limit_config becomes limits_config --- docs/sources/clients/promtail/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md index e59a589bee25d..14fb4c7c4b0d1 100644 --- a/docs/sources/clients/promtail/configuration.md +++ b/docs/sources/clients/promtail/configuration.md @@ -96,7 +96,7 @@ scrape_configs: - [] # Configures global limits for this instance of Promtail -[limit_config: ] +[limits_config: ] # Configures how tailed targets will be watched. [target_config: ] @@ -1759,9 +1759,9 @@ scrape_configs: target_label: 'container' ``` -## limit_config +## limits_config -The optional `limit_config` block configures global limits for this instance of Promtail. +The optional `limits_config` block configures global limits for this instance of Promtail. ```yaml # When true, enforces rate limiting on this instance of Promtail.