diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
index e6098594fb71..9bde176af18c 100644
--- a/operator/CHANGELOG.md
+++ b/operator/CHANGELOG.md
@@ -3,6 +3,7 @@
## Release 5.6.8
- [9830](https://github.com/grafana/loki/pull/9830) **periklis**: Expose limits config setting cardinality_limit
+- [9630](https://github.com/grafana/loki/pull/9630) **jpinsonneau**: Expose per_stream_rate_limit & burst
## Release 5.6.7
diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go
index 9067063f5204..3e77b0ff8ef8 100644
--- a/operator/apis/loki/v1/lokistack_types.go
+++ b/operator/apis/loki/v1/lokistack_types.go
@@ -610,6 +610,20 @@ type IngestionLimitSpec struct {
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Line Size"
MaxLineSize int32 `json:"maxLineSize,omitempty"`
+
+ // PerStreamRateLimit defines the maximum byte rate per second per stream. Units MB.
+ //
+ // +optional
+ // +kubebuilder:validation:Optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Maximum byte rate per second per stream (in MB)"
+ PerStreamRateLimit int32 `json:"perStreamRateLimit,omitempty"`
+
+ // PerStreamRateLimitBurst defines the maximum burst bytes per stream. Units MB.
+ //
+ // +optional
+ // +kubebuilder:validation:Optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Maximum burst bytes per stream (in MB)"
+ PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"`
}
// RetentionStreamSpec defines a log stream with separate retention time.
diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml
index e803ecc227a6..1ffba181e086 100644
--- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml
+++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml
@@ -341,6 +341,18 @@ spec:
path: limits.global.ingestion.maxLineSize
x-descriptors:
- urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimit defines the maximum byte rate per second per
+ stream. Units MB.
+ displayName: Maximum byte rate per second per stream (in MB)
+ path: limits.global.ingestion.perStreamRateLimit
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+ Units MB.
+ displayName: Maximum burst bytes per stream (in MB)
+ path: limits.global.ingestion.perStreamRateLimitBurst
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
- description: CardinalityLimit defines the cardinality limit for index queries.
displayName: Cardinality Limit
path: limits.global.queries.cardinalityLimit
@@ -413,6 +425,18 @@ spec:
path: limits.tenants.ingestion.maxLineSize
x-descriptors:
- urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimit defines the maximum byte rate per second per
+ stream. Units MB.
+ displayName: Maximum byte rate per second per stream (in MB)
+ path: limits.tenants.ingestion.perStreamRateLimit
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+ Units MB.
+ displayName: Maximum burst bytes per stream (in MB)
+ path: limits.tenants.ingestion.perStreamRateLimitBurst
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
- description: CardinalityLimit defines the cardinality limit for index queries.
displayName: Cardinality Limit
path: limits.tenants.queries.cardinalityLimit
diff --git a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml
index 5111580a87f4..bee635f8aa24 100644
--- a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml
+++ b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml
@@ -130,6 +130,16 @@ spec:
on ingestion path. Units in Bytes.
format: int32
type: integer
+ perStreamRateLimit:
+ description: PerStreamRateLimit defines the maximum byte
+ rate per second per stream. Units MB.
+ format: int32
+ type: integer
+ perStreamRateLimitBurst:
+ description: PerStreamRateLimitBurst defines the maximum
+ burst bytes per stream. Units MB.
+ format: int32
+ type: integer
type: object
queries:
description: QueryLimits defines the limit applied on querying
@@ -247,6 +257,16 @@ spec:
on ingestion path. Units in Bytes.
format: int32
type: integer
+ perStreamRateLimit:
+ description: PerStreamRateLimit defines the maximum
+ byte rate per second per stream. Units MB.
+ format: int32
+ type: integer
+ perStreamRateLimitBurst:
+ description: PerStreamRateLimitBurst defines the maximum
+ burst bytes per stream. Units MB.
+ format: int32
+ type: integer
type: object
queries:
description: QueryLimits defines the limit applied on querying
diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml
index a6615b8feba5..974268e70932 100644
--- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml
+++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml
@@ -113,6 +113,16 @@ spec:
on ingestion path. Units in Bytes.
format: int32
type: integer
+ perStreamRateLimit:
+ description: PerStreamRateLimit defines the maximum byte
+ rate per second per stream. Units MB.
+ format: int32
+ type: integer
+ perStreamRateLimitBurst:
+ description: PerStreamRateLimitBurst defines the maximum
+ burst bytes per stream. Units MB.
+ format: int32
+ type: integer
type: object
queries:
description: QueryLimits defines the limit applied on querying
@@ -230,6 +240,16 @@ spec:
on ingestion path. Units in Bytes.
format: int32
type: integer
+ perStreamRateLimit:
+ description: PerStreamRateLimit defines the maximum
+ byte rate per second per stream. Units MB.
+ format: int32
+ type: integer
+ perStreamRateLimitBurst:
+ description: PerStreamRateLimitBurst defines the maximum
+ burst bytes per stream. Units MB.
+ format: int32
+ type: integer
type: object
queries:
description: QueryLimits defines the limit applied on querying
diff --git a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml
index 80c30ad87cd7..2a578d4b907b 100644
--- a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml
+++ b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml
@@ -196,6 +196,18 @@ spec:
path: limits.global.ingestion.maxLineSize
x-descriptors:
- urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimit defines the maximum byte rate per second per
+ stream. Units MB.
+ displayName: Maximum byte rate per second per stream (in MB)
+ path: limits.global.ingestion.perStreamRateLimit
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+ Units MB.
+ displayName: Maximum burst bytes per stream (in MB)
+ path: limits.global.ingestion.perStreamRateLimitBurst
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
- description: CardinalityLimit defines the cardinality limit for index queries.
displayName: Cardinality Limit
path: limits.global.queries.cardinalityLimit
@@ -268,6 +280,18 @@ spec:
path: limits.tenants.ingestion.maxLineSize
x-descriptors:
- urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimit defines the maximum byte rate per second per
+ stream. Units MB.
+ displayName: Maximum byte rate per second per stream (in MB)
+ path: limits.tenants.ingestion.perStreamRateLimit
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
+ - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+ Units MB.
+ displayName: Maximum burst bytes per stream (in MB)
+ path: limits.tenants.ingestion.perStreamRateLimitBurst
+ x-descriptors:
+ - urn:alm:descriptor:com.tectonic.ui:number
- description: CardinalityLimit defines the cardinality limit for index queries.
displayName: Cardinality Limit
path: limits.tenants.queries.cardinalityLimit
diff --git a/operator/docs/lokistack/sop.md b/operator/docs/lokistack/sop.md
index 2e27b5a6e321..8c437bd53b67 100644
--- a/operator/docs/lokistack/sop.md
+++ b/operator/docs/lokistack/sop.md
@@ -202,6 +202,8 @@ A service(s) is rate limiting at least 10% of all incoming requests.
| `label_value_too_long` | `maxLabelValueLength` |
| `line_too_long` | `maxLineSize` |
| `max_label_names_per_series` | `maxLabelNamesPerSeries` |
+| `per_stream_rate_limit` | `perStreamRateLimit`, `perStreamRateLimitBurst` |
+
## Loki Storage Slow Write
diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md
index c300239b2941..71d5cbac55ae 100644
--- a/operator/docs/operator/api.md
+++ b/operator/docs/operator/api.md
@@ -357,6 +357,30 @@ int32
MaxLineSize defines the maximum line size on ingestion path. Units in Bytes.
+
+
+perStreamRateLimit
+
+int32
+
+ |
+
+(Optional)
+ PerStreamRateLimit defines the maximum byte rate per second per stream. Units MB.
+ |
+
+
+
+perStreamRateLimitBurst
+
+int32
+
+ |
+
+(Optional)
+ PerStreamRateLimitBurst defines the maximum burst bytes per stream. Units MB.
+ |
+
diff --git a/operator/internal/manifests/config_test.go b/operator/internal/manifests/config_test.go
index b0cebcaf3ebf..384fc81ff84e 100644
--- a/operator/internal/manifests/config_test.go
+++ b/operator/internal/manifests/config_test.go
@@ -72,6 +72,8 @@ func randomConfigOptions() Options {
MaxLabelNamesPerSeries: rand.Int31(),
MaxGlobalStreamsPerTenant: rand.Int31(),
MaxLineSize: rand.Int31(),
+ PerStreamRateLimit: rand.Int31(),
+ PerStreamRateLimitBurst: rand.Int31(),
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: rand.Int31(),
@@ -89,6 +91,8 @@ func randomConfigOptions() Options {
MaxLabelNamesPerSeries: rand.Int31(),
MaxGlobalStreamsPerTenant: rand.Int31(),
MaxLineSize: rand.Int31(),
+ PerStreamRateLimit: rand.Int31(),
+ PerStreamRateLimitBurst: rand.Int31(),
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: rand.Int31(),
diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go
index 9e990731193a..4d93a34810b0 100644
--- a/operator/internal/manifests/internal/config/build_test.go
+++ b/operator/internal/manifests/internal/config/build_test.go
@@ -187,6 +187,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -441,6 +443,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -538,6 +542,8 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) {
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
// making it nil so that the template is not generated and error is returned
QueryLimits: nil,
@@ -830,6 +836,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -1180,6 +1188,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -1544,6 +1554,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -1876,6 +1888,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -2240,6 +2254,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -2629,6 +2645,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
@@ -2911,6 +2929,8 @@ overrides:
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml
index 4709738fe6b0..007f265bcc8a 100644
--- a/operator/internal/manifests/internal/config/loki-config.yaml
+++ b/operator/internal/manifests/internal/config/loki-config.yaml
@@ -170,8 +170,8 @@ limits_config:
{{- end }}
{{- end }}{{- end }}
max_cache_freshness_per_query: 10m
- per_stream_rate_limit: 3MB
- per_stream_rate_limit_burst: 15MB
+ per_stream_rate_limit: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimit }}MB
+ per_stream_rate_limit_burst: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimitBurst }}MB
split_queries_by_interval: 30m
{{- with .GossipRing }}
memberlist:
diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml
index b551d66ae1c5..7f011fcc3ab6 100644
--- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml
+++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml
@@ -23,7 +23,13 @@ overrides:
{{- end -}}
{{ if $l.MaxGlobalStreamsPerTenant }}
max_global_streams_per_user: {{ $l.MaxGlobalStreamsPerTenant }}
- {{- end -}}
+ {{- end }}
+ {{- if $l.PerStreamRateLimit }}
+ per_stream_rate_limit: {{ $l.PerStreamRateLimit }}MB
+ {{- end }}
+ {{- if $l.PerStreamRateLimitBurst }}
+ per_stream_rate_limit_burst: {{ $l.PerStreamRateLimitBurst }}MB
+ {{- end }}
{{- end -}}
{{- if $l := $spec.QueryLimits -}}
{{ if $l.MaxEntriesLimitPerQuery }}
diff --git a/operator/internal/manifests/internal/sizes.go b/operator/internal/manifests/internal/sizes.go
index a357b0622912..4a52252da036 100644
--- a/operator/internal/manifests/internal/sizes.go
+++ b/operator/internal/manifests/internal/sizes.go
@@ -211,12 +211,14 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{
Global: &lokiv1.LimitsTemplateSpec{
IngestionLimits: &lokiv1.IngestionLimitSpec{
// Defaults from Loki docs
- IngestionRate: 4,
- IngestionBurstSize: 6,
- MaxLabelNameLength: 1024,
- MaxLabelValueLength: 2048,
- MaxLabelNamesPerSeries: 30,
- MaxLineSize: 256000,
+ IngestionRate: 4,
+ IngestionBurstSize: 6,
+ MaxLabelNameLength: 1024,
+ MaxLabelValueLength: 2048,
+ MaxLabelNamesPerSeries: 30,
+ MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
// Defaults from Loki docs
@@ -267,10 +269,12 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{
IngestionBurstSize: 20,
MaxGlobalStreamsPerTenant: 10000,
// Defaults from Loki docs
- MaxLabelNameLength: 1024,
- MaxLabelValueLength: 2048,
- MaxLabelNamesPerSeries: 30,
- MaxLineSize: 256000,
+ MaxLabelNameLength: 1024,
+ MaxLabelValueLength: 2048,
+ MaxLabelNamesPerSeries: 30,
+ MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
// Defaults from Loki docs
@@ -321,10 +325,12 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{
IngestionBurstSize: 20,
MaxGlobalStreamsPerTenant: 25000,
// Defaults from Loki docs
- MaxLabelNameLength: 1024,
- MaxLabelValueLength: 2048,
- MaxLabelNamesPerSeries: 30,
- MaxLineSize: 256000,
+ MaxLabelNameLength: 1024,
+ MaxLabelValueLength: 2048,
+ MaxLabelNamesPerSeries: 30,
+ MaxLineSize: 256000,
+ PerStreamRateLimit: 3,
+ PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
// Defaults from Loki docs