Merge pull request #159 from periklis/backport-pr-9630-to-5.6.z

[release-5.6] Backport PR grafana#9630
openshift · Jun 30, 2023 · 4bfb355 · 4bfb355
2 parents 38fa6d3 + 0aeb1a9
commit 4bfb355
Show file tree

Hide file tree

Showing 13 changed files with 182 additions and 17 deletions.
diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## Release 5.6.8
 
 - [9830](https://github.com/grafana/loki/pull/9830) **periklis**: Expose limits config setting cardinality_limit
+- [9630](https://github.com/grafana/loki/pull/9630) **jpinsonneau**: Expose per_stream_rate_limit & burst
 
 ## Release 5.6.7
 

diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go
@@ -610,6 +610,20 @@ type IngestionLimitSpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Line Size"
 	MaxLineSize int32 `json:"maxLineSize,omitempty"`
+
+	// PerStreamRateLimit defines the maximum byte rate per second per stream. Units MB.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Maximum byte rate per second per stream (in MB)"
+	PerStreamRateLimit int32 `json:"perStreamRateLimit,omitempty"`
+
+	// PerStreamRateLimitBurst defines the maximum burst bytes per stream. Units MB.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Maximum burst bytes per stream (in MB)"
+	PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"`
 }
 
 // RetentionStreamSpec defines a log stream with separate retention time.

diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml
@@ -341,6 +341,18 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimit defines the maximum byte rate per second per
+          stream. Units MB.
+        displayName: Maximum byte rate per second per stream (in MB)
+        path: limits.global.ingestion.perStreamRateLimit
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+          Units MB.
+        displayName: Maximum burst bytes per stream (in MB)
+        path: limits.global.ingestion.perStreamRateLimitBurst
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: CardinalityLimit defines the cardinality limit for index queries.
         displayName: Cardinality Limit
         path: limits.global.queries.cardinalityLimit
@@ -413,6 +425,18 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimit defines the maximum byte rate per second per
+          stream. Units MB.
+        displayName: Maximum byte rate per second per stream (in MB)
+        path: limits.tenants.ingestion.perStreamRateLimit
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+          Units MB.
+        displayName: Maximum burst bytes per stream (in MB)
+        path: limits.tenants.ingestion.perStreamRateLimitBurst
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: CardinalityLimit defines the cardinality limit for index queries.
         displayName: Cardinality Limit
         path: limits.tenants.queries.cardinalityLimit

diff --git a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml
@@ -130,6 +130,16 @@ spec:
                               on ingestion path. Units in Bytes.
                             format: int32
                             type: integer
+                          perStreamRateLimit:
+                            description: PerStreamRateLimit defines the maximum byte
+                              rate per second per stream. Units MB.
+                            format: int32
+                            type: integer
+                          perStreamRateLimitBurst:
+                            description: PerStreamRateLimitBurst defines the maximum
+                              burst bytes per stream. Units MB.
+                            format: int32
+                            type: integer
                         type: object
                       queries:
                         description: QueryLimits defines the limit applied on querying
@@ -247,6 +257,16 @@ spec:
                                 on ingestion path. Units in Bytes.
                               format: int32
                               type: integer
+                            perStreamRateLimit:
+                              description: PerStreamRateLimit defines the maximum
+                                byte rate per second per stream. Units MB.
+                              format: int32
+                              type: integer
+                            perStreamRateLimitBurst:
+                              description: PerStreamRateLimitBurst defines the maximum
+                                burst bytes per stream. Units MB.
+                              format: int32
+                              type: integer
                           type: object
                         queries:
                           description: QueryLimits defines the limit applied on querying

diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml
@@ -113,6 +113,16 @@ spec:
                               on ingestion path. Units in Bytes.
                             format: int32
                             type: integer
+                          perStreamRateLimit:
+                            description: PerStreamRateLimit defines the maximum byte
+                              rate per second per stream. Units MB.
+                            format: int32
+                            type: integer
+                          perStreamRateLimitBurst:
+                            description: PerStreamRateLimitBurst defines the maximum
+                              burst bytes per stream. Units MB.
+                            format: int32
+                            type: integer
                         type: object
                       queries:
                         description: QueryLimits defines the limit applied on querying
@@ -230,6 +240,16 @@ spec:
                                 on ingestion path. Units in Bytes.
                               format: int32
                               type: integer
+                            perStreamRateLimit:
+                              description: PerStreamRateLimit defines the maximum
+                                byte rate per second per stream. Units MB.
+                              format: int32
+                              type: integer
+                            perStreamRateLimitBurst:
+                              description: PerStreamRateLimitBurst defines the maximum
+                                burst bytes per stream. Units MB.
+                              format: int32
+                              type: integer
                           type: object
                         queries:
                           description: QueryLimits defines the limit applied on querying

diff --git a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml
@@ -196,6 +196,18 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimit defines the maximum byte rate per second per
+          stream. Units MB.
+        displayName: Maximum byte rate per second per stream (in MB)
+        path: limits.global.ingestion.perStreamRateLimit
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+          Units MB.
+        displayName: Maximum burst bytes per stream (in MB)
+        path: limits.global.ingestion.perStreamRateLimitBurst
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: CardinalityLimit defines the cardinality limit for index queries.
         displayName: Cardinality Limit
         path: limits.global.queries.cardinalityLimit
@@ -268,6 +280,18 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimit defines the maximum byte rate per second per
+          stream. Units MB.
+        displayName: Maximum byte rate per second per stream (in MB)
+        path: limits.tenants.ingestion.perStreamRateLimit
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream.
+          Units MB.
+        displayName: Maximum burst bytes per stream (in MB)
+        path: limits.tenants.ingestion.perStreamRateLimitBurst
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: CardinalityLimit defines the cardinality limit for index queries.
         displayName: Cardinality Limit
         path: limits.tenants.queries.cardinalityLimit

diff --git a/operator/docs/lokistack/sop.md b/operator/docs/lokistack/sop.md
@@ -202,6 +202,8 @@ A service(s) is rate limiting at least 10% of all incoming requests.
 | `label_value_too_long` | `maxLabelValueLength` |
 | `line_too_long` | `maxLineSize` |
 | `max_label_names_per_series` | `maxLabelNamesPerSeries` |
+| `per_stream_rate_limit` | `perStreamRateLimit`, `perStreamRateLimitBurst` |
+
 
 ## Loki Storage Slow Write
 

diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md
@@ -357,6 +357,30 @@ int32
 <p>MaxLineSize defines the maximum line size on ingestion path. Units in Bytes.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>perStreamRateLimit</code><br/>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>PerStreamRateLimit defines the maximum byte rate per second per stream. Units MB.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>perStreamRateLimitBurst</code><br/>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>PerStreamRateLimitBurst defines the maximum burst bytes per stream. Units MB.</p>
+</td>
+</tr>
 </tbody>
 </table>
 

diff --git a/operator/internal/manifests/config_test.go b/operator/internal/manifests/config_test.go
@@ -72,6 +72,8 @@ func randomConfigOptions() Options {
 						MaxLabelNamesPerSeries:    rand.Int31(),
 						MaxGlobalStreamsPerTenant: rand.Int31(),
 						MaxLineSize:               rand.Int31(),
+						PerStreamRateLimit:        rand.Int31(),
+						PerStreamRateLimitBurst:   rand.Int31(),
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: rand.Int31(),
@@ -89,6 +91,8 @@ func randomConfigOptions() Options {
 							MaxLabelNamesPerSeries:    rand.Int31(),
 							MaxGlobalStreamsPerTenant: rand.Int31(),
 							MaxLineSize:               rand.Int31(),
+							PerStreamRateLimit:        rand.Int31(),
+							PerStreamRateLimitBurst:   rand.Int31(),
 						},
 						QueryLimits: &lokiv1.QueryLimitSpec{
 							MaxEntriesLimitPerQuery: rand.Int31(),

diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go
@@ -187,6 +187,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -441,6 +443,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -538,6 +542,8 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) {
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					// making it nil so that the template is not generated and error is returned
 					QueryLimits: nil,
@@ -830,6 +836,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -1180,6 +1188,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -1544,6 +1554,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -1876,6 +1888,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -2240,6 +2254,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -2629,6 +2645,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,
@@ -2911,6 +2929,8 @@ overrides:
 						MaxLabelNamesPerSeries:    30,
 						MaxGlobalStreamsPerTenant: 0,
 						MaxLineSize:               256000,
+						PerStreamRateLimit:        3,
+						PerStreamRateLimitBurst:   15,
 					},
 					QueryLimits: &lokiv1.QueryLimitSpec{
 						MaxEntriesLimitPerQuery: 5000,

diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml
@@ -170,8 +170,8 @@ limits_config:
 {{- end }}
 {{- end }}{{- end }}
   max_cache_freshness_per_query: 10m
-  per_stream_rate_limit: 3MB
-  per_stream_rate_limit_burst: 15MB
+  per_stream_rate_limit: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimit }}MB
+  per_stream_rate_limit_burst: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimitBurst }}MB
   split_queries_by_interval: 30m
 {{- with .GossipRing }}
 memberlist:

diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml
@@ -23,7 +23,13 @@ overrides:
     {{- end -}}
     {{ if $l.MaxGlobalStreamsPerTenant }}
     max_global_streams_per_user: {{ $l.MaxGlobalStreamsPerTenant }}
-    {{- end -}}
+    {{- end }}
+    {{- if $l.PerStreamRateLimit }}
+    per_stream_rate_limit: {{ $l.PerStreamRateLimit }}MB
+    {{- end }}
+    {{- if $l.PerStreamRateLimitBurst }}
+    per_stream_rate_limit_burst: {{ $l.PerStreamRateLimitBurst }}MB
+    {{- end }}
   {{- end -}}
   {{- if $l := $spec.QueryLimits -}}
     {{ if $l.MaxEntriesLimitPerQuery }}