From 843745819b62e34f42b2abf9036b4a8b22c9b12e Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Mon, 5 Jun 2023 15:01:07 +0200 Subject: [PATCH 1/3] per_stream_rate_limit & burst limits configurable --- operator/apis/loki/v1/lokistack_types.go | 14 ++++++ .../apis/loki/v1beta1/lokistack_types_test.go | 18 +++++++ .../loki-operator.clusterserviceversion.yaml | 26 +++++++++- .../loki.grafana.com_lokistacks.yaml | 20 ++++++++ .../loki-operator.clusterserviceversion.yaml | 26 +++++++++- .../loki.grafana.com_lokistacks.yaml | 20 ++++++++ .../loki-operator.clusterserviceversion.yaml | 26 +++++++++- .../loki.grafana.com_lokistacks.yaml | 20 ++++++++ .../bases/loki.grafana.com_lokistacks.yaml | 20 ++++++++ .../loki-operator.clusterserviceversion.yaml | 24 ++++++++++ .../loki-operator.clusterserviceversion.yaml | 24 ++++++++++ .../loki-operator.clusterserviceversion.yaml | 24 ++++++++++ operator/docs/lokistack/sop.md | 2 + operator/docs/operator/api.md | 24 ++++++++++ operator/internal/manifests/config_test.go | 4 ++ .../manifests/internal/config/build_test.go | 22 +++++++++ .../internal/config/loki-config.yaml | 4 +- .../internal/config/loki-runtime-config.yaml | 6 +++ operator/internal/manifests/internal/sizes.go | 48 +++++++++++-------- 19 files changed, 347 insertions(+), 25 deletions(-) diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index ffa87248a280..1f3843739037 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -622,6 +622,20 @@ type IngestionLimitSpec struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Line Size" MaxLineSize int32 `json:"maxLineSize,omitempty"` + + // PerStreamRateLimit defines the maximum byte rate per second per stream. Units MB. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Maximum byte rate per second per stream (in MB)" + PerStreamRateLimit int32 `json:"perStreamRateLimit,omitempty"` + + // PerStreamRateLimitBurst defines the maximum burst bytes per stream. Units MB. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Maximum burst bytes per stream (in MB)" + PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"` } // RetentionStreamSpec defines a log stream with separate retention time. diff --git a/operator/apis/loki/v1beta1/lokistack_types_test.go b/operator/apis/loki/v1beta1/lokistack_types_test.go index 72fb55c6f55e..d09e12d6a759 100644 --- a/operator/apis/loki/v1beta1/lokistack_types_test.go +++ b/operator/apis/loki/v1beta1/lokistack_types_test.go @@ -379,6 +379,8 @@ func TestConvertToV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -396,6 +398,8 @@ func TestConvertToV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -412,6 +416,8 @@ func TestConvertToV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -704,6 +710,8 @@ func TestConvertFromV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -721,6 +729,8 @@ func TestConvertFromV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -737,6 +747,8 @@ func TestConvertFromV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -1001,6 +1013,8 @@ func TestConvertFromV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1beta1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -1018,6 +1032,8 @@ func TestConvertFromV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1beta1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, @@ -1034,6 +1050,8 @@ func TestConvertFromV1_LokiStack(t *testing.T) { MaxLabelNamesPerSeries: 1000, MaxGlobalStreamsPerTenant: 10000, MaxLineSize: 512, + PerStreamRateLimit: 10, + PerStreamRateLimitBurst: 20, }, QueryLimits: &v1beta1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 1000, diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index feb3bff7377d..c5f63beb3a9e 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:main-ac1c1fd - createdAt: "2023-05-24T15:10:18Z" + createdAt: "2023-06-05T12:57:30Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -329,6 +329,18 @@ spec: path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -396,6 +408,18 @@ spec: path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml index 658b0644090f..61c433c1d587 100644 --- a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -130,6 +130,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum byte + rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying @@ -242,6 +252,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum + byte rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index a2b3b69dcbaa..ad26b82868f8 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:main-ac1c1fd - createdAt: "2023-05-24T15:10:16Z" + createdAt: "2023-06-05T12:57:28Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -329,6 +329,18 @@ spec: path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -396,6 +408,18 @@ spec: path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml index 60f83f9f0746..6075ef04019b 100644 --- a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml @@ -130,6 +130,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum byte + rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying @@ -242,6 +252,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum + byte rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index dff713eb4afe..46caa77e23fa 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:v0.1.0 - createdAt: "2023-05-24T15:10:20Z" + createdAt: "2023-06-05T12:57:32Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -342,6 +342,18 @@ spec: path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -409,6 +421,18 @@ spec: path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml index ad8189744c78..86f73946b8b5 100644 --- a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -130,6 +130,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum byte + rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying @@ -242,6 +252,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum + byte rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 4821a9d89b1a..6d0f7618fc0b 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -113,6 +113,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum byte + rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying @@ -225,6 +235,16 @@ spec: on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum + byte rate per second per stream. Units MB. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. Units MB. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml index c0996d57fafd..ad8e8b303d1f 100644 --- a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml @@ -242,6 +242,18 @@ spec: path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -309,6 +321,18 @@ spec: path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml index 25b5df0291de..cef4c8043c7d 100644 --- a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml @@ -242,6 +242,18 @@ spec: path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -309,6 +321,18 @@ spec: path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml index 0b30cb6eaae0..da6266ec0a29 100644 --- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml @@ -254,6 +254,18 @@ spec: path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -321,6 +333,18 @@ spec: path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second per + stream. Units MB. + displayName: Maximum byte rate per second per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + Units MB. + displayName: Maximum burst bytes per stream (in MB) + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/operator/docs/lokistack/sop.md b/operator/docs/lokistack/sop.md index 2e27b5a6e321..a0b869bd1b6f 100644 --- a/operator/docs/lokistack/sop.md +++ b/operator/docs/lokistack/sop.md @@ -202,6 +202,8 @@ A service(s) is rate limiting at least 10% of all incoming requests. | `label_value_too_long` | `maxLabelValueLength` | | `line_too_long` | `maxLineSize` | | `max_label_names_per_series` | `maxLabelNamesPerSeries` | +| `Per stream rate limit exceeded` | `perStreamRateLimit`, `perStreamRateLimitBurst` | + ## Loki Storage Slow Write diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index e5dc35498566..8fa7cd56ae0d 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -1112,6 +1112,30 @@ int32

MaxLineSize defines the maximum line size on ingestion path. Units in Bytes.

+ + +perStreamRateLimit
+ +int32 + + + +(Optional) +

PerStreamRateLimit defines the maximum byte rate per second per stream. Units MB.

+ + + + +perStreamRateLimitBurst
+ +int32 + + + +(Optional) +

PerStreamRateLimitBurst defines the maximum burst bytes per stream. Units MB.

+ + diff --git a/operator/internal/manifests/config_test.go b/operator/internal/manifests/config_test.go index f0df0593a43d..e03cb18199d0 100644 --- a/operator/internal/manifests/config_test.go +++ b/operator/internal/manifests/config_test.go @@ -75,6 +75,8 @@ func randomConfigOptions() Options { MaxLabelNamesPerSeries: rand.Int31(), MaxGlobalStreamsPerTenant: rand.Int31(), MaxLineSize: rand.Int31(), + PerStreamRateLimit: rand.Int31(), + PerStreamRateLimitBurst: rand.Int31(), }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: rand.Int31(), @@ -92,6 +94,8 @@ func randomConfigOptions() Options { MaxLabelNamesPerSeries: rand.Int31(), MaxGlobalStreamsPerTenant: rand.Int31(), MaxLineSize: rand.Int31(), + PerStreamRateLimit: rand.Int31(), + PerStreamRateLimitBurst: rand.Int31(), }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: rand.Int31(), diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index d6e44811b641..659a9be0fd33 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -190,6 +190,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -444,6 +446,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -556,6 +560,8 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, // making it nil so that the template is not generated and error is returned QueryLimits: nil, @@ -849,6 +855,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -1199,6 +1207,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -1563,6 +1573,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -1895,6 +1907,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -2283,6 +2297,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -2672,6 +2688,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -3060,6 +3078,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, @@ -3442,6 +3462,8 @@ overrides: MaxLabelNamesPerSeries: 30, MaxGlobalStreamsPerTenant: 0, MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ MaxEntriesLimitPerQuery: 5000, diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index 4e44f081e893..fd296a18d9d1 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -177,8 +177,8 @@ limits_config: {{- end }} {{- end }}{{- end }} max_cache_freshness_per_query: 10m - per_stream_rate_limit: 3MB - per_stream_rate_limit_burst: 15MB + per_stream_rate_limit: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimit }}MB + per_stream_rate_limit_burst: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimitBurst }}MB split_queries_by_interval: 30m {{- with .GossipRing }} memberlist: diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml index b367f3bcbccc..524eb3970226 100644 --- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml +++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml @@ -26,6 +26,12 @@ overrides: {{- if $l.MaxGlobalStreamsPerTenant }} max_global_streams_per_user: {{ $l.MaxGlobalStreamsPerTenant }} {{- end }} + {{- if $l.PerStreamRateLimit }} + per_stream_rate_limit: {{ $l.PerStreamRateLimit }}MB + {{- end }} + {{- if $l.PerStreamRateLimitBurst }} + per_stream_rate_limit_burst: {{ $l.PerStreamRateLimitBurst }}MB + {{- end }} {{- end -}} {{- if $l := $spec.QueryLimits -}} {{- if $l.MaxEntriesLimitPerQuery }} diff --git a/operator/internal/manifests/internal/sizes.go b/operator/internal/manifests/internal/sizes.go index 5623da4cb7e9..8c0f4cfb6000 100644 --- a/operator/internal/manifests/internal/sizes.go +++ b/operator/internal/manifests/internal/sizes.go @@ -235,12 +235,14 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ Global: &lokiv1.LimitsTemplateSpec{ IngestionLimits: &lokiv1.IngestionLimitSpec{ // Defaults from Loki docs - IngestionRate: 4, - IngestionBurstSize: 6, - MaxLabelNameLength: 1024, - MaxLabelValueLength: 2048, - MaxLabelNamesPerSeries: 30, - MaxLineSize: 256000, + IngestionRate: 4, + IngestionBurstSize: 6, + MaxLabelNameLength: 1024, + MaxLabelValueLength: 2048, + MaxLabelNamesPerSeries: 30, + MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ // Defaults from Loki docs @@ -287,12 +289,14 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ Global: &lokiv1.LimitsTemplateSpec{ IngestionLimits: &lokiv1.IngestionLimitSpec{ // Defaults from Loki docs - IngestionRate: 4, - IngestionBurstSize: 6, - MaxLabelNameLength: 1024, - MaxLabelValueLength: 2048, - MaxLabelNamesPerSeries: 30, - MaxLineSize: 256000, + IngestionRate: 4, + IngestionBurstSize: 6, + MaxLabelNameLength: 1024, + MaxLabelValueLength: 2048, + MaxLabelNamesPerSeries: 30, + MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ // Defaults from Loki docs @@ -344,10 +348,12 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ IngestionBurstSize: 20, MaxGlobalStreamsPerTenant: 10000, // Defaults from Loki docs - MaxLabelNameLength: 1024, - MaxLabelValueLength: 2048, - MaxLabelNamesPerSeries: 30, - MaxLineSize: 256000, + MaxLabelNameLength: 1024, + MaxLabelValueLength: 2048, + MaxLabelNamesPerSeries: 30, + MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ // Defaults from Loki docs @@ -399,10 +405,12 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ IngestionBurstSize: 20, MaxGlobalStreamsPerTenant: 25000, // Defaults from Loki docs - MaxLabelNameLength: 1024, - MaxLabelValueLength: 2048, - MaxLabelNamesPerSeries: 30, - MaxLineSize: 256000, + MaxLabelNameLength: 1024, + MaxLabelValueLength: 2048, + MaxLabelNamesPerSeries: 30, + MaxLineSize: 256000, + PerStreamRateLimit: 3, + PerStreamRateLimitBurst: 15, }, QueryLimits: &lokiv1.QueryLimitSpec{ // Defaults from Loki docs From 6afa06fada5c76b89a173c29aa4628430641a624 Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Mon, 5 Jun 2023 15:12:53 +0200 Subject: [PATCH 2/3] update changelog --- operator/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 753c93d882e6..bffad9a4e34a 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,5 +1,6 @@ ## Main +- [9630](https://github.com/grafana/loki/pull/9630) **jpinsonneau**: Expose per_stream_rate_limit & burst - [9457](https://github.com/grafana/loki/pull/9457) **Red-GV**: Set seccomp profile to runtime default - [9448](https://github.com/grafana/loki/pull/9448) **btaani**: Include runtime-config in compiling the SHA1 checksum - [9511](https://github.com/grafana/loki/pull/9511) **xperimental**: Do not update status after setting degraded condition From d6971ac3516c4ae4ebce344fe02d2722b27c1f9e Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Tue, 6 Jun 2023 09:14:57 +0200 Subject: [PATCH 3/3] updated doc reason --- operator/docs/lokistack/sop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/docs/lokistack/sop.md b/operator/docs/lokistack/sop.md index a0b869bd1b6f..8c437bd53b67 100644 --- a/operator/docs/lokistack/sop.md +++ b/operator/docs/lokistack/sop.md @@ -202,7 +202,7 @@ A service(s) is rate limiting at least 10% of all incoming requests. | `label_value_too_long` | `maxLabelValueLength` | | `line_too_long` | `maxLineSize` | | `max_label_names_per_series` | `maxLabelNamesPerSeries` | -| `Per stream rate limit exceeded` | `perStreamRateLimit`, `perStreamRateLimitBurst` | +| `per_stream_rate_limit` | `perStreamRateLimit`, `perStreamRateLimitBurst` | ## Loki Storage Slow Write