From 1ff8112f2befa2c904704330a1d56d54feafdd9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Tou=C5=A1ek?= Date: Thu, 5 Sep 2024 12:09:50 +0200 Subject: [PATCH 1/5] fix: Lower slicesCount limit --- internal/pkg/service/stream/config/config_test.go | 4 ++-- .../pkg/service/stream/storage/level/target/config/config.go | 2 +- .../service/stream/storage/level/target/config/config_test.go | 2 +- .../stream/kubernetes/templates/config/config-map.yaml | 2 +- .../settings-001-all/005-get-settings/expected-response.json | 2 +- .../settings-001-all/012-get-settings/expected-response.json | 2 +- .../settings-001-all/003-get-settings/expected-response.json | 2 +- .../settings-001-all/010-get-settings/expected-response.json | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/pkg/service/stream/config/config_test.go b/internal/pkg/service/stream/config/config_test.go index bf29655b1f..235e0b0bf9 100644 --- a/internal/pkg/service/stream/config/config_test.go +++ b/internal/pkg/service/stream/config/config_test.go @@ -274,7 +274,7 @@ storage: size: 50MB # Duration from the last import to trigger the next import. Validation rules: required,minDuration=60s,maxDuration=24h interval: 15m0s - # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=50000 + # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=1000 slicesCount: 100 # Min remaining expiration to trigger file import. Validation rules: required,minDuration=5m,maxDuration=45m expiration: 30m0s @@ -583,7 +583,7 @@ func TestTableSinkConfigPatch_ToKVs(t *testing.T) { "defaultValue": 100, "overwritten": false, "protected": true, - "validation": "required,min=1,max=50000" + "validation": "required,min=1,max=1000" } ] `), strings.TrimSpace(json.MustEncodeString(kvs, true))) diff --git a/internal/pkg/service/stream/storage/level/target/config/config.go b/internal/pkg/service/stream/storage/level/target/config/config.go index ff0d6024af..52bf85294f 100644 --- a/internal/pkg/service/stream/storage/level/target/config/config.go +++ b/internal/pkg/service/stream/storage/level/target/config/config.go @@ -65,7 +65,7 @@ type ImportTrigger struct { Count uint64 `json:"count" configKey:"count" configUsage:"Records count to trigger file import." modAllowed:"true" validate:"required,min=1,max=10000000"` Size datasize.ByteSize `json:"size" configKey:"size" configUsage:"Records size to trigger file import." modAllowed:"true" validate:"required,minBytes=100B,maxBytes=500MB"` Interval duration.Duration `json:"interval" configKey:"interval" configUsage:"Duration from the last import to trigger the next import." modAllowed:"true" validate:"required,minDuration=60s,maxDuration=24h"` - SlicesCount uint64 `json:"slicesCount" configKey:"slicesCount" configUsage:"Number of slices in the file to trigger file import." validate:"required,min=1,max=50000"` + SlicesCount uint64 `json:"slicesCount" configKey:"slicesCount" configUsage:"Number of slices in the file to trigger file import." validate:"required,min=1,max=1000"` Expiration duration.Duration `json:"expiration" configKey:"expiration" configUsage:"Min remaining expiration to trigger file import." validate:"required,minDuration=5m,maxDuration=45m"` } diff --git a/internal/pkg/service/stream/storage/level/target/config/config_test.go b/internal/pkg/service/stream/storage/level/target/config/config_test.go index 2687e047c4..9c3674969d 100644 --- a/internal/pkg/service/stream/storage/level/target/config/config_test.go +++ b/internal/pkg/service/stream/storage/level/target/config/config_test.go @@ -47,7 +47,7 @@ func TestConfig_Validation(t *testing.T) { - "import.trigger.count" must be 10,000,000 or less - "import.trigger.size" must be 500MB or less - "import.trigger.interval" must be 24h0m0s or less -- "import.trigger.slicesCount" must be 50,000 or less +- "import.trigger.slicesCount" must be 1,000 or less - "import.trigger.expiration" must be 45m0s or less `, Value: overMaximumCfg, diff --git a/provisioning/stream/kubernetes/templates/config/config-map.yaml b/provisioning/stream/kubernetes/templates/config/config-map.yaml index 31221b3cf5..7756aff0c7 100644 --- a/provisioning/stream/kubernetes/templates/config/config-map.yaml +++ b/provisioning/stream/kubernetes/templates/config/config-map.yaml @@ -251,7 +251,7 @@ data: size: 50MB # Duration from the last import to trigger the next import. Validation rules: required,minDuration=60s,maxDuration=24h interval: 15m0s - # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=50000 + # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=1000 slicesCount: 100 # Min remaining expiration to trigger file import. Validation rules: required,minDuration=5m,maxDuration=45m expiration: 30m0s diff --git a/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json b/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json index fe5895ad08..03e6b3b76d 100644 --- a/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json +++ b/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json @@ -270,7 +270,7 @@ "defaultValue": 100, "overwritten": false, "protected": true, - "validation": "required,min=1,max=50000" + "validation": "required,min=1,max=1000" } ] } diff --git a/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json b/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json index 9323ad71da..4b93f8576a 100644 --- a/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json +++ b/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json @@ -270,7 +270,7 @@ "defaultValue": 100, "overwritten": false, "protected": true, - "validation": "required,min=1,max=50000" + "validation": "required,min=1,max=1000" } ] } diff --git a/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json b/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json index fe5895ad08..03e6b3b76d 100644 --- a/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json +++ b/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json @@ -270,7 +270,7 @@ "defaultValue": 100, "overwritten": false, "protected": true, - "validation": "required,min=1,max=50000" + "validation": "required,min=1,max=1000" } ] } diff --git a/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json b/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json index 9323ad71da..4b93f8576a 100644 --- a/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json +++ b/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json @@ -270,7 +270,7 @@ "defaultValue": 100, "overwritten": false, "protected": true, - "validation": "required,min=1,max=50000" + "validation": "required,min=1,max=1000" } ] } From b45a40a45dfdcbacb8ad89c8fd6ff153107d167a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Tou=C5=A1ek?= Date: Thu, 5 Sep 2024 12:10:08 +0200 Subject: [PATCH 2/5] fix: Increase ETCD_MAX_TXN_OPS --- provisioning/common/etcd/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/provisioning/common/etcd/values.yaml b/provisioning/common/etcd/values.yaml index a572f85b73..9da3cdfa9f 100644 --- a/provisioning/common/etcd/values.yaml +++ b/provisioning/common/etcd/values.yaml @@ -59,6 +59,9 @@ extraEnvVars: # GOMEMLIMIT to prevent OOM kill, update the value together with "resources" spec. - name: GOMEMLIMIT value: "256MiB" + # Increase limit of operations in transaction. + - name: ETCD_MAX_TXN_OPS + value: "1000" # The root password is used for authentication, the client gets a JWT token with short expiration. auth: From 0840dee239bd86a4f63b8844b27eeab8055f40aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Tou=C5=A1ek?= Date: Thu, 5 Sep 2024 12:11:30 +0200 Subject: [PATCH 3/5] fix: Simplify building benchmark docker image --- docker-compose.yml | 5 ++++- provisioning/stream/docker/k6/Dockerfile | 2 +- provisioning/stream/kubernetes/benchmark.sh | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 494ea5e76f..549484de15 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,7 +104,10 @@ services: k6: volumes: - ./scripts:/scripts - image: grafana/k6 + image: keboolabot/stream-benchmark + build: + context: . + dockerfile: ./provisioning/stream/docker/k6/Dockerfile network_mode: host ulimits: nofile: diff --git a/provisioning/stream/docker/k6/Dockerfile b/provisioning/stream/docker/k6/Dockerfile index 5372b16828..50f89b425c 100644 --- a/provisioning/stream/docker/k6/Dockerfile +++ b/provisioning/stream/docker/k6/Dockerfile @@ -1,3 +1,3 @@ -FROM grafana/k6:0.52.0 +FROM grafana/k6:0.53.0 COPY scripts /scripts diff --git a/provisioning/stream/kubernetes/benchmark.sh b/provisioning/stream/kubernetes/benchmark.sh index d84dce0d83..02661f1470 100644 --- a/provisioning/stream/kubernetes/benchmark.sh +++ b/provisioning/stream/kubernetes/benchmark.sh @@ -7,6 +7,7 @@ cd "$(dirname "$0")" # TODO: if we want to push #docker login --username keboolabot --password $DOCKER_REGISTRY_PASSWORD docker.io +#docker compose build k6 #docker push keboolabot/stream-benchmark:latest #docker logout From 797c09219ac0ffd6685c23ea3c7fe4fdd0453750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Tou=C5=A1ek?= Date: Thu, 5 Sep 2024 12:11:59 +0200 Subject: [PATCH 4/5] fix: Improve benchmark job configuration --- provisioning/stream/kubernetes/templates/benchmark/job.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/provisioning/stream/kubernetes/templates/benchmark/job.yaml b/provisioning/stream/kubernetes/templates/benchmark/job.yaml index 484f745c76..99e9c0e0a6 100644 --- a/provisioning/stream/kubernetes/templates/benchmark/job.yaml +++ b/provisioning/stream/kubernetes/templates/benchmark/job.yaml @@ -7,6 +7,7 @@ metadata: labels: app: stream-benchmark spec: + backoffLimit: 0 template: metadata: labels: @@ -52,7 +53,7 @@ spec: - name: K6_STATSD_ENABLE_TAGS value: "true" - name: K6_PARALLEL_REQS_PER_USER - value: "10" + value: "100" - name: K6_SCENARIO value: "ramping" - name: K6_CONST_VIRTUAL_USERS @@ -62,7 +63,7 @@ spec: - name: K6_CONST_TIMEOUT value: "20m" - name: K6_RAMPING_MAX_VIRTUAL_USERS - value: "1000" + value: "100" - name: K6_RAMPING_UP_DURATION value: "2m" - name: K6_RAMPING_STABLE_DURATION From 61596e1040b9d00aa342a69c1834af8241199700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Tou=C5=A1ek?= Date: Thu, 5 Sep 2024 12:12:38 +0200 Subject: [PATCH 5/5] fix: Increase benchmark teardown sleep --- scripts/k6/stream-api/common.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/k6/stream-api/common.js b/scripts/k6/stream-api/common.js index e0887e9852..c620d7bc68 100644 --- a/scripts/k6/stream-api/common.js +++ b/scripts/k6/stream-api/common.js @@ -176,8 +176,8 @@ export function stripUrlHost(url) { } export function teardownSource(sourceId) { - console.info("waiting 50s before source deletion") - sleep(50) + console.info("waiting 12s before source deletion") + sleep(120) const res = del(`v1/branches/default/sources/${sourceId}`); if (res.status !== 202) { console.error(res);