Merge pull request #1996 from keboola/jt-stream-benchmark-improvements

feat: Benchmark improvements

docker-compose.yml

@@ -104,7 +104,10 @@ services:
   k6:
     volumes:
       - ./scripts:/scripts
-    image: grafana/k6
+    image: keboolabot/stream-benchmark
+    build:
+      context: .
+      dockerfile: ./provisioning/stream/docker/k6/Dockerfile
     network_mode: host
     ulimits:
       nofile:

internal/pkg/service/stream/config/config_test.go

@@ -274,7 +274,7 @@ storage:
                     size: 50MB
                     # Duration from the last import to trigger the next import. Validation rules: required,minDuration=60s,maxDuration=24h
                     interval: 15m0s
-                    # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=50000
+                    # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=1000
                     slicesCount: 100
                     # Min remaining expiration to trigger file import. Validation rules: required,minDuration=5m,maxDuration=45m
                     expiration: 30m0s
@@ -583,7 +583,7 @@ func TestTableSinkConfigPatch_ToKVs(t *testing.T) {
     "defaultValue": 100,
     "overwritten": false,
     "protected": true,
-    "validation": "required,min=1,max=50000"
+    "validation": "required,min=1,max=1000"
   }
 ]
 `), strings.TrimSpace(json.MustEncodeString(kvs, true)))

internal/pkg/service/stream/storage/level/target/config/config.go

@@ -65,7 +65,7 @@ type ImportTrigger struct {
 	Count       uint64            `json:"count" configKey:"count" configUsage:"Records count to trigger file import." modAllowed:"true" validate:"required,min=1,max=10000000"`
 	Size        datasize.ByteSize `json:"size" configKey:"size" configUsage:"Records size to trigger file import." modAllowed:"true" validate:"required,minBytes=100B,maxBytes=500MB"`
 	Interval    duration.Duration `json:"interval" configKey:"interval" configUsage:"Duration from the last import to trigger the next import." modAllowed:"true" validate:"required,minDuration=60s,maxDuration=24h"`
-	SlicesCount uint64            `json:"slicesCount" configKey:"slicesCount" configUsage:"Number of slices in the file to trigger file import." validate:"required,min=1,max=50000"`
+	SlicesCount uint64            `json:"slicesCount" configKey:"slicesCount" configUsage:"Number of slices in the file to trigger file import." validate:"required,min=1,max=1000"`
 	Expiration  duration.Duration `json:"expiration" configKey:"expiration" configUsage:"Min remaining expiration to trigger file import." validate:"required,minDuration=5m,maxDuration=45m"`
 }
 

internal/pkg/service/stream/storage/level/target/config/config_test.go

@@ -47,7 +47,7 @@ func TestConfig_Validation(t *testing.T) {
 - "import.trigger.count" must be 10,000,000 or less
 - "import.trigger.size" must be 500MB or less
 - "import.trigger.interval" must be 24h0m0s or less
-- "import.trigger.slicesCount" must be 50,000 or less
+- "import.trigger.slicesCount" must be 1,000 or less
 - "import.trigger.expiration" must be 45m0s or less
 `,
 			Value: overMaximumCfg,

provisioning/common/etcd/values.yaml

@@ -59,6 +59,9 @@ extraEnvVars:
   # GOMEMLIMIT to prevent OOM kill, update the value together with "resources" spec.
   - name: GOMEMLIMIT
     value: "256MiB"
+  # Increase limit of operations in transaction.
+  - name: ETCD_MAX_TXN_OPS
+    value: "1000"
 
 # The root password is used for authentication, the client gets a JWT token with short expiration.
 auth:

provisioning/stream/docker/k6/Dockerfile

@@ -1,3 +1,3 @@
-FROM grafana/k6:0.52.0
+FROM grafana/k6:0.53.0
 
 COPY scripts /scripts

provisioning/stream/kubernetes/benchmark.sh

@@ -7,6 +7,7 @@ cd "$(dirname "$0")"
 
 # TODO: if we want to push
 #docker login --username keboolabot --password $DOCKER_REGISTRY_PASSWORD docker.io
+#docker compose build k6
 #docker push keboolabot/stream-benchmark:latest
 #docker logout
 

provisioning/stream/kubernetes/templates/benchmark/job.yaml

@@ -7,6 +7,7 @@ metadata:
   labels:
     app: stream-benchmark
 spec:
+  backoffLimit: 0
   template:
     metadata:
       labels:
@@ -52,7 +53,7 @@ spec:
             - name: K6_STATSD_ENABLE_TAGS
               value: "true"
             - name: K6_PARALLEL_REQS_PER_USER
-              value: "10"
+              value: "100"
             - name: K6_SCENARIO
               value: "ramping"
             - name: K6_CONST_VIRTUAL_USERS
@@ -62,7 +63,7 @@ spec:
             - name: K6_CONST_TIMEOUT
               value: "20m"
             - name: K6_RAMPING_MAX_VIRTUAL_USERS
-              value: "1000"
+              value: "100"
             - name: K6_RAMPING_UP_DURATION
               value: "2m"
             - name: K6_RAMPING_STABLE_DURATION

provisioning/stream/kubernetes/templates/config/config-map.yaml

@@ -251,7 +251,7 @@ data:
               size: 50MB
               # Duration from the last import to trigger the next import. Validation rules: required,minDuration=60s,maxDuration=24h
               interval: 15m0s
-              # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=50000
+              # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=1000
               slicesCount: 100
               # Min remaining expiration to trigger file import. Validation rules: required,minDuration=5m,maxDuration=45m
               expiration: 30m0s

scripts/k6/stream-api/common.js

@@ -176,8 +176,8 @@ export function stripUrlHost(url) {
 }
 
 export function teardownSource(sourceId) {
-  console.info("waiting 50s before source deletion")
-  sleep(50)
+  console.info("waiting 12s before source deletion")
+  sleep(120)
   const res = del(`v1/branches/default/sources/${sourceId}`);
   if (res.status !== 202) {
     console.error(res);

test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json

@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=50000"
+      "validation": "required,min=1,max=1000"
     }
   ]
 }

test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json

@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=50000"
+      "validation": "required,min=1,max=1000"
     }
   ]
 }

test/stream/api/source/settings-001-all/003-get-settings/expected-response.json

@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=50000"
+      "validation": "required,min=1,max=1000"
     }
   ]
 }

test/stream/api/source/settings-001-all/010-get-settings/expected-response.json

@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=50000"
+      "validation": "required,min=1,max=1000"
     }
   ]
 }