[Access-tokens][ClusterLoader2] Add standalone test

Add standalone test for ClusterLoader2 that allows benchmarking and testing access tokens thresholds. This test is based on Mateusz's code from github.com/mm4tt/k8s-util.
kubernetes · Feb 6, 2020 · f68953d · f68953d
1 parent 213b876
commit f68953d
Show file tree

Hide file tree

Showing 6 changed files with 246 additions and 0 deletions.
diff --git a/clusterloader2/testing/access-tokens/config.yaml b/clusterloader2/testing/access-tokens/config.yaml
@@ -0,0 +1,167 @@
+# Stress testing access token validation
+#
+# Targeting 2 000 tokens with 5 000 total QPS for 5k node cluster, so it's 2.5 QPS per token.
+#
+# Number of tokens does not scale with number of nodes.
+# By default, those 2 000 tokens are are assigned to 80 service accounts, with 25 tokens each.
+#
+# For smaller cluster, we scale down lineary QPS per token to 2.5 * (Number of nodes)/(5 000)
+# This results in 1 QPS per node, if there is 2 000 tokens.
+#
+# When defining your own parameters:
+# Number of tokens = ${namespaces} * ${serviceAccounts} * ${tokensPerServiceAccount}
+# Total QPS = Number of tokens * ${replicas} * ${qpsPerWorker}
+#
+# For default values in 5k cluster this means:
+# Number of tokens = 1 * 80 * 25 = 2000
+# Total QPS = 2000 * 1 * 2.5 = 5000
+
+# Size of test variables
+{{$namespaces := DefaultParam .CL2_ACCESS_TOKENS_NAMESPACES 1}}
+{{$serviceAccounts := DefaultParam .CL2_ACCESS_TOKENS_SERVICE_ACCOUNTS 80}}
+{{$tokensPerServiceAccount := DefaultParam .CL2_ACCESS_TOKENS_TOKENS_PER_SERVICE_ACCOUNT 25}}
+{{$replicas := DefaultParam .CL2_ACCESS_TOKENS_REPLICAS 1}}
+{{$qpsPerWorker := DefaultParam .CL2_ACCESS_TOKENS_QPS (MultiplyFloat 2.5 (DivideFloat .Nodes 5000))}}
+
+# TestMetrics measurement variables
+{{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}}
+{{$ENABLE_SYSTEM_POD_METRICS:= DefaultParam .ENABLE_SYSTEM_POD_METRICS true}}
+{{$ENABLE_RESTART_COUNT_CHECK := DefaultParam .ENABLE_RESTART_COUNT_CHECK false}}
+{{$RESTART_COUNT_THRESHOLD_OVERRIDES:= DefaultParam .RESTART_COUNT_THRESHOLD_OVERRIDES ""}}
+
+name: access-tokens
+automanagedNamespaces: {{$namespaces}}
+tuningSets:
+  - name: Sequence
+    parallelismLimitedLoad:
+      parallelismLimit: 1
+steps:
+- name: Starting measurements
+  measurements:
+    - Identifier: APIResponsivenessPrometheus
+      Method: APIResponsivenessPrometheus
+      Params:
+        action: start
+    - Identifier: TestMetrics
+      Method: TestMetrics
+      Params:
+        action: start
+        nodeMode: {{$NODE_MODE}}
+        systemPodMetricsEnabled: {{$ENABLE_SYSTEM_POD_METRICS}}
+        restartCountThresholdOverrides: {{YamlQuote $RESTART_COUNT_THRESHOLD_OVERRIDES 4}}
+        enableRestartCountCheck: {{$ENABLE_RESTART_COUNT_CHECK}}
+
+- name: Creating ServiceAccounts
+  phases:
+    - namespaceRange:
+        min: 1
+        max: {{$namespaces}}
+      replicasPerNamespace: 1
+      tuningSet: Sequence
+      objectBundle:
+        - basename: service-account-getter
+          objectTemplatePath: role.yaml
+    - namespaceRange:
+        min: 1
+        max: {{$namespaces}}
+      replicasPerNamespace: {{$serviceAccounts}}
+      tuningSet: Sequence
+      objectBundle:
+        - basename: account
+          objectTemplatePath: serviceAccount.yaml
+        - basename: account
+          objectTemplatePath: roleBinding.yaml
+          templateFillMap:
+            RoleName: service-account-getter
+
+- name: Creating Tokens
+  phases:
+    {{range $i := Loop $serviceAccounts}}
+    - namespaceRange:
+        min: 1
+        max: {{$namespaces}}
+      replicasPerNamespace: {{$tokensPerServiceAccount}}
+      tuningSet: Sequence
+      objectBundle:
+        - basename: account-{{$i}}
+          objectTemplatePath: token.yaml
+    {{end}}
+
+
+- name: Starting measurement for waiting for pods
+  measurements:
+    - Identifier: WaitForRunningPods
+      Method: WaitForControlledPodsRunning
+      Params:
+        action: start
+        apiVersion: apps/v1
+        kind: Deployment
+        labelSelector: group = access-tokens
+        operationTimeout: 15m
+
+- name: Creating pods
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$serviceAccounts}}
+    tuningSet: Sequence
+    objectBundle:
+    - basename: account
+      objectTemplatePath: deployment.yaml
+      templateFillMap:
+        QpsPerWorker: {{$qpsPerWorker}}
+        Replicas: {{$replicas}}
+        Tokens: {{$tokensPerServiceAccount}}
+
+- name: Waiting for pods to be running
+  measurements:
+    - Identifier: WaitForRunningPods
+      Method: WaitForControlledPodsRunning
+      Params:
+        action: gather
+
+- name: Wait 5min
+  measurements:
+    - Identifier: Wait
+      Method: Sleep
+      Params:
+        duration: 5m
+
+- name: Deleting pods
+  phases:
+    - namespaceRange:
+        min: 1
+        max: {{$namespaces}}
+      replicasPerNamespace: 0
+      tuningSet: Sequence
+      objectBundle:
+        - basename: account
+          objectTemplatePath: deployment.yaml
+          templateFillMap:
+            QpsPerWorker: {{$qpsPerWorker}}
+            Replicas: {{$replicas}}
+            Tokens: {{$tokensPerServiceAccount}}
+
+- name: Waiting for pods to be deleted
+  measurements:
+    - Identifier: WaitForRunningPods
+      Method: WaitForControlledPodsRunning
+      Params:
+        action: gather
+
+- name: Collecting measurements
+  measurements:
+    - Identifier: APIResponsivenessPrometheus
+      Method: APIResponsivenessPrometheus
+      Params:
+        action: gather
+        enableViolations: true
+    - Identifier: TestMetrics
+      Method: TestMetrics
+      Params:
+        action: gather
+        nodeMode: {{$NODE_MODE}}
+        systemPodMetricsEnabled: {{$ENABLE_SYSTEM_POD_METRICS}}
+        restartCountThresholdOverrides: {{YamlQuote $RESTART_COUNT_THRESHOLD_OVERRIDES 4}}
+        enableRestartCountCheck: {{$ENABLE_RESTART_COUNT_CHECK}}
diff --git a/clusterloader2/testing/access-tokens/deployment.yaml b/clusterloader2/testing/access-tokens/deployment.yaml
@@ -0,0 +1,45 @@
+{{$name := .Name}}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{.Name}}
+  labels:
+    group: access-tokens
+spec:
+  selector:
+    matchLabels:
+      group: access-tokens
+      name: {{.Name}}
+  replicas: {{.Replicas}}
+  template:
+    metadata:
+      labels:
+        group: access-tokens
+        name: {{.Name}}
+    spec:
+      imagePullPolicy: Always
+      containers:
+        - name: access-tokens
+          image: gcr.io/k8s-testimages/perf-tests-util/access-tokens:v0.0.6
+          args:
+          {{range $tokenId := Loop .Tokens}}
+            - --access-token-dirs=/var/tokens/{{$name}}-{{$tokenId}}
+          {{end}}
+            - --namespace={{.Namespace}}
+            - --qps-per-worker={{.QpsPerWorker}}
+          resources:
+            requests:
+              cpu: {{AddInt 10 (MultiplyFloat .Tokens .QpsPerWorker)}}m # 1mCpu per Token * per QPS
+              memory: {{AddInt 50 (MultiplyInt .Tokens 5)}}Mi
+          volumeMounts:
+          {{range $j := Loop .Tokens}}
+          - name: {{$name}}-{{$j}}
+            mountPath: /var/tokens/{{$name}}-{{$j}}
+          {{end}}
+      volumes:
+      {{range $j := Loop .Tokens}}
+      - name: {{$name}}-{{$j}}
+        secret:
+          secretName: {{$name}}-{{$j}}
+      {{end}}
diff --git a/clusterloader2/testing/access-tokens/role.yaml b/clusterloader2/testing/access-tokens/role.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{.Name}}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - serviceaccounts
+    verbs:
+      - get
diff --git a/clusterloader2/testing/access-tokens/roleBinding.yaml b/clusterloader2/testing/access-tokens/roleBinding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{.Name}}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{.RoleName}}-0
+subjects:
+  - kind: ServiceAccount
+    name: {{.Name}}
+    namespace: {{.Namespace}}
diff --git a/clusterloader2/testing/access-tokens/serviceAccount.yaml b/clusterloader2/testing/access-tokens/serviceAccount.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{.Name}}
diff --git a/clusterloader2/testing/access-tokens/token.yaml b/clusterloader2/testing/access-tokens/token.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{.Name}}
+  annotations:
+    kubernetes.io/service-account.name: {{.BaseName}}
+type: kubernetes.io/service-account-token