feat: timescaledb and grafana setup for loadtests (#367)

* feat: enable timescaledb through patroni chart

* feat: run all flows under 2 mins for testing

* feat: tags for test runs

* feat: convert datetime string for tags

* feat: updated documentation

* feat: updated github workflow on conditions

.github/workflows/publish-image-k6-runner.yml

@@ -0,0 +1,62 @@
+name: Create and publish K6 Runner Docker image
+
+on:
+ push:
+ branches:
+ - main
+ - dev
+ paths:
+ - 'k6/**'
+ - '.github/workflows/publish-image-k6-runner.yml'
+
+env:
+ GITHUB_REGISTRY: ghcr.io
+ REDHAT_REGISTRY: registry.redhat.io
+ IMAGE_NAME: bcgov/sso-k6-runner
+
+jobs:
+ build-and-push-image:
+ runs-on: ubuntu-20.04
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Log in to the GitHub Container registry
+ uses: docker/login-action@v2
+ with:
+ registry: ${{ env.GITHUB_REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+
+ - name: Cache Docker layers
+ uses: actions/cache@v2
+ with:
+ path: /tmp/.buildx-cache
+ key: ${{ runner.os }}-buildx-${{ github.sha }}
+ restore-keys: |
+ ${{ runner.os }}-buildx-
+
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v3
+ with:
+ context: k6/k6-runner
+ push: true
+ tags: ${{ env.GITHUB_REGISTRY }}/${{env.IMAGE_NAME}}:dev
+ file: k6/k6-runner/Dockerfile
+ cache-from: type=local,src=/tmp/.buildx-cache
+ cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+ # Temp fix
+ # https://github.com/docker/build-push-action/issues/252
+ # https://github.com/moby/buildkit/issues/1896
+ - name: Move cache
+ run: |
+ rm -rf /tmp/.buildx-cache
+ mv /tmp/.buildx-cache-new /tmp/.buildx-cache

helm/timescaledb/README.md

@@ -0,0 +1,48 @@
+# Installing TimescaleDB
+
+## Setup
+
+- Run below to install the patroni managed postgres instance
+
+```sh
+export NAMESPACE=
+
+helm repo add sso-charts https://bcgov.github.io/sso-helm-charts
+
+helm upgrade --install k6-patroni sso-charts/patroni -n ${NAMESPACE} --version 1.6.0 -f ${NAMESPACE}-values.yaml
+```
+
+- Create a database and enable the timescaleDB extension
+
+```sql
+CREATE DATABASE k6loadtests;
+
+\c k6loadtests;
+
+CREATE EXTENSION IF NOT EXISTS timescaledb;
+```
+
+### Grafana
+
+- Existing [grafana](https://sso-grafana-sandbox.apps.gold.devops.gov.bc.ca/) instance is being used to host the dashboards.
+- Ensure a data source exists with below config.
+
+ ```yaml
+ name: 'K6 Load Tests TimeScaleDB'
+ type: postgres
+ url: k6-patroni.xxx.svc.cluster.local:5432
+ database: k6loadtests
+ user: xxx
+ isDefault: false
+ secureJsonData:
+ password: xxx
+ jsonData:
+ sslmode: 'disable' # disable/require/verify-ca/verify-full
+ maxOpenConns: 0 # Grafana v5.4+
+ maxIdleConns: 2 # Grafana v5.4+
+ connMaxLifetime: 14400 # Grafana v5.4+
+ postgresVersion: 15
+ timescaledb: true
+ ```
+
+- Pre-built dashboards can be found [here](https://github.com/grafana/xk6-output-timescaledb/tree/main/grafana/dashboards)](https://github.com/grafana/xk6-output-timescaledb/tree/main/grafana/dashboards)

helm/timescaledb/c6af30-dev-values.yaml

@@ -0,0 +1,27 @@
+nameOverride: k6-patroni
+
+fullnameOverride: k6-patroni
+
+image:
+ # see https://github.com/zalando/spilo/releases
+ repository: ghcr.io/zalando/spilo-15
+ pullPolicy: Always
+ tag: 3.2-p1
+
+replicaCount: 2
+
+postgresMajorVersion: 15
+
+resources:
+ requests:
+ cpu: 50m
+ memory: 128Mi
+ limits:
+ cpu: 200m
+ memory: 256Mi
+
+podDisruptionBudget:
+ enabled: false
+
+persistentVolume:
+ size: 500Mi

k6/k6-runner/Dockerfile

@@ -1,8 +1,18 @@
-FROM loadimpact/k6:latest
+FROM golang:1.20-alpine3.17 as builder
+WORKDIR $GOPATH/src/go.k6.io/k6
+ADD . .
+RUN apk --no-cache add git
+RUN CGO_ENABLED=0 go install go.k6.io/xk6/cmd/xk6@latest
+RUN CGO_ENABLED=0 xk6 build --with github.com/grafana/xk6-output-timescaledb --output /tmp/k6
 
-WORKDIR /var/opt
+FROM alpine:3.17
+RUN apk add --no-cache ca-certificates && \
+ adduser -D -u 12345 -g 12345 k6
+COPY --from=builder /tmp/k6 /usr/bin/k6
 
-COPY /src/tests /var/opt/scripts
-COPY /openshift/k6/start.sh /var/opt
+USER 12345
+WORKDIR /home/k6
 
-ENTRYPOINT [ "sh", "/var/opt/start.sh" ]
+COPY /src/tests /home/k6/scripts
+
+ENTRYPOINT ["k6"]

k6/k6-runner/Makefile

@@ -8,9 +8,9 @@ config:
 
 .PHONY: run_job
 run_job:
- oc -n $(NAMESPACE) process -f ./openshift/k6/dc.yaml | oc -n $(NAMESPACE) apply -f -
+ oc -n $(NAMESPACE) process -f ./openshift/k6/dc.yaml -p DB_PASSWORD=$(DB_PASSWORD) -p DB_USER=$(DB_USER) | oc -n $(NAMESPACE) apply -f -
 
-.PHONY: delete
-delete:
+.PHONY: cleanup
+cleanup:
  oc -n $(NAMESPACE) delete job sso-k6
  oc -n $(NAMESPACE) delete configmap k6-config

k6/k6-runner/README.md

@@ -0,0 +1,27 @@
+# K6 Runner for Load Tests
+
+## Requirements
+
+- `k6-patroni` timescaleDB built on top of postgreSQL managed by patroni
+- `grafana` is the open source analytics & monitoring solution
+
+## Setup
+
+- Ensure `k6-patroni` is running in `c6af30-dev` namespace as a statefulset
+- Copy `sso-keycloak/k6/k6-runner/src/config/config.example.json` to `sso-keycloak/k6/k6-runner/src/config/config.json` and update values accordingly
+
+ ```sh
+ # initialize env vars
+ export NAMESPACE=
+ export DB_USER=
+ export DB_PASSWORD=
+
+ # create configmap with all the configurations
+ make config
+
+ # create a job to run load tests
+ make run_job
+
+ # clean up job and config map
+ make cleanup
+ ```

k6/k6-runner/openshift/k6/dc.yaml

@@ -16,6 +16,14 @@ objects:
  containers:
  - image: ${IMAGE_REPOSITORY}:${IMAGE_TAG}
  name: ${NAME}
+ args:
+ - 'run'
+ - '-e'
+ - 'CONFIG=/var/opt/config/config.json'
+ - '/home/k6/scripts/constantRateAllFlows.js'
+ env:
+ - name: K6_OUT
+ value: 'timescaledb=postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}'
  resources:
  limits:
  cpu: 2
@@ -36,9 +44,18 @@ parameters:
  - name: NAME
  value: sso-k6
  - name: IMAGE_TAG
- value: latest
+ value: dev
  - name: IMAGE_REPOSITORY
- value: ghcr.io/bcgov/sso-k6
- - name: HTTP_DEBUG
- description: enable http debug logging in the k6 pod
- value: "false"
+ value: ghcr.io/bcgov/sso-k6-runner
+ - name: DB_USER
+ value: ''
+ required: true
+ - name: DB_PASSWORD
+ value: ''
+ required: true
+ - name: DB_NAME
+ value: 'k6loadtests'
+ - name: DB_HOST
+ value: 'k6-patroni'
+ - name: DB_PORT
+ value: '5432'