Benchmarking automation with on-demand CI test

.lighthouse/jenkins-x/benchmark.yaml

@@ -0,0 +1,63 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  creationTimestamp: null
+  name: benchmark
+spec:
+  pipelineSpec:
+    tasks:
+    - name: benchmark-test-task
+      taskSpec:
+        stepTemplate:
+          name: ""
+          workingDir: /workspace/source
+        steps:
+        - image: uses:jenkins-x/jx3-pipeline-catalog/tasks/git-clone/git-clone-pr.yaml@versionStream
+          name: ""
+        - name: benchmark-step
+          image: seldonio/core-builder:0.21
+          env:
+            - name: SELDON_E2E_TESTS_TO_RUN
+              value: benchmark
+            - name: SELDON_E2E_TESTS_POD_INFORMATION
+              value: "true"
+          command:
+            - bash
+            - -c
+            - cd testing/scripts && bash kind_test_all.sh
+          volumeMounts:
+          - mountPath: /lib/modules
+            name: modules
+            readOnly: true
+          - mountPath: /sys/fs/cgroup
+            name: cgroup
+          - name: dind-storage
+            mountPath: /var/lib/docker
+          resources:
+            requests:
+              cpu: 4
+              memory: 10000Mi
+              ephemeral-storage: "150Gi"
+            limits:
+              cpu: 4
+              memory: 10000Mi
+              ephemeral-storage: "150Gi"
+          securityContext:
+            privileged: true
+          imagePullPolicy: Always
+        volumes:
+          - name: modules
+            hostPath:
+              path: /lib/modules
+              type: Directory
+          - name: cgroup
+            hostPath:
+              path: /sys/fs/cgroup
+              type: Directory
+          - name: dind-storage
+            emptyDir: {}
+  podTemplate: {}
+  serviceAccountName: tekton-bot
+  timeout: 6h0m0s
+status: {}
+

.lighthouse/jenkins-x/triggers.yaml

@@ -20,9 +20,16 @@ spec:
     context: "release-build-push"
     always_run: false
     optional: false
-    trigger: (?m)^/test( all| release.*),?(s+|$)
+    trigger: (?m)^/test( release.*),?(s+|$)
     rerun_command: "/test release"
     source: "release.yaml"
+  - name: benchmark
+    context: "benchmark"
+    always_run: false
+    optional: false
+    trigger: (?m)^/test( benchmark.*),?(s+|$)
+    rerun_command: "/test benchmark"
+    source: "benchmark.yaml"
   postsubmits:
   - name: release
     context: "release"

ci_build_and_push_images.sh

@@ -71,7 +71,15 @@ function build_push_mock {
     make \
 	-C examples/models/mean_classifier \
 	build \
-	push
+	push && \
+    make \
+    -C testing/docker/echo-model \
+    build_image \
+    push_image && \
+    make \
+    -C testing/docker/fixed-model \
+    build_images \
+    push_images
     MOCK_MODEL_EXIT_VALUE=$?
 }
 

core-builder/Dockerfile

@@ -133,6 +133,19 @@ RUN go get -u github.com/onsi/ginkgo/ginkgo
 # Helm docs
 RUN GO111MODULE=on go get github.com/norwoodj/helm-docs/cmd/helm-docs@f66fdbd6fe
 
+# Argo workflows CLI
+RUN wget https://github.com/argoproj/argo-workflows/releases/download/v3.0.8/argo-linux-amd64.gz && \
+    gunzip argo-linux-amd64.gz && \
+    mv argo-linux-amd64 argo && \
+    chmod a+x argo && \
+    mv argo /usr/local/bin/argo
+
+# Installing jx
+RUN wget https://github.com/jenkins-x/jx-cli/releases/download/v3.1.242/jx-cli-linux-amd64.tar.gz && \
+    tar -zxvf jx-cli-linux-amd64.tar.gz && \
+    chmod a+x jx && \
+    mv jx /usr/local/bin/jx
+
 WORKDIR /work
 
 # Define default command.

core-builder/Makefile

@@ -1,5 +1,5 @@
 DOCKER_IMAGE_NAME=seldonio/core-builder
-DOCKER_IMAGE_VERSION=0.20
+DOCKER_IMAGE_VERSION=0.21
 
 build_docker_image:
 	cp ../testing/scripts/dev_requirements.txt .

doc/source/examples/vegeta_bench_argo_workflows.nblink

@@ -1,3 +1,3 @@
 {
-  "path": "../../../examples/batch/benchmarking-argo-workflows/README.ipynb"
+  "path": "../../../testing/benchmarking/automated-benchmark/README.ipynb"
 }

doc/source/ingress/ambassador.md

@@ -25,7 +25,7 @@ helm install ambassador datawire/ambassador \
 ### Option 2: Ambassador API Gateway (now Emissary Ingress)
 
 The [Ambassador API Gateway](https://www.getambassador.io/products/api-gateway/) (now Emissary Ingress) is the fully open source version of Ambassador Edge Stack and provides all the functionality of a traditional ingress controller.
-Follow the [Ambassador OSS instructions](https://www.getambassador.io/docs/latest/topics/install/install-ambassador-oss/) to install it on your kubernetes cluster.
+Follow the [Ambassador OSS instructions](https://www.getambassador.io/docs/latest/topics/install/) to install it on your kubernetes cluster.
 
 Using `helm` the steps can be summarised as
 ```bash