Improve speed of execution for integration tests

jenkins-x-integration.yml

@@ -26,8 +26,8 @@ pipelineConfig:
                     mountPath: /var/lib/docker
                 resources:
                   requests:
-                    cpu: 1
-                    memory: 4000Mi
+                    cpu: 4
+                    memory: 8000Mi
                 securityContext:
                   privileged: true
                 imagePullPolicy: Always

jenkins-x.yml

@@ -57,7 +57,7 @@ pipelineConfig:
                 resources:
                   requests:
                     cpu: 1
-                    memory: 4000Mi
+                    memory: 2000Mi
                 securityContext:
                   privileged: true
                 imagePullPolicy: Always

testing/scripts/Makefile

@@ -1,5 +1,8 @@
 VERSION := $(shell cat ../../version.txt)
 
+# This can be "Auto" or a Number
+PYTEST_WORKERS ?= "4"
+
 kind_create_cluster:
 	kind create cluster --config kind_config.yaml
 
@@ -36,7 +39,7 @@ create_namespaces:
 	kubectl create namespace test1 || echo "Namespace test1 already exists"
 	kubectl config set-context $$(kubectl config current-context) --namespace=seldon
 
-kind_setup: kind_build_images install_helm install_ambassador install_seldon create_namespaces
+kind_setup: install_helm install_ambassador install_seldon create_namespaces
 
 port-forward-ambassador:
 	kubectl port-forward $$(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080
@@ -61,7 +64,21 @@ install:
 
 .PHONY: test
 test: build_protos install
-	pytest --verbose -s -W ignore 2>&1
+	pip install pytest-xdist
+	# Run the core tests in parallel
+	pytest \
+		--verbose \
+		-s \
+		-W ignore \
+		--ignore test_s2i_python.py \
+		-n $(PYTEST_WORKERS)  2>&1
+	# Then run the s2i tests in sequence (as they currently fail in parallel
+	pytest \
+		--verbose \
+		-s \
+		-W ignore \
+		-n 0 \
+		test_s2i_python.py 2>&1
 
 .PHONY: clean
 clean:

testing/scripts/conftest.py

@@ -1,15 +1,25 @@
 import pytest
-from k8s_utils import *
-from s2i_utils import *
+from seldon_e2e_utils import get_s2i_python_version
+from subprocess import run
+
+
+@pytest.fixture(scope="session", autouse=True)
+def run_pod_information_in_background(request):
+    # This command runs the pod information and prints it
+    #   in the background every time there's a new update
+    run(
+        (
+            "kubectl get pods --all-namespaces -w | "
+            + 'awk \'{print "\\nPODS UPDATE: "$0"\\n"}\' & '
+        ),
+        shell=True,
+    )
 
 
 @pytest.fixture(scope="module")
 def s2i_python_version():
     return do_s2i_python_version()
 
 
-#### Implementations below
-
-
 def do_s2i_python_version():
     return get_s2i_python_version()

testing/scripts/kind_config.yaml

@@ -6,3 +6,25 @@ nodes:
   extraPortMappings:
   - containerPort: 30080
     hostPort: 8003
+kubeadmConfigPatches:
+- |
+  apiVersion: kubelet.config.k8s.io/v1beta1
+  kind: KubeletConfiguration
+  metadata:
+    name: config
+  kubeReserved:
+      cpu: "300m"
+      memory: "300Mi"
+      ephemeral-storage: "1Gi"
+  kubeReservedCgroup: "/kube-reserved"
+  systemReserved:
+      cpu: "300m"
+      memory: "300Mi"
+      ephemeral-storage: "1Gi"
+  evictionHard:
+      memory.available:  "200Mi"
+      nodefs.available: "10%"
+  featureGates:
+      DynamicKubeletConfig: true
+      RotateKubeletServerCertificate: true
+

testing/scripts/kind_test_all.sh

@@ -34,13 +34,37 @@ export KUBECONFIG=$(kind get kubeconfig-path)
 
 # ONLY RUN THE FOLLOWING IF SUCCESS
 if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then
-    # BUILD S2I BASE IMAGES
-    make s2i_build_base_images
-    S2I_EXIT_VALUE=$?
 
-    # CREATE PROTOS
-    make build_protos
-    PROTOS_EXIT_VALUE=$?
+    echo "Files changed in python folder:"
+    git --no-pager diff --exit-code --name-only origin/master ../../python
+    PYTHON_MODIFIED=$?
+    if [[ $PYTHON_MODIFIED -gt 0 ]]; then 
+        make s2i_build_base_images
+    else
+        echo "SKIPPING PYTHON IMAGE BUILD..."
+    fi
+
+    echo "Files changed in operator folder:"
+    git --no-pager diff --exit-code --name-only origin/master ../../operator
+    OPERATOR_MODIFIED=$?
+    if [[ $OPERATOR_MODIFIED -gt 0 ]]; then
+        make kind_build_operator
+        OPERATOR_EXIT_VALUE=$?
+    else
+        echo "SKIPPING OPERATOR IMAGE BUILD..."
+    fi
+
+    echo "Files changed in engine folder:"
+    git --no-pager diff --exit-code --name-only origin/master ../../engine
+    ENGINE_MODIFIED=$?
+    if [[ $ENGINE_MODIFIED -gt 0 ]]; then
+        make build_protos
+        PROTO_EXIT_VALUE=$?
+        make kind_build_engine
+        ENGINE_EXIT_VALUE=$?
+    else
+        echo "SKIPPING ENGINE IMAGE BUILD..."
+    fi
 
     # KIND CLUSTER SETUP
     make kind_setup