diff --git a/jenkins-x-integration.yml b/jenkins-x-integration.yml index a02b8645fd..5091e3d309 100644 --- a/jenkins-x-integration.yml +++ b/jenkins-x-integration.yml @@ -26,8 +26,8 @@ pipelineConfig: mountPath: /var/lib/docker resources: requests: - cpu: 1 - memory: 4000Mi + cpu: 4 + memory: 8000Mi securityContext: privileged: true imagePullPolicy: Always diff --git a/jenkins-x.yml b/jenkins-x.yml index a347748add..3a9b0aefeb 100644 --- a/jenkins-x.yml +++ b/jenkins-x.yml @@ -57,7 +57,7 @@ pipelineConfig: resources: requests: cpu: 1 - memory: 4000Mi + memory: 2000Mi securityContext: privileged: true imagePullPolicy: Always diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 9e172231ce..ca468d933b 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,5 +1,8 @@ VERSION := $(shell cat ../../version.txt) +# This can be "Auto" or a Number +PYTEST_WORKERS ?= "4" + kind_create_cluster: kind create cluster --config kind_config.yaml @@ -36,7 +39,7 @@ create_namespaces: kubectl create namespace test1 || echo "Namespace test1 already exists" kubectl config set-context $$(kubectl config current-context) --namespace=seldon -kind_setup: kind_build_images install_helm install_ambassador install_seldon create_namespaces +kind_setup: install_helm install_ambassador install_seldon create_namespaces port-forward-ambassador: kubectl port-forward $$(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080 @@ -61,7 +64,21 @@ install: .PHONY: test test: build_protos install - pytest --verbose -s -W ignore 2>&1 + pip install pytest-xdist + # Run the core tests in parallel + pytest \ + --verbose \ + -s \ + -W ignore \ + --ignore test_s2i_python.py \ + -n $(PYTEST_WORKERS) 2>&1 + # Then run the s2i tests in sequence (as they currently fail in parallel + pytest \ + --verbose \ + -s \ + -W ignore \ + -n 0 \ + test_s2i_python.py 2>&1 .PHONY: clean clean: diff --git a/testing/scripts/conftest.py b/testing/scripts/conftest.py index 4fa3b25167..efd695470f 100644 --- a/testing/scripts/conftest.py +++ b/testing/scripts/conftest.py @@ -1,6 +1,19 @@ import pytest -from k8s_utils import * -from s2i_utils import * +from seldon_e2e_utils import get_s2i_python_version +from subprocess import run + + +@pytest.fixture(scope="session", autouse=True) +def run_pod_information_in_background(request): + # This command runs the pod information and prints it + # in the background every time there's a new update + run( + ( + "kubectl get pods --all-namespaces -w | " + + 'awk \'{print "\\nPODS UPDATE: "$0"\\n"}\' & ' + ), + shell=True, + ) @pytest.fixture(scope="module") @@ -8,8 +21,5 @@ def s2i_python_version(): return do_s2i_python_version() -#### Implementations below - - def do_s2i_python_version(): return get_s2i_python_version() diff --git a/testing/scripts/k8s_utils.py b/testing/scripts/k8s_utils.py deleted file mode 100644 index ec47cf5e2f..0000000000 --- a/testing/scripts/k8s_utils.py +++ /dev/null @@ -1,25 +0,0 @@ -from subprocess import run, Popen -import signal -import subprocess -import os -import time - -from retrying import retry - -API_AMBASSADOR = "localhost:8003" - - -def wait_for_shutdown(deploymentName): - ret = run("kubectl get deploy/" + deploymentName, shell=True) - while ret.returncode == 0: - time.sleep(1) - ret = run("kubectl get deploy/" + deploymentName, shell=True) - - -def get_seldon_version(): - completedProcess = Popen( - "cat ../../version.txt", shell=True, stdout=subprocess.PIPE - ) - output = completedProcess.stdout.readline() - version = output.decode("utf-8").strip() - return version diff --git a/testing/scripts/kind_config.yaml b/testing/scripts/kind_config.yaml index 31a5b75bba..18e8dee0cc 100644 --- a/testing/scripts/kind_config.yaml +++ b/testing/scripts/kind_config.yaml @@ -6,3 +6,25 @@ nodes: extraPortMappings: - containerPort: 30080 hostPort: 8003 +kubeadmConfigPatches: +- | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + metadata: + name: config + kubeReserved: + cpu: "300m" + memory: "300Mi" + ephemeral-storage: "1Gi" + kubeReservedCgroup: "/kube-reserved" + systemReserved: + cpu: "300m" + memory: "300Mi" + ephemeral-storage: "1Gi" + evictionHard: + memory.available: "200Mi" + nodefs.available: "10%" + featureGates: + DynamicKubeletConfig: true + RotateKubeletServerCertificate: true + diff --git a/testing/scripts/kind_test_all.sh b/testing/scripts/kind_test_all.sh index 5c4b261a24..2e69730360 100755 --- a/testing/scripts/kind_test_all.sh +++ b/testing/scripts/kind_test_all.sh @@ -34,13 +34,37 @@ export KUBECONFIG=$(kind get kubeconfig-path) # ONLY RUN THE FOLLOWING IF SUCCESS if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then - # BUILD S2I BASE IMAGES - make s2i_build_base_images - S2I_EXIT_VALUE=$? - # CREATE PROTOS - make build_protos - PROTOS_EXIT_VALUE=$? + echo "Files changed in python folder:" + git --no-pager diff --exit-code --name-only origin/master ../../python + PYTHON_MODIFIED=$? + if [[ $PYTHON_MODIFIED -gt 0 ]]; then + make s2i_build_base_images + else + echo "SKIPPING PYTHON IMAGE BUILD..." + fi + + echo "Files changed in operator folder:" + git --no-pager diff --exit-code --name-only origin/master ../../operator + OPERATOR_MODIFIED=$? + if [[ $OPERATOR_MODIFIED -gt 0 ]]; then + make kind_build_operator + OPERATOR_EXIT_VALUE=$? + else + echo "SKIPPING OPERATOR IMAGE BUILD..." + fi + + echo "Files changed in engine folder:" + git --no-pager diff --exit-code --name-only origin/master ../../engine + ENGINE_MODIFIED=$? + if [[ $ENGINE_MODIFIED -gt 0 ]]; then + make build_protos + PROTO_EXIT_VALUE=$? + make kind_build_engine + ENGINE_EXIT_VALUE=$? + else + echo "SKIPPING ENGINE IMAGE BUILD..." + fi # KIND CLUSTER SETUP make kind_setup diff --git a/testing/scripts/s2i_utils.py b/testing/scripts/s2i_utils.py deleted file mode 100644 index eb51673d2c..0000000000 --- a/testing/scripts/s2i_utils.py +++ /dev/null @@ -1,13 +0,0 @@ -import subprocess -from subprocess import run, Popen - - -def get_s2i_python_version(): - completedProcess = Popen( - "cd ../../wrappers/s2i/python && grep 'IMAGE_VERSION=' Makefile | cut -d'=' -f2", - shell=True, - stdout=subprocess.PIPE, - ) - output = completedProcess.stdout.readline() - version = output.decode("utf-8").rstrip() - return version diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py new file mode 100644 index 0000000000..46613d794d --- /dev/null +++ b/testing/scripts/seldon_e2e_utils.py @@ -0,0 +1,332 @@ +import requests +from requests.auth import HTTPBasicAuth +from seldon_core.proto import prediction_pb2 +from seldon_core.proto import prediction_pb2_grpc +import grpc +import numpy as np +import time +from subprocess import run, Popen +import subprocess +import json +from retrying import retry +import logging + +API_AMBASSADOR = "localhost:8003" + + +def get_s2i_python_version(): + completedProcess = Popen( + "cd ../../wrappers/s2i/python && grep 'IMAGE_VERSION=' Makefile | cut -d'=' -f2", + shell=True, + stdout=subprocess.PIPE, + ) + output = completedProcess.stdout.readline() + version = output.decode("utf-8").rstrip() + return version + + +def get_seldon_version(): + completedProcess = Popen( + "cat ../../version.txt", shell=True, stdout=subprocess.PIPE + ) + output = completedProcess.stdout.readline() + version = output.decode("utf-8").strip() + return version + + +def wait_for_shutdown(deploymentName, namespace): + ret = run(f"kubectl get -n {namespace} deploy/{deploymentName}", shell=True) + while ret.returncode == 0: + time.sleep(1) + ret = run(f"kubectl get -n {namespace} deploy/{deploymentName}", shell=True) + + +def wait_for_rollout(deploymentName, namespace, attempts=20, sleep=5): + for attempts in range(attempts): + ret = run( + f"kubectl rollout status -n {namespace} deploy/" + deploymentName, + shell=True, + ) + if ret.returncode == 0: + logging.warning(f"Successfully waited for deployment {deploymentName}") + break + logging.warning(f"Unsuccessful wait command but retrying for {deploymentName}") + time.sleep(sleep) + assert ret.returncode == 0 + + +def retry_run(cmd, attempts=10, sleep=5): + for i in range(attempts): + ret = run(cmd, shell=True) + if ret.returncode == 0: + logging.warning(f"Successfully ran command: {cmd}") + break + logging.warning(f"Unsuccessful command but retrying: {cmd}") + time.sleep(sleep) + assert ret.returncode == 0 + + +def wait_for_status(name, namespace): + for i in range(7): + completedProcess = run( + f"kubectl get sdep {name} -n {namespace} -o json", + shell=True, + check=True, + stdout=subprocess.PIPE, + ) + jStr = completedProcess.stdout + j = json.loads(jStr) + if "status" in j: + return j + else: + logging.warning("Failed to find status - sleeping") + time.sleep(5) + + +def rest_request( + model, + namespace, + endpoint=API_AMBASSADOR, + data_size=5, + rows=1, + data=None, + dtype="tensor", +): + try: + r = rest_request_ambassador( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + dtype=dtype, + ) + if not r.status_code == 200: + logging.warning("Bad status:", r.status_code) + return None + else: + return r + except Exception as e: + logging.warning("Failed on REST request ", str(e)) + return None + + +def initial_rest_request( + model, + namespace, + endpoint=API_AMBASSADOR, + data_size=5, + rows=1, + data=None, + dtype="tensor", +): + r = rest_request( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + dtype=dtype, + ) + if r is None or r.status_code != 200: + logging.warning("Sleeping 1 sec and trying again") + time.sleep(1) + r = rest_request( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + dtype=dtype, + ) + if r is None or r.status_code != 200: + logging.warning("Sleeping 5 sec and trying again") + time.sleep(5) + r = rest_request( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + dtype=dtype, + ) + if r is None or r.status_code != 200: + logging.warning("Sleeping 10 sec and trying again") + time.sleep(10) + r = rest_request( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + dtype=dtype, + ) + return r + + +def create_random_data(data_size, rows=1): + shape = [rows, data_size] + arr = np.random.rand(rows * data_size) + return (shape, arr) + + +@retry( + wait_exponential_multiplier=1000, + wait_exponential_max=10000, + stop_max_attempt_number=5, +) +def rest_request_ambassador( + deploymentName, + namespace, + endpoint=API_AMBASSADOR, + data_size=5, + rows=1, + data=None, + dtype="tensor", +): + if data is None: + shape, arr = create_random_data(data_size, rows) + elif dtype == "tensor": + shape = data.shape + arr = data.flatten() + else: + arr = data + + if dtype == "tensor": + payload = { + "data": { + "names": ["a", "b"], + "tensor": {"shape": shape, "values": arr.tolist()}, + } + } + else: + payload = {"data": {"names": ["a", "b"], "ndarray": arr}} + + if namespace is None: + response = requests.post( + "http://" + + endpoint + + "/seldon/" + + deploymentName + + "/api/v0.1/predictions", + json=payload, + ) + else: + response = requests.post( + "http://" + + endpoint + + "/seldon/" + + namespace + + "/" + + deploymentName + + "/api/v0.1/predictions", + json=payload, + ) + return response + + +@retry( + wait_exponential_multiplier=1000, + wait_exponential_max=10000, + stop_max_attempt_number=5, +) +def rest_request_ambassador_auth( + deploymentName, + namespace, + username, + password, + endpoint="localhost:8003", + data_size=5, + rows=1, + data=None, +): + if data is None: + shape, arr = create_random_data(data_size, rows) + else: + shape = data.shape + arr = data.flatten() + payload = { + "data": { + "names": ["a", "b"], + "tensor": {"shape": shape, "values": arr.tolist()}, + } + } + if namespace is None: + response = requests.post( + "http://" + + endpoint + + "/seldon/" + + deploymentName + + "/api/v0.1/predictions", + json=payload, + auth=HTTPBasicAuth(username, password), + ) + else: + response = requests.post( + "http://" + + endpoint + + "/seldon/" + + namespace + + "/" + + deploymentName + + "/api/v0.1/predictions", + json=payload, + auth=HTTPBasicAuth(username, password), + ) + return response + + +@retry( + wait_exponential_multiplier=1000, + wait_exponential_max=10000, + stop_max_attempt_number=5, +) +def grpc_request_ambassador( + deploymentName, namespace, endpoint="localhost:8004", data_size=5, rows=1, data=None +): + if data is None: + shape, arr = create_random_data(data_size, rows) + else: + shape = data.shape + arr = data.flatten() + datadef = prediction_pb2.DefaultData( + tensor=prediction_pb2.Tensor(shape=shape, values=arr) + ) + request = prediction_pb2.SeldonMessage(data=datadef) + channel = grpc.insecure_channel(endpoint) + stub = prediction_pb2_grpc.SeldonStub(channel) + if namespace is None: + metadata = [("seldon", deploymentName)] + else: + metadata = [("seldon", deploymentName), ("namespace", namespace)] + response = stub.Predict(request=request, metadata=metadata) + return response + + +def grpc_request_ambassador2( + deploymentName, namespace, endpoint="localhost:8004", data_size=5, rows=1, data=None +): + try: + return grpc_request_ambassador( + deploymentName, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + ) + except: + logging.warning("Warning - caught exception") + return grpc_request_ambassador( + deploymentName, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + ) diff --git a/testing/scripts/seldon_utils.py b/testing/scripts/seldon_utils.py deleted file mode 100644 index ba29639804..0000000000 --- a/testing/scripts/seldon_utils.py +++ /dev/null @@ -1,196 +0,0 @@ -import requests -from requests.auth import HTTPBasicAuth -from seldon_core.proto import prediction_pb2 -from seldon_core.proto import prediction_pb2_grpc -import grpc -import numpy as np -from k8s_utils import * - - -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status -n test1 deploy/" + deploymentName, shell=True) - while ret.returncode > 0: - time.sleep(1) - ret = run( - "kubectl rollout status -n test1 deploy/" + deploymentName, shell=True - ) - - -def rest_request(model, namespace): - try: - r = rest_request_ambassador(model, namespace, API_AMBASSADOR) - if not r.status_code == 200: - print("Bad status:", r.status_code) - return None - else: - return r - except Exception as e: - print("Failed on REST request ", str(e)) - return None - - -def initial_rest_request(model, namespace): - r = rest_request(model, namespace) - if r is None: - print("Sleeping 1 sec and trying again") - time.sleep(1) - r = rest_request(model, namespace) - if r is None: - print("Sleeping 5 sec and trying again") - time.sleep(5) - r = rest_request(model, namespace) - if r is None: - print("Sleeping 10 sec and trying again") - time.sleep(10) - r = rest_request(model, namespace) - return r - - -def create_random_data(data_size, rows=1): - shape = [rows, data_size] - arr = np.random.rand(rows * data_size) - return (shape, arr) - - -@retry( - wait_exponential_multiplier=1000, - wait_exponential_max=10000, - stop_max_attempt_number=5, -) -def rest_request_ambassador( - deploymentName, namespace, endpoint="localhost:8003", data_size=5, rows=1, data=None -): - if data is None: - shape, arr = create_random_data(data_size, rows) - else: - shape = data.shape - arr = data.flatten() - payload = { - "data": { - "names": ["a", "b"], - "tensor": {"shape": shape, "values": arr.tolist()}, - } - } - if namespace is None: - response = requests.post( - "http://" - + endpoint - + "/seldon/" - + deploymentName - + "/api/v0.1/predictions", - json=payload, - ) - else: - response = requests.post( - "http://" - + endpoint - + "/seldon/" - + namespace - + "/" - + deploymentName - + "/api/v0.1/predictions", - json=payload, - ) - return response - - -@retry( - wait_exponential_multiplier=1000, - wait_exponential_max=10000, - stop_max_attempt_number=5, -) -def rest_request_ambassador_auth( - deploymentName, - namespace, - username, - password, - endpoint="localhost:8003", - data_size=5, - rows=1, - data=None, -): - if data is None: - shape, arr = create_random_data(data_size, rows) - else: - shape = data.shape - arr = data.flatten() - payload = { - "data": { - "names": ["a", "b"], - "tensor": {"shape": shape, "values": arr.tolist()}, - } - } - if namespace is None: - response = requests.post( - "http://" - + endpoint - + "/seldon/" - + deploymentName - + "/api/v0.1/predictions", - json=payload, - auth=HTTPBasicAuth(username, password), - ) - else: - response = requests.post( - "http://" - + endpoint - + "/seldon/" - + namespace - + "/" - + deploymentName - + "/api/v0.1/predictions", - json=payload, - auth=HTTPBasicAuth(username, password), - ) - return response - - -@retry( - wait_exponential_multiplier=1000, - wait_exponential_max=10000, - stop_max_attempt_number=5, -) -def grpc_request_ambassador( - deploymentName, namespace, endpoint="localhost:8004", data_size=5, rows=1, data=None -): - if data is None: - shape, arr = create_random_data(data_size, rows) - else: - shape = data.shape - arr = data.flatten() - datadef = prediction_pb2.DefaultData( - tensor=prediction_pb2.Tensor(shape=shape, values=arr) - ) - request = prediction_pb2.SeldonMessage(data=datadef) - channel = grpc.insecure_channel(endpoint) - stub = prediction_pb2_grpc.SeldonStub(channel) - if namespace is None: - metadata = [("seldon", deploymentName)] - else: - metadata = [("seldon", deploymentName), ("namespace", namespace)] - response = stub.Predict(request=request, metadata=metadata) - return response - - -def grpc_request_ambassador2( - deploymentName, namespace, endpoint="localhost:8004", data_size=5, rows=1, data=None -): - try: - return grpc_request_ambassador( - deploymentName, - namespace, - endpoint=endpoint, - data_size=data_size, - rows=rows, - data=data, - ) - except: - print("Warning - caught exception") - return grpc_request_ambassador( - deploymentName, - namespace, - endpoint=endpoint, - data_size=data_size, - rows=rows, - data=data, - ) diff --git a/testing/scripts/test_bad_graphs.py b/testing/scripts/test_bad_graphs.py index 21fd3ce46c..2ec80d1099 100644 --- a/testing/scripts/test_bad_graphs.py +++ b/testing/scripts/test_bad_graphs.py @@ -1,23 +1,6 @@ import subprocess import json -from seldon_utils import * - - -def wait_for_status(name): - for attempts in range(7): - completedProcess = run( - "kubectl get sdep " + name + " -o json -n seldon", - shell=True, - check=True, - stdout=subprocess.PIPE, - ) - jStr = completedProcess.stdout - j = json.loads(jStr) - if "status" in j: - return j - else: - print("Failed to find status - sleeping") - time.sleep(5) +from subprocess import run class TestBadGraphs(object): diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py index 6372465272..d92e57210b 100644 --- a/testing/scripts/test_helm_charts_clusterwide.py +++ b/testing/scripts/test_helm_charts_clusterwide.py @@ -1,82 +1,85 @@ import pytest -from seldon_utils import * -from k8s_utils import * - - -def wait_for_shutdown(deploymentName): - ret = run("kubectl get -n test1 deploy/" + deploymentName, shell=True) - while ret.returncode == 0: - time.sleep(1) - ret = run("kubectl get -n test1 deploy/" + deploymentName, shell=True) +from seldon_e2e_utils import ( + wait_for_rollout, + initial_rest_request, + rest_request_ambassador, + grpc_request_ambassador2, + retry_run, + API_AMBASSADOR, +) +from subprocess import run +import logging class TestClusterWide(object): # Test singe model helm script with 4 API methods def test_single_model(self): - run("helm delete mymodel --purge", shell=True) + namespace = "test-single-model" + retry_run(f"kubectl create namespace {namespace}") run( - "helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", + f"helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) - wait_for_rollout("mymodel-mymodel-7cd068f") - initial_rest_request("mymodel", "test1") - print("Test Ambassador REST gateway") - r = rest_request_ambassador("mymodel", "test1", API_AMBASSADOR) - print(r.json()) + wait_for_rollout(f"mymodel-mymodel-7cd068f", namespace) + initial_rest_request("mymodel", namespace) + logging.warning("Test Ambassador REST gateway") + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) + logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 - print("Test Ambassador gRPC gateway") - r = grpc_request_ambassador2("mymodel", "test1", API_AMBASSADOR) - print(r) - run("helm delete mymodel --purge", shell=True) + logging.warning("Test Ambassador gRPC gateway") + r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR) + logging.warning(r) + run(f"helm delete mymodel --purge", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test AB Test model helm script with 4 API methods def test_abtest_model(self): - run("helm delete myabtest --purge", shell=True) + namespace = "test-abtest-model" + retry_run(f"kubectl create namespace {namespace}") run( - "helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", + f"helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) - wait_for_rollout("myabtest-myabtest-41de5b8") - wait_for_rollout("myabtest-myabtest-df66c5c") - initial_rest_request("myabtest", "test1") - print("Test Ambassador REST gateway") - r = rest_request_ambassador("myabtest", "test1", API_AMBASSADOR) - print(r.json()) + wait_for_rollout("myabtest-myabtest-41de5b8", namespace) + wait_for_rollout("myabtest-myabtest-df66c5c", namespace) + initial_rest_request("myabtest", namespace) + logging.warning("Test Ambassador REST gateway") + r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR) + logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 - print("Test Ambassador gRPC gateway") - print( + logging.warning("Test Ambassador gRPC gateway") + logging.warning( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) - # r = grpc_request_ambassador2("myabtest", "test1", API_AMBASSADOR) - # print(r) - run("helm delete myabtest --purge", shell=True) + run(f"helm delete myabtest --purge", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test MAB Test model helm script with 4 API methods def test_mab_model(self): - run("helm delete mymab --purge", shell=True) + namespace = "test-mab-model" + retry_run(f"kubectl create namespace {namespace}") run( - "helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", + f"helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) - wait_for_rollout("mymab-mymab-41de5b8") - wait_for_rollout("mymab-mymab-b8038b2") - wait_for_rollout("mymab-mymab-df66c5c") - initial_rest_request("mymab", "test1") - print("Test Ambassador REST gateway") - r = rest_request_ambassador("mymab", "test1", API_AMBASSADOR) - print(r.json()) + wait_for_rollout("mymab-mymab-41de5b8", namespace) + wait_for_rollout("mymab-mymab-b8038b2", namespace) + wait_for_rollout("mymab-mymab-df66c5c", namespace) + initial_rest_request("mymab", namespace) + logging.warning("Test Ambassador REST gateway") + r = rest_request_ambassador("mymab", namespace, API_AMBASSADOR) + logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 - print("Test Ambassador gRPC gateway") - print( + logging.warning("Test Ambassador gRPC gateway") + logging.warning( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) - # r = grpc_request_ambassador2("mymab", "test1", API_AMBASSADOR) - # print(r) - run("helm delete mymab --purge", shell=True) + run(f"helm delete mymab --purge", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index dfb9a76061..0e35c4ac86 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -1,79 +1,84 @@ import subprocess import json -from seldon_utils import * -from seldon_core.seldon_client import SeldonClient - - -def wait_for_status(name): - for attempts in range(7): - completedProcess = run( - "kubectl get sdep " + name + " -o json -n seldon", - shell=True, - check=True, - stdout=subprocess.PIPE, - ) - jStr = completedProcess.stdout - j = json.loads(jStr) - if "status" in j and j["status"] == "Available": - return j - else: - print("Failed to find status - sleeping") - time.sleep(5) - - -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) - while ret.returncode > 0: - time.sleep(1) - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) +from seldon_e2e_utils import ( + wait_for_rollout, + initial_rest_request, + retry_run, + create_random_data, + wait_for_status, +) +from subprocess import run +import time +import logging class TestPrepack(object): # Test prepackaged server for sklearn def test_sklearn(self): - run("kubectl delete sdep --all", shell=True) + namespace = "test-sklearn" + retry_run(f"kubectl create namespace {namespace}") + retry_run( + f"kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}" + ) + wait_for_rollout("iris-default-4903e3c", namespace) + wait_for_status("sklearn", namespace) + time.sleep(1) + logging.warning("Initial request") + r = initial_rest_request( + "sklearn", namespace, data=[[0.1, 0.2, 0.3, 0.4]], dtype="ndarray" + ) + assert r.status_code == 200 + logging.warning("Success for test_prepack_sklearn") run( - "kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml", + f"kubectl delete -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", shell=True, - check=True, ) - wait_for_rollout("iris-default-4903e3c") - wait_for_status("sklearn") - print("Initial request") - sc = SeldonClient(deployment_name="sklearn", namespace="seldon") - r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) - assert r.success - print("Success for test_prepack_sklearn") + run(f"kubectl delete namespace {namespace}", shell=True) # Test prepackaged server for tfserving def test_tfserving(self): - run("kubectl delete sdep --all", shell=True) + namespace = "test-tfserving" + retry_run(f"kubectl create namespace {namespace}") + retry_run( + f"kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}" + ) + wait_for_rollout("mnist-default-725903e", namespace) + wait_for_status("tfserving", namespace) + time.sleep(1) + logging.warning("Initial request") + r = initial_rest_request( + "tfserving", + namespace, + data=[create_random_data(784)[1].tolist()], + dtype="ndarray", + ) + assert r.status_code == 200 + logging.warning("Success for test_prepack_tfserving") run( - "kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml", + f"kubectl delete -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", shell=True, - check=True, ) - wait_for_rollout("mnist-default-725903e") - wait_for_status("tfserving") - print("Initial request") - sc = SeldonClient(deployment_name="tfserving", namespace="seldon") - r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 784)) - assert r.success - print("Success for test_prepack_tfserving") + run(f"kubectl delete namespace {namespace}", shell=True) # Test prepackaged server for xgboost def test_xgboost(self): - run("kubectl delete sdep --all", shell=True) + namespace = "test-xgboost" + retry_run(f"kubectl create namespace {namespace}") + retry_run( + f"kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}" + ) + wait_for_rollout("iris-default-af1783b", namespace) + wait_for_status("xgboost", namespace) + time.sleep(1) + logging.warning("Initial request") + r = initial_rest_request( + "xgboost", namespace, data=[[0.1, 0.2, 0.3, 0.4]], dtype="ndarray" + ) + assert r.status_code == 200 + logging.warning("Success for test_prepack_xgboost") run( - "kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml", + f"kubectl delete -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", shell=True, - check=True, ) - wait_for_rollout("iris-default-af1783b") - wait_for_status("xgboost") - print("Initial request") - sc = SeldonClient(deployment_name="xgboost", namespace="seldon") - r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) - assert r.success - print("Success for test_prepack_xgboost") + run(f"kubectl delete namespace {namespace}", shell=True) diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index 8f76c3fad9..8fb8cdb9c1 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -1,40 +1,36 @@ -from seldon_utils import * -from k8s_utils import * - - -def wait_for_shutdown(deploymentName): - ret = run("kubectl get deploy/" + deploymentName, shell=True) - while ret.returncode == 0: - time.sleep(1) - ret = run("kubectl get deploy/" + deploymentName, shell=True) - - -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) - while ret.returncode > 0: - time.sleep(1) - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) +from subprocess import run +from seldon_e2e_utils import ( + wait_for_rollout, + rest_request_ambassador, + initial_rest_request, + retry_run, + API_AMBASSADOR, +) +import time +import logging class TestRollingHttp(object): # Test updating a model with a new image version as the only change def test_rolling_update1(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-1" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph2.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph2.json", shell=True, check=True) i = 0 for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.1" @@ -50,25 +46,30 @@ def test_rolling_update1(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update1") + logging.warning("Success for test_rolling_update1") + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph2.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # test changing the image version and the name of its container def test_rolling_update2(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-2" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph3.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph3.json", shell=True, check=True) i = 0 for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -85,25 +86,30 @@ def test_rolling_update2(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update2") + logging.warning("Success for test_rolling_update2") + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph3.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a new resource request but same image def test_rolling_update3(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-updates-3" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph4.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph4.json", shell=True, check=True) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert res["meta"]["requestPath"][ "complex-model" ] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [ @@ -114,25 +120,30 @@ def test_rolling_update3(self): ] time.sleep(1) assert i == 49 - print("Success for test_rolling_update3") + logging.warning("Success for test_rolling_update3") + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph4.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a multi deployment new model def test_rolling_update4(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-4" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph5.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph5.json", shell=True, check=True) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -147,26 +158,30 @@ def test_rolling_update4(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update4") + logging.warning("Success for test_rolling_update4") + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph5.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model to a multi predictor model def test_rolling_update5(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-5" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph6.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph6.json", shell=True, check=True) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -183,28 +198,31 @@ def test_rolling_update5(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update5") + logging.warning("Success for test_rolling_update5") + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph6.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a new image version as the only change def test_rolling_update6(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-6" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph2svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph2svc.json", shell=True, check=True) i = 0 for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.1" @@ -220,28 +238,31 @@ def test_rolling_update6(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update6") + logging.warning("Success for test_rolling_update6") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph2svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # test changing the image version and the name of its container def test_rolling_update7(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-7" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph3svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph3svc.json", shell=True, check=True) i = 0 for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -258,27 +279,30 @@ def test_rolling_update7(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update7") + logging.warning("Success for test_rolling_update7") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph3svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a new resource request but same image def test_rolling_update8(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-8" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph4svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph4svc.json", shell=True, check=True) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert res["meta"]["requestPath"][ "complex-model" ] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [ @@ -289,27 +313,30 @@ def test_rolling_update8(self): ] time.sleep(1) assert i == 49 - print("Success for test_rolling_update8") + logging.warning("Success for test_rolling_update8") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph4svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a multi deployment new model def test_rolling_update9(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-9" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph5svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph5svc.json", shell=True, check=True) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -324,27 +351,30 @@ def test_rolling_update9(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update9") + logging.warning("Success for test_rolling_update9") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph5svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model to a multi predictor model def test_rolling_update10(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - r = initial_rest_request("mymodel", "seldon") + namespace = "test-rolling-update-10" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph6svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph6svc.json", shell=True, check=True) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -361,4 +391,7 @@ def test_rolling_update10(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update10") + logging.warning("Success for test_rolling_update10") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph6svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index d41a91037c..64b0ac5385 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -1,10 +1,15 @@ import pytest import time -import subprocess -from subprocess import run, Popen -from seldon_utils import * -from k8s_utils import * +from subprocess import run import numpy as np +from seldon_e2e_utils import ( + wait_for_rollout, + rest_request_ambassador, + initial_rest_request, + retry_run, + API_AMBASSADOR, +) +import logging S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" @@ -16,14 +21,14 @@ def create_s2I_image(s2i_python_version, component_type, api_type): .replace("#API#", api_type) .replace("#VERSION#", s2i_python_version) ) - print(cmd) + logging.warning(cmd) run(cmd, shell=True, check=True) def kind_push_s2i_image(component_type, api_type): img = get_image_name(component_type, api_type) cmd = "kind load docker-image " + img + " --loglevel trace" - print(cmd) + logging.warning(cmd) run(cmd, shell=True, check=True) @@ -41,66 +46,67 @@ class TestPythonS2i(object): def test_build_router_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "router", "rest") img = get_image_name("router", "rest") - run("docker run -d --rm --name 'router' " + img, shell=True, check=True) + run("docker run -d --rm --name 'router-rest' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f router", shell=True, check=True) + run("docker rm -f router-rest", shell=True, check=True) def test_build_router_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "router", "grpc") img = get_image_name("router", "grpc") - run("docker run -d --rm --name 'router' " + img, shell=True, check=True) + run("docker run -d --rm --name 'router-grpc' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f router", shell=True, check=True) + run("docker rm -f router-grpc", shell=True, check=True) def test_build_model_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "model", "rest") img = get_image_name("model", "rest") - run("docker run -d --rm --name 'model' " + img, shell=True, check=True) + run("docker run -d --rm --name 'model-rest' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f model", shell=True, check=True) + run("docker rm -f model-rest", shell=True, check=True) def test_build_model_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "model", "grpc") img = get_image_name("model", "grpc") - run("docker run -d --rm --name 'model' " + img, shell=True, check=True) + run("docker run -d --rm --name 'model-grpc' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f model", shell=True, check=True) + run("docker rm -f model-grpc", shell=True, check=True) def test_build_transformer_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "transformer", "rest") img = get_image_name("transformer", "rest") - run("docker run -d --rm --name 'transformer' " + img, shell=True, check=True) + run( + "docker run -d --rm --name 'transformer-rest' " + img, + shell=True, + check=True, + ) time.sleep(2) - run("docker rm -f transformer", shell=True, check=True) + run("docker rm -f transformer-rest", shell=True, check=True) def test_build_transformer_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "transformer", "grpc") img = get_image_name("transformer", "grpc") - run("docker run -d --rm --name 'transformer' " + img, shell=True, check=True) + run( + "docker run -d --rm --name 'transformer-grpc' " + img, + shell=True, + check=True, + ) time.sleep(2) - run("docker rm -f transformer", shell=True, check=True) + run("docker rm -f transformer-grpc", shell=True, check=True) def test_build_combiner_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "combiner", "rest") img = get_image_name("combiner", "rest") - print(img) - run("docker run -d --rm --name 'combiner' " + img, shell=True, check=True) + logging.warning(img) + run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f combiner", shell=True, check=True) + run("docker rm -f combiner-rest", shell=True, check=True) def test_build_combiner_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "combiner", "grpc") img = get_image_name("combiner", "grpc") - run("docker run -d --rm --name 'combiner' " + img, shell=True, check=True) + run("docker run -d --rm --name 'combiner-grpc' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f combiner", shell=True, check=True) - - -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) - while ret.returncode > 0: - time.sleep(1) - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) + run("docker rm -f combiner-grpc", shell=True, check=True) @pytest.mark.usefixtures("s2i_python_version") @@ -128,98 +134,111 @@ def test_combiner_rest(self, s2i_python_version): class S2IK8S(object): def test_model_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-model-rest" + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest") - run( - "kubectl apply -f ../resources/s2i_python_model.json", - shell=True, - check=True, - ) - wait_for_rollout("mymodel-mymodel-8715075") - r = initial_rest_request("mymodel", "seldon") + retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-8715075", namespace) + r = initial_rest_request("mymodel", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_model.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True) def test_input_transformer_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-input-transformer-rest" + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "transformer", "rest") - run( - "kubectl apply -f ../resources/s2i_python_transformer.json", - shell=True, - check=True, + retry_run( + f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" ) - wait_for_rollout("mytrans-mytrans-1f278ae") - r = initial_rest_request("mytrans", "seldon") + wait_for_rollout("mytrans-mytrans-1f278ae", namespace) + r = initial_rest_request("mytrans", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mytrans", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True) def test_output_transformer_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-output-transformer-rest" + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "transformer", "rest") - run( - "kubectl apply -f ../resources/s2i_python_output_transformer.json", - shell=True, - check=True, + retry_run( + f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" ) - wait_for_rollout("mytrans-mytrans-52996cb") - r = initial_rest_request("mytrans", "seldon") + wait_for_rollout("mytrans-mytrans-52996cb", namespace) + r = initial_rest_request("mytrans", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mytrans", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_output_transformer.json -n {namespace}", + shell=True, + ) + run(f"kubectl create namespace {namespace}", shell=True) def test_router_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-router-rest" + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "router", "rest") - run( - "kubectl apply -f ../resources/s2i_python_router.json", - shell=True, - check=True, + retry_run( + f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}" ) - wait_for_rollout("myrouter-myrouter-340ed69") - r = initial_rest_request("myrouter", "seldon") + wait_for_rollout("myrouter-myrouter-340ed69", namespace) + r = initial_rest_request("myrouter", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("myrouter", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("myrouter", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_router.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True) def test_combiner_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-combiner-rest" + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "combiner", "rest") - run( - "kubectl apply -f ../resources/s2i_python_combiner.json", - shell=True, - check=True, + retry_run( + f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" ) - wait_for_rollout("mycombiner-mycombiner-acc7c4d") - r = initial_rest_request("mycombiner", "seldon") + wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) + r = initial_rest_request("mycombiner", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mycombiner", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True)