From 6337728287bf24a5a49f085946fb47e264edb14d Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sun, 10 Nov 2019 16:11:38 +0000 Subject: [PATCH 01/40] Testing parallel tests --- testing/scripts/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 9e172231ce..8e94e470f3 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -61,7 +61,8 @@ install: .PHONY: test test: build_protos install - pytest --verbose -s -W ignore 2>&1 + pip install pytest-xdist + pytest --verbose -s -W ignore -n8 2>&1 .PHONY: clean clean: From 945847d81bf3c8e59f1f170b59b9faca1b490a6d Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sun, 10 Nov 2019 18:33:18 +0000 Subject: [PATCH 02/40] Added unique namespace per test --- testing/scripts/seldon_utils.py | 9 +- .../scripts/test_helm_charts_clusterwide.py | 54 +-- testing/scripts/test_prepackaged_servers.py | 65 +++- testing/scripts/test_rolling_updates.py | 362 ++++++------------ testing/scripts/test_s2i_python.py | 90 +++-- 5 files changed, 255 insertions(+), 325 deletions(-) diff --git a/testing/scripts/seldon_utils.py b/testing/scripts/seldon_utils.py index ba29639804..928a258012 100644 --- a/testing/scripts/seldon_utils.py +++ b/testing/scripts/seldon_utils.py @@ -7,12 +7,15 @@ from k8s_utils import * -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status -n test1 deploy/" + deploymentName, shell=True) +def wait_for_rollout(deploymentName, namespace): + ret = run( + f"kubectl rollout status -n {namespace} deploy/" + deploymentName, shell=True + ) while ret.returncode > 0: time.sleep(1) ret = run( - "kubectl rollout status -n test1 deploy/" + deploymentName, shell=True + f"kubectl rollout status -n {namespace} deploy/" + deploymentName, + shell=True, ) diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py index 6372465272..0bb07ed388 100644 --- a/testing/scripts/test_helm_charts_clusterwide.py +++ b/testing/scripts/test_helm_charts_clusterwide.py @@ -3,48 +3,51 @@ from k8s_utils import * -def wait_for_shutdown(deploymentName): - ret = run("kubectl get -n test1 deploy/" + deploymentName, shell=True) +def wait_for_shutdown(deploymentName, namespace): + ret = run(f"kubectl get -n {namespace} deploy/" + deploymentName, shell=True) while ret.returncode == 0: time.sleep(1) - ret = run("kubectl get -n test1 deploy/" + deploymentName, shell=True) + ret = run(f"kubectl get -n {namespace} deploy/" + deploymentName, shell=True) class TestClusterWide(object): # Test singe model helm script with 4 API methods def test_single_model(self): - run("helm delete mymodel --purge", shell=True) + namespace = "test-single-model" + run(f"kubectl create namespace {namespace}", shell=true, check=true) run( - "helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", + f"helm install ../../helm-charts/seldon-single-model --name mymodel-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) - wait_for_rollout("mymodel-mymodel-7cd068f") - initial_rest_request("mymodel", "test1") + wait_for_rollout("mymodel-mymodel-7cd068f", namespace) + initial_rest_request("mymodel", namespace) print("Test Ambassador REST gateway") - r = rest_request_ambassador("mymodel", "test1", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) print(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 print("Test Ambassador gRPC gateway") - r = grpc_request_ambassador2("mymodel", "test1", API_AMBASSADOR) + r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR) print(r) - run("helm delete mymodel --purge", shell=True) + run(f"helm delete mymodel-{namespace} --purge", shell=True) + run(f"kubectl delete namespace {namespace}", shell=true, check=true) # Test AB Test model helm script with 4 API methods def test_abtest_model(self): - run("helm delete myabtest --purge", shell=True) + namespace = "test-abtest-model" + run(f"kubectl create namespace {namespace}", shell=true, check=true) run( - "helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", + f"helm install ../../helm-charts/seldon-abtest --name myabtest-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) - wait_for_rollout("myabtest-myabtest-41de5b8") - wait_for_rollout("myabtest-myabtest-df66c5c") - initial_rest_request("myabtest", "test1") + wait_for_rollout("myabtest-myabtest-41de5b8", namespace) + wait_for_rollout("myabtest-myabtest-df66c5c", namespace) + initial_rest_request("myabtest", namespace) print("Test Ambassador REST gateway") - r = rest_request_ambassador("myabtest", "test1", API_AMBASSADOR) + r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR) print(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 @@ -54,22 +57,24 @@ def test_abtest_model(self): ) # r = grpc_request_ambassador2("myabtest", "test1", API_AMBASSADOR) # print(r) - run("helm delete myabtest --purge", shell=True) + run(f"helm delete myabtest-{namespace} --purge", shell=True) + run(f"kubectl delete namespace {namespace}", shell=true, check=true) # Test MAB Test model helm script with 4 API methods def test_mab_model(self): - run("helm delete mymab --purge", shell=True) + namespace = "test-mab-model" + run(f"kubectl create namespace {namespace}", shell=true, check=true) run( - "helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", + f"helm install ../../helm-charts/seldon-mab --name mymab-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) - wait_for_rollout("mymab-mymab-41de5b8") - wait_for_rollout("mymab-mymab-b8038b2") - wait_for_rollout("mymab-mymab-df66c5c") + wait_for_rollout("mymab-mymab-41de5b8", namespace) + wait_for_rollout("mymab-mymab-b8038b2", namespace) + wait_for_rollout("mymab-mymab-df66c5c", namespace) initial_rest_request("mymab", "test1") print("Test Ambassador REST gateway") - r = rest_request_ambassador("mymab", "test1", API_AMBASSADOR) + r = rest_request_ambassador("mymab", namespace, API_AMBASSADOR) print(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 @@ -79,4 +84,5 @@ def test_mab_model(self): ) # r = grpc_request_ambassador2("mymab", "test1", API_AMBASSADOR) # print(r) - run("helm delete mymab --purge", shell=True) + run(f"helm delete mymab-{namespace} --purge", shell=True) + run(f"kubectl delete namespace {namespace}", shell=true, check=true) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index dfb9a76061..c47ef7fd4b 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -4,10 +4,10 @@ from seldon_core.seldon_client import SeldonClient -def wait_for_status(name): +def wait_for_status(name, namespace): for attempts in range(7): completedProcess = run( - "kubectl get sdep " + name + " -o json -n seldon", + f"kubectl get sdep " + name + " -o json -n {namespace}", shell=True, check=True, stdout=subprocess.PIPE, @@ -21,59 +21,84 @@ def wait_for_status(name): time.sleep(5) -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) +def wait_for_rollout(deploymentName, namespace): + ret = run( + f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True + ) while ret.returncode > 0: time.sleep(1) - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) + ret = run( + f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True + ) class TestPrepack(object): # Test prepackaged server for sklearn def test_sklearn(self): - run("kubectl delete sdep --all", shell=True) + namespace = "test-sklearn" + run(f"kubectl create namespace {namespace}", shell=True, check=True) run( - "kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml", + f"kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", shell=True, check=True, ) - wait_for_rollout("iris-default-4903e3c") - wait_for_status("sklearn") + wait_for_rollout("iris-default-4903e3c", namespace) + wait_for_status("sklearn", namespace) print("Initial request") - sc = SeldonClient(deployment_name="sklearn", namespace="seldon") + sc = SeldonClient(deployment_name="sklearn", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) assert r.success print("Success for test_prepack_sklearn") + run( + f"kubectl delete -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test prepackaged server for tfserving def test_tfserving(self): - run("kubectl delete sdep --all", shell=True) + namespace = "test-tfserving" + run(f"kubectl create namespace {namespace}", shell=True, check=True) run( - "kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml", + f"kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", shell=True, check=True, ) - wait_for_rollout("mnist-default-725903e") - wait_for_status("tfserving") + wait_for_rollout("mnist-default-725903e", namespace) + wait_for_status("tfserving", namespace) print("Initial request") - sc = SeldonClient(deployment_name="tfserving", namespace="seldon") + sc = SeldonClient(deployment_name="tfserving", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 784)) assert r.success print("Success for test_prepack_tfserving") + run( + f"kubectl delete -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test prepackaged server for xgboost def test_xgboost(self): - run("kubectl delete sdep --all", shell=True) + namespace = "test-xgboost" + run(f"kubectl create namespace {namespace}", shell=True, check=True) run( - "kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml", + f"kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", shell=True, check=True, ) - wait_for_rollout("iris-default-af1783b") - wait_for_status("xgboost") + wait_for_rollout("iris-default-af1783b", namespace) + wait_for_status("xgboost", namespace) print("Initial request") - sc = SeldonClient(deployment_name="xgboost", namespace="seldon") + sc = SeldonClient(deployment_name="xgboost", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) assert r.success print("Success for test_prepack_xgboost") + run( + f"kubectl delete -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index 8f76c3fad9..373fbee51c 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -2,36 +2,48 @@ from k8s_utils import * -def wait_for_shutdown(deploymentName): - ret = run("kubectl get deploy/" + deploymentName, shell=True) +def wait_for_shutdown(deploymentName, namespace): + ret = run(f"kubectl get deploy/{deploymentName} -n {namespace}", shell=True) while ret.returncode == 0: time.sleep(1) - ret = run("kubectl get deploy/" + deploymentName, shell=True) + ret = run(f"kubectl get deploy/{deploymentName} -n {namespace}", shell=True) -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) +def wait_for_rollout(deploymentName, namespace): + ret = run( + f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True + ) while ret.returncode > 0: time.sleep(1) - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) + ret = run( + f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True + ) class TestRollingHttp(object): # Test updating a model with a new image version as the only change def test_rolling_update1(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") + namespace = "test-rolling-update-1" + run(f"kubectl create namespace {namespace}", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") - r = initial_rest_request("mymodel", "seldon") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph2.json", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph2.json -n {namespace}", + shell=True, + check=True, + ) i = 0 for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() print(res) @@ -51,21 +63,40 @@ def test_rolling_update1(self): time.sleep(1) assert i < 100 print("Success for test_rolling_update1") + run( + f"kubectl delete -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + run( + f"kubectl delete -f ../resources/graph2.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) # test changing the image version and the name of its container def test_rolling_update2(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") + namespace = "test-rolling-update-2" + run(f"kubectl create namespace {namespace}", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") - r = initial_rest_request("mymodel", "seldon") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph3.json", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph3.json -n {namespace}", + shell=True, + check=True, + ) i = 0 for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() print(res) @@ -86,21 +117,40 @@ def test_rolling_update2(self): time.sleep(1) assert i < 100 print("Success for test_rolling_update2") + run( + f"kubectl delete -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + run( + f"kubectl delete -f ../resources/graph3.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test updating a model with a new resource request but same image def test_rolling_update3(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") + namespace = "test-rolling-updates-3" + run(f"kubectl create namespace {namespace}", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") - r = initial_rest_request("mymodel", "seldon") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph4.json", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph4.json -n {namespace}", + shell=True, + check=True, + ) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() print(res) @@ -115,21 +165,40 @@ def test_rolling_update3(self): time.sleep(1) assert i == 49 print("Success for test_rolling_update3") + run( + f"kubectl delete -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + run( + f"kubectl delete -f ../resources/graph4.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test updating a model with a multi deployment new model def test_rolling_update4(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") + namespace = "test-rolling-update-4" + run(f"kubectl create namespace {namespace}", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") - r = initial_rest_request("mymodel", "seldon") + r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph5.json", shell=True, check=True) + run( + f"kubectl apply -f ../resources/graph5.json -n {namespace}", + shell=True, + check=True, + ) i = 0 for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() print(res) @@ -148,217 +217,14 @@ def test_rolling_update4(self): time.sleep(1) assert i < 100 print("Success for test_rolling_update4") - - # Test updating a model to a multi predictor model - def test_rolling_update5(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - run("kubectl apply -f ../resources/graph1.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph6.json", shell=True, check=True) - i = 0 - for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) - assert r.status_code == 200 - res = r.json() - print(res) - assert ( - "complex-model" in res["meta"]["requestPath"] - and res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.1" - and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - ) or ( - res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.2" - and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ) - if (not r.status_code == 200) or ( - res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ): - break - time.sleep(1) - assert i < 100 - print("Success for test_rolling_update5") - - # Test updating a model with a new image version as the only change - def test_rolling_update6(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph2svc.json", shell=True, check=True) - i = 0 - for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) - assert r.status_code == 200 - res = r.json() - print(res) - assert ( - res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.1" - and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - ) or ( - res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.2" - and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ) - if (not r.status_code == 200) or ( - res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ): - break - time.sleep(1) - assert i < 100 - print("Success for test_rolling_update6") - - # test changing the image version and the name of its container - def test_rolling_update7(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - print("Initial request") - r = initial_rest_request("mymodel", "seldon") - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph3svc.json", shell=True, check=True) - i = 0 - for i in range(100): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) - assert r.status_code == 200 - res = r.json() - print(res) - assert ( - "complex-model" in res["meta"]["requestPath"] - and res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.1" - and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - ) or ( - res["meta"]["requestPath"]["complex-model2"] - == "seldonio/fixed-model:0.2" - and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ) - if (not r.status_code == 200) or ( - res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ): - break - time.sleep(1) - assert i < 100 - print("Success for test_rolling_update7") - - # Test updating a model with a new resource request but same image - def test_rolling_update8(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - r = initial_rest_request("mymodel", "seldon") - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph4svc.json", shell=True, check=True) - i = 0 - for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) - assert r.status_code == 200 - res = r.json() - print(res) - assert res["meta"]["requestPath"][ - "complex-model" - ] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [ - 1.0, - 2.0, - 3.0, - 4.0, - ] - time.sleep(1) - assert i == 49 - print("Success for test_rolling_update8") - - # Test updating a model with a multi deployment new model - def test_rolling_update9(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - r = initial_rest_request("mymodel", "seldon") - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph5svc.json", shell=True, check=True) - i = 0 - for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) - assert r.status_code == 200 - res = r.json() - print(res) - assert ( - "complex-model" in res["meta"]["requestPath"] - and res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.1" - and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - ) or ( - res["meta"]["requestPath"]["model1"] == "seldonio/fixed-model:0.1" - and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - and res["meta"]["requestPath"]["model2"] == "seldonio/fixed-model:0.1" - ) - if (not r.status_code == 200) or ("model1" in res["meta"]["requestPath"]): - break - time.sleep(1) - assert i < 100 - print("Success for test_rolling_update9") - - # Test updating a model to a multi predictor model - def test_rolling_update10(self): - run("kubectl delete sdep --all", shell=True) - wait_for_shutdown("mymodel-mymodel-e2eb561") - wait_for_shutdown("mymodel-mymodel-svc-orch-8e2a24b") - run("kubectl apply -f ../resources/graph1svc.json", shell=True, check=True) - wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b") - wait_for_rollout("mymodel-mymodel-e2eb561") - r = initial_rest_request("mymodel", "seldon") - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run("kubectl apply -f ../resources/graph6svc.json", shell=True, check=True) - i = 0 - for i in range(50): - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR) - print("Status code", r.status_code) - assert r.status_code == 200 - res = r.json() - print(res) - assert ( - "complex-model" in res["meta"]["requestPath"] - and res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.1" - and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - ) or ( - res["meta"]["requestPath"]["complex-model"] - == "seldonio/fixed-model:0.2" - and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ) - if (not r.status_code == 200) or ( - res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] - ): - break - time.sleep(1) - assert i < 100 - print("Success for test_rolling_update10") + run( + f"kubectl apply -f ../resources/graph1.json -n {namespace}", + shell=True, + check=True, + ) + run( + f"kubectl delete -f ../resources/graph5.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index d41a91037c..6fe543db28 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -128,98 +128,128 @@ def test_combiner_rest(self, s2i_python_version): class S2IK8S(object): def test_model_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-model-rest" + run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "model", "rest") run( - "kubectl apply -f ../resources/s2i_python_model.json", + f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}", shell=True, check=True, ) - wait_for_rollout("mymodel-mymodel-8715075") - r = initial_rest_request("mymodel", "seldon") + wait_for_rollout("mymodel-mymodel-8715075", namespace) + r = initial_rest_request("mymodel", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) res = r.json() print(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_model.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) def test_input_transformer_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-input-transformer-rest" + run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "transformer", "rest") run( - "kubectl apply -f ../resources/s2i_python_transformer.json", + f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}", shell=True, check=True, ) - wait_for_rollout("mytrans-mytrans-1f278ae") - r = initial_rest_request("mytrans", "seldon") + wait_for_rollout("mytrans-mytrans-1f278ae", namespace) + r = initial_rest_request("mytrans", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mytrans", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) res = r.json() print(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl create namespace {namespace}", shell=True, check=True) def test_output_transformer_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-output-transformer-rest" + run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "transformer", "rest") run( - "kubectl apply -f ../resources/s2i_python_output_transformer.json", + f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}", shell=True, check=True, ) - wait_for_rollout("mytrans-mytrans-52996cb") - r = initial_rest_request("mytrans", "seldon") + wait_for_rollout("mytrans-mytrans-52996cb", namespace) + r = initial_rest_request("mytrans", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mytrans", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) res = r.json() print(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_output_transformer.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl create namespace {namespace}", shell=True, check=True) def test_router_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-router-rest" + run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "router", "rest") run( - "kubectl apply -f ../resources/s2i_python_router.json", + f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}", shell=True, check=True, ) - wait_for_rollout("myrouter-myrouter-340ed69") - r = initial_rest_request("myrouter", "seldon") + wait_for_rollout("myrouter-myrouter-340ed69", namespace) + r = initial_rest_request("myrouter", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("myrouter", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("myrouter", namespace, API_AMBASSADOR, data=arr) res = r.json() print(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_router.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) def test_combiner_rest(self, s2i_python_version): - run("kubectl delete sdep --all", shell=True) + namespace = "s2i-test-combiner-rest" + run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "combiner", "rest") run( - "kubectl apply -f ../resources/s2i_python_combiner.json", + f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}", shell=True, check=True, ) - wait_for_rollout("mycombiner-mycombiner-acc7c4d") - r = initial_rest_request("mycombiner", "seldon") + wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) + r = initial_rest_request("mycombiner", namespace) arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mycombiner", "seldon", API_AMBASSADOR, data=arr) + r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) res = r.json() print(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] - run("kubectl delete sdep --all", shell=True) + run( + f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", + shell=True, + check=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) From b06b9cd5aec3afed32a34d3acf6cfbc9ddbec541 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sun, 10 Nov 2019 19:28:37 +0000 Subject: [PATCH 03/40] Cleaned and fixed namespaces --- testing/scripts/Makefile | 2 +- testing/scripts/k8s_utils.py | 25 -------- testing/scripts/s2i_utils.py | 13 ----- .../{seldon_utils.py => seldon_e2e_utils.py} | 52 ++++++++++++++++- testing/scripts/test_bad_graphs.py | 19 +----- .../scripts/test_helm_charts_clusterwide.py | 31 +++++----- testing/scripts/test_prepackaged_servers.py | 32 +--------- testing/scripts/test_rolling_updates.py | 28 +++------ testing/scripts/test_s2i_python.py | 58 ++++++++++--------- 9 files changed, 109 insertions(+), 151 deletions(-) delete mode 100644 testing/scripts/k8s_utils.py delete mode 100644 testing/scripts/s2i_utils.py rename testing/scripts/{seldon_utils.py => seldon_e2e_utils.py} (78%) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 8e94e470f3..a5eb9d46e3 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -62,7 +62,7 @@ install: .PHONY: test test: build_protos install pip install pytest-xdist - pytest --verbose -s -W ignore -n8 2>&1 + pytest --verbose -s -W ignore -n auto 2>&1 .PHONY: clean clean: diff --git a/testing/scripts/k8s_utils.py b/testing/scripts/k8s_utils.py deleted file mode 100644 index ec47cf5e2f..0000000000 --- a/testing/scripts/k8s_utils.py +++ /dev/null @@ -1,25 +0,0 @@ -from subprocess import run, Popen -import signal -import subprocess -import os -import time - -from retrying import retry - -API_AMBASSADOR = "localhost:8003" - - -def wait_for_shutdown(deploymentName): - ret = run("kubectl get deploy/" + deploymentName, shell=True) - while ret.returncode == 0: - time.sleep(1) - ret = run("kubectl get deploy/" + deploymentName, shell=True) - - -def get_seldon_version(): - completedProcess = Popen( - "cat ../../version.txt", shell=True, stdout=subprocess.PIPE - ) - output = completedProcess.stdout.readline() - version = output.decode("utf-8").strip() - return version diff --git a/testing/scripts/s2i_utils.py b/testing/scripts/s2i_utils.py deleted file mode 100644 index eb51673d2c..0000000000 --- a/testing/scripts/s2i_utils.py +++ /dev/null @@ -1,13 +0,0 @@ -import subprocess -from subprocess import run, Popen - - -def get_s2i_python_version(): - completedProcess = Popen( - "cd ../../wrappers/s2i/python && grep 'IMAGE_VERSION=' Makefile | cut -d'=' -f2", - shell=True, - stdout=subprocess.PIPE, - ) - output = completedProcess.stdout.readline() - version = output.decode("utf-8").rstrip() - return version diff --git a/testing/scripts/seldon_utils.py b/testing/scripts/seldon_e2e_utils.py similarity index 78% rename from testing/scripts/seldon_utils.py rename to testing/scripts/seldon_e2e_utils.py index 928a258012..545115f4f4 100644 --- a/testing/scripts/seldon_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -4,7 +4,40 @@ from seldon_core.proto import prediction_pb2_grpc import grpc import numpy as np -from k8s_utils import * +import time +from subprocess import run, Popen +import subprocess +import json +from retrying import retry + +API_AMBASSADOR = "localhost:8003" + + +def get_s2i_python_version(): + completedProcess = Popen( + "cd ../../wrappers/s2i/python && grep 'IMAGE_VERSION=' Makefile | cut -d'=' -f2", + shell=True, + stdout=subprocess.PIPE, + ) + output = completedProcess.stdout.readline() + version = output.decode("utf-8").rstrip() + return version + + +def get_seldon_version(): + completedProcess = Popen( + "cat ../../version.txt", shell=True, stdout=subprocess.PIPE + ) + output = completedProcess.stdout.readline() + version = output.decode("utf-8").strip() + return version + + +def wait_for_shutdown(deploymentName, namespace): + ret = run(f"kubectl get -n {namespace} deploy/{deploymentName}", shell=True) + while ret.returncode == 0: + time.sleep(1) + ret = run(f"kubectl get -n {namespace} deploy/{deploymentName}", shell=True) def wait_for_rollout(deploymentName, namespace): @@ -19,6 +52,23 @@ def wait_for_rollout(deploymentName, namespace): ) +def wait_for_status(name, namespace): + for attempts in range(7): + completedProcess = run( + f"kubectl get sdep {name} -n {namespace} -o json", + shell=True, + check=True, + stdout=subprocess.PIPE, + ) + jStr = completedProcess.stdout + j = json.loads(jStr) + if "status" in j: + return j + else: + print("Failed to find status - sleeping") + time.sleep(5) + + def rest_request(model, namespace): try: r = rest_request_ambassador(model, namespace, API_AMBASSADOR) diff --git a/testing/scripts/test_bad_graphs.py b/testing/scripts/test_bad_graphs.py index 21fd3ce46c..2ec80d1099 100644 --- a/testing/scripts/test_bad_graphs.py +++ b/testing/scripts/test_bad_graphs.py @@ -1,23 +1,6 @@ import subprocess import json -from seldon_utils import * - - -def wait_for_status(name): - for attempts in range(7): - completedProcess = run( - "kubectl get sdep " + name + " -o json -n seldon", - shell=True, - check=True, - stdout=subprocess.PIPE, - ) - jStr = completedProcess.stdout - j = json.loads(jStr) - if "status" in j: - return j - else: - print("Failed to find status - sleeping") - time.sleep(5) +from subprocess import run class TestBadGraphs(object): diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py index 0bb07ed388..1f5762215d 100644 --- a/testing/scripts/test_helm_charts_clusterwide.py +++ b/testing/scripts/test_helm_charts_clusterwide.py @@ -1,13 +1,12 @@ import pytest -from seldon_utils import * -from k8s_utils import * - - -def wait_for_shutdown(deploymentName, namespace): - ret = run(f"kubectl get -n {namespace} deploy/" + deploymentName, shell=True) - while ret.returncode == 0: - time.sleep(1) - ret = run(f"kubectl get -n {namespace} deploy/" + deploymentName, shell=True) +from seldon_e2e_utils import ( + wait_for_rollout, + initial_rest_request, + rest_request_ambassador, + grpc_request_ambassador2, + API_AMBASSADOR, +) +from subprocess import run class TestClusterWide(object): @@ -15,7 +14,7 @@ class TestClusterWide(object): # Test singe model helm script with 4 API methods def test_single_model(self): namespace = "test-single-model" - run(f"kubectl create namespace {namespace}", shell=true, check=true) + run(f"kubectl create namespace {namespace}", shell=True, check=True) run( f"helm install ../../helm-charts/seldon-single-model --name mymodel-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, @@ -32,12 +31,12 @@ def test_single_model(self): r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR) print(r) run(f"helm delete mymodel-{namespace} --purge", shell=True) - run(f"kubectl delete namespace {namespace}", shell=true, check=true) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test AB Test model helm script with 4 API methods def test_abtest_model(self): namespace = "test-abtest-model" - run(f"kubectl create namespace {namespace}", shell=true, check=true) + run(f"kubectl create namespace {namespace}", shell=True, check=True) run( f"helm install ../../helm-charts/seldon-abtest --name myabtest-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, @@ -55,15 +54,13 @@ def test_abtest_model(self): print( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) - # r = grpc_request_ambassador2("myabtest", "test1", API_AMBASSADOR) - # print(r) run(f"helm delete myabtest-{namespace} --purge", shell=True) - run(f"kubectl delete namespace {namespace}", shell=true, check=true) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test MAB Test model helm script with 4 API methods def test_mab_model(self): namespace = "test-mab-model" - run(f"kubectl create namespace {namespace}", shell=true, check=true) + run(f"kubectl create namespace {namespace}", shell=True, check=True) run( f"helm install ../../helm-charts/seldon-mab --name mymab-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, @@ -85,4 +82,4 @@ def test_mab_model(self): # r = grpc_request_ambassador2("mymab", "test1", API_AMBASSADOR) # print(r) run(f"helm delete mymab-{namespace} --purge", shell=True) - run(f"kubectl delete namespace {namespace}", shell=true, check=true) + run(f"kubectl delete namespace {namespace}", shell=True, check=True) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index c47ef7fd4b..2b702da113 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -1,35 +1,9 @@ import subprocess import json -from seldon_utils import * from seldon_core.seldon_client import SeldonClient - - -def wait_for_status(name, namespace): - for attempts in range(7): - completedProcess = run( - f"kubectl get sdep " + name + " -o json -n {namespace}", - shell=True, - check=True, - stdout=subprocess.PIPE, - ) - jStr = completedProcess.stdout - j = json.loads(jStr) - if "status" in j and j["status"] == "Available": - return j - else: - print("Failed to find status - sleeping") - time.sleep(5) - - -def wait_for_rollout(deploymentName, namespace): - ret = run( - f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True - ) - while ret.returncode > 0: - time.sleep(1) - ret = run( - f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True - ) +from seldon_e2e_utils import wait_for_rollout, wait_for_status +from subprocess import run +import time class TestPrepack(object): diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index 373fbee51c..58cb69dff9 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -1,23 +1,11 @@ -from seldon_utils import * -from k8s_utils import * - - -def wait_for_shutdown(deploymentName, namespace): - ret = run(f"kubectl get deploy/{deploymentName} -n {namespace}", shell=True) - while ret.returncode == 0: - time.sleep(1) - ret = run(f"kubectl get deploy/{deploymentName} -n {namespace}", shell=True) - - -def wait_for_rollout(deploymentName, namespace): - ret = run( - f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True - ) - while ret.returncode > 0: - time.sleep(1) - ret = run( - f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True - ) +from subprocess import run +from seldon_e2e_utils import ( + wait_for_rollout, + rest_request_ambassador, + initial_rest_request, + API_AMBASSADOR, +) +import time class TestRollingHttp(object): diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index 6fe543db28..a4534efea3 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -1,10 +1,13 @@ import pytest import time -import subprocess -from subprocess import run, Popen -from seldon_utils import * -from k8s_utils import * +from subprocess import run import numpy as np +from seldon_e2e_utils import ( + wait_for_rollout, + rest_request_ambassador, + initial_rest_request, + API_AMBASSADOR, +) S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" @@ -41,66 +44,67 @@ class TestPythonS2i(object): def test_build_router_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "router", "rest") img = get_image_name("router", "rest") - run("docker run -d --rm --name 'router' " + img, shell=True, check=True) + run("docker run -d --rm --name 'router-rest' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f router", shell=True, check=True) + run("docker rm -f router-rest", shell=True, check=True) def test_build_router_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "router", "grpc") img = get_image_name("router", "grpc") - run("docker run -d --rm --name 'router' " + img, shell=True, check=True) + run("docker run -d --rm --name 'router-grpc' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f router", shell=True, check=True) + run("docker rm -f router-grpc", shell=True, check=True) def test_build_model_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "model", "rest") img = get_image_name("model", "rest") - run("docker run -d --rm --name 'model' " + img, shell=True, check=True) + run("docker run -d --rm --name 'model-rest' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f model", shell=True, check=True) + run("docker rm -f model-rest", shell=True, check=True) def test_build_model_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "model", "grpc") img = get_image_name("model", "grpc") - run("docker run -d --rm --name 'model' " + img, shell=True, check=True) + run("docker run -d --rm --name 'model-grpc' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f model", shell=True, check=True) + run("docker rm -f model-grpc", shell=True, check=True) def test_build_transformer_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "transformer", "rest") img = get_image_name("transformer", "rest") - run("docker run -d --rm --name 'transformer' " + img, shell=True, check=True) + run( + "docker run -d --rm --name 'transformer-rest' " + img, + shell=True, + check=True, + ) time.sleep(2) - run("docker rm -f transformer", shell=True, check=True) + run("docker rm -f transformer-rest", shell=True, check=True) def test_build_transformer_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "transformer", "grpc") img = get_image_name("transformer", "grpc") - run("docker run -d --rm --name 'transformer' " + img, shell=True, check=True) + run( + "docker run -d --rm --name 'transformer-grpc' " + img, + shell=True, + check=True, + ) time.sleep(2) - run("docker rm -f transformer", shell=True, check=True) + run("docker rm -f transformer-grpc", shell=True, check=True) def test_build_combiner_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "combiner", "rest") img = get_image_name("combiner", "rest") print(img) - run("docker run -d --rm --name 'combiner' " + img, shell=True, check=True) + run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f combiner", shell=True, check=True) + run("docker rm -f combiner-rest", shell=True, check=True) def test_build_combiner_grpc(self, s2i_python_version): create_s2I_image(s2i_python_version, "combiner", "grpc") img = get_image_name("combiner", "grpc") - run("docker run -d --rm --name 'combiner' " + img, shell=True, check=True) + run("docker run -d --rm --name 'combiner-grpc' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f combiner", shell=True, check=True) - - -def wait_for_rollout(deploymentName): - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) - while ret.returncode > 0: - time.sleep(1) - ret = run("kubectl rollout status deploy/" + deploymentName, shell=True) + run("docker rm -f combiner-grpc", shell=True, check=True) @pytest.mark.usefixtures("s2i_python_version") From e90be22afe26e719df061e96f9886cb66fc58235 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sun, 10 Nov 2019 19:43:33 +0000 Subject: [PATCH 04/40] Updated conftest typo --- testing/scripts/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/scripts/conftest.py b/testing/scripts/conftest.py index 4fa3b25167..fd31208ba0 100644 --- a/testing/scripts/conftest.py +++ b/testing/scripts/conftest.py @@ -1,6 +1,5 @@ import pytest -from k8s_utils import * -from s2i_utils import * +from seldon_e2e_utils import get_s2i_python_version @pytest.fixture(scope="module") From 3231b979ec764565a6a40725cd936df58ebd8be5 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sun, 10 Nov 2019 20:55:29 +0000 Subject: [PATCH 05/40] Updated to support retries on apply --- testing/scripts/seldon_e2e_utils.py | 25 ++++-- .../scripts/test_helm_charts_clusterwide.py | 4 +- testing/scripts/test_prepackaged_servers.py | 25 +++--- testing/scripts/test_rolling_updates.py | 89 +++++-------------- testing/scripts/test_s2i_python.py | 36 +++----- 5 files changed, 62 insertions(+), 117 deletions(-) diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index 545115f4f4..bbacc890e7 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -40,16 +40,29 @@ def wait_for_shutdown(deploymentName, namespace): ret = run(f"kubectl get -n {namespace} deploy/{deploymentName}", shell=True) -def wait_for_rollout(deploymentName, namespace): - ret = run( - f"kubectl rollout status -n {namespace} deploy/" + deploymentName, shell=True - ) - while ret.returncode > 0: - time.sleep(1) +def wait_for_rollout(deploymentName, namespace, attempts=50, sleep=5): + for attempts in range(attempts): ret = run( f"kubectl rollout status -n {namespace} deploy/" + deploymentName, shell=True, ) + if ret.returncode == 0: + print(f"Successfully waited for deployment {deploymentName}") + break + print(f"Unsuccessful wait command but retrying for {deploymentName}") + time.sleep(sleep) + assert ret.returncode == 0 + + +def retry_run(cmd, attempts=3, sleep=5): + for attempts in range(attempts): + ret = run(cmd, shell=True) + if ret.returncode == 0: + print(f"Successfully ran command: {cmd}") + break + print(f"Unsuccessful command but retrying: {cmd}") + time.sleep(sleep) + assert ret.returncode == 0 def wait_for_status(name, namespace): diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py index 1f5762215d..f7d3143dd5 100644 --- a/testing/scripts/test_helm_charts_clusterwide.py +++ b/testing/scripts/test_helm_charts_clusterwide.py @@ -69,7 +69,7 @@ def test_mab_model(self): wait_for_rollout("mymab-mymab-41de5b8", namespace) wait_for_rollout("mymab-mymab-b8038b2", namespace) wait_for_rollout("mymab-mymab-df66c5c", namespace) - initial_rest_request("mymab", "test1") + initial_rest_request("mymab", namespace) print("Test Ambassador REST gateway") r = rest_request_ambassador("mymab", namespace, API_AMBASSADOR) print(r.json()) @@ -79,7 +79,5 @@ def test_mab_model(self): print( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) - # r = grpc_request_ambassador2("mymab", "test1", API_AMBASSADOR) - # print(r) run(f"helm delete mymab-{namespace} --purge", shell=True) run(f"kubectl delete namespace {namespace}", shell=True, check=True) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index 2b702da113..f0a5dcaa58 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -1,7 +1,11 @@ import subprocess import json from seldon_core.seldon_client import SeldonClient -from seldon_e2e_utils import wait_for_rollout, wait_for_status +from seldon_e2e_utils import ( + wait_for_rollout, + retry_run, + wait_for_status, +) from subprocess import run import time @@ -12,10 +16,8 @@ class TestPrepack(object): def test_sklearn(self): namespace = "test-sklearn" run(f"kubectl create namespace {namespace}", shell=True, check=True) - run( + retry_run( f"kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("iris-default-4903e3c", namespace) wait_for_status("sklearn", namespace) @@ -27,18 +29,15 @@ def test_sklearn(self): run( f"kubectl delete -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", shell=True, - check=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test prepackaged server for tfserving def test_tfserving(self): namespace = "test-tfserving" run(f"kubectl create namespace {namespace}", shell=True, check=True) - run( + retry_run( f"kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("mnist-default-725903e", namespace) wait_for_status("tfserving", namespace) @@ -50,18 +49,15 @@ def test_tfserving(self): run( f"kubectl delete -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", shell=True, - check=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test prepackaged server for xgboost def test_xgboost(self): namespace = "test-xgboost" run(f"kubectl create namespace {namespace}", shell=True, check=True) - run( + retry_run( f"kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("iris-default-af1783b", namespace) wait_for_status("xgboost", namespace) @@ -73,6 +69,5 @@ def test_xgboost(self): run( f"kubectl delete -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", shell=True, - check=True, ) run(f"kubectl delete namespace {namespace}", shell=True, check=True) diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index 58cb69dff9..3480d91031 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -3,6 +3,7 @@ wait_for_rollout, rest_request_ambassador, initial_rest_request, + retry_run, API_AMBASSADOR, ) import time @@ -14,21 +15,13 @@ class TestRollingHttp(object): def test_rolling_update1(self): namespace = "test-rolling-update-1" run(f"kubectl create namespace {namespace}", shell=True, check=True) - run( - f"kubectl apply -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run( - f"kubectl apply -f ../resources/graph2.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph2.json -n {namespace}",) i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -52,36 +45,24 @@ def test_rolling_update1(self): assert i < 100 print("Success for test_rolling_update1") run( - f"kubectl delete -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, ) run( - f"kubectl delete -f ../resources/graph2.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph2.json -n {namespace}", shell=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) # test changing the image version and the name of its container def test_rolling_update2(self): namespace = "test-rolling-update-2" run(f"kubectl create namespace {namespace}", shell=True, check=True) - run( - f"kubectl apply -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run( - f"kubectl apply -f ../resources/graph3.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph3.json -n {namespace}",) i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -106,36 +87,24 @@ def test_rolling_update2(self): assert i < 100 print("Success for test_rolling_update2") run( - f"kubectl delete -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, ) run( - f"kubectl delete -f ../resources/graph3.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph3.json -n {namespace}", shell=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a new resource request but same image def test_rolling_update3(self): namespace = "test-rolling-updates-3" run(f"kubectl create namespace {namespace}", shell=True, check=True) - run( - f"kubectl apply -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run( - f"kubectl apply -f ../resources/graph4.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph4.json -n {namespace}",) i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -154,36 +123,24 @@ def test_rolling_update3(self): assert i == 49 print("Success for test_rolling_update3") run( - f"kubectl delete -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, ) run( - f"kubectl delete -f ../resources/graph4.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph4.json -n {namespace}", shell=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a multi deployment new model def test_rolling_update4(self): namespace = "test-rolling-update-4" run(f"kubectl create namespace {namespace}", shell=True, check=True) - run( - f"kubectl apply -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - run( - f"kubectl apply -f ../resources/graph5.json -n {namespace}", - shell=True, - check=True, - ) + retry_run(f"kubectl apply -f ../resources/graph5.json -n {namespace}",) i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -206,13 +163,9 @@ def test_rolling_update4(self): assert i < 100 print("Success for test_rolling_update4") run( - f"kubectl apply -f ../resources/graph1.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, ) run( - f"kubectl delete -f ../resources/graph5.json -n {namespace}", - shell=True, - check=True, + f"kubectl delete -f ../resources/graph5.json -n {namespace}", shell=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index a4534efea3..2cfd985d7d 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -6,6 +6,7 @@ wait_for_rollout, rest_request_ambassador, initial_rest_request, + retry_run, API_AMBASSADOR, ) @@ -135,10 +136,8 @@ def test_model_rest(self, s2i_python_version): namespace = "s2i-test-model-rest" run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "model", "rest") - run( + retry_run( f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("mymodel-mymodel-8715075", namespace) r = initial_rest_request("mymodel", namespace) @@ -152,18 +151,15 @@ def test_model_rest(self, s2i_python_version): run( f"kubectl delete -f ../resources/s2i_python_model.json -n {namespace}", shell=True, - check=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) def test_input_transformer_rest(self, s2i_python_version): namespace = "s2i-test-input-transformer-rest" run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "transformer", "rest") - run( + retry_run( f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("mytrans-mytrans-1f278ae", namespace) r = initial_rest_request("mytrans", namespace) @@ -177,18 +173,15 @@ def test_input_transformer_rest(self, s2i_python_version): run( f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", shell=True, - check=True, ) - run(f"kubectl create namespace {namespace}", shell=True, check=True) + run(f"kubectl create namespace {namespace}", shell=True) def test_output_transformer_rest(self, s2i_python_version): namespace = "s2i-test-output-transformer-rest" run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "transformer", "rest") - run( + retry_run( f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("mytrans-mytrans-52996cb", namespace) r = initial_rest_request("mytrans", namespace) @@ -202,19 +195,16 @@ def test_output_transformer_rest(self, s2i_python_version): run( f"kubectl delete -f ../resources/s2i_python_output_transformer.json -n {namespace}", shell=True, - check=True, ) - run(f"kubectl create namespace {namespace}", shell=True, check=True) + run(f"kubectl create namespace {namespace}", shell=True) def test_router_rest(self, s2i_python_version): namespace = "s2i-test-router-rest" run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "router", "rest") - run( + retry_run( f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("myrouter-myrouter-340ed69", namespace) r = initial_rest_request("myrouter", namespace) @@ -228,19 +218,16 @@ def test_router_rest(self, s2i_python_version): run( f"kubectl delete -f ../resources/s2i_python_router.json -n {namespace}", shell=True, - check=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) def test_combiner_rest(self, s2i_python_version): namespace = "s2i-test-combiner-rest" run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "combiner", "rest") - run( + retry_run( f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}", - shell=True, - check=True, ) wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) r = initial_rest_request("mycombiner", namespace) @@ -254,6 +241,5 @@ def test_combiner_rest(self, s2i_python_version): run( f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", shell=True, - check=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) From 2680d01877efdb49963ca0f4035841f8cb14f71f Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sun, 10 Nov 2019 22:04:04 +0000 Subject: [PATCH 06/40] Updated to support retries on apply --- testing/scripts/seldon_e2e_utils.py | 6 +++--- testing/scripts/test_prepackaged_servers.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index bbacc890e7..b914a2f424 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -54,8 +54,8 @@ def wait_for_rollout(deploymentName, namespace, attempts=50, sleep=5): assert ret.returncode == 0 -def retry_run(cmd, attempts=3, sleep=5): - for attempts in range(attempts): +def retry_run(cmd, attempts=10, sleep=5): + for i in range(attempts): ret = run(cmd, shell=True) if ret.returncode == 0: print(f"Successfully ran command: {cmd}") @@ -66,7 +66,7 @@ def retry_run(cmd, attempts=3, sleep=5): def wait_for_status(name, namespace): - for attempts in range(7): + for i in range(7): completedProcess = run( f"kubectl get sdep {name} -n {namespace} -o json", shell=True, diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index f0a5dcaa58..e2205065b6 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -70,4 +70,4 @@ def test_xgboost(self): f"kubectl delete -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", shell=True, ) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) From cf6d0550c3a1dc4943fc1db93c703e14a77811ed Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 05:44:49 +0000 Subject: [PATCH 07/40] Changed make to have optional number of workers --- testing/scripts/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index a5eb9d46e3..c0de724188 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,4 +1,5 @@ VERSION := $(shell cat ../../version.txt) +PYTEST_WORKERS ?= "auto" kind_create_cluster: kind create cluster --config kind_config.yaml @@ -62,7 +63,7 @@ install: .PHONY: test test: build_protos install pip install pytest-xdist - pytest --verbose -s -W ignore -n auto 2>&1 + pytest --verbose -s -W ignore -n $$PYTEST_WORKERS 2>&1 .PHONY: clean clean: From a0bca4342b82f2f4dbf35d863f78eb96bc995784 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 06:14:13 +0000 Subject: [PATCH 08/40] Added limits to ensure enough memory for kube system --- testing/scripts/kind_config.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/testing/scripts/kind_config.yaml b/testing/scripts/kind_config.yaml index 31a5b75bba..18e8dee0cc 100644 --- a/testing/scripts/kind_config.yaml +++ b/testing/scripts/kind_config.yaml @@ -6,3 +6,25 @@ nodes: extraPortMappings: - containerPort: 30080 hostPort: 8003 +kubeadmConfigPatches: +- | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + metadata: + name: config + kubeReserved: + cpu: "300m" + memory: "300Mi" + ephemeral-storage: "1Gi" + kubeReservedCgroup: "/kube-reserved" + systemReserved: + cpu: "300m" + memory: "300Mi" + ephemeral-storage: "1Gi" + evictionHard: + memory.available: "200Mi" + nodefs.available: "10%" + featureGates: + DynamicKubeletConfig: true + RotateKubeletServerCertificate: true + From 16a3dd6c4071e06c8d6fb6821f65b0a1c5cb3bfa Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 06:45:01 +0000 Subject: [PATCH 09/40] Fixed helm charts --- testing/scripts/Makefile | 12 +++++++++--- testing/scripts/test_helm_charts_clusterwide.py | 14 +++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index c0de724188..cee9dc1d8a 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,5 +1,11 @@ VERSION := $(shell cat ../../version.txt) -PYTEST_WORKERS ?= "auto" + +PYTEST_WORKERS ?= auto +#ifdef $$PYTEST_WORKERS +#PYTEST_WORKERS := $$PYTEST_WORKERS +#else +#PYTEST_WORKERS := auto +#endif kind_create_cluster: kind create cluster --config kind_config.yaml @@ -61,9 +67,9 @@ install: pip install -r dev_requirements.txt .PHONY: test -test: build_protos install +test: pip install pytest-xdist - pytest --verbose -s -W ignore -n $$PYTEST_WORKERS 2>&1 + pytest --verbose -s -W ignore -n $(PYTEST_WORKERS) 2>&1 .PHONY: clean clean: diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py index f7d3143dd5..b3d351ddea 100644 --- a/testing/scripts/test_helm_charts_clusterwide.py +++ b/testing/scripts/test_helm_charts_clusterwide.py @@ -16,11 +16,11 @@ def test_single_model(self): namespace = "test-single-model" run(f"kubectl create namespace {namespace}", shell=True, check=True) run( - f"helm install ../../helm-charts/seldon-single-model --name mymodel-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", + f"helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) - wait_for_rollout("mymodel-mymodel-7cd068f", namespace) + wait_for_rollout(f"mymodel-mymodel-7cd068f", namespace) initial_rest_request("mymodel", namespace) print("Test Ambassador REST gateway") r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -30,7 +30,7 @@ def test_single_model(self): print("Test Ambassador gRPC gateway") r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR) print(r) - run(f"helm delete mymodel-{namespace} --purge", shell=True) + run(f"helm delete mymodel --purge", shell=True) run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test AB Test model helm script with 4 API methods @@ -38,7 +38,7 @@ def test_abtest_model(self): namespace = "test-abtest-model" run(f"kubectl create namespace {namespace}", shell=True, check=True) run( - f"helm install ../../helm-charts/seldon-abtest --name myabtest-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", + f"helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) @@ -54,7 +54,7 @@ def test_abtest_model(self): print( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) - run(f"helm delete myabtest-{namespace} --purge", shell=True) + run(f"helm delete myabtest --purge", shell=True) run(f"kubectl delete namespace {namespace}", shell=True, check=True) # Test MAB Test model helm script with 4 API methods @@ -62,7 +62,7 @@ def test_mab_model(self): namespace = "test-mab-model" run(f"kubectl create namespace {namespace}", shell=True, check=True) run( - f"helm install ../../helm-charts/seldon-mab --name mymab-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", + f"helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) @@ -79,5 +79,5 @@ def test_mab_model(self): print( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) - run(f"helm delete mymab-{namespace} --purge", shell=True) + run(f"helm delete mymab --purge", shell=True) run(f"kubectl delete namespace {namespace}", shell=True, check=True) From 285e0588f7a1529ca44321228a86a5b308e3ce39 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 06:50:20 +0000 Subject: [PATCH 10/40] Added wait before initial request --- testing/scripts/test_prepackaged_servers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index e2205065b6..72812c04d4 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -21,6 +21,7 @@ def test_sklearn(self): ) wait_for_rollout("iris-default-4903e3c", namespace) wait_for_status("sklearn", namespace) + time.sleep(1) print("Initial request") sc = SeldonClient(deployment_name="sklearn", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) @@ -41,6 +42,7 @@ def test_tfserving(self): ) wait_for_rollout("mnist-default-725903e", namespace) wait_for_status("tfserving", namespace) + time.sleep(1) print("Initial request") sc = SeldonClient(deployment_name="tfserving", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 784)) @@ -61,6 +63,7 @@ def test_xgboost(self): ) wait_for_rollout("iris-default-af1783b", namespace) wait_for_status("xgboost", namespace) + time.sleep(1) print("Initial request") sc = SeldonClient(deployment_name="xgboost", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) From 9d9c93754dfc08ab9806f899e0dcb2f4dd7ed617 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 06:54:14 +0000 Subject: [PATCH 11/40] Re-added make dependencies --- testing/scripts/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index cee9dc1d8a..7f37ff282b 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,6 +1,6 @@ VERSION := $(shell cat ../../version.txt) -PYTEST_WORKERS ?= auto +PYTEST_WORKERS ?= 0 #ifdef $$PYTEST_WORKERS #PYTEST_WORKERS := $$PYTEST_WORKERS #else @@ -67,7 +67,7 @@ install: pip install -r dev_requirements.txt .PHONY: test -test: +test: build_protos install pip install pytest-xdist pytest --verbose -s -W ignore -n $(PYTEST_WORKERS) 2>&1 From 4d071cc378eda3b9d9bbd83ff573882606b85ce0 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 07:30:52 +0000 Subject: [PATCH 12/40] Updated kind script to only run builds for images that have been updated --- testing/scripts/Makefile | 2 +- testing/scripts/kind_test_all.sh | 29 ++++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 7f37ff282b..7f2d8107f6 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -43,7 +43,7 @@ create_namespaces: kubectl create namespace test1 || echo "Namespace test1 already exists" kubectl config set-context $$(kubectl config current-context) --namespace=seldon -kind_setup: kind_build_images install_helm install_ambassador install_seldon create_namespaces +kind_setup: install_helm install_ambassador install_seldon create_namespaces port-forward-ambassador: kubectl port-forward $$(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080 diff --git a/testing/scripts/kind_test_all.sh b/testing/scripts/kind_test_all.sh index cf2ad04d5e..9ef80676c3 100755 --- a/testing/scripts/kind_test_all.sh +++ b/testing/scripts/kind_test_all.sh @@ -35,12 +35,31 @@ export KUBECONFIG=$(kind get kubeconfig-path) # ONLY RUN THE FOLLOWING IF SUCCESS if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then # BUILD S2I BASE IMAGES - make s2i_build_base_images - S2I_EXIT_VALUE=$? + PYTHON_MODIFIED=`git diff --exit-code --quiet master ../../python` + if [[ $PYTHON_MODIFIED -gt 0 ]]; then + make s2i_build_base_images + else + echo "SKIPPING PYTHON IMAGE BUILD..." + fi - # CREATE PROTOS - make build_protos - PROTOS_EXIT_VALUE=$? + # MORE EFFICIENT CLUSTER SETUP + OPERATOR_MODIFIED=`git diff --exit-code --quiet master ../../operator` + if [[ $OPERATOR_MODIFIED -gt 0 ]]; then + make kind_build_operator + OPERATOR_EXIT_VALUE=$? + else + echo "SKIPPING OPERATOR IMAGE BUILD..." + fi + + ENGINE_MODIFIED=`git diff --exit-code --quiet master ../../engine` + if [[ $ENGINE_MODIFIED -gt 0 ]]; then + make build_protos + PROTO_EXIT_VALUE=$? + make kind_build_engine + ENGINE_EXIT_VALUE=$? + else + echo "SKIPPING ENGINE IMAGE BUILD..." + fi # KIND CLUSTER SETUP make kind_setup From 611c19a230b1ddf6082afb5f7e23b92a0ed79274 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 08:01:35 +0000 Subject: [PATCH 13/40] Updated for linter --- testing/scripts/test_prepackaged_servers.py | 12 ++---- testing/scripts/test_rolling_updates.py | 48 +++++++-------------- testing/scripts/test_s2i_python.py | 12 +++--- 3 files changed, 25 insertions(+), 47 deletions(-) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index 72812c04d4..0dd45235b0 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -1,11 +1,7 @@ import subprocess import json from seldon_core.seldon_client import SeldonClient -from seldon_e2e_utils import ( - wait_for_rollout, - retry_run, - wait_for_status, -) +from seldon_e2e_utils import wait_for_rollout, retry_run, wait_for_status from subprocess import run import time @@ -17,7 +13,7 @@ def test_sklearn(self): namespace = "test-sklearn" run(f"kubectl create namespace {namespace}", shell=True, check=True) retry_run( - f"kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", + f"kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}" ) wait_for_rollout("iris-default-4903e3c", namespace) wait_for_status("sklearn", namespace) @@ -38,7 +34,7 @@ def test_tfserving(self): namespace = "test-tfserving" run(f"kubectl create namespace {namespace}", shell=True, check=True) retry_run( - f"kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", + f"kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}" ) wait_for_rollout("mnist-default-725903e", namespace) wait_for_status("tfserving", namespace) @@ -59,7 +55,7 @@ def test_xgboost(self): namespace = "test-xgboost" run(f"kubectl create namespace {namespace}", shell=True, check=True) retry_run( - f"kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", + f"kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}" ) wait_for_rollout("iris-default-af1783b", namespace) wait_for_status("xgboost", namespace) diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index 3480d91031..a303606a2d 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -15,13 +15,13 @@ class TestRollingHttp(object): def test_rolling_update1(self): namespace = "test-rolling-update-1" run(f"kubectl create namespace {namespace}", shell=True, check=True) - retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - retry_run(f"kubectl apply -f ../resources/graph2.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph2.json -n {namespace}") i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -44,25 +44,21 @@ def test_rolling_update1(self): time.sleep(1) assert i < 100 print("Success for test_rolling_update1") - run( - f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, - ) - run( - f"kubectl delete -f ../resources/graph2.json -n {namespace}", shell=True, - ) + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph2.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) # test changing the image version and the name of its container def test_rolling_update2(self): namespace = "test-rolling-update-2" run(f"kubectl create namespace {namespace}", shell=True, check=True) - retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - retry_run(f"kubectl apply -f ../resources/graph3.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph3.json -n {namespace}") i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -86,25 +82,21 @@ def test_rolling_update2(self): time.sleep(1) assert i < 100 print("Success for test_rolling_update2") - run( - f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, - ) - run( - f"kubectl delete -f ../resources/graph3.json -n {namespace}", shell=True, - ) + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph3.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a new resource request but same image def test_rolling_update3(self): namespace = "test-rolling-updates-3" run(f"kubectl create namespace {namespace}", shell=True, check=True) - retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - retry_run(f"kubectl apply -f ../resources/graph4.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph4.json -n {namespace}") i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -122,25 +114,21 @@ def test_rolling_update3(self): time.sleep(1) assert i == 49 print("Success for test_rolling_update3") - run( - f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, - ) - run( - f"kubectl delete -f ../resources/graph4.json -n {namespace}", shell=True, - ) + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph4.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) # Test updating a model with a multi deployment new model def test_rolling_update4(self): namespace = "test-rolling-update-4" run(f"kubectl create namespace {namespace}", shell=True, check=True) - retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) print("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] - retry_run(f"kubectl apply -f ../resources/graph5.json -n {namespace}",) + retry_run(f"kubectl apply -f ../resources/graph5.json -n {namespace}") i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -162,10 +150,6 @@ def test_rolling_update4(self): time.sleep(1) assert i < 100 print("Success for test_rolling_update4") - run( - f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True, - ) - run( - f"kubectl delete -f ../resources/graph5.json -n {namespace}", shell=True, - ) + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph5.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index 2cfd985d7d..88afa80e2a 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -136,9 +136,7 @@ def test_model_rest(self, s2i_python_version): namespace = "s2i-test-model-rest" run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "model", "rest") - retry_run( - f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}", - ) + retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") wait_for_rollout("mymodel-mymodel-8715075", namespace) r = initial_rest_request("mymodel", namespace) arr = np.array([[1, 2, 3]]) @@ -159,7 +157,7 @@ def test_input_transformer_rest(self, s2i_python_version): run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "transformer", "rest") retry_run( - f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}", + f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" ) wait_for_rollout("mytrans-mytrans-1f278ae", namespace) r = initial_rest_request("mytrans", namespace) @@ -181,7 +179,7 @@ def test_output_transformer_rest(self, s2i_python_version): run(f"kubectl create namespace {namespace}", shell=True, check=True) create_push_s2i_image(s2i_python_version, "transformer", "rest") retry_run( - f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}", + f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" ) wait_for_rollout("mytrans-mytrans-52996cb", namespace) r = initial_rest_request("mytrans", namespace) @@ -204,7 +202,7 @@ def test_router_rest(self, s2i_python_version): create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "router", "rest") retry_run( - f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}", + f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}" ) wait_for_rollout("myrouter-myrouter-340ed69", namespace) r = initial_rest_request("myrouter", namespace) @@ -227,7 +225,7 @@ def test_combiner_rest(self, s2i_python_version): create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "combiner", "rest") retry_run( - f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}", + f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" ) wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) r = initial_rest_request("mycombiner", namespace) From df0b07df97b53da5d678dae678b02c9944843b3c Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 08:02:34 +0000 Subject: [PATCH 14/40] Removed commented code --- testing/scripts/Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 7f2d8107f6..40419c83fb 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,11 +1,6 @@ VERSION := $(shell cat ../../version.txt) PYTEST_WORKERS ?= 0 -#ifdef $$PYTEST_WORKERS -#PYTEST_WORKERS := $$PYTEST_WORKERS -#else -#PYTEST_WORKERS := auto -#endif kind_create_cluster: kind create cluster --config kind_config.yaml From 8c725514e678e8e947a18d2a99a13ef7f5c2335b Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 10:49:42 +0000 Subject: [PATCH 15/40] Added multithreading by default --- testing/scripts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 40419c83fb..9bef5f2103 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,6 +1,6 @@ VERSION := $(shell cat ../../version.txt) -PYTEST_WORKERS ?= 0 +PYTEST_WORKERS ?= "auto" kind_create_cluster: kind create cluster --config kind_config.yaml From 06fabef79519ace1930f8facd436e2aeb100fc69 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 14:23:17 +0000 Subject: [PATCH 16/40] Updated script to run when files are modified and show modified files --- testing/scripts/kind_test_all.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/testing/scripts/kind_test_all.sh b/testing/scripts/kind_test_all.sh index 9ef80676c3..619cc00520 100755 --- a/testing/scripts/kind_test_all.sh +++ b/testing/scripts/kind_test_all.sh @@ -34,16 +34,19 @@ export KUBECONFIG=$(kind get kubeconfig-path) # ONLY RUN THE FOLLOWING IF SUCCESS if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then - # BUILD S2I BASE IMAGES - PYTHON_MODIFIED=`git diff --exit-code --quiet master ../../python` + + echo "Files changed in python folder:" + git --no-pager diff --exit-code --name-only origin/master ../../python + PYTHON_MODIFIED=$? if [[ $PYTHON_MODIFIED -gt 0 ]]; then make s2i_build_base_images else echo "SKIPPING PYTHON IMAGE BUILD..." fi - # MORE EFFICIENT CLUSTER SETUP - OPERATOR_MODIFIED=`git diff --exit-code --quiet master ../../operator` + echo "Files changed in operator folder:" + git --no-pager diff --exit-code --name-only origin/master ../../operator + OPERATOR_MODIFIED=$? if [[ $OPERATOR_MODIFIED -gt 0 ]]; then make kind_build_operator OPERATOR_EXIT_VALUE=$? @@ -51,7 +54,9 @@ if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then echo "SKIPPING OPERATOR IMAGE BUILD..." fi - ENGINE_MODIFIED=`git diff --exit-code --quiet master ../../engine` + echo "Files changed in engine folder:" + git --no-pager diff --exit-code --name-only origin/master ../../engine + ENGINE_MODIFIED=$? if [[ $ENGINE_MODIFIED -gt 0 ]]; then make build_protos PROTO_EXIT_VALUE=$? From a68f8b50f54adc66fa463f3b2a754335fafbd7a1 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 17:58:27 +0000 Subject: [PATCH 17/40] Added parallel execution of python tests --- python/setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/setup.cfg b/python/setup.cfg index a0aa6426bc..f2a3a37f8c 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -13,6 +13,7 @@ deps = pytest pytest-cov Pillow + pytest-xdist commands = pytest {posargs} [testenv:tensorflow] @@ -29,4 +30,5 @@ addopts = --tb native -W ignore --cov=seldon_core + -n auto From 66763501a16ef20b31f80a53e16b44bb53e07c4c Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 18:10:48 +0000 Subject: [PATCH 18/40] Removed concurrent tests --- python/setup.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/setup.cfg b/python/setup.cfg index f2a3a37f8c..a0aa6426bc 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -13,7 +13,6 @@ deps = pytest pytest-cov Pillow - pytest-xdist commands = pytest {posargs} [testenv:tensorflow] @@ -30,5 +29,4 @@ addopts = --tb native -W ignore --cov=seldon_core - -n auto From 366741c2d308db9917e747c9c227962bf20e1182 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 19:19:30 +0000 Subject: [PATCH 19/40] Added requests --- jenkins-x-integration.yml | 4 ++-- jenkins-x.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-x-integration.yml b/jenkins-x-integration.yml index a02b8645fd..3601df0bda 100644 --- a/jenkins-x-integration.yml +++ b/jenkins-x-integration.yml @@ -26,8 +26,8 @@ pipelineConfig: mountPath: /var/lib/docker resources: requests: - cpu: 1 - memory: 4000Mi + cpu: 6 + memory: 8000Mi securityContext: privileged: true imagePullPolicy: Always diff --git a/jenkins-x.yml b/jenkins-x.yml index a347748add..3a9b0aefeb 100644 --- a/jenkins-x.yml +++ b/jenkins-x.yml @@ -57,7 +57,7 @@ pipelineConfig: resources: requests: cpu: 1 - memory: 4000Mi + memory: 2000Mi securityContext: privileged: true imagePullPolicy: Always From 3f56f3bafefefc426f27a3e315224ca6787f7e12 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 19:34:19 +0000 Subject: [PATCH 20/40] Added less cores --- jenkins-x-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-x-integration.yml b/jenkins-x-integration.yml index 3601df0bda..5091e3d309 100644 --- a/jenkins-x-integration.yml +++ b/jenkins-x-integration.yml @@ -26,7 +26,7 @@ pipelineConfig: mountPath: /var/lib/docker resources: requests: - cpu: 6 + cpu: 4 memory: 8000Mi securityContext: privileged: true From 65b69609b927e15279d9e4e4a58332603e956f37 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 20:43:44 +0000 Subject: [PATCH 21/40] Moved prints to logs and added extra guards on namespace creation --- testing/scripts/seldon_e2e_utils.py | 25 +- .../scripts/test_helm_charts_clusterwide.py | 38 +-- testing/scripts/test_prepackaged_servers.py | 19 +- testing/scripts/test_rolling_updates.py | 260 ++++++++++++++++-- testing/scripts/test_s2i_python.py | 29 +- 5 files changed, 302 insertions(+), 69 deletions(-) diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index b914a2f424..37cc161789 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -9,6 +9,7 @@ import subprocess import json from retrying import retry +import logging API_AMBASSADOR = "localhost:8003" @@ -47,20 +48,20 @@ def wait_for_rollout(deploymentName, namespace, attempts=50, sleep=5): shell=True, ) if ret.returncode == 0: - print(f"Successfully waited for deployment {deploymentName}") + logging.warning(f"Successfully waited for deployment {deploymentName}") break - print(f"Unsuccessful wait command but retrying for {deploymentName}") + logging.warning(f"Unsuccessful wait command but retrying for {deploymentName}") time.sleep(sleep) assert ret.returncode == 0 -def retry_run(cmd, attempts=10, sleep=5): +def retry_run(cmd, attempts=50, sleep=5): for i in range(attempts): ret = run(cmd, shell=True) if ret.returncode == 0: - print(f"Successfully ran command: {cmd}") + logging.warning(f"Successfully ran command: {cmd}") break - print(f"Unsuccessful command but retrying: {cmd}") + logging.warning(f"Unsuccessful command but retrying: {cmd}") time.sleep(sleep) assert ret.returncode == 0 @@ -78,7 +79,7 @@ def wait_for_status(name, namespace): if "status" in j: return j else: - print("Failed to find status - sleeping") + logging.warning("Failed to find status - sleeping") time.sleep(5) @@ -86,27 +87,27 @@ def rest_request(model, namespace): try: r = rest_request_ambassador(model, namespace, API_AMBASSADOR) if not r.status_code == 200: - print("Bad status:", r.status_code) + logging.warning("Bad status:", r.status_code) return None else: return r except Exception as e: - print("Failed on REST request ", str(e)) + logging.warning("Failed on REST request ", str(e)) return None def initial_rest_request(model, namespace): r = rest_request(model, namespace) if r is None: - print("Sleeping 1 sec and trying again") + logging.warning("Sleeping 1 sec and trying again") time.sleep(1) r = rest_request(model, namespace) if r is None: - print("Sleeping 5 sec and trying again") + logging.warning("Sleeping 5 sec and trying again") time.sleep(5) r = rest_request(model, namespace) if r is None: - print("Sleeping 10 sec and trying again") + logging.warning("Sleeping 10 sec and trying again") time.sleep(10) r = rest_request(model, namespace) return r @@ -251,7 +252,7 @@ def grpc_request_ambassador2( data=data, ) except: - print("Warning - caught exception") + logging.warning("Warning - caught exception") return grpc_request_ambassador( deploymentName, namespace, diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py index b3d351ddea..d92e57210b 100644 --- a/testing/scripts/test_helm_charts_clusterwide.py +++ b/testing/scripts/test_helm_charts_clusterwide.py @@ -4,9 +4,11 @@ initial_rest_request, rest_request_ambassador, grpc_request_ambassador2, + retry_run, API_AMBASSADOR, ) from subprocess import run +import logging class TestClusterWide(object): @@ -14,7 +16,7 @@ class TestClusterWide(object): # Test singe model helm script with 4 API methods def test_single_model(self): namespace = "test-single-model" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") run( f"helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, @@ -22,21 +24,21 @@ def test_single_model(self): ) wait_for_rollout(f"mymodel-mymodel-7cd068f", namespace) initial_rest_request("mymodel", namespace) - print("Test Ambassador REST gateway") + logging.warning("Test Ambassador REST gateway") r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) - print(r.json()) + logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 - print("Test Ambassador gRPC gateway") + logging.warning("Test Ambassador gRPC gateway") r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR) - print(r) + logging.warning(r) run(f"helm delete mymodel --purge", shell=True) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test AB Test model helm script with 4 API methods def test_abtest_model(self): namespace = "test-abtest-model" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") run( f"helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, @@ -45,22 +47,22 @@ def test_abtest_model(self): wait_for_rollout("myabtest-myabtest-41de5b8", namespace) wait_for_rollout("myabtest-myabtest-df66c5c", namespace) initial_rest_request("myabtest", namespace) - print("Test Ambassador REST gateway") + logging.warning("Test Ambassador REST gateway") r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR) - print(r.json()) + logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 - print("Test Ambassador gRPC gateway") - print( + logging.warning("Test Ambassador gRPC gateway") + logging.warning( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) run(f"helm delete myabtest --purge", shell=True) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) # Test MAB Test model helm script with 4 API methods def test_mab_model(self): namespace = "test-mab-model" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") run( f"helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, @@ -70,14 +72,14 @@ def test_mab_model(self): wait_for_rollout("mymab-mymab-b8038b2", namespace) wait_for_rollout("mymab-mymab-df66c5c", namespace) initial_rest_request("mymab", namespace) - print("Test Ambassador REST gateway") + logging.warning("Test Ambassador REST gateway") r = rest_request_ambassador("mymab", namespace, API_AMBASSADOR) - print(r.json()) + logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 - print("Test Ambassador gRPC gateway") - print( + logging.warning("Test Ambassador gRPC gateway") + logging.warning( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) run(f"helm delete mymab --purge", shell=True) - run(f"kubectl delete namespace {namespace}", shell=True, check=True) + run(f"kubectl delete namespace {namespace}", shell=True) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index 0dd45235b0..e4b960b162 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -4,6 +4,7 @@ from seldon_e2e_utils import wait_for_rollout, retry_run, wait_for_status from subprocess import run import time +import logging class TestPrepack(object): @@ -11,18 +12,18 @@ class TestPrepack(object): # Test prepackaged server for sklearn def test_sklearn(self): namespace = "test-sklearn" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") retry_run( f"kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}" ) wait_for_rollout("iris-default-4903e3c", namespace) wait_for_status("sklearn", namespace) time.sleep(1) - print("Initial request") + logging.warning("Initial request") sc = SeldonClient(deployment_name="sklearn", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) assert r.success - print("Success for test_prepack_sklearn") + logging.warning("Success for test_prepack_sklearn") run( f"kubectl delete -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", shell=True, @@ -32,18 +33,18 @@ def test_sklearn(self): # Test prepackaged server for tfserving def test_tfserving(self): namespace = "test-tfserving" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") retry_run( f"kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}" ) wait_for_rollout("mnist-default-725903e", namespace) wait_for_status("tfserving", namespace) time.sleep(1) - print("Initial request") + logging.warning("Initial request") sc = SeldonClient(deployment_name="tfserving", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 784)) assert r.success - print("Success for test_prepack_tfserving") + logging.warning("Success for test_prepack_tfserving") run( f"kubectl delete -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", shell=True, @@ -53,18 +54,18 @@ def test_tfserving(self): # Test prepackaged server for xgboost def test_xgboost(self): namespace = "test-xgboost" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") retry_run( f"kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}" ) wait_for_rollout("iris-default-af1783b", namespace) wait_for_status("xgboost", namespace) time.sleep(1) - print("Initial request") + logging.warning("Initial request") sc = SeldonClient(deployment_name="xgboost", namespace=namespace) r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) assert r.success - print("Success for test_prepack_xgboost") + logging.warning("Success for test_prepack_xgboost") run( f"kubectl delete -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", shell=True, diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index a303606a2d..3365e26260 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -7,6 +7,7 @@ API_AMBASSADOR, ) import time +import logging class TestRollingHttp(object): @@ -14,10 +15,10 @@ class TestRollingHttp(object): # Test updating a model with a new image version as the only change def test_rolling_update1(self): namespace = "test-rolling-update-1" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) - print("Initial request") + logging.warning("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] @@ -27,7 +28,7 @@ def test_rolling_update1(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) + logging.warning(res) assert ( res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.1" @@ -43,7 +44,7 @@ def test_rolling_update1(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update1") + logging.warning("Success for test_rolling_update1") run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph2.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) @@ -51,10 +52,10 @@ def test_rolling_update1(self): # test changing the image version and the name of its container def test_rolling_update2(self): namespace = "test-rolling-update-2" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) - print("Initial request") + logging.warning("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] @@ -64,7 +65,7 @@ def test_rolling_update2(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) + logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -81,7 +82,7 @@ def test_rolling_update2(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update2") + logging.warning("Success for test_rolling_update2") run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph3.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) @@ -89,10 +90,10 @@ def test_rolling_update2(self): # Test updating a model with a new resource request but same image def test_rolling_update3(self): namespace = "test-rolling-updates-3" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) - print("Initial request") + logging.warning("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] @@ -102,7 +103,7 @@ def test_rolling_update3(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) + logging.warning(res) assert res["meta"]["requestPath"][ "complex-model" ] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [ @@ -113,7 +114,7 @@ def test_rolling_update3(self): ] time.sleep(1) assert i == 49 - print("Success for test_rolling_update3") + logging.warning("Success for test_rolling_update3") run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph4.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) @@ -121,10 +122,10 @@ def test_rolling_update3(self): # Test updating a model with a multi deployment new model def test_rolling_update4(self): namespace = "test-rolling-update-4" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_rollout("mymodel-mymodel-e2eb561", namespace) - print("Initial request") + logging.warning("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] @@ -134,7 +135,7 @@ def test_rolling_update4(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - print(res) + logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -149,7 +150,234 @@ def test_rolling_update4(self): break time.sleep(1) assert i < 100 - print("Success for test_rolling_update4") + logging.warning("Success for test_rolling_update4") run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph5.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True) + + # Test updating a model to a multi predictor model + def test_rolling_update5(self): + namespace = "test-rolling-update-5" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph6.json -n {namespace}") + i = 0 + for i in range(50): + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) + logging.warning("Status code", r.status_code) + assert r.status_code == 200 + res = r.json() + logging.warning(res) + assert ( + "complex-model" in res["meta"]["requestPath"] + and res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.1" + and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + ) or ( + res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.2" + and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ) + if (not r.status_code == 200) or ( + res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ): + break + time.sleep(1) + assert i < 100 + logging.warning("Success for test_rolling_update5") + run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph6.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) + + # Test updating a model with a new image version as the only change + def test_rolling_update6(self): + namespace = "test-rolling-update-6" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph2svc.json -n {namespace}") + i = 0 + for i in range(100): + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) + logging.warning("Status code", r.status_code) + assert r.status_code == 200 + res = r.json() + logging.warning(res) + assert ( + res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.1" + and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + ) or ( + res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.2" + and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ) + if (not r.status_code == 200) or ( + res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ): + break + time.sleep(1) + assert i < 100 + logging.warning("Success for test_rolling_update6") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph2svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) + + # test changing the image version and the name of its container + def test_rolling_update7(self): + namespace = "test-rolling-update-7" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + logging.warning("Initial request") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph3svc.json -n {namespace}") + i = 0 + for i in range(100): + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) + logging.warning("Status code", r.status_code) + assert r.status_code == 200 + res = r.json() + logging.warning(res) + assert ( + "complex-model" in res["meta"]["requestPath"] + and res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.1" + and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + ) or ( + res["meta"]["requestPath"]["complex-model2"] + == "seldonio/fixed-model:0.2" + and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ) + if (not r.status_code == 200) or ( + res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ): + break + time.sleep(1) + assert i < 100 + logging.warning("Success for test_rolling_update7") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph3svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) + + # Test updating a model with a new resource request but same image + def test_rolling_update8(self): + namespace = "test-rolling-update-8" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph4svc.json -n {namespace}") + i = 0 + for i in range(50): + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) + logging.warning("Status code", r.status_code) + assert r.status_code == 200 + res = r.json() + logging.warning(res) + assert res["meta"]["requestPath"][ + "complex-model" + ] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + time.sleep(1) + assert i == 49 + logging.warning("Success for test_rolling_update8") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph4svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) + + # Test updating a model with a multi deployment new model + def test_rolling_update9(self): + namespace = "test-rolling-update-9" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph5svc.json -n {namespace}") + i = 0 + for i in range(50): + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) + logging.warning("Status code", r.status_code) + assert r.status_code == 200 + res = r.json() + logging.warning(res) + assert ( + "complex-model" in res["meta"]["requestPath"] + and res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.1" + and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + ) or ( + res["meta"]["requestPath"]["model1"] == "seldonio/fixed-model:0.1" + and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + and res["meta"]["requestPath"]["model2"] == "seldonio/fixed-model:0.1" + ) + if (not r.status_code == 200) or ("model1" in res["meta"]["requestPath"]): + break + time.sleep(1) + assert i < 100 + logging.warning("Success for test_rolling_update9") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph5svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) + + # Test updating a model to a multi predictor model + def test_rolling_update10(self): + namespace = "test-rolling-update-10" + retry_run(f"kubectl create namespace {namespace}") + retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) + wait_for_rollout("mymodel-mymodel-e2eb561", namespace) + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + retry_run(f"kubectl apply -f ../resources/graph6svc.json -n {namespace}") + i = 0 + for i in range(50): + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) + logging.warning("Status code", r.status_code) + assert r.status_code == 200 + res = r.json() + logging.warning(res) + assert ( + "complex-model" in res["meta"]["requestPath"] + and res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.1" + and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] + ) or ( + res["meta"]["requestPath"]["complex-model"] + == "seldonio/fixed-model:0.2" + and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ) + if (not r.status_code == 200) or ( + res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] + ): + break + time.sleep(1) + assert i < 100 + logging.warning("Success for test_rolling_update10") + run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) + run(f"kubectl delete -f ../resources/graph6svc.json -n {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index 88afa80e2a..64b0ac5385 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -9,6 +9,7 @@ retry_run, API_AMBASSADOR, ) +import logging S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" @@ -20,14 +21,14 @@ def create_s2I_image(s2i_python_version, component_type, api_type): .replace("#API#", api_type) .replace("#VERSION#", s2i_python_version) ) - print(cmd) + logging.warning(cmd) run(cmd, shell=True, check=True) def kind_push_s2i_image(component_type, api_type): img = get_image_name(component_type, api_type) cmd = "kind load docker-image " + img + " --loglevel trace" - print(cmd) + logging.warning(cmd) run(cmd, shell=True, check=True) @@ -95,7 +96,7 @@ def test_build_transformer_grpc(self, s2i_python_version): def test_build_combiner_rest(self, s2i_python_version): create_s2I_image(s2i_python_version, "combiner", "rest") img = get_image_name("combiner", "rest") - print(img) + logging.warning(img) run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) time.sleep(2) run("docker rm -f combiner-rest", shell=True, check=True) @@ -134,7 +135,7 @@ def test_combiner_rest(self, s2i_python_version): class S2IK8S(object): def test_model_rest(self, s2i_python_version): namespace = "s2i-test-model-rest" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest") retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") wait_for_rollout("mymodel-mymodel-8715075", namespace) @@ -142,7 +143,7 @@ def test_model_rest(self, s2i_python_version): arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] @@ -154,7 +155,7 @@ def test_model_rest(self, s2i_python_version): def test_input_transformer_rest(self, s2i_python_version): namespace = "s2i-test-input-transformer-rest" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "transformer", "rest") retry_run( f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" @@ -164,7 +165,7 @@ def test_input_transformer_rest(self, s2i_python_version): arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] @@ -172,11 +173,11 @@ def test_input_transformer_rest(self, s2i_python_version): f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", shell=True, ) - run(f"kubectl create namespace {namespace}", shell=True) + run(f"kubectl delete namespace {namespace}", shell=True) def test_output_transformer_rest(self, s2i_python_version): namespace = "s2i-test-output-transformer-rest" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "transformer", "rest") retry_run( f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" @@ -186,7 +187,7 @@ def test_output_transformer_rest(self, s2i_python_version): arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] @@ -198,7 +199,7 @@ def test_output_transformer_rest(self, s2i_python_version): def test_router_rest(self, s2i_python_version): namespace = "s2i-test-router-rest" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "router", "rest") retry_run( @@ -209,7 +210,7 @@ def test_router_rest(self, s2i_python_version): arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("myrouter", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] @@ -221,7 +222,7 @@ def test_router_rest(self, s2i_python_version): def test_combiner_rest(self, s2i_python_version): namespace = "s2i-test-combiner-rest" - run(f"kubectl create namespace {namespace}", shell=True, check=True) + retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "combiner", "rest") retry_run( @@ -232,7 +233,7 @@ def test_combiner_rest(self, s2i_python_version): arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) res = r.json() - print(res) + logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] From 73c9077cd0e5131911b4b0d10e67355d7eee2e34 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 11 Nov 2019 21:13:46 +0000 Subject: [PATCH 22/40] Removed logging from request loops --- testing/scripts/test_rolling_updates.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index 3365e26260..2068df0c9a 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -28,7 +28,6 @@ def test_rolling_update1(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.1" @@ -65,7 +64,6 @@ def test_rolling_update2(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -103,7 +101,6 @@ def test_rolling_update3(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - logging.warning(res) assert res["meta"]["requestPath"][ "complex-model" ] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [ @@ -135,7 +132,6 @@ def test_rolling_update4(self): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -169,10 +165,8 @@ def test_rolling_update5(self): i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) - logging.warning("Status code", r.status_code) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -209,10 +203,8 @@ def test_rolling_update6(self): i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) - logging.warning("Status code", r.status_code) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.1" @@ -248,10 +240,8 @@ def test_rolling_update7(self): i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) - logging.warning("Status code", r.status_code) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -287,10 +277,8 @@ def test_rolling_update8(self): i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) - logging.warning("Status code", r.status_code) assert r.status_code == 200 res = r.json() - logging.warning(res) assert res["meta"]["requestPath"][ "complex-model" ] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [ @@ -320,10 +308,8 @@ def test_rolling_update9(self): i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) - logging.warning("Status code", r.status_code) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] @@ -357,10 +343,8 @@ def test_rolling_update10(self): i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) - logging.warning("Status code", r.status_code) assert r.status_code == 200 res = r.json() - logging.warning(res) assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] From adf933c22428a3bb3dbd420d3ccfa993dfb78a74 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 06:03:48 +0000 Subject: [PATCH 23/40] Added networking config --- testing/scripts/kind_config.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/testing/scripts/kind_config.yaml b/testing/scripts/kind_config.yaml index 18e8dee0cc..dcfe68afb4 100644 --- a/testing/scripts/kind_config.yaml +++ b/testing/scripts/kind_config.yaml @@ -6,6 +6,11 @@ nodes: extraPortMappings: - containerPort: 30080 hostPort: 8003 +networking: + disableDefaultCNI: true + podSubnet: "10.15.240.0/20" + apiServerAddress: "1.0.0.1" + apiServerPort: 26443 kubeadmConfigPatches: - | apiVersion: kubelet.config.k8s.io/v1beta1 @@ -27,4 +32,11 @@ kubeadmConfigPatches: featureGates: DynamicKubeletConfig: true RotateKubeletServerCertificate: true - +--- + apiVersion: kubeadm.k8s.io/v1beta2 + metadata: + name: config + kind: ClusterConfiguration + networking: + serviceSubnet: "10.12.0.0/14" + podSubnet: "10.15.240.0/20" From 9b5b9d736486bc2cda82c45043a62b9a9d45195a Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 06:07:45 +0000 Subject: [PATCH 24/40] Added networking config --- testing/scripts/kind_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scripts/kind_config.yaml b/testing/scripts/kind_config.yaml index dcfe68afb4..963d7163a2 100644 --- a/testing/scripts/kind_config.yaml +++ b/testing/scripts/kind_config.yaml @@ -9,7 +9,7 @@ nodes: networking: disableDefaultCNI: true podSubnet: "10.15.240.0/20" - apiServerAddress: "1.0.0.1" + apiServerAddress: "0.0.0.0" apiServerPort: 26443 kubeadmConfigPatches: - | From 95f1f68e014ba4c813d3d54b229a3b43a2a5d80f Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 06:19:41 +0000 Subject: [PATCH 25/40] Removing networking --- testing/scripts/kind_config.yaml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/testing/scripts/kind_config.yaml b/testing/scripts/kind_config.yaml index 963d7163a2..18e8dee0cc 100644 --- a/testing/scripts/kind_config.yaml +++ b/testing/scripts/kind_config.yaml @@ -6,11 +6,6 @@ nodes: extraPortMappings: - containerPort: 30080 hostPort: 8003 -networking: - disableDefaultCNI: true - podSubnet: "10.15.240.0/20" - apiServerAddress: "0.0.0.0" - apiServerPort: 26443 kubeadmConfigPatches: - | apiVersion: kubelet.config.k8s.io/v1beta1 @@ -32,11 +27,4 @@ kubeadmConfigPatches: featureGates: DynamicKubeletConfig: true RotateKubeletServerCertificate: true ---- - apiVersion: kubeadm.k8s.io/v1beta2 - metadata: - name: config - kind: ClusterConfiguration - networking: - serviceSubnet: "10.12.0.0/14" - podSubnet: "10.15.240.0/20" + From 1b2cf7c1598a800643027eb3388547e98019b0a9 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 06:52:46 +0000 Subject: [PATCH 26/40] Disabling CNI --- testing/scripts/kind_config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/scripts/kind_config.yaml b/testing/scripts/kind_config.yaml index 18e8dee0cc..8aeb79601f 100644 --- a/testing/scripts/kind_config.yaml +++ b/testing/scripts/kind_config.yaml @@ -6,6 +6,8 @@ nodes: extraPortMappings: - containerPort: 30080 hostPort: 8003 +networking: + disableDefaultCNI: true kubeadmConfigPatches: - | apiVersion: kubelet.config.k8s.io/v1beta1 From 8322dfff8020cab956e1c767adcd42702f6ec626 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 07:15:40 +0000 Subject: [PATCH 27/40] Disabling s2i tests --- testing/scripts/kind_config.yaml | 2 - testing/scripts/test_s2i_python.py | 488 ++++++++++++++--------------- 2 files changed, 244 insertions(+), 246 deletions(-) diff --git a/testing/scripts/kind_config.yaml b/testing/scripts/kind_config.yaml index 8aeb79601f..18e8dee0cc 100644 --- a/testing/scripts/kind_config.yaml +++ b/testing/scripts/kind_config.yaml @@ -6,8 +6,6 @@ nodes: extraPortMappings: - containerPort: 30080 hostPort: 8003 -networking: - disableDefaultCNI: true kubeadmConfigPatches: - | apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index 64b0ac5385..ecedb95f9c 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -1,244 +1,244 @@ -import pytest -import time -from subprocess import run -import numpy as np -from seldon_e2e_utils import ( - wait_for_rollout, - rest_request_ambassador, - initial_rest_request, - retry_run, - API_AMBASSADOR, -) -import logging - -S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" -IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" - - -def create_s2I_image(s2i_python_version, component_type, api_type): - cmd = ( - S2I_CREATE.replace("#TYPE#", component_type) - .replace("#API#", api_type) - .replace("#VERSION#", s2i_python_version) - ) - logging.warning(cmd) - run(cmd, shell=True, check=True) - - -def kind_push_s2i_image(component_type, api_type): - img = get_image_name(component_type, api_type) - cmd = "kind load docker-image " + img + " --loglevel trace" - logging.warning(cmd) - run(cmd, shell=True, check=True) - - -def get_image_name(component_type, api_type): - return IMAGE_NAME.replace("#TYPE#", component_type).replace("#API#", api_type) - - -def create_push_s2i_image(s2i_python_version, component_type, api_type): - create_s2I_image(s2i_python_version, component_type, api_type) - kind_push_s2i_image(component_type, api_type) - - -@pytest.mark.usefixtures("s2i_python_version") -class TestPythonS2i(object): - def test_build_router_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "router", "rest") - img = get_image_name("router", "rest") - run("docker run -d --rm --name 'router-rest' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f router-rest", shell=True, check=True) - - def test_build_router_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "router", "grpc") - img = get_image_name("router", "grpc") - run("docker run -d --rm --name 'router-grpc' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f router-grpc", shell=True, check=True) - - def test_build_model_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "model", "rest") - img = get_image_name("model", "rest") - run("docker run -d --rm --name 'model-rest' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f model-rest", shell=True, check=True) - - def test_build_model_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "model", "grpc") - img = get_image_name("model", "grpc") - run("docker run -d --rm --name 'model-grpc' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f model-grpc", shell=True, check=True) - - def test_build_transformer_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "transformer", "rest") - img = get_image_name("transformer", "rest") - run( - "docker run -d --rm --name 'transformer-rest' " + img, - shell=True, - check=True, - ) - time.sleep(2) - run("docker rm -f transformer-rest", shell=True, check=True) - - def test_build_transformer_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "transformer", "grpc") - img = get_image_name("transformer", "grpc") - run( - "docker run -d --rm --name 'transformer-grpc' " + img, - shell=True, - check=True, - ) - time.sleep(2) - run("docker rm -f transformer-grpc", shell=True, check=True) - - def test_build_combiner_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "combiner", "rest") - img = get_image_name("combiner", "rest") - logging.warning(img) - run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f combiner-rest", shell=True, check=True) - - def test_build_combiner_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "combiner", "grpc") - img = get_image_name("combiner", "grpc") - run("docker run -d --rm --name 'combiner-grpc' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f combiner-grpc", shell=True, check=True) - - -@pytest.mark.usefixtures("s2i_python_version") -class TestPythonS2iK8s(object): - def test_model_rest(self, s2i_python_version): - tester = S2IK8S() - tester.test_model_rest(s2i_python_version) - - def test_input_transformer_rest(self, s2i_python_version): - tester = S2IK8S() - tester.test_input_transformer_rest(s2i_python_version) - - def test_output_transformer_rest(self, s2i_python_version): - tester = S2IK8S() - tester.test_output_transformer_rest(s2i_python_version) - - def test_router_rest(self, s2i_python_version): - tester = S2IK8S() - tester.test_router_rest(s2i_python_version) - - def test_combiner_rest(self, s2i_python_version): - tester = S2IK8S() - tester.test_combiner_rest(s2i_python_version) - - -class S2IK8S(object): - def test_model_rest(self, s2i_python_version): - namespace = "s2i-test-model-rest" - retry_run(f"kubectl create namespace {namespace}") - create_push_s2i_image(s2i_python_version, "model", "rest") - retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") - wait_for_rollout("mymodel-mymodel-8715075", namespace) - r = initial_rest_request("mymodel", namespace) - arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) - res = r.json() - logging.warning(res) - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["shape"] == [1, 3] - assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run( - f"kubectl delete -f ../resources/s2i_python_model.json -n {namespace}", - shell=True, - ) - run(f"kubectl delete namespace {namespace}", shell=True) - - def test_input_transformer_rest(self, s2i_python_version): - namespace = "s2i-test-input-transformer-rest" - retry_run(f"kubectl create namespace {namespace}") - create_push_s2i_image(s2i_python_version, "transformer", "rest") - retry_run( - f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" - ) - wait_for_rollout("mytrans-mytrans-1f278ae", namespace) - r = initial_rest_request("mytrans", namespace) - arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) - res = r.json() - logging.warning(res) - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["shape"] == [1, 3] - assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run( - f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", - shell=True, - ) - run(f"kubectl delete namespace {namespace}", shell=True) - - def test_output_transformer_rest(self, s2i_python_version): - namespace = "s2i-test-output-transformer-rest" - retry_run(f"kubectl create namespace {namespace}") - create_push_s2i_image(s2i_python_version, "transformer", "rest") - retry_run( - f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" - ) - wait_for_rollout("mytrans-mytrans-52996cb", namespace) - r = initial_rest_request("mytrans", namespace) - arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) - res = r.json() - logging.warning(res) - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["shape"] == [1, 3] - assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] - run( - f"kubectl delete -f ../resources/s2i_python_output_transformer.json -n {namespace}", - shell=True, - ) - run(f"kubectl create namespace {namespace}", shell=True) - - def test_router_rest(self, s2i_python_version): - namespace = "s2i-test-router-rest" - retry_run(f"kubectl create namespace {namespace}") - create_push_s2i_image(s2i_python_version, "model", "rest") - create_push_s2i_image(s2i_python_version, "router", "rest") - retry_run( - f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}" - ) - wait_for_rollout("myrouter-myrouter-340ed69", namespace) - r = initial_rest_request("myrouter", namespace) - arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("myrouter", namespace, API_AMBASSADOR, data=arr) - res = r.json() - logging.warning(res) - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["shape"] == [1, 3] - assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] - run( - f"kubectl delete -f ../resources/s2i_python_router.json -n {namespace}", - shell=True, - ) - run(f"kubectl delete namespace {namespace}", shell=True) - - def test_combiner_rest(self, s2i_python_version): - namespace = "s2i-test-combiner-rest" - retry_run(f"kubectl create namespace {namespace}") - create_push_s2i_image(s2i_python_version, "model", "rest") - create_push_s2i_image(s2i_python_version, "combiner", "rest") - retry_run( - f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" - ) - wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) - r = initial_rest_request("mycombiner", namespace) - arr = np.array([[1, 2, 3]]) - r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) - res = r.json() - logging.warning(res) - assert r.status_code == 200 - assert r.json()["data"]["tensor"]["shape"] == [1, 3] - assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] - run( - f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", - shell=True, - ) - run(f"kubectl delete namespace {namespace}", shell=True) +# import pytest +# import time +# from subprocess import run +# import numpy as np +# from seldon_e2e_utils import ( +# wait_for_rollout, +# rest_request_ambassador, +# initial_rest_request, +# retry_run, +# API_AMBASSADOR, +# ) +# import logging +# +# S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" +# IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" +# +# +# def create_s2I_image(s2i_python_version, component_type, api_type): +# cmd = ( +# S2I_CREATE.replace("#TYPE#", component_type) +# .replace("#API#", api_type) +# .replace("#VERSION#", s2i_python_version) +# ) +# logging.warning(cmd) +# run(cmd, shell=True, check=True) +# +# +# def kind_push_s2i_image(component_type, api_type): +# img = get_image_name(component_type, api_type) +# cmd = "kind load docker-image " + img + " --loglevel trace" +# logging.warning(cmd) +# run(cmd, shell=True, check=True) +# +# +# def get_image_name(component_type, api_type): +# return IMAGE_NAME.replace("#TYPE#", component_type).replace("#API#", api_type) +# +# +# def create_push_s2i_image(s2i_python_version, component_type, api_type): +# create_s2I_image(s2i_python_version, component_type, api_type) +# kind_push_s2i_image(component_type, api_type) +# +# +# @pytest.mark.usefixtures("s2i_python_version") +# class TestPythonS2i(object): +# def test_build_router_rest(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "router", "rest") +# img = get_image_name("router", "rest") +# run("docker run -d --rm --name 'router-rest' " + img, shell=True, check=True) +# time.sleep(2) +# run("docker rm -f router-rest", shell=True, check=True) +# +# def test_build_router_grpc(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "router", "grpc") +# img = get_image_name("router", "grpc") +# run("docker run -d --rm --name 'router-grpc' " + img, shell=True, check=True) +# time.sleep(2) +# run("docker rm -f router-grpc", shell=True, check=True) +# +# def test_build_model_rest(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "model", "rest") +# img = get_image_name("model", "rest") +# run("docker run -d --rm --name 'model-rest' " + img, shell=True, check=True) +# time.sleep(2) +# run("docker rm -f model-rest", shell=True, check=True) +# +# def test_build_model_grpc(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "model", "grpc") +# img = get_image_name("model", "grpc") +# run("docker run -d --rm --name 'model-grpc' " + img, shell=True, check=True) +# time.sleep(2) +# run("docker rm -f model-grpc", shell=True, check=True) +# +# def test_build_transformer_rest(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "transformer", "rest") +# img = get_image_name("transformer", "rest") +# run( +# "docker run -d --rm --name 'transformer-rest' " + img, +# shell=True, +# check=True, +# ) +# time.sleep(2) +# run("docker rm -f transformer-rest", shell=True, check=True) +# +# def test_build_transformer_grpc(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "transformer", "grpc") +# img = get_image_name("transformer", "grpc") +# run( +# "docker run -d --rm --name 'transformer-grpc' " + img, +# shell=True, +# check=True, +# ) +# time.sleep(2) +# run("docker rm -f transformer-grpc", shell=True, check=True) +# +# def test_build_combiner_rest(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "combiner", "rest") +# img = get_image_name("combiner", "rest") +# logging.warning(img) +# run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) +# time.sleep(2) +# run("docker rm -f combiner-rest", shell=True, check=True) +# +# def test_build_combiner_grpc(self, s2i_python_version): +# create_s2I_image(s2i_python_version, "combiner", "grpc") +# img = get_image_name("combiner", "grpc") +# run("docker run -d --rm --name 'combiner-grpc' " + img, shell=True, check=True) +# time.sleep(2) +# run("docker rm -f combiner-grpc", shell=True, check=True) +# +# +# @pytest.mark.usefixtures("s2i_python_version") +# class TestPythonS2iK8s(object): +# def test_model_rest(self, s2i_python_version): +# tester = S2IK8S() +# tester.test_model_rest(s2i_python_version) +# +# def test_input_transformer_rest(self, s2i_python_version): +# tester = S2IK8S() +# tester.test_input_transformer_rest(s2i_python_version) +# +# def test_output_transformer_rest(self, s2i_python_version): +# tester = S2IK8S() +# tester.test_output_transformer_rest(s2i_python_version) +# +# def test_router_rest(self, s2i_python_version): +# tester = S2IK8S() +# tester.test_router_rest(s2i_python_version) +# +# def test_combiner_rest(self, s2i_python_version): +# tester = S2IK8S() +# tester.test_combiner_rest(s2i_python_version) +# +# +# class S2IK8S(object): +# def test_model_rest(self, s2i_python_version): +# namespace = "s2i-test-model-rest" +# retry_run(f"kubectl create namespace {namespace}") +# create_push_s2i_image(s2i_python_version, "model", "rest") +# retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") +# wait_for_rollout("mymodel-mymodel-8715075", namespace) +# r = initial_rest_request("mymodel", namespace) +# arr = np.array([[1, 2, 3]]) +# r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) +# res = r.json() +# logging.warning(res) +# assert r.status_code == 200 +# assert r.json()["data"]["tensor"]["shape"] == [1, 3] +# assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] +# run( +# f"kubectl delete -f ../resources/s2i_python_model.json -n {namespace}", +# shell=True, +# ) +# run(f"kubectl delete namespace {namespace}", shell=True) +# +# def test_input_transformer_rest(self, s2i_python_version): +# namespace = "s2i-test-input-transformer-rest" +# retry_run(f"kubectl create namespace {namespace}") +# create_push_s2i_image(s2i_python_version, "transformer", "rest") +# retry_run( +# f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" +# ) +# wait_for_rollout("mytrans-mytrans-1f278ae", namespace) +# r = initial_rest_request("mytrans", namespace) +# arr = np.array([[1, 2, 3]]) +# r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) +# res = r.json() +# logging.warning(res) +# assert r.status_code == 200 +# assert r.json()["data"]["tensor"]["shape"] == [1, 3] +# assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] +# run( +# f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", +# shell=True, +# ) +# run(f"kubectl delete namespace {namespace}", shell=True) +# +# def test_output_transformer_rest(self, s2i_python_version): +# namespace = "s2i-test-output-transformer-rest" +# retry_run(f"kubectl create namespace {namespace}") +# create_push_s2i_image(s2i_python_version, "transformer", "rest") +# retry_run( +# f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" +# ) +# wait_for_rollout("mytrans-mytrans-52996cb", namespace) +# r = initial_rest_request("mytrans", namespace) +# arr = np.array([[1, 2, 3]]) +# r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) +# res = r.json() +# logging.warning(res) +# assert r.status_code == 200 +# assert r.json()["data"]["tensor"]["shape"] == [1, 3] +# assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] +# run( +# f"kubectl delete -f ../resources/s2i_python_output_transformer.json -n {namespace}", +# shell=True, +# ) +# run(f"kubectl create namespace {namespace}", shell=True) +# +# def test_router_rest(self, s2i_python_version): +# namespace = "s2i-test-router-rest" +# retry_run(f"kubectl create namespace {namespace}") +# create_push_s2i_image(s2i_python_version, "model", "rest") +# create_push_s2i_image(s2i_python_version, "router", "rest") +# retry_run( +# f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}" +# ) +# wait_for_rollout("myrouter-myrouter-340ed69", namespace) +# r = initial_rest_request("myrouter", namespace) +# arr = np.array([[1, 2, 3]]) +# r = rest_request_ambassador("myrouter", namespace, API_AMBASSADOR, data=arr) +# res = r.json() +# logging.warning(res) +# assert r.status_code == 200 +# assert r.json()["data"]["tensor"]["shape"] == [1, 3] +# assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] +# run( +# f"kubectl delete -f ../resources/s2i_python_router.json -n {namespace}", +# shell=True, +# ) +# run(f"kubectl delete namespace {namespace}", shell=True) +# +# def test_combiner_rest(self, s2i_python_version): +# namespace = "s2i-test-combiner-rest" +# retry_run(f"kubectl create namespace {namespace}") +# create_push_s2i_image(s2i_python_version, "model", "rest") +# create_push_s2i_image(s2i_python_version, "combiner", "rest") +# retry_run( +# f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" +# ) +# wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) +# r = initial_rest_request("mycombiner", namespace) +# arr = np.array([[1, 2, 3]]) +# r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) +# res = r.json() +# logging.warning(res) +# assert r.status_code == 200 +# assert r.json()["data"]["tensor"]["shape"] == [1, 3] +# assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] +# run( +# f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", +# shell=True, +# ) +# run(f"kubectl delete namespace {namespace}", shell=True) From b7238cb072cb3eb551ffbb9c3de447cd8b3a1674 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 07:43:47 +0000 Subject: [PATCH 28/40] Updated testing scripts --- testing/scripts/test_s2i_python.py | 488 ++++++++++++++--------------- 1 file changed, 244 insertions(+), 244 deletions(-) diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index ecedb95f9c..64b0ac5385 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -1,244 +1,244 @@ -# import pytest -# import time -# from subprocess import run -# import numpy as np -# from seldon_e2e_utils import ( -# wait_for_rollout, -# rest_request_ambassador, -# initial_rest_request, -# retry_run, -# API_AMBASSADOR, -# ) -# import logging -# -# S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" -# IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" -# -# -# def create_s2I_image(s2i_python_version, component_type, api_type): -# cmd = ( -# S2I_CREATE.replace("#TYPE#", component_type) -# .replace("#API#", api_type) -# .replace("#VERSION#", s2i_python_version) -# ) -# logging.warning(cmd) -# run(cmd, shell=True, check=True) -# -# -# def kind_push_s2i_image(component_type, api_type): -# img = get_image_name(component_type, api_type) -# cmd = "kind load docker-image " + img + " --loglevel trace" -# logging.warning(cmd) -# run(cmd, shell=True, check=True) -# -# -# def get_image_name(component_type, api_type): -# return IMAGE_NAME.replace("#TYPE#", component_type).replace("#API#", api_type) -# -# -# def create_push_s2i_image(s2i_python_version, component_type, api_type): -# create_s2I_image(s2i_python_version, component_type, api_type) -# kind_push_s2i_image(component_type, api_type) -# -# -# @pytest.mark.usefixtures("s2i_python_version") -# class TestPythonS2i(object): -# def test_build_router_rest(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "router", "rest") -# img = get_image_name("router", "rest") -# run("docker run -d --rm --name 'router-rest' " + img, shell=True, check=True) -# time.sleep(2) -# run("docker rm -f router-rest", shell=True, check=True) -# -# def test_build_router_grpc(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "router", "grpc") -# img = get_image_name("router", "grpc") -# run("docker run -d --rm --name 'router-grpc' " + img, shell=True, check=True) -# time.sleep(2) -# run("docker rm -f router-grpc", shell=True, check=True) -# -# def test_build_model_rest(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "model", "rest") -# img = get_image_name("model", "rest") -# run("docker run -d --rm --name 'model-rest' " + img, shell=True, check=True) -# time.sleep(2) -# run("docker rm -f model-rest", shell=True, check=True) -# -# def test_build_model_grpc(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "model", "grpc") -# img = get_image_name("model", "grpc") -# run("docker run -d --rm --name 'model-grpc' " + img, shell=True, check=True) -# time.sleep(2) -# run("docker rm -f model-grpc", shell=True, check=True) -# -# def test_build_transformer_rest(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "transformer", "rest") -# img = get_image_name("transformer", "rest") -# run( -# "docker run -d --rm --name 'transformer-rest' " + img, -# shell=True, -# check=True, -# ) -# time.sleep(2) -# run("docker rm -f transformer-rest", shell=True, check=True) -# -# def test_build_transformer_grpc(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "transformer", "grpc") -# img = get_image_name("transformer", "grpc") -# run( -# "docker run -d --rm --name 'transformer-grpc' " + img, -# shell=True, -# check=True, -# ) -# time.sleep(2) -# run("docker rm -f transformer-grpc", shell=True, check=True) -# -# def test_build_combiner_rest(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "combiner", "rest") -# img = get_image_name("combiner", "rest") -# logging.warning(img) -# run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) -# time.sleep(2) -# run("docker rm -f combiner-rest", shell=True, check=True) -# -# def test_build_combiner_grpc(self, s2i_python_version): -# create_s2I_image(s2i_python_version, "combiner", "grpc") -# img = get_image_name("combiner", "grpc") -# run("docker run -d --rm --name 'combiner-grpc' " + img, shell=True, check=True) -# time.sleep(2) -# run("docker rm -f combiner-grpc", shell=True, check=True) -# -# -# @pytest.mark.usefixtures("s2i_python_version") -# class TestPythonS2iK8s(object): -# def test_model_rest(self, s2i_python_version): -# tester = S2IK8S() -# tester.test_model_rest(s2i_python_version) -# -# def test_input_transformer_rest(self, s2i_python_version): -# tester = S2IK8S() -# tester.test_input_transformer_rest(s2i_python_version) -# -# def test_output_transformer_rest(self, s2i_python_version): -# tester = S2IK8S() -# tester.test_output_transformer_rest(s2i_python_version) -# -# def test_router_rest(self, s2i_python_version): -# tester = S2IK8S() -# tester.test_router_rest(s2i_python_version) -# -# def test_combiner_rest(self, s2i_python_version): -# tester = S2IK8S() -# tester.test_combiner_rest(s2i_python_version) -# -# -# class S2IK8S(object): -# def test_model_rest(self, s2i_python_version): -# namespace = "s2i-test-model-rest" -# retry_run(f"kubectl create namespace {namespace}") -# create_push_s2i_image(s2i_python_version, "model", "rest") -# retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") -# wait_for_rollout("mymodel-mymodel-8715075", namespace) -# r = initial_rest_request("mymodel", namespace) -# arr = np.array([[1, 2, 3]]) -# r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) -# res = r.json() -# logging.warning(res) -# assert r.status_code == 200 -# assert r.json()["data"]["tensor"]["shape"] == [1, 3] -# assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] -# run( -# f"kubectl delete -f ../resources/s2i_python_model.json -n {namespace}", -# shell=True, -# ) -# run(f"kubectl delete namespace {namespace}", shell=True) -# -# def test_input_transformer_rest(self, s2i_python_version): -# namespace = "s2i-test-input-transformer-rest" -# retry_run(f"kubectl create namespace {namespace}") -# create_push_s2i_image(s2i_python_version, "transformer", "rest") -# retry_run( -# f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" -# ) -# wait_for_rollout("mytrans-mytrans-1f278ae", namespace) -# r = initial_rest_request("mytrans", namespace) -# arr = np.array([[1, 2, 3]]) -# r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) -# res = r.json() -# logging.warning(res) -# assert r.status_code == 200 -# assert r.json()["data"]["tensor"]["shape"] == [1, 3] -# assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] -# run( -# f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", -# shell=True, -# ) -# run(f"kubectl delete namespace {namespace}", shell=True) -# -# def test_output_transformer_rest(self, s2i_python_version): -# namespace = "s2i-test-output-transformer-rest" -# retry_run(f"kubectl create namespace {namespace}") -# create_push_s2i_image(s2i_python_version, "transformer", "rest") -# retry_run( -# f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" -# ) -# wait_for_rollout("mytrans-mytrans-52996cb", namespace) -# r = initial_rest_request("mytrans", namespace) -# arr = np.array([[1, 2, 3]]) -# r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) -# res = r.json() -# logging.warning(res) -# assert r.status_code == 200 -# assert r.json()["data"]["tensor"]["shape"] == [1, 3] -# assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] -# run( -# f"kubectl delete -f ../resources/s2i_python_output_transformer.json -n {namespace}", -# shell=True, -# ) -# run(f"kubectl create namespace {namespace}", shell=True) -# -# def test_router_rest(self, s2i_python_version): -# namespace = "s2i-test-router-rest" -# retry_run(f"kubectl create namespace {namespace}") -# create_push_s2i_image(s2i_python_version, "model", "rest") -# create_push_s2i_image(s2i_python_version, "router", "rest") -# retry_run( -# f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}" -# ) -# wait_for_rollout("myrouter-myrouter-340ed69", namespace) -# r = initial_rest_request("myrouter", namespace) -# arr = np.array([[1, 2, 3]]) -# r = rest_request_ambassador("myrouter", namespace, API_AMBASSADOR, data=arr) -# res = r.json() -# logging.warning(res) -# assert r.status_code == 200 -# assert r.json()["data"]["tensor"]["shape"] == [1, 3] -# assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] -# run( -# f"kubectl delete -f ../resources/s2i_python_router.json -n {namespace}", -# shell=True, -# ) -# run(f"kubectl delete namespace {namespace}", shell=True) -# -# def test_combiner_rest(self, s2i_python_version): -# namespace = "s2i-test-combiner-rest" -# retry_run(f"kubectl create namespace {namespace}") -# create_push_s2i_image(s2i_python_version, "model", "rest") -# create_push_s2i_image(s2i_python_version, "combiner", "rest") -# retry_run( -# f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" -# ) -# wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) -# r = initial_rest_request("mycombiner", namespace) -# arr = np.array([[1, 2, 3]]) -# r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) -# res = r.json() -# logging.warning(res) -# assert r.status_code == 200 -# assert r.json()["data"]["tensor"]["shape"] == [1, 3] -# assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] -# run( -# f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", -# shell=True, -# ) -# run(f"kubectl delete namespace {namespace}", shell=True) +import pytest +import time +from subprocess import run +import numpy as np +from seldon_e2e_utils import ( + wait_for_rollout, + rest_request_ambassador, + initial_rest_request, + retry_run, + API_AMBASSADOR, +) +import logging + +S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" +IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" + + +def create_s2I_image(s2i_python_version, component_type, api_type): + cmd = ( + S2I_CREATE.replace("#TYPE#", component_type) + .replace("#API#", api_type) + .replace("#VERSION#", s2i_python_version) + ) + logging.warning(cmd) + run(cmd, shell=True, check=True) + + +def kind_push_s2i_image(component_type, api_type): + img = get_image_name(component_type, api_type) + cmd = "kind load docker-image " + img + " --loglevel trace" + logging.warning(cmd) + run(cmd, shell=True, check=True) + + +def get_image_name(component_type, api_type): + return IMAGE_NAME.replace("#TYPE#", component_type).replace("#API#", api_type) + + +def create_push_s2i_image(s2i_python_version, component_type, api_type): + create_s2I_image(s2i_python_version, component_type, api_type) + kind_push_s2i_image(component_type, api_type) + + +@pytest.mark.usefixtures("s2i_python_version") +class TestPythonS2i(object): + def test_build_router_rest(self, s2i_python_version): + create_s2I_image(s2i_python_version, "router", "rest") + img = get_image_name("router", "rest") + run("docker run -d --rm --name 'router-rest' " + img, shell=True, check=True) + time.sleep(2) + run("docker rm -f router-rest", shell=True, check=True) + + def test_build_router_grpc(self, s2i_python_version): + create_s2I_image(s2i_python_version, "router", "grpc") + img = get_image_name("router", "grpc") + run("docker run -d --rm --name 'router-grpc' " + img, shell=True, check=True) + time.sleep(2) + run("docker rm -f router-grpc", shell=True, check=True) + + def test_build_model_rest(self, s2i_python_version): + create_s2I_image(s2i_python_version, "model", "rest") + img = get_image_name("model", "rest") + run("docker run -d --rm --name 'model-rest' " + img, shell=True, check=True) + time.sleep(2) + run("docker rm -f model-rest", shell=True, check=True) + + def test_build_model_grpc(self, s2i_python_version): + create_s2I_image(s2i_python_version, "model", "grpc") + img = get_image_name("model", "grpc") + run("docker run -d --rm --name 'model-grpc' " + img, shell=True, check=True) + time.sleep(2) + run("docker rm -f model-grpc", shell=True, check=True) + + def test_build_transformer_rest(self, s2i_python_version): + create_s2I_image(s2i_python_version, "transformer", "rest") + img = get_image_name("transformer", "rest") + run( + "docker run -d --rm --name 'transformer-rest' " + img, + shell=True, + check=True, + ) + time.sleep(2) + run("docker rm -f transformer-rest", shell=True, check=True) + + def test_build_transformer_grpc(self, s2i_python_version): + create_s2I_image(s2i_python_version, "transformer", "grpc") + img = get_image_name("transformer", "grpc") + run( + "docker run -d --rm --name 'transformer-grpc' " + img, + shell=True, + check=True, + ) + time.sleep(2) + run("docker rm -f transformer-grpc", shell=True, check=True) + + def test_build_combiner_rest(self, s2i_python_version): + create_s2I_image(s2i_python_version, "combiner", "rest") + img = get_image_name("combiner", "rest") + logging.warning(img) + run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) + time.sleep(2) + run("docker rm -f combiner-rest", shell=True, check=True) + + def test_build_combiner_grpc(self, s2i_python_version): + create_s2I_image(s2i_python_version, "combiner", "grpc") + img = get_image_name("combiner", "grpc") + run("docker run -d --rm --name 'combiner-grpc' " + img, shell=True, check=True) + time.sleep(2) + run("docker rm -f combiner-grpc", shell=True, check=True) + + +@pytest.mark.usefixtures("s2i_python_version") +class TestPythonS2iK8s(object): + def test_model_rest(self, s2i_python_version): + tester = S2IK8S() + tester.test_model_rest(s2i_python_version) + + def test_input_transformer_rest(self, s2i_python_version): + tester = S2IK8S() + tester.test_input_transformer_rest(s2i_python_version) + + def test_output_transformer_rest(self, s2i_python_version): + tester = S2IK8S() + tester.test_output_transformer_rest(s2i_python_version) + + def test_router_rest(self, s2i_python_version): + tester = S2IK8S() + tester.test_router_rest(s2i_python_version) + + def test_combiner_rest(self, s2i_python_version): + tester = S2IK8S() + tester.test_combiner_rest(s2i_python_version) + + +class S2IK8S(object): + def test_model_rest(self, s2i_python_version): + namespace = "s2i-test-model-rest" + retry_run(f"kubectl create namespace {namespace}") + create_push_s2i_image(s2i_python_version, "model", "rest") + retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") + wait_for_rollout("mymodel-mymodel-8715075", namespace) + r = initial_rest_request("mymodel", namespace) + arr = np.array([[1, 2, 3]]) + r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) + res = r.json() + logging.warning(res) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["shape"] == [1, 3] + assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] + run( + f"kubectl delete -f ../resources/s2i_python_model.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True) + + def test_input_transformer_rest(self, s2i_python_version): + namespace = "s2i-test-input-transformer-rest" + retry_run(f"kubectl create namespace {namespace}") + create_push_s2i_image(s2i_python_version, "transformer", "rest") + retry_run( + f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" + ) + wait_for_rollout("mytrans-mytrans-1f278ae", namespace) + r = initial_rest_request("mytrans", namespace) + arr = np.array([[1, 2, 3]]) + r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) + res = r.json() + logging.warning(res) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["shape"] == [1, 3] + assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] + run( + f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True) + + def test_output_transformer_rest(self, s2i_python_version): + namespace = "s2i-test-output-transformer-rest" + retry_run(f"kubectl create namespace {namespace}") + create_push_s2i_image(s2i_python_version, "transformer", "rest") + retry_run( + f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" + ) + wait_for_rollout("mytrans-mytrans-52996cb", namespace) + r = initial_rest_request("mytrans", namespace) + arr = np.array([[1, 2, 3]]) + r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) + res = r.json() + logging.warning(res) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["shape"] == [1, 3] + assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] + run( + f"kubectl delete -f ../resources/s2i_python_output_transformer.json -n {namespace}", + shell=True, + ) + run(f"kubectl create namespace {namespace}", shell=True) + + def test_router_rest(self, s2i_python_version): + namespace = "s2i-test-router-rest" + retry_run(f"kubectl create namespace {namespace}") + create_push_s2i_image(s2i_python_version, "model", "rest") + create_push_s2i_image(s2i_python_version, "router", "rest") + retry_run( + f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}" + ) + wait_for_rollout("myrouter-myrouter-340ed69", namespace) + r = initial_rest_request("myrouter", namespace) + arr = np.array([[1, 2, 3]]) + r = rest_request_ambassador("myrouter", namespace, API_AMBASSADOR, data=arr) + res = r.json() + logging.warning(res) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["shape"] == [1, 3] + assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] + run( + f"kubectl delete -f ../resources/s2i_python_router.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True) + + def test_combiner_rest(self, s2i_python_version): + namespace = "s2i-test-combiner-rest" + retry_run(f"kubectl create namespace {namespace}") + create_push_s2i_image(s2i_python_version, "model", "rest") + create_push_s2i_image(s2i_python_version, "combiner", "rest") + retry_run( + f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" + ) + wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace) + r = initial_rest_request("mycombiner", namespace) + arr = np.array([[1, 2, 3]]) + r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) + res = r.json() + logging.warning(res) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["shape"] == [1, 3] + assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] + run( + f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", + shell=True, + ) + run(f"kubectl delete namespace {namespace}", shell=True) From 1aa5727339156dd3a733f4fbef1693c6e7f750ba Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 11:10:04 +0000 Subject: [PATCH 29/40] Added initial rest request --- testing/scripts/test_prepackaged_servers.py | 21 +++++++++------ testing/scripts/test_rolling_updates.py | 30 +++++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index e4b960b162..ef98d2f5ca 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -1,7 +1,11 @@ import subprocess import json -from seldon_core.seldon_client import SeldonClient -from seldon_e2e_utils import wait_for_rollout, retry_run, wait_for_status +from seldon_e2e_utils import ( + wait_for_rollout, + initial_rest_request, + retry_run, + wait_for_status, +) from subprocess import run import time import logging @@ -20,8 +24,8 @@ def test_sklearn(self): wait_for_status("sklearn", namespace) time.sleep(1) logging.warning("Initial request") - sc = SeldonClient(deployment_name="sklearn", namespace=namespace) - r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) + r = initial_rest_request("sklearn", namespace) + assert r.status_code == 200 assert r.success logging.warning("Success for test_prepack_sklearn") run( @@ -41,8 +45,8 @@ def test_tfserving(self): wait_for_status("tfserving", namespace) time.sleep(1) logging.warning("Initial request") - sc = SeldonClient(deployment_name="tfserving", namespace=namespace) - r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 784)) + r = initial_rest_request("tfserving", namespace) + assert r.status_code == 200 assert r.success logging.warning("Success for test_prepack_tfserving") run( @@ -62,8 +66,9 @@ def test_xgboost(self): wait_for_status("xgboost", namespace) time.sleep(1) logging.warning("Initial request") - sc = SeldonClient(deployment_name="xgboost", namespace=namespace) - r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4)) + r = initial_rest_request("xgboost", namespace) + assert r.status_code == 200 + assert r.success assert r.success logging.warning("Success for test_prepack_xgboost") run( diff --git a/testing/scripts/test_rolling_updates.py b/testing/scripts/test_rolling_updates.py index 2068df0c9a..8fb8cdb9c1 100644 --- a/testing/scripts/test_rolling_updates.py +++ b/testing/scripts/test_rolling_updates.py @@ -23,6 +23,9 @@ def test_rolling_update1(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph2.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -59,6 +62,9 @@ def test_rolling_update2(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph3.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -96,6 +102,9 @@ def test_rolling_update3(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph4.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -127,6 +136,9 @@ def test_rolling_update4(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph5.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -162,6 +174,9 @@ def test_rolling_update5(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph6.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -200,6 +215,9 @@ def test_rolling_update6(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph2svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -237,6 +255,9 @@ def test_rolling_update7(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph3svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -274,6 +295,9 @@ def test_rolling_update8(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph4svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -305,6 +329,9 @@ def test_rolling_update9(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph5svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) @@ -340,6 +367,9 @@ def test_rolling_update10(self): assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph6svc.json -n {namespace}") + r = initial_rest_request("mymodel", namespace) + assert r.status_code == 200 + assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) From 125e92544dd8b4d948a3b840a1de0e676afc03a5 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 11:30:53 +0000 Subject: [PATCH 30/40] Added support for initial requests with data --- testing/scripts/seldon_e2e_utils.py | 48 ++++++++++++++++++--- testing/scripts/test_prepackaged_servers.py | 6 +-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index 37cc161789..03282a2c7d 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -83,9 +83,18 @@ def wait_for_status(name, namespace): time.sleep(5) -def rest_request(model, namespace): +def rest_request( + model, namespace, endpoint=API_AMBASSADOR, data_size=5, rows=1, data=None +): try: - r = rest_request_ambassador(model, namespace, API_AMBASSADOR) + r = rest_request_ambassador( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + ) if not r.status_code == 200: logging.warning("Bad status:", r.status_code) return None @@ -96,20 +105,45 @@ def rest_request(model, namespace): return None -def initial_rest_request(model, namespace): - r = rest_request(model, namespace) +def initial_rest_request( + model, namespace, endpoint=API_AMBASSADOR, data_size=5, rows=1, data=None +): + r = rest_request( + model, namespace, endpoint=endpoint, data_size=data_size, rows=rows, data=data + ) if r is None: logging.warning("Sleeping 1 sec and trying again") time.sleep(1) - r = rest_request(model, namespace) + r = rest_request( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + ) if r is None: logging.warning("Sleeping 5 sec and trying again") time.sleep(5) - r = rest_request(model, namespace) + r = rest_request( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + ) if r is None: logging.warning("Sleeping 10 sec and trying again") time.sleep(10) - r = rest_request(model, namespace) + r = rest_request( + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + ) return r diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index ef98d2f5ca..b7c19b37b4 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -24,7 +24,7 @@ def test_sklearn(self): wait_for_status("sklearn", namespace) time.sleep(1) logging.warning("Initial request") - r = initial_rest_request("sklearn", namespace) + r = initial_rest_request("sklearn", namespace, rows=1, data_size=4) assert r.status_code == 200 assert r.success logging.warning("Success for test_prepack_sklearn") @@ -45,7 +45,7 @@ def test_tfserving(self): wait_for_status("tfserving", namespace) time.sleep(1) logging.warning("Initial request") - r = initial_rest_request("tfserving", namespace) + r = initial_rest_request("tfserving", namespace, rows=1, data_size=784) assert r.status_code == 200 assert r.success logging.warning("Success for test_prepack_tfserving") @@ -66,7 +66,7 @@ def test_xgboost(self): wait_for_status("xgboost", namespace) time.sleep(1) logging.warning("Initial request") - r = initial_rest_request("xgboost", namespace) + r = initial_rest_request("xgboost", namespace, rows=1, data_size=4) assert r.status_code == 200 assert r.success assert r.success From b44b0382737e4d438be23ed8f8ba445fab656491 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 12:15:57 +0000 Subject: [PATCH 31/40] Added support for initial requests with data --- testing/scripts/seldon_e2e_utils.py | 51 +++++++++++++++++---- testing/scripts/test_prepackaged_servers.py | 12 +++-- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index 03282a2c7d..741c860fed 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -84,7 +84,13 @@ def wait_for_status(name, namespace): def rest_request( - model, namespace, endpoint=API_AMBASSADOR, data_size=5, rows=1, data=None + model, + namespace, + endpoint=API_AMBASSADOR, + data_size=5, + rows=1, + data=None, + dtype="tensor", ): try: r = rest_request_ambassador( @@ -94,6 +100,7 @@ def rest_request( data_size=data_size, rows=rows, data=data, + dtype=dtype, ) if not r.status_code == 200: logging.warning("Bad status:", r.status_code) @@ -106,10 +113,22 @@ def rest_request( def initial_rest_request( - model, namespace, endpoint=API_AMBASSADOR, data_size=5, rows=1, data=None + model, + namespace, + endpoint=API_AMBASSADOR, + data_size=5, + rows=1, + data=None, + dtype="tensor", ): r = rest_request( - model, namespace, endpoint=endpoint, data_size=data_size, rows=rows, data=data + model, + namespace, + endpoint=endpoint, + data_size=data_size, + rows=rows, + data=data, + dtype=dtype, ) if r is None: logging.warning("Sleeping 1 sec and trying again") @@ -121,6 +140,7 @@ def initial_rest_request( data_size=data_size, rows=rows, data=data, + dtype=dtype, ) if r is None: logging.warning("Sleeping 5 sec and trying again") @@ -132,6 +152,7 @@ def initial_rest_request( data_size=data_size, rows=rows, data=data, + dtype=dtype, ) if r is None: logging.warning("Sleeping 10 sec and trying again") @@ -143,6 +164,7 @@ def initial_rest_request( data_size=data_size, rows=rows, data=data, + dtype=dtype, ) return r @@ -159,19 +181,30 @@ def create_random_data(data_size, rows=1): stop_max_attempt_number=5, ) def rest_request_ambassador( - deploymentName, namespace, endpoint="localhost:8003", data_size=5, rows=1, data=None + deploymentName, + namespace, + endpoint="localhost:8003", + data_size=5, + rows=1, + data=None, + dtype="tensor", ): if data is None: shape, arr = create_random_data(data_size, rows) else: shape = data.shape arr = data.flatten() - payload = { - "data": { - "names": ["a", "b"], - "tensor": {"shape": shape, "values": arr.tolist()}, + + if dtype == "tensor": + payload = { + "data": { + "names": ["a", "b"], + "tensor": {"shape": shape, "values": arr.tolist()}, + } } - } + else: + payload = {"data": {"names": ["a", "b"], "ndarray": arr.tolist(),}} + if namespace is None: response = requests.post( "http://" diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index b7c19b37b4..d04e3a1042 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -24,7 +24,9 @@ def test_sklearn(self): wait_for_status("sklearn", namespace) time.sleep(1) logging.warning("Initial request") - r = initial_rest_request("sklearn", namespace, rows=1, data_size=4) + r = initial_rest_request( + "sklearn", namespace, rows=1, data_size=4, dtype="ndarray" + ) assert r.status_code == 200 assert r.success logging.warning("Success for test_prepack_sklearn") @@ -45,7 +47,9 @@ def test_tfserving(self): wait_for_status("tfserving", namespace) time.sleep(1) logging.warning("Initial request") - r = initial_rest_request("tfserving", namespace, rows=1, data_size=784) + r = initial_rest_request( + "tfserving", namespace, rows=1, data_size=784, dtype="ndarray" + ) assert r.status_code == 200 assert r.success logging.warning("Success for test_prepack_tfserving") @@ -66,7 +70,9 @@ def test_xgboost(self): wait_for_status("xgboost", namespace) time.sleep(1) logging.warning("Initial request") - r = initial_rest_request("xgboost", namespace, rows=1, data_size=4) + r = initial_rest_request( + "xgboost", namespace, rows=1, data_size=4, dtype="ndarray" + ) assert r.status_code == 200 assert r.success assert r.success From f211d0dd81e6ac8762c2f7ca8a68425768cce0f2 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 12 Nov 2019 13:07:45 +0000 Subject: [PATCH 32/40] Added support for initial requests with data --- testing/scripts/test_prepackaged_servers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index d04e3a1042..c3caac4cef 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -28,7 +28,6 @@ def test_sklearn(self): "sklearn", namespace, rows=1, data_size=4, dtype="ndarray" ) assert r.status_code == 200 - assert r.success logging.warning("Success for test_prepack_sklearn") run( f"kubectl delete -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}", @@ -51,7 +50,6 @@ def test_tfserving(self): "tfserving", namespace, rows=1, data_size=784, dtype="ndarray" ) assert r.status_code == 200 - assert r.success logging.warning("Success for test_prepack_tfserving") run( f"kubectl delete -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}", @@ -74,8 +72,6 @@ def test_xgboost(self): "xgboost", namespace, rows=1, data_size=4, dtype="ndarray" ) assert r.status_code == 200 - assert r.success - assert r.success logging.warning("Success for test_prepack_xgboost") run( f"kubectl delete -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}", From 10a9c8c6bf304fb52890454d36a13917146d6940 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 08:20:03 +0000 Subject: [PATCH 33/40] Adding explicit run of script on background --- testing/scripts/conftest.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/testing/scripts/conftest.py b/testing/scripts/conftest.py index fd31208ba0..21063a38ee 100644 --- a/testing/scripts/conftest.py +++ b/testing/scripts/conftest.py @@ -1,5 +1,16 @@ import pytest from seldon_e2e_utils import get_s2i_python_version +from subprocess import run + + +@pytest.fixture(scope="session", autouse=True) +def run_pod_information_in_background(request): + # This command runs the pod information and prints it + # in the background every time there's a new update + run( + "kubectl get pods --all-namespaces -w | awk '{print \"\nPODS UPDATE: \"$0}\n' &", + shell=True, + ) @pytest.fixture(scope="module") @@ -7,8 +18,5 @@ def s2i_python_version(): return do_s2i_python_version() -#### Implementations below - - def do_s2i_python_version(): return get_s2i_python_version() From b6fe47a9be8df4a558c0dbc239b90b56eda3ee30 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 08:41:07 +0000 Subject: [PATCH 34/40] Updated pods info --- testing/scripts/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/scripts/conftest.py b/testing/scripts/conftest.py index 21063a38ee..efd695470f 100644 --- a/testing/scripts/conftest.py +++ b/testing/scripts/conftest.py @@ -8,7 +8,10 @@ def run_pod_information_in_background(request): # This command runs the pod information and prints it # in the background every time there's a new update run( - "kubectl get pods --all-namespaces -w | awk '{print \"\nPODS UPDATE: \"$0}\n' &", + ( + "kubectl get pods --all-namespaces -w | " + + 'awk \'{print "\\nPODS UPDATE: "$0"\\n"}\' & ' + ), shell=True, ) From 6f85ebd7fb717f7e5d463aad370de39b4c210d1a Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 11:02:50 +0000 Subject: [PATCH 35/40] Updated pods to pass all tests --- testing/scripts/seldon_e2e_utils.py | 8 +++++--- testing/scripts/test_prepackaged_servers.py | 8 ++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index 741c860fed..8c8795c48f 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -183,7 +183,7 @@ def create_random_data(data_size, rows=1): def rest_request_ambassador( deploymentName, namespace, - endpoint="localhost:8003", + endpoint=API_AMBASSADOR, data_size=5, rows=1, data=None, @@ -191,9 +191,11 @@ def rest_request_ambassador( ): if data is None: shape, arr = create_random_data(data_size, rows) - else: + elif dtype == "tensor": shape = data.shape arr = data.flatten() + else: + arr = data if dtype == "tensor": payload = { @@ -203,7 +205,7 @@ def rest_request_ambassador( } } else: - payload = {"data": {"names": ["a", "b"], "ndarray": arr.tolist(),}} + payload = {"data": {"names": ["a", "b"], "ndarray": arr}} if namespace is None: response = requests.post( diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index c3caac4cef..d2e93691b8 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -4,6 +4,7 @@ wait_for_rollout, initial_rest_request, retry_run, + create_random_data, wait_for_status, ) from subprocess import run @@ -47,7 +48,10 @@ def test_tfserving(self): time.sleep(1) logging.warning("Initial request") r = initial_rest_request( - "tfserving", namespace, rows=1, data_size=784, dtype="ndarray" + "tfserving", + namespace, + data=[create_random_data(784)[1].tolist()], + dtype="ndarray", ) assert r.status_code == 200 logging.warning("Success for test_prepack_tfserving") @@ -69,7 +73,7 @@ def test_xgboost(self): time.sleep(1) logging.warning("Initial request") r = initial_rest_request( - "xgboost", namespace, rows=1, data_size=4, dtype="ndarray" + "xgboost", namespace, data=[[0.1, 0.2, 0.3, 0.4]], dtype="ndarray" ) assert r.status_code == 200 logging.warning("Success for test_prepack_xgboost") From 1abde693819497676d39bc222a6c0abb9422711f Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 11:31:04 +0000 Subject: [PATCH 36/40] Updated prepackaged model --- testing/scripts/test_prepackaged_servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index d2e93691b8..0e35c4ac86 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -26,7 +26,7 @@ def test_sklearn(self): time.sleep(1) logging.warning("Initial request") r = initial_rest_request( - "sklearn", namespace, rows=1, data_size=4, dtype="ndarray" + "sklearn", namespace, data=[[0.1, 0.2, 0.3, 0.4]], dtype="ndarray" ) assert r.status_code == 200 logging.warning("Success for test_prepack_sklearn") From 7f50210bd8e43d6d94f5bd11021cbd25380fd778 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 13:39:23 +0000 Subject: [PATCH 37/40] Added lower attempts and lower cores --- testing/scripts/Makefile | 2 +- testing/scripts/seldon_e2e_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 9bef5f2103..32095de114 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,6 +1,6 @@ VERSION := $(shell cat ../../version.txt) -PYTEST_WORKERS ?= "auto" +PYTEST_WORKERS ?= "4" kind_create_cluster: kind create cluster --config kind_config.yaml diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index 8c8795c48f..c0ccca6436 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -41,7 +41,7 @@ def wait_for_shutdown(deploymentName, namespace): ret = run(f"kubectl get -n {namespace} deploy/{deploymentName}", shell=True) -def wait_for_rollout(deploymentName, namespace, attempts=50, sleep=5): +def wait_for_rollout(deploymentName, namespace, attempts=20, sleep=5): for attempts in range(attempts): ret = run( f"kubectl rollout status -n {namespace} deploy/" + deploymentName, @@ -55,7 +55,7 @@ def wait_for_rollout(deploymentName, namespace, attempts=50, sleep=5): assert ret.returncode == 0 -def retry_run(cmd, attempts=50, sleep=5): +def retry_run(cmd, attempts=10, sleep=5): for i in range(attempts): ret = run(cmd, shell=True) if ret.returncode == 0: From b1df57d540e7ce9b4e551e70f2f1637c949ac520 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 16:00:05 +0000 Subject: [PATCH 38/40] Separated s2i tests from the rest --- testing/scripts/Makefile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 32095de114..19df7c396f 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,6 +1,7 @@ VERSION := $(shell cat ../../version.txt) -PYTEST_WORKERS ?= "4" +# This can be "Auto" or a Number +PYTEST_WORKERS ?= "auto" kind_create_cluster: kind create cluster --config kind_config.yaml @@ -64,7 +65,20 @@ install: .PHONY: test test: build_protos install pip install pytest-xdist - pytest --verbose -s -W ignore -n $(PYTEST_WORKERS) 2>&1 + # Run the core tests in parallel + pytest \ + --verbose \ + -s \ + -W ignore \ + -n $(PYTEST_WORKERS) \ + --ignore test_s2i_python 2>&1 + # Then run the s2i tests in sequence (as they currently fail in parallel + pytest \ + --verbose \ + -s \ + -W ignore \ + -n 0 \ + test_s2i_python.py 2>&1 .PHONY: clean clean: From cc865bc737780ed2153ae96caa325a6e009c658c Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 16:33:57 +0000 Subject: [PATCH 39/40] Separated s2i tests from the rest --- testing/scripts/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 19df7c396f..ca468d933b 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -1,7 +1,7 @@ VERSION := $(shell cat ../../version.txt) # This can be "Auto" or a Number -PYTEST_WORKERS ?= "auto" +PYTEST_WORKERS ?= "4" kind_create_cluster: kind create cluster --config kind_config.yaml @@ -70,8 +70,8 @@ test: build_protos install --verbose \ -s \ -W ignore \ - -n $(PYTEST_WORKERS) \ - --ignore test_s2i_python 2>&1 + --ignore test_s2i_python.py \ + -n $(PYTEST_WORKERS) 2>&1 # Then run the s2i tests in sequence (as they currently fail in parallel pytest \ --verbose \ From c4025e4fd9f0032aaf7060143ca7b49805e808d6 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 13 Nov 2019 17:12:09 +0000 Subject: [PATCH 40/40] Added check for 200 check --- testing/scripts/seldon_e2e_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index c0ccca6436..46613d794d 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -130,7 +130,7 @@ def initial_rest_request( data=data, dtype=dtype, ) - if r is None: + if r is None or r.status_code != 200: logging.warning("Sleeping 1 sec and trying again") time.sleep(1) r = rest_request( @@ -142,7 +142,7 @@ def initial_rest_request( data=data, dtype=dtype, ) - if r is None: + if r is None or r.status_code != 200: logging.warning("Sleeping 5 sec and trying again") time.sleep(5) r = rest_request( @@ -154,7 +154,7 @@ def initial_rest_request( data=data, dtype=dtype, ) - if r is None: + if r is None or r.status_code != 200: logging.warning("Sleeping 10 sec and trying again") time.sleep(10) r = rest_request(