Skip to content

Commit

Permalink
[#133] run edi one model tests in new enclave
Browse files Browse the repository at this point in the history
  • Loading branch information
@ablatov
ablatov committed Oct 7, 2018
1 parent cec1fd2 commit 1456523
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 48 deletions.
20 changes: 9 additions & 11 deletions legion/legion/k8s/services.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,9 @@ def _load_deployment_data(self):
self._deployment = ensure_function_succeed(self._load_deployment_data_logic,
LOAD_DATA_ITERATIONS, LOAD_DATA_TIMEOUT)
if not self._deployment:
self._deployment_data_loaded = True
raise Exception('Failed to load deployment for {!r}'.format(self))

self._deployment_data_loaded = True

def reload_cache(self):
"""
Expand All @@ -318,7 +320,8 @@ def scale(self):
:return: int -- current model scale
"""
if self.deployment and self.deployment.status.available_replicas:
self._load_deployment_data()
if self.deployment.status.available_replicas:
return self.deployment.status.available_replicas
else:
return 0
Expand All @@ -335,9 +338,7 @@ def scale(self, new_scale):
if new_scale < 1:
raise Exception('Invalid scale parameter: should be greater then 0')

if not self.deployment:
raise Exception('Unable to find desired deployment.')

self._load_deployment_data()
client = legion.k8s.utils.build_client()

extension_api = kubernetes.client.ExtensionsV1beta1Api(client)
Expand All @@ -362,9 +363,6 @@ def delete(self, grace_period_seconds=0):
:type grace_period_seconds: int
:return: None
"""
if not self.deployment:
raise Exception('Unable to find desired deployment.')

client = legion.k8s.utils.build_client()

api_instance = kubernetes.client.AppsV1beta1Api(client)
Expand Down Expand Up @@ -392,7 +390,7 @@ def desired_scale(self):
:return: int -- desired model scale
"""
self._load_deployment_data()
return 0 if not self.deployment else self.deployment.status.replicas if self.deployment.status.replicas else 0
return self.deployment.status.replicas if self.deployment.status.replicas else 0

@property
def status(self):
Expand All @@ -416,10 +414,10 @@ def image(self):
"""
Get model image
:return: str -- model image, or None if deployment isn't found
:return: str -- model image
"""
self._load_deployment_data()
return None if not self.deployment else self.deployment.spec.template.spec.containers[0].image
return self.deployment.spec.template.spec.containers[0].image

@property
def metrics_name(self):
Expand Down
39 changes: 39 additions & 0 deletions legion_test/legion_test/robot/k8s.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import kubernetes.config
import kubernetes.config.config_exception
import urllib3
from kubernetes.client.rest import ApiException

import legion.k8s.utils
import legion.k8s.properties
Expand Down Expand Up @@ -231,3 +232,41 @@ def update_model_property_key(self, namespace, model_id, model_version, key, val
property.load()
property[key] = value
property.save()

def create_enclave(self, enclave_name):
"""
Generate which returns events for enclaves
:return: A tuple (event type [ADDED, DELETED, MODIFIED], service object)
:rtype: (str, Enclave)
"""
client = self.build_client()
core_api = kubernetes.client.CoreV1Api(client)

body = kubernetes.client.V1Namespace(metadata={'name': enclave_name}) # V1Namespace |
pretty = 'pretty_example' # str | If 'true', then the output is pretty printed. (optional)

try:
api_response = core_api.create_namespace(body, pretty=pretty)
print(api_response)
except ApiException as e:
print("Exception when calling CoreV1Api->create_namespace: %s\n" % e)

def delete_enclave(self, enclave_name):
"""
Generate which returns events for enclaves
:return: A tuple (event type [ADDED, DELETED, MODIFIED], service object)
:rtype: (str, Enclave)
"""
client = self.build_client()
core_api = kubernetes.client.CoreV1Api(client)

body = kubernetes.client.V1DeleteOptions()
pretty = 'pretty_example'

try:
api_response = core_api.delete_namespace(enclave_name, body, pretty=pretty)
print(api_response)
except ApiException as e:
print("Exception when calling CoreV1Api->create_namespace: %s\n" % e)
3 changes: 2 additions & 1 deletion tests/robot/resources/variables.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,5 @@ ${TEST_MODEL_2_VERSION} 1.1
${TEST_MODEL_3_VERSION} 1.2
${TEST_MODEL_VERSION} 1.0
${FEEDBACK_TAG} tag1
${FEEDBACK_PREFIX} events
${FEEDBACK_PREFIX} events
${MODEL_TEST_ENCLAVE_2} company-autotest
76 changes: 40 additions & 36 deletions tests/robot/tests/3_edi_one_model/3.0_check_edi_one_model.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@ Resource ../../resources/keywords.robot
Resource ../../resources/variables.robot
Variables ../../load_variables_from_profiles.py ${PATH_TO_PROFILES_DIR}
Library legion_test.robot.Utils
Library legion_test.robot.K8s
Library Collections
Suite Setup Choose cluster context ${CLUSTER_NAME}
Test Setup Run EDI deploy and check model started ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
Test Teardown Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
Suite Setup Run keywords
... Create enclave ${MODEL_TEST_ENCLAVE_2} AND
... Choose cluster context ${CLUSTER_NAME}
Suite Teardown Delete enclave ${MODEL_TEST_ENCLAVE_2}
Test Setup Run EDI deploy and check model started ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
Test Teardown Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}

*** Test Cases ***
Check EDI availability in all enclaves
Expand All @@ -21,139 +25,139 @@ Check EDI availability in all enclaves
[Teardown] NONE

Check EDI deploy procedure
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}
[Documentation] Try to deploy dummy model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3}
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3}
Should Be Equal As Integers ${resp.rc} 0
${response}= Check model started ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
${response}= Check model started ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
Should contain ${response} "model_version": "${TEST_MODEL_3_VERSION}"

Check EDI deploy with scale to 0
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}
[Documentation] Try to deploy dummy model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI deploy with scale ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3} 0
${resp}= Run EDI deploy with scale ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3} 0
Should Be Equal As Integers ${resp.rc} 2
Should contain ${resp.stderr} Invalid scale parameter: should be greater then 0

Check EDI deploy with scale to 1
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}
[Documentation] Try to deploy dummy model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI deploy with scale ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3} 1
${resp}= Run EDI deploy with scale ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3} 1
Should Be Equal As Integers ${resp.rc} 0
${response}= Check model started ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
${response}= Check model started ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
Should contain ${response} "model_version": "${TEST_MODEL_3_VERSION}"

${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE_2}
${model}= Find model information in edi ${resp} ${TEST_EDI_MODEL_ID}
Log ${model}
Verify model info from edi ${model} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_IMAGE_3} ${TEST_MODEL_3_VERSION} 1

Check EDI deploy with scale to 2
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}
[Documentation] Try to deploy dummy model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI deploy with scale ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3} 2
${resp}= Run EDI deploy with scale ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3} 2
Should Be Equal As Integers ${resp.rc} 0
${response}= Check model started ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
${response}= Check model started ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
Should contain ${response} "model_version": "${TEST_MODEL_3_VERSION}"

${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE_2}
${model}= Find model information in edi ${resp} ${TEST_EDI_MODEL_ID}
Log ${model}
Verify model info from edi ${model} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_IMAGE_3} ${TEST_MODEL_3_VERSION} 2

Check EDI invalid model name deploy procedure
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}
[Documentation] Try to deploy dummy invalid model name through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3}test
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3}test
Should Be Equal As Integers ${resp.rc} 2
Should Contain ${resp.stderr} Can't get image labels for ${TEST_MODEL_IMAGE_3}test

Check EDI double deploy procedure for the same model
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
[Setup] Run EDI undeploy model without version and check ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}
[Documentation] Try to deploy twice the same dummy model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3}
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3}
Should Be Equal As Integers ${resp.rc} 0
${response}= Check model started ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
${response}= Check model started ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
Should contain ${response} "model_version": "${TEST_MODEL_3_VERSION}"
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE} ${TEST_MODEL_IMAGE_3}
${resp}= Run EDI deploy ${MODEL_TEST_ENCLAVE_2} ${TEST_MODEL_IMAGE_3}
Should Be Equal As Integers ${resp.rc} 0
${response}= Check model started ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
${response}= Check model started ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_3_VERSION}
Should contain ${response} "model_version": "${TEST_MODEL_3_VERSION}"

Check EDI undeploy procedure
[Documentation] Try to undeploy dummy valid model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI undeploy without version ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}
${resp}= Run EDI undeploy without version ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}
Should Be Equal As Integers ${resp.rc} 0
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE_2}
Should Be Equal As Integers ${resp.rc} 0
Should not contain ${resp.stdout} ${TEST_EDI_MODEL_ID}

Check EDI scale up procedure
[Documentation] Try to scale up model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} 2
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} 2
Should Be Equal As Integers ${resp.rc} 0
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE_2}
${model}= Find model information in edi ${resp} ${TEST_EDI_MODEL_ID}
Log ${model}
Verify model info from edi ${model} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_IMAGE_3} ${TEST_MODEL_3_VERSION} 2

Check EDI scale down procedure
[Documentation] Try to scale up model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} 2
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} 2
Should Be Equal As Integers ${resp.rc} 0
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE_2}
${model}= Find model information in edi ${resp} ${TEST_EDI_MODEL_ID}
Log ${model}
Verify model info from edi ${model} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_IMAGE_3} ${TEST_MODEL_3_VERSION} 2

${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} 1
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} 1
Should Be Equal As Integers ${resp.rc} 0
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect with parse ${MODEL_TEST_ENCLAVE_2}
${model}= Find model information in edi ${resp} ${TEST_EDI_MODEL_ID}
Log ${model}
Verify model info from edi ${model} ${TEST_EDI_MODEL_ID} ${TEST_MODEL_IMAGE_3} ${TEST_MODEL_3_VERSION} 1

Check EDI scale to 0 procedure
[Documentation] Try to scale to 0 model through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID} 0
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID} 0
Should Be Equal As Integers ${resp.rc} 2
Should contain ${resp.stderr} Invalid scale parameter: should be greater then 0

Check EDI invalid model id scale up procedure
[Documentation] Try to scale up dummy model with invalid name through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE} ${TEST_EDI_MODEL_ID}test 2
${resp}= Run EDI scale ${MODEL_TEST_ENCLAVE_2} ${TEST_EDI_MODEL_ID}test 2
Should Be Equal As Integers ${resp.rc} 2
Should contain ${resp.stderr} No one model can be found

Check EDI enclave inspect procedure
[Documentation] Try to inspect enclave through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE_2}
Should Be Equal As Integers ${resp.rc} 0
Should contain ${resp.stdout} ${TEST_EDI_MODEL_ID}

Check EDI invalid enclave name inspect procedure
[Documentation] Try to inspect enclave through EDI console
[Tags] edi cli enclave one_version
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE}test
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE_2}test
Should Be Equal As Integers ${resp.rc} 2
Should contain ${resp.stderr} ERROR - Failed to connect

Check EDI enclave inspect procedure without deployed model
[Setup] NONE
[Documentation] Try inspect through EDI console on empty enclave
[Tags] edi cli enclave one_version
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE}
${resp}= Run EDI inspect ${MODEL_TEST_ENCLAVE_2}
Should Be Equal As Integers ${resp.rc} 0
Should Not Contain ${resp.stdout} ${TEST_EDI_MODEL_ID}

0 comments on commit 1456523

Please sign in to comment.