[#133] Rearrange tests

Jenkinsfile

@@ -301,7 +301,12 @@ node {
                     cd k8s/test-bare-model-api/model-2
                     docker build --build-arg version="${Globals.buildVersion}" -t legion/test-bare-model-api-model-2:${Globals.buildVersion} ${Globals.dockerLabels} .
                     """
-                }, 'Build Edi Docker image': {
+                }, 'Build Bare model 3': {
+                    sh """
+                    cd k8s/test-bare-model-api/model-3
+                    docker build --build-arg version="${Globals.buildVersion}" -t legion/test-bare-model-api-model-3:${Globals.buildVersion} ${Globals.dockerLabels} .
+                    """
+                },'Build Edi Docker image': {
                     sh """
                     cd k8s/edi
                     docker build --build-arg version="${Globals.buildVersion}" --build-arg pip_extra_index_params="--extra-index-url ${params.PyPiRepository}" --build-arg pip_legion_version_string="==${Globals.buildVersion}" -t legion/k8s-edi:${Globals.buildVersion} ${Globals.dockerLabels} .
@@ -334,6 +339,8 @@ node {
                     UploadDockerImage('test-bare-model-api-model-1')
                 }, 'Upload Bare model 2': {
                     UploadDockerImage('test-bare-model-api-model-2')
+                }, 'Upload Bare model 3': {
+                    UploadDockerImage('test-bare-model-api-model-3')
                 }, 'Upload Edi Docker image': {
                     UploadDockerImage('k8s-edi')
                 }, 'Upload Airflow Docker image': {

deploy/legionPipeline.groovy

@@ -173,6 +173,7 @@ def runRobotTests(tags="") {
 
             PATH_TO_PROFILES_DIR="${PROFILES_PATH:-../../deploy/profiles}/"
             PATH_TO_PROFILE_FILE="${PATH_TO_PROFILES_DIR}$Profile.yml"
+            PATH_TO_COOKIES="${PATH_TO_PROFILES_DIR}cookies.dat"
             CLUSTER_NAME=$(yq -r .cluster_name $PATH_TO_PROFILE_FILE)
             CLUSTER_STATE_STORE=$(yq -r .state_store $PATH_TO_PROFILE_FILE)
             echo "Loading kubectl config from $CLUSTER_STATE_STORE for cluster $CLUSTER_NAME"
@@ -184,7 +185,12 @@ def runRobotTests(tags="") {
             kops export kubecfg --name $CLUSTER_NAME --state $CLUSTER_STATE_STORE
             PATH=../../.venv/bin:$PATH DISPLAY=:99 \
             PROFILE=$Profile LEGION_VERSION=$LegionVersion \
-            ../../.venv/bin/python3 -m robot.run --listener legion_test.process_reporter --variable PATH_TO_PROFILES_DIR:$PATH_TO_PROFILES_DIR $robot_tags *.robot || true
+            jenkins_dex_client --path_to_profiles $PATH_TO_PROFILES_DIR > $PATH_TO_COOKIES &
+            wait
+            cat $PATH_TO_COOKIES
+            PATH=../../.venv/bin:$PATH DISPLAY=:99 \
+            PROFILE=$Profile LEGION_VERSION=$LegionVersion PATH_TO_COOKIES=$PATH_TO_COOKIES \
+            ../../.venv/bin/python3 ../../.venv/bin/pabot --verbose --processes 4 --variable PATH_TO_PROFILES_DIR:$PATH_TO_PROFILES_DIR --listener legion_test.process_reporter $robot_tags --outputdir . tests/**/*.robot || true
 
             echo "Starting python tests"
             cd ../python

k8s/test-bare-model-api/model-3/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.6-alpine3.7
+
+COPY . /app
+WORKDIR /app
+
+RUN pip install -r requirements.txt
+
+LABEL com.epam.legion.model.id="edi-test-model" \
+      com.epam.legion.model.version="1.2" \
+      com.epam.legion.container_type="model"
+
+ENTRYPOINT ["python"]
+CMD ["app.py"]

k8s/test-bare-model-api/model-3/app.py

@@ -0,0 +1,40 @@
+from flask import Flask, jsonify
+
+app = Flask(__name__)
+
+
+@app.route('/')
+def root():
+    return jsonify(demo=True)
+
+
+@app.route('/healthcheck')
+def healthcheck():
+    return jsonify(demo=True)
+
+
+@app.route('/api/model/<model>/<version>/info', methods=['GET', 'POST'])
+def model_info(model, version):
+    return jsonify(
+        model_version=version,
+        model_id=model,
+        endpoints={
+            'default': {
+                'name': 'default',
+                'use_df': False,
+                'input_params': {'b': {'numpy_type': 'int64', 'type': 'Integer'},
+                                 'a': {'numpy_type': 'int64', 'type': 'Integer'}}
+
+            }
+        }
+    )
+
+
+@app.route('/api/model/<model>/<version>/invoke', methods=['GET', 'POST'])
+@app.route('/api/model/<model>/<version>/invoke/<endpoint>', methods=['GET', 'POST'])
+def model_invoke(model, version, endpoint=None):
+    return jsonify(result=42.0)
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0')

k8s/test-bare-model-api/model-3/requirements.txt

@@ -0,0 +1 @@
+Flask==0.10.1

legion_test/bin/jenkins_dex_client

@@ -22,8 +22,13 @@ from legion_test.robot import dex_client
 from legion_test import profiler_loader
 import os
 
-def work():
-    profiler_loader.get_variables(os.getenv('PATH_TO_PROFILES_DIR'))
+
+def work(path: str):
+    if path == 'PATH_TO_PROFILES_DIR':
+        path = os.getenv(path)
+
+    profiler_loader.get_variables(path)
+
     if dex_client.get_jenkins_credentials():
         username, password = dex_client.get_jenkins_credentials()
         cookies = ';'.join(['{}={}'.format(k, v) for (k, v) in dex_client.get_session_cookies().items()])
@@ -35,7 +40,9 @@ if __name__ == '__main__':
                                      description='Jenkins Dex client authorizes in Jenkins via Dex and '
                                                  'grabs Cookies and API Token from Jenkins UI, that are '
                                                  'required for testing.')
+    parser.add_argument("--path_to_profiles", nargs='?', default="PATH_TO_PROFILES_DIR",
+                        const='path_to_profiles', help='Path to the dir with profiles')
 
     args = parser.parse_args()
 
-    work()
+    work(args.path_to_profiles)

legion_test/legion_test/profiler_loader.py

@@ -20,10 +20,11 @@
 import os
 
 import yaml
-from legion_test.robot.dex_client import init_session_id
+from legion_test.robot.dex_client import init_session_id, init_session_id_from_data
 
 PROFILE_ENVIRON_KEY = 'PROFILE'
 PATH_TO_PROFILES_DIR = 'PATH_TO_PROFILES_DIR'
+PATH_TO_COOKIES_FILE = 'PATH_TO_COOKIES'
 CREDENTIAL_SECRETS_ENVIRONMENT_KEY = 'CREDENTIAL_SECRETS'
 
 
@@ -85,9 +86,28 @@ def get_variables(arg=None):
     variables['HOST_PROTOCOL'] = 'https' if variables['USE_HTTPS_FOR_TESTS'] else 'http'
     variables['MODEL_TEST_ENCLAVE'] = variables['ENCLAVES'][0] if len(variables['ENCLAVES']) > 0 else 'UNKNOWN_ENCLAVE'
 
+    cookies = os.getenv(PATH_TO_COOKIES_FILE)
+    if cookies:
+        try:
+            with open(cookies, 'r') as stream:
+                lines = stream.readlines()
+                data['jenkins_user'] = lines[0].rstrip()
+                data['jenkins_password'] = lines[1].rstrip()
+                data['cookies'] = lines[2].rstrip()
+        except IOError:
+            pass
+
     if 'dex' in data and data['dex']['enabled'] and 'staticPasswords' in data['dex']['config'] and \
             data['dex']['config']['staticPasswords']:
         static_user = data['dex']['config']['staticPasswords'][0]
-        init_session_id(static_user['email'], static_user['password'], data.get('test_base_domain', data['base_domain']))
+        if not data.get('cookies', ''):
+            init_session_id(static_user['email'], static_user['password'], data.get('test_base_domain', data['base_domain']))
+        else:
+            init_session_id_from_data(data)
+        variables['STATIC_USER_EMAIL'] = static_user['email']
+        variables['STATIC_USER_PASS'] = static_user['password']
+    else:
+        variables['STATIC_USER_EMAIL'] = ''
+        variables['STATIC_USER_PASS'] = ''
 
     return variables

legion_test/legion_test/robot/dex_client.py

@@ -30,6 +30,19 @@
 _session_cookies = {}
 _jenkins_credentials = None
 
+
+def init_session_id_from_data(data: dict):
+    global _session_cookies, _jenkins_credentials
+    if not _session_cookies and not _jenkins_credentials:
+        cookies = data['cookies'].split(';')
+        for cookie in cookies:
+            if len(cookie.split('=')) > 2:
+                _session_cookies[cookie.split('=')[0]] = '{}='.format(cookie.split('=')[1])
+            else:
+                _session_cookies[cookie.split('=')[0]] = cookie.split('=')[1]
+        _jenkins_credentials = (data['jenkins_user'], data['jenkins_password'])
+
+
 def init_session_id(login: str, password: str, cluster_host: str) -> None:
     """Initialize Session ID value from a Cookie after authentication.
 
@@ -58,8 +71,9 @@ def init_session_id(login: str, password: str, cluster_host: str) -> None:
             response = session.post(AUTHENTICATION_PATH.format(cluster_host, request_id),
                                 {PARAM_NAME_LOGIN: login, PARAM_NAME_PASSWORD: password})
             if response.status_code != 200:
-                raise IOError('Unable to authorise, got {} http code'
-                              .format(response.status_code))
+                raise IOError('Unable to authorise, got {} http code from {} for the query {}, response {}'
+                              .format(response.status_code, auth_endpoint_url.format(cluster_host), AUTHENTICATION_PATH.format(cluster_host, request_id),
+                                {PARAM_NAME_LOGIN: login, PARAM_NAME_PASSWORD: password}, response.text))
 
         for cookie_name in session.cookies.keys():
             if cookie_name.startswith(SESSION_ID_COOKIE_NAMES):
@@ -81,6 +95,7 @@ def get_session_cookies():
     """
     return _session_cookies
 
+
 def get_jenkins_credentials():
     """Get credentials (username and API Token) for Jenkins API,
     if they are found.

legion_test/legion_test/robot/utils.py

@@ -227,29 +227,3 @@ def post_credentials_to_auth(component_url, cluster_host, creds, jenkins=False):
         else:
             response = session.post(AUTHENTICATION_PATH.format(cluster_host, request_id), creds)
         return {"response_code": response.status_code, "response_text": response.text}
-
-    @staticmethod
-    def get_static_user_data():
-        """
-        Get static user email and password form secrets
-
-        :return: user email and password
-        :rtype: dict
-        """
-        import os
-
-        import yaml
-        from legion_test.profiler_loader import CREDENTIAL_SECRETS_ENVIRONMENT_KEY
-        secrets = os.getenv(CREDENTIAL_SECRETS_ENVIRONMENT_KEY)
-        if not secrets:
-            raise Exception(
-                'Cannot get secrets - {} env variable is not set'.format(CREDENTIAL_SECRETS_ENVIRONMENT_KEY))
-
-        if not os.path.exists(secrets):
-            raise Exception('Cannot get secrets - file not found {}'.format(secrets))
-
-        with open(secrets, 'r') as stream:
-            data = yaml.load(stream)
-
-        static_user = data['dex']['config']['staticPasswords'][0]
-        return {"login": static_user['email'], "password": static_user['password']}

legion_test/requirements/base.txt

@@ -2,6 +2,7 @@ python-jenkins==0.4.15
 Jinja2==2.10
 robotframework==3.0.2
 robotframework-seleniumlibrary==3.1.1
+robotframework-pabot==0.44
 requests==2.9.1
 PyYAML==3.12
 kubernetes==5.0.0

tests/robot/load_variables_from_profiles.py

@@ -32,5 +32,6 @@ def get_variables(arg):
     variables = legion_test.profiler_loader.get_variables(arg)
     variables['TEST_MODEL_IMAGE_1'] = legion_test.test_assets.get_test_bare_model_api_image(variables, 1)
     variables['TEST_MODEL_IMAGE_2'] = legion_test.test_assets.get_test_bare_model_api_image(variables, 2)
+    variables['TEST_MODEL_IMAGE_3'] = legion_test.test_assets.get_test_bare_model_api_image(variables, 3)
 
     return variables

tests/robot/resources/keywords.robot

@@ -191,44 +191,46 @@ Verify model info from edi
     Should Be Equal  ${target_model[4]}    ${scale_num}       invalid desired scale
 #    Should Be Empty  ${target_model[5]}                       got some errors ${target_model[5]}
 
-Run and wait Jenkins job
-    [Arguments]          ${model_name}                      ${enclave}
-    Log                  Start running model: ${model_name}
-    Run Jenkins job                                         DYNAMIC MODEL ${model_name}   Enclave=${enclave}
-    Log                  Waiting for running model: ${model_name}
-    Wait Jenkins job                                        DYNAMIC MODEL ${model_name}   600
-
-Test model pipeline
-    [Arguments]          ${model_name}                      ${enclave}=${CLUSTER_NAMESPACE}
-    Run and wait Jenkins job                                ${model_name}        ${enclave}
-    Last Jenkins job is successful                          DYNAMIC MODEL ${model_name}
-    Jenkins artifact present                                DYNAMIC MODEL ${model_name}   notebook.html
-    ${model_meta} =      Jenkins log meta information       DYNAMIC MODEL ${model_name}
-    Log                  Model meta is ${model_meta}
-    ${model_path} =      Get From Dictionary                ${model_meta}                 modelPath
-    ${model_id} =        Get From Dictionary                ${model_meta}                 modelId
-    ${model_version} =   Get From Dictionary                ${model_meta}                 modelVersion
-    ${model_path} =	     Get Regexp Matches	                ${model_path}                 (.*)://[^/]+/(?P<path>.*)   path
-    ${model_url} =       Set Variable                       ${HOST_PROTOCOL}://nexus.${HOST_BASE_DOMAIN}/${model_path[0]}
-    Log                  External model URL is ${model_url}
-    Check remote file exists                                ${model_url}                  ${SERVICE_ACCOUNT}          jonny
-    Connect to enclave Grafana                              ${enclave}
-    Dashboard should exists                                 ${model_id}
-    Sleep                15s
-    Metric should be presented                              ${model_id}                   ${model_version}
-    ${edi_state}=        Run      legionctl inspect --model-id ${model_id} --format column --edi ${HOST_PROTOCOL}://edi.${HOST_BASE_DOMAIN} --user ${SERVICE_ACCOUNT} --password ${SERVICE_PASSWORD}
-    Log                  State of ${model_id} is ${edi_state}
+Run all test Jenkins jobs for enclave
+    [Arguments]             ${enclave}
+    :FOR  ${model_name}  IN  @{JENKINS_JOBS}
+    \   Connect to Jenkins endpoint
+    \   Log                  Start running model: ${model_name}
+    \   Run Jenkins job      DYNAMIC MODEL ${model_name}   Enclave=${enclave}
+    \   Sleep   10
+
+Wait all test Jenkins jobs are finished
+    :FOR  ${model_name}  IN  @{JENKINS_JOBS}
+    \   Log                  Waiting for running model: ${model_name}
+    \   Wait Jenkins job     DYNAMIC MODEL ${model_name}   600
+
+Check all test models are successful and have metrics
+    [Arguments]             ${enclave}
+    :FOR  ${model_name}  IN  @{JENKINS_JOBS}
+    \   Log                  Checking model: ${model_name}
+    \   Last Jenkins job is successful                          DYNAMIC MODEL ${model_name}
+    \   Jenkins artifact present                                DYNAMIC MODEL ${model_name}   notebook.html
+    \   ${model_meta} =      Jenkins log meta information       DYNAMIC MODEL ${model_name}
+    \   Log                  Model meta is ${model_meta}
+    \   ${model_path} =      Get From Dictionary                ${model_meta}                 modelPath
+    \   ${model_id} =        Get From Dictionary                ${model_meta}                 modelId
+    \   ${model_version} =   Get From Dictionary                ${model_meta}                 modelVersion
+    \   ${model_path} =	     Get Regexp Matches	                ${model_path}                 (.*)://[^/]+/(?P<path>.*)   path
+    \   ${model_url} =       Set Variable                       ${HOST_PROTOCOL}://nexus.${HOST_BASE_DOMAIN}/${model_path[0]}
+    \   Log                  External model URL is ${model_url}
+    \   Check remote file exists                                ${model_url}                  ${SERVICE_ACCOUNT}          jonny
+    \   Connect to enclave Grafana                              ${enclave}
+    \   Dashboard should exists                                 ${model_id}
+    \   Sleep                15s
+    \   Metric should be presented                              ${model_id}                   ${model_version}
+    \   ${edi_state}=        Run      legionctl inspect --model-id ${model_id} --format column --edi ${HOST_PROTOCOL}://edi.${HOST_BASE_DOMAIN} --user ${SERVICE_ACCOUNT} --password ${SERVICE_PASSWORD}
+    \   Log                  State of ${model_id} is ${edi_state}
 
 Check if all enclave domains are registered
     [Arguments]             ${enclave}
     :FOR    ${enclave_subdomain}    IN    @{ENCLAVE_SUBDOMAINS}
     \   Check domain exists  ${enclave_subdomain}-${enclave}.${HOST_BASE_DOMAIN}
 
-Run, wait and check jenkins jobs for enclave
-    [Arguments]             ${enclave}
-    :FOR  ${model_name}  IN  @{JENKINS_JOBS}
-    \    Test model pipeline    ${model_name}    ${enclave}
-
     # --------- TEMPLATE KEYWORDS SECTION -----------
 
 Check if component domain has been secured
@@ -266,7 +268,7 @@ Secured component domain should be accessible by valid credentials
     ${jenkins} =     Run Keyword If   '${component}' == 'jenkins'    Set Variable    True
     ...    ELSE      Set Variable    False
     ${boolean} =     Convert To Boolean    ${jenkins}
-    &{creds} =       Get static user data
+    &{creds} =       Create Dictionary    login=${STATIC_USER_EMAIL}    password=${STATIC_USER_PASS}
     &{response} =    Run Keyword If   '${enclave}' == '${EMPTY}'    Post credentials to auth    ${HOST_PROTOCOL}://${component}    ${HOST_BASE_DOMAIN}    ${creds}    ${boolean}
     ...    ELSE      Post credentials to auth    ${HOST_PROTOCOL}://${component}-${enclave}     ${HOST_BASE_DOMAIN}    ${creds}    ${boolean}
     Log              Bad auth page for ${component} is ${response}

tests/robot/resources/variables.robot

@@ -2,5 +2,7 @@
 @{SUBDOMAINS}                       jenkins  nexus
 @{ENCLAVE_SUBDOMAINS}               edi  edge  airflow  flower
 ${TEST_MODEL_ID}                    demo-abc-model
+${TEST_EDI_MODEL_ID}                edi-test-model
 ${TEST_MODEL_1_VERSION}             1.0
-${TEST_MODEL_2_VERSION}             1.1
+${TEST_MODEL_2_VERSION}             1.1
+${TEST_MODEL_3_VERSION}             1.2

tests/robot/0.1_check_components_auth.robot → tests/robot/tests/1_common/1.0_check_components_auth.robot

@@ -1,9 +1,8 @@
 *** Settings ***
 Documentation       Check if all core components are secured
-Resource            resources/browser.robot
-Resource            resources/keywords.robot
-Resource            resources/variables.robot
-Variables           load_variables_from_profiles.py   ${PATH_TO_PROFILES_DIR}
+Resource            ../../resources/browser.robot
+Resource            ../../resources/keywords.robot
+Variables           ../../load_variables_from_profiles.py    ${PATH_TO_PROFILES_DIR}
 Library             Collections
 Library             legion_test.robot.K8s
 Library             legion_test.robot.Utils