diff --git a/Tests/kaas/plugin/README.md b/Tests/kaas/README.md similarity index 89% rename from Tests/kaas/plugin/README.md rename to Tests/kaas/README.md index e54cf1864..16697d3fd 100644 --- a/Tests/kaas/plugin/README.md +++ b/Tests/kaas/README.md @@ -1,4 +1,4 @@ -# Plugin for provisioning k8s clusters and performing conformance tests on these clusters +# Test suite for SCS-compatible KaaS ## Development environment @@ -6,6 +6,7 @@ * [docker](https://docs.docker.com/engine/install/) * [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) +* [sonobuoy](https://sonobuoy.io/docs/v0.57.1/#installation) ### setup for development @@ -19,7 +20,6 @@ (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 (venv) python3.10 -m pip install --upgrade pip (venv) python3.10 -m pip --version - ``` 2. Install dependencies: diff --git a/Tests/kaas/plugin/requirements.in b/Tests/kaas/requirements.in similarity index 65% rename from Tests/kaas/plugin/requirements.in rename to Tests/kaas/requirements.in index 0a60c3c3c..640831e54 100644 --- a/Tests/kaas/plugin/requirements.in +++ b/Tests/kaas/requirements.in @@ -1,2 +1,3 @@ pytest-kind kubernetes +junitparser diff --git a/Tests/kaas/plugin/requirements.txt b/Tests/kaas/requirements.txt similarity index 95% rename from Tests/kaas/plugin/requirements.txt rename to Tests/kaas/requirements.txt index a04a03167..c36ca21d1 100644 --- a/Tests/kaas/plugin/requirements.txt +++ b/Tests/kaas/requirements.txt @@ -16,6 +16,8 @@ google-auth==2.34.0 # via kubernetes idna==3.8 # via requests +junitparser==3.2.0 + # via -r requirements.in kubernetes==30.1.0 # via -r requirements.in oauthlib==3.2.2 diff --git a/Tests/kaas/sonobuoy_handler/run_sonobuoy.py b/Tests/kaas/sonobuoy_handler/run_sonobuoy.py new file mode 100755 index 000000000..50ef4249c --- /dev/null +++ b/Tests/kaas/sonobuoy_handler/run_sonobuoy.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +# +import logging +import sys + +import click + +from sonobuoy_handler import SonobuoyHandler + +logger = logging.getLogger(__name__) + + +@click.command() +@click.option("-k", "--kubeconfig", "kubeconfig", required=True, type=click.Path(exists=True), help="path/to/kubeconfig_file.yaml",) +@click.option("-r", "--result_dir_name", "result_dir_name", type=str, default="sonobuoy_results", help="directory name to store results at",) +@click.option("-c", "--check", "check_name", type=str, default="sonobuoy_executor", help="this MUST be the same name as the id in 'scs-compatible-kaas.yaml'",) +@click.option("-a", "--arg", "args", multiple=True) +def sonobuoy_run(kubeconfig, result_dir_name, check_name, args): + sonobuoy_handler = SonobuoyHandler(check_name, kubeconfig, result_dir_name, args) + sys.exit(sonobuoy_handler.run()) + + +if __name__ == "__main__": + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) + sonobuoy_run() diff --git a/Tests/kaas/sonobuoy_handler/sonobuoy_handler.py b/Tests/kaas/sonobuoy_handler/sonobuoy_handler.py new file mode 100644 index 000000000..65593a411 --- /dev/null +++ b/Tests/kaas/sonobuoy_handler/sonobuoy_handler.py @@ -0,0 +1,133 @@ +from collections import Counter +import json +import logging +import os +import shlex +import shutil +import subprocess + +from junitparser import JUnitXml + +logger = logging.getLogger(__name__) + + +class SonobuoyHandler: + """ + A class that handles both the execution of sonobuoy and + the generation of the results for a test report + """ + + kubeconfig_path = None + working_directory = None + + def __init__( + self, + check_name="sonobuoy_handler", + kubeconfig=None, + result_dir_name="sonobuoy_results", + args=(), + ): + self.check_name = check_name + logger.debug(f"kubeconfig: {kubeconfig} ") + if kubeconfig is None: + raise RuntimeError("No kubeconfig provided") + self.kubeconfig_path = kubeconfig + self.working_directory = os.getcwd() + self.result_dir_name = result_dir_name + self.sonobuoy = shutil.which('sonobuoy') + logger.debug(f"working from {self.working_directory}") + logger.debug(f"placing results at {self.result_dir_name}") + logger.debug(f"sonobuoy executable at {self.sonobuoy}") + self.args = (arg0 for arg in args for arg0 in shlex.split(str(arg))) + + def _invoke_sonobuoy(self, *args, **kwargs): + inv_args = (self.sonobuoy, "--kubeconfig", self.kubeconfig_path) + args + logger.debug(f'invoking {" ".join(inv_args)}') + return subprocess.run(args=inv_args, capture_output=True, check=True, **kwargs) + + def _sonobuoy_run(self): + self._invoke_sonobuoy("run", "--wait", *self.args) + + def _sonobuoy_delete(self): + self._invoke_sonobuoy("delete", "--wait") + + def _sonobuoy_status_result(self): + process = self._invoke_sonobuoy("status", "--json") + json_data = json.loads(process.stdout) + counter = Counter() + for entry in json_data["plugins"]: + logger.debug(f"plugin:{entry['plugin']}:{entry['result-status']}") + for result, count in entry["result-counts"].items(): + counter[result] += count + return counter + + def _eval_result(self, counter): + """evaluate test results and return return code""" + result_str = ', '.join(f"{counter[key]} {key}" for key in ('passed', 'failed', 'skipped')) + result_message = f"sonobuoy reports {result_str}" + if counter['failed']: + logger.error(result_message) + return 3 + logger.info(result_message) + return 0 + + def _preflight_check(self): + """ + Preflight test to ensure that everything is set up correctly for execution + """ + if not self.sonobuoy: + raise RuntimeError("sonobuoy executable not found; is it in PATH?") + + def _sonobuoy_retrieve_result(self): + """ + This method invokes sonobuoy to store the results in a subdirectory of + the working directory. The Junit results file contained in it is then + analyzed in order to interpret the relevant information it containes + """ + logger.debug(f"retrieving results to {self.result_dir_name}") + result_dir = os.path.join(self.working_directory, self.result_dir_name) + if os.path.exists(result_dir): + raise Exception("result directory already existing") + os.mkdir(result_dir) + + # XXX use self._invoke_sonobuoy + os.system( + # ~ f"sonobuoy retrieve {result_dir} -x --filename='{result_dir}' --kubeconfig='{self.kubeconfig_path}'" + f"sonobuoy retrieve {result_dir} --kubeconfig='{self.kubeconfig_path}'" + ) + logger.debug( + f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} " + ) + xml = JUnitXml.fromfile(result_dir + "/plugins/e2e/results/global/junit_01.xml") + counter = Counter() + for suite in xml: + for case in suite: + if case.is_passed is True: # XXX why `is True`??? + counter['passed'] += 1 + elif case.is_skipped is True: + counter['skipped'] += 1 + else: + counter['failed'] += 1 + logger.error(f"{case.name}") + return counter + + def run(self): + """ + This method is to be called to run the plugin + """ + logger.info(f"running sonobuoy for testcase {self.check_name}") + self._preflight_check() + try: + self._sonobuoy_run() + return_code = self._eval_result(self._sonobuoy_status_result()) + print(self.check_name + ": " + ("PASS", "FAIL")[min(1, return_code)]) + return return_code + + # ERROR: currently disabled due to: "error retrieving results: unexpected EOF" + # might be related to following bug: https://github.com/vmware-tanzu/sonobuoy/issues/1633 + # self._sonobuoy_retrieve_result(self) + except BaseException: + logger.exception("something went wrong") + return 112 + finally: + self._sonobuoy_delete() diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index a4010c64e..7cb2fbd58 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -9,6 +9,10 @@ modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance url: https://github.com/cncf/k8s-conformance/tree/master + run: + - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py + args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'cncf-k8s-conformance' -a '--mode=certified-conformance' + #~ args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'cncf-k8s-conformance' -a '--plugin-env e2e.E2E_DRYRUN=true' testcases: - id: cncf-k8s-conformance tags: [mandatory] @@ -30,6 +34,15 @@ modules: testcases: - id: node-distribution-check tags: [mandatory] + - id: scs-0219-v1 + name: KaaS networking + url: https://docs.scs.community/standards/scs-0219-v1-kaas-networking + run: + - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py + args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"' + testcases: + - id: kaas-networking-check + tags: [mandatory] timeline: - date: 2024-02-28 versions: @@ -40,5 +53,6 @@ versions: - cncf-k8s-conformance - scs-0210-v2 - scs-0214-v2 + - scs-0219-v1 targets: main: mandatory