Feat/sonobuoy integration

Tests/kaas/plugin/README.md → Tests/kaas/README.md

@@ -1,11 +1,12 @@
-# Plugin for provisioning k8s clusters and performing conformance tests on these clusters
+# Test suite for SCS-compatible KaaS
 
 ## Development environment
 
 ### requirements
 
 * [docker](https://docs.docker.com/engine/install/)
 * [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
+* [sonobuoy](https://sonobuoy.io/docs/v0.57.1/#installation)
 
 ### setup for development
 
@@ -19,7 +20,6 @@
    (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
    (venv) python3.10 -m pip install --upgrade pip
    (venv) python3.10 -m pip --version
-
    ```
 
 2. Install dependencies:

Tests/kaas/plugin/requirements.in → Tests/kaas/requirements.in

@@ -1,2 +1,3 @@
 pytest-kind
 kubernetes
+junitparser

Tests/kaas/plugin/requirements.txt → Tests/kaas/requirements.txt

@@ -16,6 +16,8 @@ google-auth==2.34.0
     # via kubernetes
 idna==3.8
     # via requests
+junitparser==3.2.0
+    # via -r requirements.in
 kubernetes==30.1.0
     # via -r requirements.in
 oauthlib==3.2.2

Tests/kaas/sonobuoy_handler/run_sonobuoy.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# vim: set ts=4 sw=4 et:
+#
+import logging
+import sys
+
+import click
+
+from sonobuoy_handler import SonobuoyHandler
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option("-k", "--kubeconfig", "kubeconfig", required=True, type=click.Path(exists=True), help="path/to/kubeconfig_file.yaml",)
+@click.option("-r", "--result_dir_name", "result_dir_name", type=str, default="sonobuoy_results", help="directory name to store results at",)
+@click.option("-c", "--check", "check_name", type=str, default="sonobuoy_executor", help="this MUST be the same name as the id in 'scs-compatible-kaas.yaml'",)
+@click.option("-a", "--arg", "args", multiple=True)
+def sonobuoy_run(kubeconfig, result_dir_name, check_name, args):
+    sonobuoy_handler = SonobuoyHandler(check_name, kubeconfig, result_dir_name, args)
+    sys.exit(sonobuoy_handler.run())
+
+
+if __name__ == "__main__":
+    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
+    sonobuoy_run()

Tests/kaas/sonobuoy_handler/sonobuoy_handler.py

@@ -0,0 +1,133 @@
+from collections import Counter
+import json
+import logging
+import os
+import shlex
+import shutil
+import subprocess
+
+from junitparser import JUnitXml
+
+logger = logging.getLogger(__name__)
+
+
+class SonobuoyHandler:
+    """
+    A class that handles both the execution of sonobuoy and
+    the generation of the results for a test report
+    """
+
+    kubeconfig_path = None
+    working_directory = None
+
+    def __init__(
+        self,
+        check_name="sonobuoy_handler",
+        kubeconfig=None,
+        result_dir_name="sonobuoy_results",
+        args=(),
+    ):
+        self.check_name = check_name
+        logger.debug(f"kubeconfig: {kubeconfig} ")
+        if kubeconfig is None:
+            raise RuntimeError("No kubeconfig provided")
+        self.kubeconfig_path = kubeconfig
+        self.working_directory = os.getcwd()
+        self.result_dir_name = result_dir_name
+        self.sonobuoy = shutil.which('sonobuoy')
+        logger.debug(f"working from {self.working_directory}")
+        logger.debug(f"placing results at {self.result_dir_name}")
+        logger.debug(f"sonobuoy executable at {self.sonobuoy}")
+        self.args = (arg0 for arg in args for arg0 in shlex.split(str(arg)))
+
+    def _invoke_sonobuoy(self, *args, **kwargs):
+        inv_args = (self.sonobuoy, "--kubeconfig", self.kubeconfig_path) + args
+        logger.debug(f'invoking {" ".join(inv_args)}')
+        return subprocess.run(args=inv_args, capture_output=True, check=True, **kwargs)
+
+    def _sonobuoy_run(self):
+        self._invoke_sonobuoy("run", "--wait", *self.args)
+
+    def _sonobuoy_delete(self):
+        self._invoke_sonobuoy("delete", "--wait")
+
+    def _sonobuoy_status_result(self):
+        process = self._invoke_sonobuoy("status", "--json")
+        json_data = json.loads(process.stdout)
+        counter = Counter()
+        for entry in json_data["plugins"]:
+            logger.debug(f"plugin:{entry['plugin']}:{entry['result-status']}")
+            for result, count in entry["result-counts"].items():
+                counter[result] += count
+        return counter
+
+    def _eval_result(self, counter):
+        """evaluate test results and return return code"""
+        result_str = ', '.join(f"{counter[key]} {key}" for key in ('passed', 'failed', 'skipped'))
+        result_message = f"sonobuoy reports {result_str}"
+        if counter['failed']:
+            logger.error(result_message)
+            return 3
+        logger.info(result_message)
+        return 0
+
+    def _preflight_check(self):
+        """
+        Preflight test to ensure that everything is set up correctly for execution
+        """
+        if not self.sonobuoy:
+            raise RuntimeError("sonobuoy executable not found; is it in PATH?")
+
+    def _sonobuoy_retrieve_result(self):
+        """
+        This method invokes sonobuoy to store the results in a subdirectory of
+        the working directory. The Junit results file contained in it is then
+        analyzed in order to interpret the relevant information it containes
+        """
+        logger.debug(f"retrieving results to {self.result_dir_name}")
+        result_dir = os.path.join(self.working_directory, self.result_dir_name)
+        if os.path.exists(result_dir):
+            raise Exception("result directory already existing")
+        os.mkdir(result_dir)
+
+        # XXX use self._invoke_sonobuoy
+        os.system(
+            # ~ f"sonobuoy retrieve {result_dir} -x --filename='{result_dir}' --kubeconfig='{self.kubeconfig_path}'"
+            f"sonobuoy retrieve {result_dir} --kubeconfig='{self.kubeconfig_path}'"
+        )
+        logger.debug(
+            f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} "
+        )
+        xml = JUnitXml.fromfile(result_dir + "/plugins/e2e/results/global/junit_01.xml")
+        counter = Counter()
+        for suite in xml:
+            for case in suite:
+                if case.is_passed is True:  # XXX why `is True`???
+                    counter['passed'] += 1
+                elif case.is_skipped is True:
+                    counter['skipped'] += 1
+                else:
+                    counter['failed'] += 1
+                    logger.error(f"{case.name}")
+        return counter
+
+    def run(self):
+        """
+        This method is to be called to run the plugin
+        """
+        logger.info(f"running sonobuoy for testcase {self.check_name}")
+        self._preflight_check()
+        try:
+            self._sonobuoy_run()
+            return_code = self._eval_result(self._sonobuoy_status_result())
+            print(self.check_name + ": " + ("PASS", "FAIL")[min(1, return_code)])
+            return return_code
+
+            # ERROR: currently disabled due to: "error retrieving results: unexpected EOF"
+            #  might be related to following bug: https://github.com/vmware-tanzu/sonobuoy/issues/1633
+            # self._sonobuoy_retrieve_result(self)
+        except BaseException:
+            logger.exception("something went wrong")
+            return 112
+        finally:
+            self._sonobuoy_delete()

Tests/scs-compatible-kaas.yaml

@@ -9,6 +9,10 @@ modules:
   - id: cncf-k8s-conformance
     name: CNCF Kubernetes conformance
     url: https://github.com/cncf/k8s-conformance/tree/master
+    run:
+      - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py
+        args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'cncf-k8s-conformance'  -a '--mode=certified-conformance'
+        #~ args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'cncf-k8s-conformance'  -a '--plugin-env e2e.E2E_DRYRUN=true'
     testcases:
       - id: cncf-k8s-conformance
         tags: [mandatory]
@@ -30,6 +34,15 @@ modules:
     testcases:
       - id: node-distribution-check
         tags: [mandatory]
+  - id: scs-0219-v1
+    name: KaaS networking
+    url: https://docs.scs.community/standards/scs-0219-v1-kaas-networking
+    run:
+      - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py
+        args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"'
+    testcases:
+      - id: kaas-networking-check
+        tags: [mandatory]
 timeline:
   - date: 2024-02-28
     versions:
@@ -40,5 +53,6 @@ versions:
       - cncf-k8s-conformance
       - scs-0210-v2
       - scs-0214-v2
+      - scs-0219-v1
     targets:
       main: mandatory