BWC test workflow (opensearch-project#1603)

* BWC test workflow

Adding backwards compatability workflow for OpenSearch and OpenSearch
Dashboards.

Issue:
opensearch-project#705

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* [BWC] Update docs and return cmd

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

ONBOARDING.md

@@ -51,3 +51,5 @@ Add the new plugin to the [opensearch-plugins meta](https://github.com/opensearc
     2. If your plugin is dependent on `job-scheduler` zip, you can define that in `build-dependencies` in the config. Currently, the test workflow only supports `job-scheduler` as build dependency. Please create an issue if your plugin needs more support.
 
 3. For backward compatibility testing, the `test-workflow` runs backward compatibility tests available in the plugin repository, (see [reference]((https://github.com/opensearch-project/anomaly-detection/blob/d9a122d05282f7efc1e24c61d64f18dec0fd47af/build.gradle#L428))). Like integration test, it has a set of configurable options defined in opensearch-1.3.0-test.yml, [example](manifests/1.3.0/opensearch-1.3.0-test.yml).
+
+    1. It supports two test configs - `with-security` and `without-security`, which runs test with security plugin enabled and disabled respectively. Choose one or both depending on what your plugin integration tests support.

manifests/2.0.0/opensearch-dashboards-2.0.0-test.yml

@@ -0,0 +1,13 @@
+---
+name: OpenSearch Dashboards
+components:
+  - name: OpenSearch-Dashboards
+    bwc-test:
+      test-configs:
+        - without-security
+  - name: functionalTestDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+schema-version: '1.0'

src/run_bwc_test.py

@@ -5,22 +5,28 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
-import os
 import sys
 
-from manifests.bundle_manifest import BundleManifest
+from manifests.test_manifest import TestManifest
 from system import console
-from system.temporary_directory import TemporaryDirectory
-from test_workflow.bwc_test.bwc_test_suite import BwcTestSuite
+from test_workflow.bwc_test.bwc_test_runners import BwcTestRunners
 from test_workflow.test_args import TestArgs
 
 
 def main():
     args = TestArgs()
+
+    # Any logging.info call preceding to next line in the execution chain will make the console output not displaying logs in console.
     console.configure(level=args.logging_level)
-    with TemporaryDirectory(keep=args.keep) as work_dir:
-        bundle_manifest = BundleManifest.from_urlpath(args.paths.get("opensearch", os.getcwd()))
-        BwcTestSuite(bundle_manifest, work_dir.name, args.component, args.keep).execute()
+
+    test_manifest = TestManifest.from_path(args.test_manifest_path)
+
+    all_results = BwcTestRunners.from_test_manifest(args, test_manifest).run()
+
+    all_results.log()
+
+    if all_results.failed():
+        sys.exit(1)
 
 
 if __name__ == "__main__":

src/test_workflow/README.md

@@ -76,6 +76,27 @@ Usage:
 ./test.sh bwc-test <test-manifest-path> <target>
 ```
 
+For example, build locally and run BWC tests.
+
+```bash
+./build.sh manifests/1.3.0/opensearch-1.3.0.yml
+./assemble.sh builds/opensearch/manifest.yml
+./test.sh bwc-test manifests/1.3.0/opensearch-1.3.0-test.yml . # looks for "./builds/opensearch/manifest.yml" and "./dist/opensearch/manifest.yml"
+```
+
+Or run BWC tests against an existing build.
+
+```bash
+./test.sh bwc-test manifests/1.3.0/opensearch-1.3.0-test.yml --paths opensearch=https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/869/linux/x64 # looks for https://.../builds/opensearch/manifest.yml and https://.../dist/opensearch/manifest.yml
+```
+
+To run OpenSearch Dashboards BWC tests.
+
+```bash
+./test.sh bwc-test manifests/1.3.0/opensearch-dashboards-1.3.0-test.yml --paths 
+opensearch-dashboards=https://ci.opensearch.org/ci/dbc/bundle-build-dashboards/1.2.0/869/linux/x64 
+```
+
 ### Performance Tests
 
 TODO

src/test_workflow/bwc_test/bwc_test_runner.py

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import abc
+import logging
+import os
+
+from system.temporary_directory import TemporaryDirectory
+from test_workflow.test_recorder.test_recorder import TestRecorder
+from test_workflow.test_result.test_suite_results import TestSuiteResults
+
+
+class BwcTestRunner(abc.ABC):
+    def __init__(self, args, test_manifest):
+        self.args = args
+        self.test_manifest = test_manifest
+
+        self.tests_dir = os.path.join(os.getcwd(), "test-results")
+        os.makedirs(self.tests_dir, exist_ok=True)
+        self.test_recorder = TestRecorder(self.args.test_run_id, "bwc-test", self.tests_dir)
+
+    def run(self):
+        with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
+            all_results = TestSuiteResults()
+            for component in self.components.select(focus=self.args.component):
+                if component.name in self.test_manifest.components:
+                    test_config = self.test_manifest.components[component.name]
+                    if test_config.bwc_test:
+                        test_suite = self.__create_test_suite__(component, test_config, work_dir)
+                        test_results = test_suite.execute_tests()
+                        all_results.append(component.name, test_results)
+                    else:
+                        logging.info(f"Skipping bwc-tests for {component.name}, as it is currently not supported")
+                else:
+                    logging.info(f"Skipping bwc-tests for {component.name}, as it is currently not declared in the test manifest")
+
+        return all_results

src/test_workflow/bwc_test/bwc_test_runner_opensearch.py

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+
+from manifests.test_manifest import TestManifest
+from test_workflow.bwc_test.bwc_test_runner import BwcTestRunner
+from test_workflow.bwc_test.bwc_test_start_properties_opensearch import BwcTestStartPropertiesOpenSearch
+from test_workflow.bwc_test.bwc_test_suite_opensearch import BwcTestSuiteOpenSearch
+from test_workflow.test_args import TestArgs
+
+
+class BwcTestRunnerOpenSearch(BwcTestRunner):
+
+    def __init__(self, args: TestArgs, test_manifest: TestManifest):
+        super().__init__(args, test_manifest)
+        self.properties = BwcTestStartPropertiesOpenSearch(args.paths.get("opensearch", os.getcwd()))
+
+        self.components = self.properties.build_manifest.components
+
+        logging.info("Entering BWC test for OpenSearch")
+
+    def __create_test_suite__(self, component, test_config, work_dir):
+        return BwcTestSuiteOpenSearch(
+            work_dir.name,
+            component,
+            test_config,
+            self.test_recorder,
+            self.properties.bundle_manifest,
+        )

src/test_workflow/bwc_test/bwc_test_runner_opensearch_dashboards.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+
+from manifests.test_manifest import TestManifest
+from test_workflow.bwc_test.bwc_test_runner import BwcTestRunner
+from test_workflow.bwc_test.bwc_test_start_properties_opensearch_dashboards import BwcTestStartPropertiesOpenSearchDashboards
+from test_workflow.bwc_test.bwc_test_suite_opensearch_dashboards import BwcTestSuiteOpenSearchDashboards
+from test_workflow.test_args import TestArgs
+
+
+class BwcTestRunnerOpenSearchDashboards(BwcTestRunner):
+
+    def __init__(self, args: TestArgs, test_manifest: TestManifest):
+        super().__init__(args, test_manifest)
+        self.properties = BwcTestStartPropertiesOpenSearchDashboards(args.paths.get("opensearch-dashboards", os.getcwd()))
+
+        self.components = self.properties.build_manifest.components
+        logging.info("Entering BWC test for OpenSearch Dashboards")
+
+    def __create_test_suite__(self, component, test_config, work_dir):
+        return BwcTestSuiteOpenSearchDashboards(
+            work_dir.name,
+            component,
+            test_config,
+            self.test_recorder,
+            self.properties.bundle_manifest,
+        )

src/test_workflow/bwc_test/bwc_test_runners.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+
+from manifests.test_manifest import TestManifest
+from test_workflow.bwc_test.bwc_test_runner_opensearch import BwcTestRunnerOpenSearch
+from test_workflow.bwc_test.bwc_test_runner_opensearch_dashboards import BwcTestRunnerOpenSearchDashboards
+from test_workflow.test_args import TestArgs
+
+
+class BwcTestRunners:
+    RUNNERS = {
+        "OpenSearch": BwcTestRunnerOpenSearch,
+        "OpenSearch Dashboards": BwcTestRunnerOpenSearchDashboards
+    }
+
+    @classmethod
+    def from_test_manifest(cls, args: TestArgs, test_manifest: TestManifest):
+        return cls.RUNNERS[test_manifest.name](args, test_manifest)

src/test_workflow/bwc_test/bwc_test_start_properties.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+
+import abc
+
+from manifests.build_manifest import BuildManifest
+from manifests.bundle_manifest import BundleManifest
+
+
+class BwcTestStartProperties(abc.ABC):
+    def __init__(self, path, build_dir, bundle_dir):
+        self.path = path
+        self.build_dir = build_dir
+        self.bundle_dir = bundle_dir
+
+        self.bundle_manifest = BundleManifest.from_urlpath("/".join([self.path.rstrip("/"), self.bundle_dir]))
+        self.build_manifest = BuildManifest.from_urlpath("/".join([self.path.rstrip("/"), self.build_dir]))

src/test_workflow/bwc_test/bwc_test_start_properties_opensearch.py

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from test_workflow.bwc_test.bwc_test_start_properties import BwcTestStartProperties
+
+
+class BwcTestStartPropertiesOpenSearch(BwcTestStartProperties):
+    def __init__(self, path):
+        super().__init__(path, "builds/opensearch/manifest.yml", "dist/opensearch/manifest.yml")

src/test_workflow/bwc_test/bwc_test_start_properties_opensearch_dashboards.py

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from test_workflow.bwc_test.bwc_test_start_properties import BwcTestStartProperties
+
+
+class BwcTestStartPropertiesOpenSearchDashboards(BwcTestStartProperties):
+    def __init__(self, path):
+        super().__init__(path, "builds/opensearch-dashboards/manifest.yml", "dist/opensearch-dashboards/manifest.yml")

src/test_workflow/bwc_test/bwc_test_suite.py

@@ -7,43 +7,97 @@
 # Modifications Copyright OpenSearch Contributors. See
 # GitHub history for details.
 
+import abc
 import logging
 import os
 
+from git.git_repository import GitRepository
 from paths.script_finder import ScriptFinder
 from system.execute import execute
-from test_workflow.test_component import TestComponent
+from test_workflow.test_recorder.test_result_data import TestResultData
+from test_workflow.test_result.test_component_results import TestComponentResults
+from test_workflow.test_result.test_result import TestResult
 
 
-class BwcTestSuite:
-    manifest: str
-    work_dir: str
-    component: str
-    keep: bool
+class BwcTestSuite(abc.ABC):
 
-    def __init__(self, manifest, work_dir, component=None, keep=False):
-        self.manifest = manifest
+    def __init__(
+        self,
+        work_dir,
+        component,
+        test_config,
+        test_recorder,
+        manifest
+    ):
         self.work_dir = work_dir
         self.component = component
-        self.keep = keep
-
-    def run_tests(self, work_dir, component_name):
-        script = ScriptFinder.find_bwc_test_script(component_name, work_dir)
-        (status, stdout, stderr) = execute(script, work_dir, True, False)
-        return (status, stdout, stderr)
-
-    def component_bwc_tests(self, component):
-        test_component = TestComponent(component.repository, component.commit_id)
-        test_component.checkout(os.path.join(self.work_dir, component.name))
-        try:
-            console_output = self.run_tests(os.path.join(self.work_dir, component.name), component.name)
-            return console_output
-        except:
-            # TODO: Store and report test failures for {component}
-            logging.info(f"Exception while running BWC tests for {component.name}")
-
-    def execute(self):
-        # For each component, check out the git repo and run `bwctest.sh`
-        for component in self.manifest.components.select(focus=self.component):
-            # TODO: Store and report test results, send notification via {console_output}
-            self.component_bwc_tests(component)
+        self.test_config = test_config
+        self.test_recorder = test_recorder
+        self.manifest = manifest
+
+        self.repo = GitRepository(
+            self.component.repository,
+            self.component.commit_id,
+            os.path.join(self.work_dir, self.component.name),
+            test_config.working_directory
+        )
+
+        self.save_logs = test_recorder.test_results_logs
+
+    def execute_tests(self):
+        test_results = TestComponentResults()
+
+        for config in self.test_config.bwc_test["test-configs"]:
+            status = self.execute_bwctest_sh(config)
+
+            test_results.append(TestResult(self.component.name, config, status))
+        return test_results
+
+    def execute_bwctest_sh(self, config):
+        security = self.is_security_enabled(config)
+        script = ScriptFinder.find_bwc_test_script(self.component.name, self.repo.working_directory)
+        if os.path.exists(script):
+            cmd = self.get_cmd(script, security, self.manifest.build.location)
+            self.repo_work_dir = os.path.join(
+                self.repo.dir, self.test_config.working_directory) if self.test_config.working_directory is not None else self.repo.dir
+            (status, stdout, stderr) = execute(cmd, self.repo_work_dir, True, False)
+
+            test_result_data = TestResultData(
+                self.component.name,
+                self.test_config,
+                status,
+                stdout,
+                stderr,
+                self.test_artifact_files
+            )
+            self.save_logs.save_test_result_data(test_result_data)
+            if stderr:
+                logging.info("BWC test run failed for component " + self.component.name)
+                logging.info(stderr)
+            return status
+        else:
+            logging.info(f"{script} does not exist. Skipping integ tests for {self.component.name}")
+
+    def is_security_enabled(self, config):
+        if config in ["with-security", "without-security"]:
+            return True if config == "with-security" else False
+        else:
+            raise InvalidTestConfigError("Unsupported test config: " + config)
+
+    def pretty_print_message(self, message):
+        logging.info("===============================================")
+        logging.info(message)
+        logging.info("===============================================")
+
+    @abc.abstractmethod
+    def get_cmd(self):
+        pass
+
+    @property
+    @abc.abstractmethod
+    def test_artifact_files(self):
+        pass
+
+
+class InvalidTestConfigError(Exception):
+    pass