BWC test workflow

manifests/2.0.0/opensearch-dashboards-2.0.0-test.yml

@@ -0,0 +1,13 @@
+---
+name: OpenSearch Dashboards
+components:
+ - name: OpenSearch-Dashboards
+ bwc-test:
+ test-configs:
+ - without-security
+ - name: functionalTestDashboards
+ integ-test:
+ test-configs:
+ - with-security
+ - without-security
+schema-version: '1.0'

src/run_bwc_test.py

@@ -5,22 +5,28 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
-import os
 import sys
 
-from manifests.bundle_manifest import BundleManifest
+from manifests.test_manifest import TestManifest
 from system import console
-from system.temporary_directory import TemporaryDirectory
-from test_workflow.bwc_test.bwc_test_suite import BwcTestSuite
+from test_workflow.bwc_test.bwc_test_runners import BwcTestRunners
 from test_workflow.test_args import TestArgs
 
 
 def main():
  args = TestArgs()
+
+ # Any logging.info call preceding to next line in the execution chain will make the console output not displaying logs in console.
  console.configure(level=args.logging_level)
- with TemporaryDirectory(keep=args.keep) as work_dir:
- bundle_manifest = BundleManifest.from_urlpath(args.paths.get("opensearch", os.getcwd()))
- BwcTestSuite(bundle_manifest, work_dir.name, args.component, args.keep).execute()
+
+ test_manifest = TestManifest.from_path(args.test_manifest_path)
+
+ all_results = BwcTestRunners.from_test_manifest(args, test_manifest).run()
+
+ all_results.log()
+
+ if all_results.failed():
+ sys.exit(1)
 
 
 if __name__ == "__main__":

src/test_workflow/bwc_test/bwc_test_runner.py

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import abc
+import logging
+import os
+
+from system.temporary_directory import TemporaryDirectory
+from test_workflow.test_recorder.test_recorder import TestRecorder
+from test_workflow.test_result.test_suite_results import TestSuiteResults
+
+
+class BwcTestRunner(abc.ABC):
+ def __init__(self, args, test_manifest):
+ self.args = args
+ self.test_manifest = test_manifest
+
+ self.tests_dir = os.path.join(os.getcwd(), "test-results")
+ os.makedirs(self.tests_dir, exist_ok=True)
+ self.test_recorder = TestRecorder(self.args.test_run_id, "bwc-test", self.tests_dir)
+
+ def run(self):
+ with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
+ all_results = TestSuiteResults()
+ for component in self.components.select(focus=self.args.component):
+ if component.name in self.test_manifest.components:
+ test_config = self.test_manifest.components[component.name]
+ if test_config.bwc_test:
+ test_suite = self.__create_test_suite__(component, test_config, work_dir)
+ test_results = test_suite.execute_tests()
+ all_results.append(component.name, test_results)
+ else:
+ logging.info(f"Skipping bwc-tests for {component.name}, as it is currently not supported")
+ else:
+ logging.info(f"Skipping bwc-tests for {component.name}, as it is currently not declared in the test manifest")
+
+ return all_results

src/test_workflow/bwc_test/bwc_test_runner_opensearch.py

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+
+from manifests.test_manifest import TestManifest
+from test_workflow.bwc_test.bwc_test_runner import BwcTestRunner
+from test_workflow.bwc_test.bwc_test_start_properties_opensearch import BwcTestStartPropertiesOpenSearch
+from test_workflow.bwc_test.bwc_test_suite_opensearch import BwcTestSuiteOpenSearch
+from test_workflow.test_args import TestArgs
+
+
+class BwcTestRunnerOpenSearch(BwcTestRunner):
+
+ def __init__(self, args: TestArgs, test_manifest: TestManifest):
+ super().__init__(args, test_manifest)
+ self.properties = BwcTestStartPropertiesOpenSearch(args.paths.get("opensearch", os.getcwd()))
+
+ self.components = self.properties.build_manifest.components
+
+ logging.info("Entering BWC test for OpenSearch")
+
+ def __create_test_suite__(self, component, test_config, work_dir):
+ return BwcTestSuiteOpenSearch(
+ work_dir.name,
+ component,
+ test_config,
+ self.test_recorder,
+ self.properties.bundle_manifest,
+ )

src/test_workflow/bwc_test/bwc_test_runner_opensearch_dashboards.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+
+from manifests.test_manifest import TestManifest
+from test_workflow.bwc_test.bwc_test_runner import BwcTestRunner
+from test_workflow.bwc_test.bwc_test_start_properties_opensearch_dashboards import BwcTestStartPropertiesOpenSearchDashboards
+from test_workflow.bwc_test.bwc_test_suite_opensearch_dashboards import BwcTestSuiteOpenSearchDashboards
+from test_workflow.test_args import TestArgs
+
+
+class BwcTestRunnerOpenSearchDashboards(BwcTestRunner):
+
+ def __init__(self, args: TestArgs, test_manifest: TestManifest):
+ super().__init__(args, test_manifest)
+ self.properties = BwcTestStartPropertiesOpenSearchDashboards(args.paths.get("opensearch-dashboards", os.getcwd()))
+
+ self.components = self.properties.build_manifest.components
+ logging.info("Entering BWC test for OpenSearch Dashboards")
+
+ def __create_test_suite__(self, component, test_config, work_dir):
+ return BwcTestSuiteOpenSearchDashboards(
+ work_dir.name,
+ component,
+ test_config,
+ self.test_recorder,
+ self.properties.bundle_manifest,
+ )

src/test_workflow/bwc_test/bwc_test_runners.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+
+from manifests.test_manifest import TestManifest
+from test_workflow.bwc_test.bwc_test_runner_opensearch import BwcTestRunnerOpenSearch
+from test_workflow.bwc_test.bwc_test_runner_opensearch_dashboards import BwcTestRunnerOpenSearchDashboards
+from test_workflow.test_args import TestArgs
+
+
+class BwcTestRunners:
+ RUNNERS = {
+ "OpenSearch": BwcTestRunnerOpenSearch,
+ "OpenSearch Dashboards": BwcTestRunnerOpenSearchDashboards
+ }
+
+ @classmethod
+ def from_test_manifest(cls, args: TestArgs, test_manifest: TestManifest):
+ return cls.RUNNERS[test_manifest.name](args, test_manifest)

src/test_workflow/bwc_test/bwc_test_start_properties.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+
+import abc
+
+from manifests.build_manifest import BuildManifest
+from manifests.bundle_manifest import BundleManifest
+
+
+class BwcTestStartProperties(abc.ABC):
+ def __init__(self, path, build_dir, bundle_dir):
+ self.path = path
+ self.build_dir = build_dir
+ self.bundle_dir = bundle_dir
+
+ self.bundle_manifest = BundleManifest.from_urlpath("/".join([self.path.rstrip("/"), self.bundle_dir]))
+ self.build_manifest = BuildManifest.from_urlpath("/".join([self.path.rstrip("/"), self.build_dir]))

src/test_workflow/bwc_test/bwc_test_start_properties_opensearch.py

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from test_workflow.bwc_test.bwc_test_start_properties import BwcTestStartProperties
+
+
+class BwcTestStartPropertiesOpenSearch(BwcTestStartProperties):
+ def __init__(self, path):
+ super().__init__(path, "builds/opensearch/manifest.yml", "dist/opensearch/manifest.yml")

src/test_workflow/bwc_test/bwc_test_start_properties_opensearch_dashboards.py

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from test_workflow.bwc_test.bwc_test_start_properties import BwcTestStartProperties
+
+
+class BwcTestStartPropertiesOpenSearchDashboards(BwcTestStartProperties):
+ def __init__(self, path):
+ super().__init__(path, "builds/opensearch-dashboards/manifest.yml", "dist/opensearch-dashboards/manifest.yml")

src/test_workflow/bwc_test/bwc_test_suite.py

@@ -7,43 +7,97 @@
 # Modifications Copyright OpenSearch Contributors. See
 # GitHub history for details.
 
+import abc
 import logging
 import os
 
+from git.git_repository import GitRepository
 from paths.script_finder import ScriptFinder
 from system.execute import execute
-from test_workflow.test_component import TestComponent
+from test_workflow.test_recorder.test_result_data import TestResultData
+from test_workflow.test_result.test_component_results import TestComponentResults
+from test_workflow.test_result.test_result import TestResult
 
 
-class BwcTestSuite:
- manifest: str
- work_dir: str
- component: str
- keep: bool
+class BwcTestSuite(abc.ABC):
 
- def __init__(self, manifest, work_dir, component=None, keep=False):
- self.manifest = manifest
+ def __init__(
+ self,
+ work_dir,
+ component,
+ test_config,
+ test_recorder,
+ manifest
+ ):
  self.work_dir = work_dir
  self.component = component
- self.keep = keep
-
- def run_tests(self, work_dir, component_name):
- script = ScriptFinder.find_bwc_test_script(component_name, work_dir)
- (status, stdout, stderr) = execute(script, work_dir, True, False)
- return (status, stdout, stderr)
-
- def component_bwc_tests(self, component):
- test_component = TestComponent(component.repository, component.commit_id)
- test_component.checkout(os.path.join(self.work_dir, component.name))
- try:
- console_output = self.run_tests(os.path.join(self.work_dir, component.name), component.name)
- return console_output
- except:
- # TODO: Store and report test failures for {component}
- logging.info(f"Exception while running BWC tests for {component.name}")
-
- def execute(self):
- # For each component, check out the git repo and run `bwctest.sh`
- for component in self.manifest.components.select(focus=self.component):
- # TODO: Store and report test results, send notification via {console_output}
- self.component_bwc_tests(component)
+ self.test_config = test_config
+ self.test_recorder = test_recorder
+ self.manifest = manifest
+
+ self.repo = GitRepository(
+ self.component.repository,
+ self.component.commit_id,
+ os.path.join(self.work_dir, self.component.name),
+ test_config.working_directory
+ )
+
+ self.save_logs = test_recorder.test_results_logs
+
+ def execute_tests(self):
+ test_results = TestComponentResults()
+
+ for config in self.test_config.bwc_test["test-configs"]:
+ status = self.execute_bwctest_sh(config)
+
+ test_results.append(TestResult(self.component.name, config, status))
+ return test_results
+
+ def execute_bwctest_sh(self, config):
+ security = self.is_security_enabled(config)
+ script = ScriptFinder.find_bwc_test_script(self.component.name, self.repo.working_directory)
+ if os.path.exists(script):
+ cmd = self.get_cmd(script, security, self.manifest.build.location)
+ self.repo_work_dir = os.path.join(
+ self.repo.dir, self.test_config.working_directory) if self.test_config.working_directory is not None else self.repo.dir
+ (status, stdout, stderr) = execute(cmd, self.repo_work_dir, True, False)
+
+ test_result_data = TestResultData(
+ self.component.name,
+ self.test_config,
+ status,
+ stdout,
+ stderr,
+ self.test_artifact_files
+ )
+ self.save_logs.save_test_result_data(test_result_data)
+ if stderr:
+ logging.info("BWC test run failed for component " + self.component.name)
+ logging.info(stderr)
+ return status
+ else:
+ logging.info(f"{script} does not exist. Skipping integ tests for {self.component.name}")
+
+ def is_security_enabled(self, config):
+ if config in ["with-security", "without-security"]:
+ return True if config == "with-security" else False
+ else:
+ raise InvalidTestConfigError("Unsupported test config: " + config)
+
+ def pretty_print_message(self, message):
+ logging.info("===============================================")
+ logging.info(message)
+ logging.info("===============================================")
+
+ @abc.abstractmethod
+ def get_cmd(self):
+ pass
+
+ @property
+ @abc.abstractmethod
+ def test_artifact_files(self):
+ pass
+
+
+class InvalidTestConfigError(Exception):
+ pass

src/test_workflow/bwc_test/bwc_test_suite_opensearch.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import os
+
+from test_workflow.bwc_test.bwc_test_suite import BwcTestSuite
+
+
+class BwcTestSuiteOpenSearch(BwcTestSuite):
+
+ def __init__(
+ self,
+ work_dir,
+ component,
+ test_config,
+ test_recorder,
+ manifest
+ ):
+
+ super().__init__(
+ work_dir,
+ component,
+ test_config,
+ test_recorder,
+ manifest
+ )
+
+ def get_cmd(self, script, security, manifest_build_location):
+ return f"{script}"
+
+ @property
+ def test_artifact_files(self):
+ return {
+ "opensearch-bwc-test": os.path.join(self.repo_work_dir, "build", "reports", "tests", "bwcTest")
+ }