From 51322761d8377d1db7af5767991b0bcd422e26e5 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Fri, 23 Jun 2023 17:08:58 -0700 Subject: [PATCH] update location schema and added skip clouds in suite yml (#2852) * update location schema in suite yml * address comments * . * pylint warn * comment --- .../orchestrator/lib/agent_test_loader.py | 52 ++++++++++++++----- .../lib/agent_test_suite_combinator.py | 12 ++++- tests_e2e/test_suites/agent_update.yml | 7 ++- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_loader.py b/tests_e2e/orchestrator/lib/agent_test_loader.py index a2576f9b6b..31c6e52a40 100644 --- a/tests_e2e/orchestrator/lib/agent_test_loader.py +++ b/tests_e2e/orchestrator/lib/agent_test_loader.py @@ -53,14 +53,16 @@ class TestSuiteInfo(object): tests: List[TestInfo] # Images or image sets (as defined in images.yml) on which the suite must run. images: List[str] - # The location (region) on which the suite must run; if empty, the suite can run on any location - location: str + # The locations (regions) on which the suite must run; if empty, the suite can run on any location + locations: List[str] # Whether this suite must run on its own test VM owns_vm: bool # Whether to install the test Agent on the test VM install_test_agent: bool # Customization for the ARM template used when creating the test VM template: str + # skip test suite if the test not supposed to run on specific clouds + skip_on_clouds: List[str] def __str__(self): return self.name @@ -137,15 +139,24 @@ def _validate(self): if image not in self.images: raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {image} in images.yml") - # If the suite specifies a location, validate that the images it uses are available in that location - if suite.location != '': + # If the suite specifies a cloud and it's location, validate that location string is start with and then validate that the images it uses are available in that location + for suite_location in suite.locations: + if suite_location.startswith(self.__cloud + ":"): + suite_location = suite_location.split(":")[1] + else: + continue for suite_image in suite.images: for image in self.images[suite_image]: # If the image has a location restriction, validate that it is available on the location the suite must run on if image.locations: locations = image.locations.get(self.__cloud) - if locations is not None and not any(suite.location in l for l in locations): - raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location") + if locations is not None and not any(suite_location in l for l in locations): + raise Exception(f"Test suite {suite.name} must be executed in {suite_location}, but <{image.urn}> is not available in that location") + + # if the suite specifies skip clouds, validate that cloud used in our tests + for suite_skip_cloud in suite.skip_on_clouds: + if suite_skip_cloud not in ["AzureCloud", "AzureChinaCloud", "AzureUSGovernment"]: + raise Exception(f"Invalid cloud {suite_skip_cloud} for in {suite.name}") @staticmethod def _load_test_suites(test_suites: str) -> List[TestSuiteInfo]: @@ -180,7 +191,7 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: - "bvts/run_command.py" - "bvts/vm_access.py" images: "endorsed" - location: "eastuseaup" + locations: "AzureCloud:eastuseaup" owns_vm: true install_test_agent: true template: "bvts/template.py" @@ -195,8 +206,8 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: * images - A string, or a list of strings, specifying the images on which the test suite must be executed. Each value can be the name of a single image (e.g."ubuntu_2004"), or the name of an image set (e.g. "endorsed"). The names for images and image sets are defined in WALinuxAgent/tests_e2e/tests_suites/images.yml. - * location - [Optional; string] If given, the test suite must be executed on that location. If not specified, - or set to an empty string, the test suite will be executed in the default location. This is useful + * locations - [Optional; string or list of strings] If given, the test suite must be executed on that cloud location(e.g. "AzureCloud:eastus2euap"). + If not specified, or set to an empty string, the test suite will be executed in the default location. This is useful for test suites that exercise a feature that is enabled only in certain regions. * owns_vm - [Optional; boolean] By default all suites in a test run are executed on the same test VMs; if this value is set to True, new test VMs will be created and will be used exclusively for this test suite. @@ -206,6 +217,9 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: * install_test_agent - [Optional; boolean] By default the setup process installs the test Agent on the test VMs; set this property to False to skip the installation. * template - [Optional; string] If given, the ARM template for the test VM is customized using the given Python module. + * skip_on_clouds - [Optional; string or list of strings] If given, the test suite will be skipped in the specified cloud(e.g. "AzureCloud"). + If not specified, the test suite will be executed in all the clouds that we use. This is useful + if you want to skip a test suite validation in a particular cloud when certain feature is not available in that cloud. """ test_suite: Dict[str, Any] = AgentTestLoader._load_file(description_file) @@ -234,14 +248,28 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: else: test_suite_info.images = images - test_suite_info.location = test_suite.get("location") - if test_suite_info.location is None: - test_suite_info.location = "" + locations = test_suite.get("locations") + if locations is None: + test_suite_info.locations = [] + else: + if isinstance(locations, str): + test_suite_info.locations = [locations] + else: + test_suite_info.locations = locations test_suite_info.owns_vm = "owns_vm" in test_suite and test_suite["owns_vm"] test_suite_info.install_test_agent = "install_test_agent" not in test_suite or test_suite["install_test_agent"] test_suite_info.template = test_suite.get("template", "") + skip_on_clouds = test_suite.get("skip_on_clouds") + if skip_on_clouds is not None: + if isinstance(skip_on_clouds, str): + test_suite_info.skip_on_clouds = [skip_on_clouds] + else: + test_suite_info.skip_on_clouds = skip_on_clouds + else: + test_suite_info.skip_on_clouds = [] + return test_suite_info @staticmethod diff --git a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py index 82915dcfb0..5efdf54bf5 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py @@ -149,7 +149,11 @@ def create_environment_list(self) -> List[Dict[str, Any]]: runbook_images = self._get_runbook_images(loader) + skip_test_suites: List[str] = [] for suite_info in loader.test_suites: + if self.runbook.cloud in suite_info.skip_on_clouds: + skip_test_suites.append(suite_info.name) + continue if len(runbook_images) > 0: images_info: List[VmImageInfo] = runbook_images else: @@ -216,6 +220,9 @@ def create_environment(c_env_name: str) -> Dict[str, Any]: raise Exception("No VM images were found to execute the test suites.") log: logging.Logger = logging.getLogger("lisa") + if len(skip_test_suites) > 0: + log.info("") + log.info("Test suites skipped on %s:\n\n\t%s\n", self.runbook.cloud, '\n\t'.join(skip_test_suites)) log.info("") log.info("******** Waagent: Test Environments *****") log.info("") @@ -282,8 +289,9 @@ def _get_location(self, suite_info: TestSuiteInfo, image: VmImageInfo) -> str: return self.runbook.location # Then try the suite location, if any. - if suite_info.location != '': - return suite_info.location + for location in suite_info.locations: + if location.startswith(self.runbook.cloud + ":"): + return location.split(":")[1] # If the image has a location restriction, use any location where it is available. # However, if it is not available on any location, skip the image (return None) diff --git a/tests_e2e/test_suites/agent_update.yml b/tests_e2e/test_suites/agent_update.yml index 77a0144d57..865fa89ca5 100644 --- a/tests_e2e/test_suites/agent_update.yml +++ b/tests_e2e/test_suites/agent_update.yml @@ -2,5 +2,8 @@ name: "AgentUpdate" tests: - "agent_update/rsm_update.py" images: "endorsed" -location: "eastus2euap" -owns_vm: true \ No newline at end of file +locations: "AzureCloud:eastus2euap" +owns_vm: true +skip_on_clouds: + - "AzureChinaCloud" + - "AzureUSGovernment" \ No newline at end of file