From 3737d87bd8b3311f9f46e4473ff0b6c6b4adb201 Mon Sep 17 00:00:00 2001 From: nnandigam Date: Sun, 18 Jun 2023 21:13:35 -0700 Subject: [PATCH 1/5] update location schema in suite yml --- .../orchestrator/lib/agent_test_loader.py | 61 ++++++++++++++----- .../lib/agent_test_suite_combinator.py | 12 +++- tests_e2e/test_suites/agent_update.yml | 7 ++- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_loader.py b/tests_e2e/orchestrator/lib/agent_test_loader.py index a2576f9b6b..034ba4d87d 100644 --- a/tests_e2e/orchestrator/lib/agent_test_loader.py +++ b/tests_e2e/orchestrator/lib/agent_test_loader.py @@ -54,13 +54,15 @@ class TestSuiteInfo(object): # Images or image sets (as defined in images.yml) on which the suite must run. images: List[str] # The location (region) on which the suite must run; if empty, the suite can run on any location - location: str + locations: List[str] # Whether this suite must run on its own test VM owns_vm: bool # Whether to install the test Agent on the test VM install_test_agent: bool # Customization for the ARM template used when creating the test VM template: str + # skip test suite if the test not suppose to run on specific clouds + skip_on_clouds: List[str] def __str__(self): return self.name @@ -137,15 +139,25 @@ def _validate(self): if image not in self.images: raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {image} in images.yml") - # If the suite specifies a location, validate that the images it uses are available in that location - if suite.location != '': - for suite_image in suite.images: - for image in self.images[suite_image]: - # If the image has a location restriction, validate that it is available on the location the suite must run on - if image.locations: - locations = image.locations.get(self.__cloud) - if locations is not None and not any(suite.location in l for l in locations): - raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location") + # If the suite specifies a cloud specific location, validate that the images it uses are available in that location + for suite_location in suite.locations: + if self.__cloud in suite_location: + suite_location = suite_location.split(":")[1] + else: + suite_location = "" + if suite_location != "": + for suite_image in suite.images: + for image in self.images[suite_image]: + # If the image has a location restriction, validate that it is available on the location the suite must run on + if image.locations: + locations = image.locations.get(self.__cloud) + if locations is not None and not any(suite_location in l for l in locations): + raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location") + + # if the suite specifies skip clouds, validate that cloud used in our tests + for suite_skip_cloud in suite.skip_on_clouds: + if suite_skip_cloud not in ["AzureCloud", "AzureChinaCloud", "AzureUSGovernment"]: + raise Exception(f"Invalid cloud {suite_skip_cloud} for in {suite.name}") @staticmethod def _load_test_suites(test_suites: str) -> List[TestSuiteInfo]: @@ -180,7 +192,7 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: - "bvts/run_command.py" - "bvts/vm_access.py" images: "endorsed" - location: "eastuseaup" + locations: "AzureCloud:eastuseaup" owns_vm: true install_test_agent: true template: "bvts/template.py" @@ -195,8 +207,8 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: * images - A string, or a list of strings, specifying the images on which the test suite must be executed. Each value can be the name of a single image (e.g."ubuntu_2004"), or the name of an image set (e.g. "endorsed"). The names for images and image sets are defined in WALinuxAgent/tests_e2e/tests_suites/images.yml. - * location - [Optional; string] If given, the test suite must be executed on that location. If not specified, - or set to an empty string, the test suite will be executed in the default location. This is useful + * location - [Optional; string or list of strings] If given, the test suite must be executed on that cloud location(e.g. "AzureCloud:eastus2euap"). + If not specified, or set to an empty string, the test suite will be executed in the default location. This is useful for test suites that exercise a feature that is enabled only in certain regions. * owns_vm - [Optional; boolean] By default all suites in a test run are executed on the same test VMs; if this value is set to True, new test VMs will be created and will be used exclusively for this test suite. @@ -206,6 +218,9 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: * install_test_agent - [Optional; boolean] By default the setup process installs the test Agent on the test VMs; set this property to False to skip the installation. * template - [Optional; string] If given, the ARM template for the test VM is customized using the given Python module. + * skip_on_clouds - [Optional; string or list of strings] If given, the test suite will be skipped in the specified cloud(e.g. "AzureCloud"). + If not specified, the test suite will be executed in all the clouds that we use. This is useful + if you want to skip a test suite validation in a particular cloud when certain feature is not available in that cloud. """ test_suite: Dict[str, Any] = AgentTestLoader._load_file(description_file) @@ -234,14 +249,28 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: else: test_suite_info.images = images - test_suite_info.location = test_suite.get("location") - if test_suite_info.location is None: - test_suite_info.location = "" + locations = test_suite.get("locations") + if locations is None: + test_suite_info.locations = [] + else: + if isinstance(locations, str): + test_suite_info.locations = [locations] + else: + test_suite_info.locations = locations test_suite_info.owns_vm = "owns_vm" in test_suite and test_suite["owns_vm"] test_suite_info.install_test_agent = "install_test_agent" not in test_suite or test_suite["install_test_agent"] test_suite_info.template = test_suite.get("template", "") + skip_on_clouds = test_suite.get("skip_on_clouds") + if skip_on_clouds is not None: + if isinstance(skip_on_clouds, str): + test_suite_info.skip_on_clouds = [skip_on_clouds] + else: + test_suite_info.skip_on_clouds = skip_on_clouds + else: + test_suite_info.skip_on_clouds = [] + return test_suite_info @staticmethod diff --git a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py index 82915dcfb0..bec1d5993a 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py @@ -149,7 +149,11 @@ def create_environment_list(self) -> List[Dict[str, Any]]: runbook_images = self._get_runbook_images(loader) + skip_test_suites: List[str] = [] for suite_info in loader.test_suites: + if self.runbook.cloud in suite_info.skip_on_clouds: + skip_test_suites.append(suite_info.name) + continue if len(runbook_images) > 0: images_info: List[VmImageInfo] = runbook_images else: @@ -216,6 +220,9 @@ def create_environment(c_env_name: str) -> Dict[str, Any]: raise Exception("No VM images were found to execute the test suites.") log: logging.Logger = logging.getLogger("lisa") + if len(skip_test_suites) > 0: + log.info("") + log.info("Test suites skipped on %s:\n\n\t%s\n", self.runbook.cloud, '\n\t'.join(skip_test_suites)) log.info("") log.info("******** Waagent: Test Environments *****") log.info("") @@ -282,8 +289,9 @@ def _get_location(self, suite_info: TestSuiteInfo, image: VmImageInfo) -> str: return self.runbook.location # Then try the suite location, if any. - if suite_info.location != '': - return suite_info.location + for location in suite_info.locations: + if self.runbook.cloud in location: + return location.split(":")[1] # If the image has a location restriction, use any location where it is available. # However, if it is not available on any location, skip the image (return None) diff --git a/tests_e2e/test_suites/agent_update.yml b/tests_e2e/test_suites/agent_update.yml index 77a0144d57..865fa89ca5 100644 --- a/tests_e2e/test_suites/agent_update.yml +++ b/tests_e2e/test_suites/agent_update.yml @@ -2,5 +2,8 @@ name: "AgentUpdate" tests: - "agent_update/rsm_update.py" images: "endorsed" -location: "eastus2euap" -owns_vm: true \ No newline at end of file +locations: "AzureCloud:eastus2euap" +owns_vm: true +skip_on_clouds: + - "AzureChinaCloud" + - "AzureUSGovernment" \ No newline at end of file From 182f898fab61a50ef40aa2e87d967cf883946285 Mon Sep 17 00:00:00 2001 From: nnandigam Date: Thu, 22 Jun 2023 14:58:37 -0700 Subject: [PATCH 2/5] address comments --- .../orchestrator/lib/agent_test_loader.py | 25 +++++++++---------- .../lib/agent_test_suite_combinator.py | 2 +- tests_e2e/test_suites/agent_update.yml | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_loader.py b/tests_e2e/orchestrator/lib/agent_test_loader.py index 034ba4d87d..485b63a252 100644 --- a/tests_e2e/orchestrator/lib/agent_test_loader.py +++ b/tests_e2e/orchestrator/lib/agent_test_loader.py @@ -53,7 +53,7 @@ class TestSuiteInfo(object): tests: List[TestInfo] # Images or image sets (as defined in images.yml) on which the suite must run. images: List[str] - # The location (region) on which the suite must run; if empty, the suite can run on any location + # The locations (regions) on which the suite must run; if empty, the suite can run on any location locations: List[str] # Whether this suite must run on its own test VM owns_vm: bool @@ -61,7 +61,7 @@ class TestSuiteInfo(object): install_test_agent: bool # Customization for the ARM template used when creating the test VM template: str - # skip test suite if the test not suppose to run on specific clouds + # skip test suite if the test not supposed to run on specific clouds skip_on_clouds: List[str] def __str__(self): @@ -139,20 +139,19 @@ def _validate(self): if image not in self.images: raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {image} in images.yml") - # If the suite specifies a cloud specific location, validate that the images it uses are available in that location + # If the suite specifies a cloud and it's location, validate that location string is start with and then validate that the images it uses are available in that location for suite_location in suite.locations: - if self.__cloud in suite_location: + if suite_location.startswith(self.__cloud + ":"): suite_location = suite_location.split(":")[1] else: - suite_location = "" - if suite_location != "": - for suite_image in suite.images: - for image in self.images[suite_image]: - # If the image has a location restriction, validate that it is available on the location the suite must run on - if image.locations: - locations = image.locations.get(self.__cloud) - if locations is not None and not any(suite_location in l for l in locations): - raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location") + continue + for suite_image in suite.images: + for image in self.images[suite_image]: + # If the image has a location restriction, validate that it is available on the location the suite must run on + if image.locations: + locations = image.locations.get(self.__cloud) + if locations is not None and not any(suite_location in l for l in locations): + raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location") # if the suite specifies skip clouds, validate that cloud used in our tests for suite_skip_cloud in suite.skip_on_clouds: diff --git a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py index bec1d5993a..5efdf54bf5 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py @@ -290,7 +290,7 @@ def _get_location(self, suite_info: TestSuiteInfo, image: VmImageInfo) -> str: # Then try the suite location, if any. for location in suite_info.locations: - if self.runbook.cloud in location: + if location.startswith(self.runbook.cloud + ":"): return location.split(":")[1] # If the image has a location restriction, use any location where it is available. diff --git a/tests_e2e/test_suites/agent_update.yml b/tests_e2e/test_suites/agent_update.yml index 865fa89ca5..7b9bb988ce 100644 --- a/tests_e2e/test_suites/agent_update.yml +++ b/tests_e2e/test_suites/agent_update.yml @@ -1,7 +1,7 @@ name: "AgentUpdate" tests: - "agent_update/rsm_update.py" -images: "endorsed" +images: "ubuntu_2004" locations: "AzureCloud:eastus2euap" owns_vm: true skip_on_clouds: From 355a70f8aa9ca97c90a7f48e6435574d4563bff7 Mon Sep 17 00:00:00 2001 From: nnandigam Date: Thu, 22 Jun 2023 15:05:11 -0700 Subject: [PATCH 3/5] . --- tests_e2e/test_suites/agent_update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_e2e/test_suites/agent_update.yml b/tests_e2e/test_suites/agent_update.yml index 7b9bb988ce..865fa89ca5 100644 --- a/tests_e2e/test_suites/agent_update.yml +++ b/tests_e2e/test_suites/agent_update.yml @@ -1,7 +1,7 @@ name: "AgentUpdate" tests: - "agent_update/rsm_update.py" -images: "ubuntu_2004" +images: "endorsed" locations: "AzureCloud:eastus2euap" owns_vm: true skip_on_clouds: From 7e0d5c40c486dd5011b88f1dce256ca5ca365157 Mon Sep 17 00:00:00 2001 From: nnandigam Date: Thu, 22 Jun 2023 15:10:26 -0700 Subject: [PATCH 4/5] pylint warn --- tests_e2e/orchestrator/lib/agent_test_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_loader.py b/tests_e2e/orchestrator/lib/agent_test_loader.py index 485b63a252..b7ffe14dff 100644 --- a/tests_e2e/orchestrator/lib/agent_test_loader.py +++ b/tests_e2e/orchestrator/lib/agent_test_loader.py @@ -151,7 +151,7 @@ def _validate(self): if image.locations: locations = image.locations.get(self.__cloud) if locations is not None and not any(suite_location in l for l in locations): - raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location") + raise Exception(f"Test suite {suite.name} must be executed in {suite_location}, but <{image.urn}> is not available in that location") # if the suite specifies skip clouds, validate that cloud used in our tests for suite_skip_cloud in suite.skip_on_clouds: From 26bc1508af7eaf7bc4b5516d0b8d6fba769df35c Mon Sep 17 00:00:00 2001 From: nnandigam Date: Thu, 22 Jun 2023 15:16:22 -0700 Subject: [PATCH 5/5] comment --- tests_e2e/orchestrator/lib/agent_test_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_loader.py b/tests_e2e/orchestrator/lib/agent_test_loader.py index b7ffe14dff..31c6e52a40 100644 --- a/tests_e2e/orchestrator/lib/agent_test_loader.py +++ b/tests_e2e/orchestrator/lib/agent_test_loader.py @@ -206,7 +206,7 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo: * images - A string, or a list of strings, specifying the images on which the test suite must be executed. Each value can be the name of a single image (e.g."ubuntu_2004"), or the name of an image set (e.g. "endorsed"). The names for images and image sets are defined in WALinuxAgent/tests_e2e/tests_suites/images.yml. - * location - [Optional; string or list of strings] If given, the test suite must be executed on that cloud location(e.g. "AzureCloud:eastus2euap"). + * locations - [Optional; string or list of strings] If given, the test suite must be executed on that cloud location(e.g. "AzureCloud:eastus2euap"). If not specified, or set to an empty string, the test suite will be executed in the default location. This is useful for test suites that exercise a feature that is enabled only in certain regions. * owns_vm - [Optional; boolean] By default all suites in a test run are executed on the same test VMs; if this