update location schema and added skip clouds in suite yml (#2852)

* update location schema in suite yml

* address comments

* .

* pylint warn

* comment

tests_e2e/orchestrator/lib/agent_test_loader.py

@@ -53,14 +53,16 @@ class TestSuiteInfo(object):
     tests: List[TestInfo]
     # Images or image sets (as defined in images.yml) on which the suite must run.
     images: List[str]
-    # The location (region) on which the suite must run; if empty, the suite can run on any location
-    location: str
+    # The locations (regions) on which the suite must run; if empty, the suite can run on any location
+    locations: List[str]
     # Whether this suite must run on its own test VM
     owns_vm: bool
     # Whether to install the test Agent on the test VM
     install_test_agent: bool
     # Customization for the ARM template used when creating the test VM
     template: str
+    # skip test suite if the test not supposed to run on specific clouds
+    skip_on_clouds: List[str]
 
     def __str__(self):
         return self.name
@@ -137,15 +139,24 @@ def _validate(self):
                 if image not in self.images:
                     raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {image} in images.yml")
 
-            # If the suite specifies a location, validate that the images it uses are available in that location
-            if suite.location != '':
+            # If the suite specifies a cloud and it's location<cloud:location>, validate that location string is start with <cloud:> and then validate that the images it uses are available in that location
+            for suite_location in suite.locations:
+                if suite_location.startswith(self.__cloud + ":"):
+                    suite_location = suite_location.split(":")[1]
+                else:
+                    continue
                 for suite_image in suite.images:
                     for image in self.images[suite_image]:
                         # If the image has a location restriction, validate that it is available on the location the suite must run on
                         if image.locations:
                             locations = image.locations.get(self.__cloud)
-                            if locations is not None and not any(suite.location in l for l in locations):
-                                raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location")
+                            if locations is not None and not any(suite_location in l for l in locations):
+                                raise Exception(f"Test suite {suite.name} must be executed in {suite_location}, but <{image.urn}> is not available in that location")
+
+            # if the suite specifies skip clouds, validate that cloud used in our tests
+            for suite_skip_cloud in suite.skip_on_clouds:
+                if suite_skip_cloud not in ["AzureCloud", "AzureChinaCloud", "AzureUSGovernment"]:
+                    raise Exception(f"Invalid cloud {suite_skip_cloud} for in {suite.name}")
 
     @staticmethod
     def _load_test_suites(test_suites: str) -> List[TestSuiteInfo]:
@@ -180,7 +191,7 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
               - "bvts/run_command.py"
               - "bvts/vm_access.py"
             images: "endorsed"
-            location: "eastuseaup"
+            locations: "AzureCloud:eastuseaup"
             owns_vm: true
             install_test_agent: true
             template: "bvts/template.py"
@@ -195,8 +206,8 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
         * images   - A string, or a list of strings, specifying the images on which the test suite must be executed. Each value
                      can be the name of a single image (e.g."ubuntu_2004"), or the name of an image set (e.g. "endorsed"). The
                      names for images and image sets are defined in WALinuxAgent/tests_e2e/tests_suites/images.yml.
-        * location - [Optional; string] If given, the test suite must be executed on that location. If not specified,
-                     or set to an empty string, the test suite will be executed in the default location. This is useful
+        * locations - [Optional; string or list of strings] If given, the test suite must be executed on that cloud location(e.g. "AzureCloud:eastus2euap").
+                     If not specified, or set to an empty string, the test suite will be executed in the default location. This is useful
                      for test suites that exercise a feature that is enabled only in certain regions.
         * owns_vm - [Optional; boolean] By default all suites in a test run are executed on the same test VMs; if this
                     value is set to True, new test VMs will be created and will be used exclusively for this test suite.
@@ -206,6 +217,9 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
         * install_test_agent - [Optional; boolean] By default the setup process installs the test Agent on the test VMs; set this property
                     to False to skip the installation.
         * template - [Optional; string] If given, the ARM template for the test VM is customized using the given Python module.
+        * skip_on_clouds - [Optional; string or list of strings] If given, the test suite will be skipped in the specified cloud(e.g. "AzureCloud").
+                    If not specified, the test suite will be executed in all the clouds that we use. This is useful
+                    if you want to skip a test suite validation in a particular cloud when certain feature is not available in that cloud.
 
         """
         test_suite: Dict[str, Any] = AgentTestLoader._load_file(description_file)
@@ -234,14 +248,28 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
         else:
             test_suite_info.images = images
 
-        test_suite_info.location = test_suite.get("location")
-        if test_suite_info.location is None:
-            test_suite_info.location = ""
+        locations = test_suite.get("locations")
+        if locations is None:
+            test_suite_info.locations = []
+        else:
+            if isinstance(locations, str):
+                test_suite_info.locations = [locations]
+            else:
+                test_suite_info.locations = locations
 
         test_suite_info.owns_vm = "owns_vm" in test_suite and test_suite["owns_vm"]
         test_suite_info.install_test_agent = "install_test_agent" not in test_suite or test_suite["install_test_agent"]
         test_suite_info.template = test_suite.get("template", "")
 
+        skip_on_clouds = test_suite.get("skip_on_clouds")
+        if skip_on_clouds is not None:
+            if isinstance(skip_on_clouds, str):
+                test_suite_info.skip_on_clouds = [skip_on_clouds]
+            else:
+                test_suite_info.skip_on_clouds = skip_on_clouds
+        else:
+            test_suite_info.skip_on_clouds = []
+
         return test_suite_info
 
     @staticmethod

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -149,7 +149,11 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
 
         runbook_images = self._get_runbook_images(loader)
 
+        skip_test_suites: List[str] = []
         for suite_info in loader.test_suites:
+            if self.runbook.cloud in suite_info.skip_on_clouds:
+                skip_test_suites.append(suite_info.name)
+                continue
             if len(runbook_images) > 0:
                 images_info: List[VmImageInfo] = runbook_images
             else:
@@ -216,6 +220,9 @@ def create_environment(c_env_name: str) -> Dict[str, Any]:
             raise Exception("No VM images were found to execute the test suites.")
 
         log: logging.Logger = logging.getLogger("lisa")
+        if len(skip_test_suites) > 0:
+            log.info("")
+            log.info("Test suites skipped on %s:\n\n\t%s\n", self.runbook.cloud, '\n\t'.join(skip_test_suites))
         log.info("")
         log.info("******** Waagent: Test Environments *****")
         log.info("")
@@ -282,8 +289,9 @@ def _get_location(self, suite_info: TestSuiteInfo, image: VmImageInfo) -> str:
             return self.runbook.location
 
         #  Then try the suite location, if any.
-        if suite_info.location != '':
-            return suite_info.location
+        for location in suite_info.locations:
+            if location.startswith(self.runbook.cloud + ":"):
+                return location.split(":")[1]
 
         # If the image has a location restriction, use any location where it is available.
         # However, if it is not available on any location, skip the image (return None)

tests_e2e/test_suites/agent_update.yml

@@ -2,5 +2,8 @@ name: "AgentUpdate"
 tests:
   - "agent_update/rsm_update.py"
 images: "endorsed"
-location: "eastus2euap"
-owns_vm: true
+locations: "AzureCloud:eastus2euap"
+owns_vm: true
+skip_on_clouds:
+  - "AzureChinaCloud"
+  - "AzureUSGovernment"