Add support for VM Scale Sets to end-to-end tests

test-requirements.txt

@@ -17,6 +17,7 @@ assertpy
 azure-core
 azure-identity
 azure-mgmt-compute>=22.1.0
+azure-mgmt-network>=19.3.0
 azure-mgmt-resource>=15.0.0
 msrestazure
 pytz

tests_e2e/orchestrator/lib/agent_test_loader.py

@@ -23,15 +23,15 @@
 from typing import Any, Dict, List, Type
 
 import tests_e2e
-from tests_e2e.tests.lib.agent_test import AgentTest
+from tests_e2e.tests.lib.agent_test import AgentTest, AgentVmTest, AgentVmssTest
 
 
 class TestInfo(object):
     """
     Description of a test
     """
     # The class that implements the test
-    test_class: Type[AgentTest]
+    test_class: Type[AgentVmTest]
     # If True, an error in the test blocks the execution of the test suite (defaults to False)
     blocks_suite: bool
 
@@ -57,6 +57,8 @@ class TestSuiteInfo(object):
     locations: List[str]
     # Whether this suite must run on its own test VM
     owns_vm: bool
+    # If True, the suite must run on a scale set (instead of a single VM)
+    executes_on_scale_set: bool
     # Whether to install the test Agent on the test VM
     install_test_agent: bool
     # Customization for the ARM template used when creating the test VM
@@ -222,6 +224,7 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
                     This is useful for suites that modify the test VMs in such a way that the setup may cause problems
                     in other test suites (for example, some tests targeted to the HGAP block internet access in order to
                     force the agent to use the HGAP).
+        * executes_on_scale_set - [Optional; boolean] True indicates that the test runs on a scale set. 
         * install_test_agent - [Optional; boolean] By default the setup process installs the test Agent on the test VMs; set this property
                     to False to skip the installation.
         * template - [Optional; string] If given, the ARM template for the test VM is customized using the given Python module.
@@ -267,8 +270,13 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
 
         test_suite_info.owns_vm = "owns_vm" in test_suite and test_suite["owns_vm"]
         test_suite_info.install_test_agent = "install_test_agent" not in test_suite or test_suite["install_test_agent"]
+        test_suite_info.executes_on_scale_set = "executes_on_scale_set" in test_suite and test_suite["executes_on_scale_set"]
         test_suite_info.template = test_suite.get("template", "")
 
+        # TODO: Add support for custom templates
+        if test_suite_info.executes_on_scale_set and test_suite_info.template != '':
+            raise Exception(f"Currently custom templates are not supported on scale sets. [Test suite: {test_suite_info.name}]")
+
         skip_on_clouds = test_suite.get("skip_on_clouds")
         if skip_on_clouds is not None:
             if isinstance(skip_on_clouds, str):
@@ -281,16 +289,16 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
         return test_suite_info
 
     @staticmethod
-    def _load_test_class(relative_path: str) -> Type[AgentTest]:
+    def _load_test_class(relative_path: str) -> Type[AgentVmTest]:
         """
         Loads an AgentTest from its source code file, which is given as a path relative to WALinuxAgent/tests_e2e/tests.
         """
         full_path: Path = AgentTestLoader._SOURCE_CODE_ROOT/"tests"/relative_path
         spec = importlib.util.spec_from_file_location(f"tests_e2e.tests.{relative_path.replace('/', '.').replace('.py', '')}", str(full_path))
         module = importlib.util.module_from_spec(spec)
         spec.loader.exec_module(module)
-        # return all the classes in the module that are subclasses of AgentTest but are not AgentTest itself.
-        matches = [v for v in module.__dict__.values() if isinstance(v, type) and issubclass(v, AgentTest) and v != AgentTest]
+        # return all the classes in the module that are subclasses of AgentTest but are not AgentVmTest or AgentVmssTest themselves.
+        matches = [v for v in module.__dict__.values() if isinstance(v, type) and issubclass(v, AgentTest) and v != AgentVmTest and v != AgentVmssTest]
         if len(matches) != 1:
             raise Exception(f"Error in {full_path} (each test file must contain exactly one class derived from AgentTest)")
         return matches[0]

tests_e2e/orchestrator/lib/update_arm_template_hook.py

@@ -46,7 +46,7 @@ def azure_update_arm_template(self, template: Any, environment: Environment) ->
         # Add the network security group for the test VM. This group includes a rule allowing SSH access from the current machine.
         #
         log.info("******** Waagent: Adding network security rule to the ARM template")
-        AddNetworkSecurityGroup().update(template)
+        AddNetworkSecurityGroup().update(template, is_lisa_template=True)
 
         #
         # Apply any template customizations provided by the tests.
@@ -60,7 +60,7 @@ def azure_update_arm_template(self, template: Any, environment: Environment) ->
 
             for t in test_templates.split(","):
                 update_arm_template = self._get_update_arm_template(t)
-                update_arm_template().update(template)
+                update_arm_template().update(template, is_lisa_template=True)
 
     _SOURCE_CODE_ROOT: Path = Path(tests_e2e.__path__[0])
 

tests_e2e/orchestrator/runbook.yml

@@ -8,138 +8,187 @@ extension:
   - "./lib"
 
 variable:
+  #
+  # The test environments are generated dynamically by the AgentTestSuitesCombinator using the 'platform' and 'environment' variables.
+  # Most of the variables below are parameters for the combinator and/or the AgentTestSuite (marked as 'is_case_visible'), but a few of
+  # them, such as the runbook name and the SSH proxy variables, are handled by LISA.
+  #
+  # Many of these variables are optional, depending on the scenario. An empty values indicates that the variable has not been specified.
+  #
+
+  #
+  # The name of the runbook, it is added as a prefix ("lisa-<name>") to ARM resources created by the test run.
+  #
+  # Set the name to your email alias when doing developer runs.
+  #
   - name: name
     value: "WALinuxAgent"
+    is_case_visible: true
+
+  #
+  # Test suites to execute
+  #
+  - name: test_suites
+    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_update, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline"
+
   #
-  # These variables define parameters handled by LISA.
+  # Parameters used to create test VMs
   #
   - name: subscription_id
     value: ""
-  - name: user
-    value: "waagent"
-  - name: identity_file
+    is_case_visible: true
+  - name: cloud
+    value: "AzureCloud"
+    is_case_visible: true
+  - name: location
     value: ""
-    is_secret: true
-  - name: admin_password
+  - name: image
     value: ""
-    is_secret: true
+  - name: vm_size
+    value: ""
+
+  #
+  # Whether to skip deletion of the test VMs after the test run completes.
+  #
+  # Possible values: always, no, failed
+  #
   - name: keep_environment
     value: "no"
+    is_case_visible: true
+
+  #
+  # Username and SSH public key for the admin user on the test VMs
+  #
+  - name: user
+    value: "waagent"
+    is_case_visible: true
+  - name: identity_file
+    value: ""
+    is_case_visible: true
+
   #
-  # These variables define parameters for the AgentTestSuite; see the test wiki for details.
+  # Set the resource group and vm, or the group and the vmss, to execute the test run on an existing VM or VMSS.
   #
-  # NOTE: c_test_suites, generated by the AgentTestSuitesCombinator, is also a parameter
-  #       for the AgentTestSuite
+  - name: resource_group_name
+    value: ""
+    is_case_visible: true
+  - name: vm_name
+    value: ""
+    is_case_visible: true
+  - name: vmss_name
+    value: ""
+    is_case_visible: true
+
+  #
+  # Directory for test logs
   #
-  # Root directory for log files (optional)
   - name: log_path
     value: ""
     is_case_visible: true
 
+  #
   # Whether to collect logs from the test VM
+  #
+  # Possible values: always, no, failed
+  #
   - name: collect_logs
     value: "failed"
     is_case_visible: true
 
-  # Whether to skip setup of the test VM
+  #
+  # Whether to skip setup of the test VMs. This is useful in developer runs when using existing VMs to save initialization time.
+  #
   - name: skip_setup
     value: false
     is_case_visible: true
 
   #
-  # These variables are parameters for the AgentTestSuitesCombinator
+  # These variables are handled by LISA to use an SSH proxy when executing the runbook
   #
-  # The test suites to execute
-  - name: test_suites
-    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_update, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline"
-  - name: cloud
-    value: "AzureCloud"
-    is_case_visible: true
-  - name: image
-    value: ""
-  - name: location
+  - name: proxy
+    value: False
+  - name: proxy_host
     value: ""
-  - name: vm_size
+  - name: proxy_user
+    value: "foo"
+  - name: proxy_identity_file
     value: ""
+    is_secret: true
 
   #
-  # The values for these variables are generated by the AgentTestSuitesCombinator combinator. They are
+  # The variables below are generated by the AgentTestSuitesCombinator combinator. They are
   # prefixed with "c_" to distinguish them from the rest of the variables, whose value can be set from
   # the command line.
   #
-  # Most of these variables are handled by LISA and are used to define the set of test VMs that need to be
-  # created. The variables marked with 'is_case_visible' are also referenced by the AgentTestSuite.
+
+  #
+  # The combinator generates the test environments using these two variables, which are passed to LISA
+  #
+  - name: c_environment
+    value: {}
+  - name: c_platform
+    value: []
+
   #
-  # 'c_vm_tags' is a special case: it is used by the azure_update_arm_template hook. This hook does not
-  # have access to the runbook variables, so instead we use a dummy VM tag named "template" to pass the
-  # name of the custom ARM template that the hook needs to use (see wiki for more details).
+  # Name of the test environment, used for mainly for logging purposes
   #
   - name: c_env_name
     value: ""
     is_case_visible: true
-  - name: c_marketplace_image
-    value: ""
-  - name: c_marketplace_image_information_location
-    value: ""
-  - name: c_shared_resource_group_location
+
+  #
+  # Test suites assigned for execution in the current test environment.
+  #
+  # The combinator splits the test suites specified in the 'test_suites' variable in subsets and assigns each subset
+  # to a test environment. The AgentTestSuite uses 'c_test_suites' to execute the suites assigned to the current environment.
+  #
+  - name: c_test_suites
+    value: []
+    is_case_visible: true
+
+  #
+  # These parameters are used by the AgentTestSuite to create the test scale sets.
+  #
+  # Note that there are other 3 variables named 'image', 'vm_size' and 'location', which can be passed
+  # from the command line. The combinator generates the values for these parameters using test metadata,
+  # but they can be overriden with these command line variables. The final values are passed to the
+  # AgentTestSuite in the corresponding 'c_*' variables.
+  #
+  - name: c_image
     value: ""
+    is_case_visible: true
   - name: c_vm_size
     value: ""
-  - name: c_location
-    value: ""
     is_case_visible: true
-  - name: c_vhd
+  - name: c_location
     value: ""
     is_case_visible: true
-  - name: c_test_suites
-    value: []
-    is_case_visible: true
-  - name: c_vm_tags
-    value: {}
 
   #
-  # Set these variables to use an SSH proxy when executing the runbook
+  # True if the image is a VHD (instead of a URN)
   #
-  - name: proxy
-    value: False
-  - name: proxy_host
-    value: ""
-  - name: proxy_user
-    value: "foo"
-  - name: proxy_identity_file
-    value: ""
-    is_secret: true
+  - name: c_is_vhd
+    value: false
+    is_case_visible: true
 
-platform:
-  - type: azure
-    admin_username: $(user)
-    admin_private_key_file: $(identity_file)
-    admin_password: $(admin_password)
-    keep_environment: $(keep_environment)
-    azure:
-      deploy: True
-      cloud: $(cloud)
-      marketplace_image_information_location: $(c_marketplace_image_information_location)
-      shared_resource_group_location: $(c_shared_resource_group_location)
-      subscription_id: $(subscription_id)
-      wait_delete: false
-      vm_tags: $(c_vm_tags)
-    requirement:
-      core_count:
-        min: 2
-      azure:
-        marketplace: $(c_marketplace_image)
-        vhd: $(c_vhd)
-        location: $(c_location)
-        vm_size: $(c_vm_size)
+environment: $(c_environment)
+
+platform: $(c_platform)
 
 combinator:
   type: agent_test_suites
-  test_suites: $(test_suites)
   cloud: $(cloud)
+  identity_file: $(identity_file)
   image: $(image)
+  keep_environment: $(keep_environment)
   location: $(location)
+  resource_group_name: $(resource_group_name)
+  subscription_id: $(subscription_id)
+  test_suites: $(test_suites)
+  user: $(user)
+  vm_name: $(vm_name)
   vm_size: $(vm_size)
+  vmss_name: $(vmss_name)
 
 concurrency: 32