Merge branch 'develop' into log-setup

Azure · Nov 8, 2023 · de43b56 · de43b56
2 parents 45a36c5 + 5542f84
commit de43b56
Show file tree

Hide file tree

Showing 12 changed files with 808 additions and 112 deletions.
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
@@ -53,8 +53,7 @@
 from azurelinuxagent.common.utils import textutil
 from azurelinuxagent.common.utils.archive import ARCHIVE_DIRECTORY_NAME
 from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
-from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION, \
-    PY_VERSION_MAJOR, PY_VERSION_MICRO, PY_VERSION_MINOR
+from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION
 
 _HANDLER_NAME_PATTERN = r'^([^-]+)'
 _HANDLER_VERSION_PATTERN = r'(\d+(?:\.\d+)*)'
@@ -964,33 +963,6 @@ def report_ext_handlers_status(self, goal_state_changed=False, vm_agent_update_s
                       message=msg)
             return None
 
-    def get_ext_handlers_status_debug_info(self, vm_status):
-        status_blob_text = self.protocol.get_status_blob_data()
-        if status_blob_text is None:
-            status_blob_text = ""
-
-        support_multi_config = {}
-        vm_status_data = get_properties(vm_status)
-        vm_handler_statuses = vm_status_data.get('vmAgent', {}).get('extensionHandlers')
-        for handler_status in vm_handler_statuses:
-            if handler_status.get('name') is not None:
-                support_multi_config[handler_status.get('name')] = handler_status.get('supports_multi_config')
-
-        debug_text = json.dumps({
-            "agentName": AGENT_NAME,
-            "daemonVersion": str(version.get_daemon_version()),
-            "pythonVersion": "Python: {0}.{1}.{2}".format(PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO),
-            "extensionSupportedFeatures": [name for name, _ in get_agent_supported_features_list_for_extensions().items()],
-            "supportsMultiConfig": support_multi_config
-        })
-
-        return '''{{
-    "__comment__": "The __status__ property is the actual status reported to CRP",
-    "__status__": {0},
-    "__debug__": {1}
-}}
-'''.format(status_blob_text, debug_text)
-
     def report_ext_handler_status(self, vm_status, ext_handler, goal_state_changed):
         ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
 

diff --git a/azurelinuxagent/ga/logcollector_manifests.py b/azurelinuxagent/ga/logcollector_manifests.py
@@ -119,4 +119,8 @@
 
 echo,### Gathering Disk Info ###
 diskinfo,
+
+echo,### Gathering Guest ProxyAgent Log Files ###
+copy,/var/log/proxyagent/*
+echo,
 """
diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py
@@ -617,8 +617,10 @@ def _report_status(self, exthandlers_handler, agent_update_handler):
         if vm_status is not None:
             self._report_extensions_summary(vm_status)
             if self._goal_state is not None:
-                agent_status = exthandlers_handler.get_ext_handlers_status_debug_info(vm_status)
-                self._goal_state.save_to_history(agent_status, AGENT_STATUS_FILE)
+                status_blob_text = exthandlers_handler.protocol.get_status_blob_data()
+                if status_blob_text is None:
+                    status_blob_text = "{}"
+                self._goal_state.save_to_history(status_blob_text, AGENT_STATUS_FILE)
                 if self._goal_state.extensions_goal_state.is_outdated:
                     exthandlers_handler.protocol.client.get_host_plugin().clear_fast_track_state()
 

diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py
@@ -28,16 +28,14 @@
 import unittest
 
 from azurelinuxagent.common import conf
-from azurelinuxagent.common.agent_supported_feature import get_agent_supported_features_list_for_extensions, \
-    get_agent_supported_features_list_for_crp
+from azurelinuxagent.common.agent_supported_feature import get_agent_supported_features_list_for_crp
 from azurelinuxagent.ga.cgroupconfigurator import CGroupConfigurator
 from azurelinuxagent.common.datacontract import get_properties
 from azurelinuxagent.common.event import WALAEventOperation
 from azurelinuxagent.common.utils import fileutil
 from azurelinuxagent.common.utils.fileutil import read_file
 from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
-from azurelinuxagent.common.version import PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO, AGENT_NAME, \
-    AGENT_VERSION
+from azurelinuxagent.common.version import AGENT_VERSION
 from azurelinuxagent.common.exception import ResourceGoneError, ExtensionDownloadError, ProtocolError, \
     ExtensionErrorCodes, ExtensionError, GoalStateAggregateStatusCodes
 from azurelinuxagent.common.protocol.restapi import ExtensionSettings, Extension, ExtHandlerStatus, \
@@ -3247,91 +3245,75 @@ def mock_http_put(url, *args, **_):
                 )
 
             expected_status = {
-                "__comment__": "The __status__ property is the actual status reported to CRP",
-                "__status__": {
-                    "version": "1.1",
-                    "timestampUTC": "1970-01-01T00:00:00Z",
-                    "aggregateStatus": {
-                        "guestAgentStatus": {
-                            "version": AGENT_VERSION,
+                "version": "1.1",
+                "timestampUTC": "1970-01-01T00:00:00Z",
+                "aggregateStatus": {
+                    "guestAgentStatus": {
+                        "version": AGENT_VERSION,
+                        "status": "Ready",
+                        "formattedMessage": {
+                            "lang": "en-US",
+                            "message": "Guest Agent is running"
+                        }
+                    },
+                    "handlerAggregateStatus": [
+                        {
+                            "handlerVersion": "1.0.0",
+                            "handlerName": "OSTCExtensions.ExampleHandlerLinux",
                             "status": "Ready",
+                            "code": 0,
+                            "useExactVersion": True,
                             "formattedMessage": {
                                 "lang": "en-US",
-                                "message": "Guest Agent is running"
-                            }
-                        },
-                        "handlerAggregateStatus": [
-                            {
-                                "handlerVersion": "1.0.0",
-                                "handlerName": "OSTCExtensions.ExampleHandlerLinux",
-                                "status": "Ready",
-                                "code": 0,
-                                "useExactVersion": True,
-                                "formattedMessage": {
-                                    "lang": "en-US",
-                                    "message": "Plugin enabled"
-                                },
-                                "runtimeSettingsStatus": {
-                                    "settingsStatus": {
-                                        "status": {
-                                            "name": "OSTCExtensions.ExampleHandlerLinux",
-                                            "configurationAppliedTime": None,
-                                            "operation": None,
-                                            "status": "success",
-                                            "code": 0,
-                                            "formattedMessage": {
-                                                "lang": "en-US",
-                                                "message": None
-                                            }
-                                        },
-                                        "version": 1.0,
-                                        "timestampUTC": "1970-01-01T00:00:00Z"
+                                "message": "Plugin enabled"
+                            },
+                            "runtimeSettingsStatus": {
+                                "settingsStatus": {
+                                    "status": {
+                                        "name": "OSTCExtensions.ExampleHandlerLinux",
+                                        "configurationAppliedTime": None,
+                                        "operation": None,
+                                        "status": "success",
+                                        "code": 0,
+                                        "formattedMessage": {
+                                            "lang": "en-US",
+                                            "message": None
+                                        }
                                     },
-                                    "sequenceNumber": 0
-                                }
-                            }
-                        ],
-                        "vmArtifactsAggregateStatus": {
-                            "goalStateAggregateStatus": {
-                                "formattedMessage": {
-                                    "lang": "en-US",
-                                    "message": "GoalState executed successfully"
+                                    "version": 1.0,
+                                    "timestampUTC": "1970-01-01T00:00:00Z"
                                 },
-                                "timestampUTC": "1970-01-01T00:00:00Z",
-                                "inSvdSeqNo": "1",
-                                "status": "Success",
-                                "code": 0
+                                "sequenceNumber": 0
                             }
                         }
-                    },
-                    "guestOSInfo": None,
-                    "supportedFeatures": supported_features
-                },
-                "__debug__": {
-                    "agentName": AGENT_NAME,
-                    "daemonVersion": "0.0.0.0",
-                    "pythonVersion": "Python: {0}.{1}.{2}".format(PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO),
-                    "extensionSupportedFeatures": [name for name, _ in get_agent_supported_features_list_for_extensions().items()],
-                    "supportsMultiConfig": {
-                        "OSTCExtensions.ExampleHandlerLinux": False
+                    ],
+                    "vmArtifactsAggregateStatus": {
+                        "goalStateAggregateStatus": {
+                            "formattedMessage": {
+                                "lang": "en-US",
+                                "message": "GoalState executed successfully"
+                            },
+                            "timestampUTC": "1970-01-01T00:00:00Z",
+                            "inSvdSeqNo": "1",
+                            "status": "Success",
+                            "code": 0
+                        }
                     }
-                }
+                },
+                "guestOSInfo": None,
+                "supportedFeatures": supported_features
             }
 
-
             exthandlers_handler.run()
-            vm_status = exthandlers_handler.report_ext_handlers_status()
-            actual_status_json = json.loads(exthandlers_handler.get_ext_handlers_status_debug_info(vm_status))
+            exthandlers_handler.report_ext_handlers_status()
 
-            # Don't compare the guestOSInfo
-            status_property = actual_status_json.get("__status__")
-            self.assertIsNotNone(status_property, "The status file is missing the __status__ property")
-            self.assertIsNotNone(status_property.get("guestOSInfo"), "The status file is missing the guestOSInfo property")
-            status_property["guestOSInfo"] = None
+            actual_status = json.loads(protocol.get_status_blob_data())
 
-            actual_status_json.pop('guestOSInfo', None)
+            # Don't compare the guestOSInfo
+            self.assertIsNotNone(actual_status.get("guestOSInfo"), "The status file is missing the guestOSInfo property")
+            actual_status["guestOSInfo"] = None
 
-            self.assertEqual(expected_status, actual_status_json)
+            self.assertEqual(expected_status, actual_status)
 
     def test_it_should_process_extensions_only_if_allowed(self):
         def assert_extensions_called(exthandlers_handler, expected_call_count=0):

diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py
@@ -2198,7 +2198,6 @@ def create_vm_status(extension_status):
             exthandlers_handler.report_ext_handlers_status = Mock(return_value=create_vm_status(ExtensionStatusValue.success))
         else:
             exthandlers_handler.report_ext_handlers_status = Mock(side_effect=[create_vm_status(s) for s in extension_statuses])
-        exthandlers_handler.get_ext_handlers_status_debug_info = Mock(return_value='')
         yield exthandlers_handler
 
 

diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py
@@ -169,6 +169,14 @@ def __init__(self, metadata: TestSuiteMetadata) -> None:
         self._create_scale_set: bool
         self._delete_scale_set: bool
 
+    #
+    # Test suites within the same runbook may be executed concurrently, and we need to keep track of how many resource
+    # groups are being created. We use this lock and counter to allow only 1 thread to increment the resource group
+    # count.
+    #
+    _rg_count_lock = RLock()
+    _rg_count = 0
+
     def _initialize(self, environment: Environment, variables: Dict[str, Any], lisa_working_path: str, lisa_log_path: str, lisa_log: Logger):
         """
         Initializes the AgentTestSuite from the data passed as arguments by LISA.
@@ -231,9 +239,16 @@ def _initialize(self, environment: Environment, variables: Dict[str, Any], lisa_
 
         if isinstance(environment.nodes[0], LocalNode):
             # We need to create a new VMSS.
-            # Use the same naming convention as LISA for the scale set name: lisa-<runbook name>-<run id>-e0-n0. Note that we hardcode the resource group
-            # id to "e0" and the scale set name to "n0" since we are creating a single scale set.
-            self._resource_group_name = f"lisa-{self._runbook_name}-{RUN_ID}-e0"
+            # Use the same naming convention as LISA for the scale set name: lisa-<runbook name>-<run id>-e<rg count>-n0
+            # Note that we hardcode the scale set name to "n0" since we are creating a single scale set.
+            # Resource group name cannot have any uppercase characters, because the publicIP cannot have uppercase
+            # characters in its domain name label.
+            AgentTestSuite._rg_count_lock.acquire()
+            try:
+                self._resource_group_name = f"lisa-{self._runbook_name.lower()}-{RUN_ID}-e{AgentTestSuite._rg_count}"
+                AgentTestSuite._rg_count += 1
+            finally:
+                AgentTestSuite._rg_count_lock.release()
             self._vmss_name = f"{self._resource_group_name}-n0"
             self._test_nodes = []  # we'll fill this up when the scale set is created
             self._create_scale_set = True

diff --git a/tests_e2e/orchestrator/runbook.yml b/tests_e2e/orchestrator/runbook.yml
@@ -29,7 +29,7 @@ variable:
   # Test suites to execute
   #
   - name: test_suites
-    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline"
+    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing"
 
   #
   # Parameters used to create test VMs

diff --git a/tests_e2e/test_suites/ext_sequencing.yml b/tests_e2e/test_suites/ext_sequencing.yml
@@ -0,0 +1,10 @@
+#
+# Adds extensions with multiple dependencies to VMSS using 'provisionAfterExtensions' property and validates they are
+# enabled in order of dependencies.
+#
+name: "ExtSequencing"
+tests:
+  - "ext_sequencing/ext_sequencing.py"
+images: "endorsed"
+# This scenario is executed on instances of a scaleset created by the agent test suite.
+executes_on_scale_set: true