Merge branch 'develop' into skip-cloud-location

.github/workflows/ci_pr.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  test-legacy-python-versions:
+  test-python-2_6-and-3_4-versions:
 
     strategy:
       fail-fast: false
@@ -50,16 +50,42 @@ jobs:
         ./ci/nosetests.sh
         exit $?
 
+  test-python-2_7:
+
+    strategy:
+      fail-fast: false
+
+    name: "Python 2.7 Unit Tests"
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      NOSEOPTS: "--verbose"
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Install Python 2.7
+      run: |
+        apt-get update
+        apt-get install -y curl bzip2 sudo
+        curl https://dcrdata.blob.core.windows.net/python/python-2.7.tar.bz2 -o python-2.7.tar.bz2
+        sudo tar xjvf python-2.7.tar.bz2 --directory /
+
+    - name: Test with nosetests
+      run: |
+        source /home/waagent/virtualenv/python2.7.16/bin/activate
+        ./ci/nosetests.sh
+        exit $?
+
   test-current-python-versions:
 
     strategy:
       fail-fast: false
       matrix:
         include:
-
-          - python-version: 2.7
-            PYLINTOPTS: "--rcfile=ci/2.7.pylintrc --ignore=tests_e2e,makepkg.py"
-
           - python-version: 3.5
             PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e,makepkg.py"
 
@@ -125,4 +151,4 @@ jobs:
       if: matrix.python-version == 3.9
       uses: codecov/codecov-action@v3
       with:
-        file: ./coverage.xml
+        file: ./coverage.xml

azurelinuxagent/common/exception.py

@@ -75,6 +75,15 @@ def __init__(self, msg=None, inner=None):
         super(AgentNetworkError, self).__init__(msg, inner)
 
 
+class AgentUpdateError(AgentError):
+    """
+    When agent failed to update.
+    """
+
+    def __init__(self, msg=None, inner=None):
+        super(AgentUpdateError, self).__init__(msg, inner)
+
+
 class CGroupsException(AgentError):
     """
     Exception to classify any cgroups related issue.

azurelinuxagent/ga/agent_update_handler.py

@@ -5,14 +5,14 @@
 
 from azurelinuxagent.common import conf, logger
 from azurelinuxagent.common.event import add_event, WALAEventOperation
-from azurelinuxagent.common.exception import AgentUpgradeExitException
+from azurelinuxagent.common.exception import AgentUpgradeExitException, AgentUpdateError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.logger import LogLevel
 from azurelinuxagent.common.protocol.extensions_goal_state import GoalStateSource
-from azurelinuxagent.common.protocol.restapi import VMAgentUpdateStatuses, VMAgentUpdateStatus
+from azurelinuxagent.common.protocol.restapi import VERSION_0, VMAgentUpdateStatuses, VMAgentUpdateStatus
 from azurelinuxagent.common.utils import fileutil, textutil
 from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
-from azurelinuxagent.common.version import CURRENT_VERSION, AGENT_NAME, AGENT_DIR_PATTERN
+from azurelinuxagent.common.version import get_daemon_version, CURRENT_VERSION, AGENT_NAME, AGENT_DIR_PATTERN
 from azurelinuxagent.ga.guestagent import GuestAgent, GAUpdateReportState
 
 
@@ -37,8 +37,6 @@ def __init__(self):
         self.last_attempted_requested_version_update_time = datetime.datetime.min
         self.last_attempted_hotfix_update_time = datetime.datetime.min
         self.last_attempted_normal_update_time = datetime.datetime.min
-        self.last_warning = ""
-        self.last_warning_time = datetime.datetime.min
 
 
 class AgentUpdateHandler(object):
@@ -130,10 +128,10 @@ def __get_agent_family_manifests(self, goal_state):
                     agent_family_manifests.append(m)
 
         if not family_found:
-            raise Exception(u"Agent family: {0} not found in the goal state, skipping agent update".format(family))
+            raise AgentUpdateError(u"Agent family: {0} not found in the goal state, skipping agent update".format(family))
 
         if len(agent_family_manifests) == 0:
-            raise Exception(
+            raise AgentUpdateError(
                 u"No manifest links found for agent family: {0} for incarnation: {1}, skipping agent update".format(
                     self._ga_family, self._gs_id))
         return agent_family_manifests[0]
@@ -179,7 +177,7 @@ def __get_agent_package_to_download(self, agent_manifest, version):
                 # Found a matching package, only download that one
                 return pkg
 
-        raise Exception("No matching package found in the agent manifest for requested version: {0} in goal state incarnation: {1}, "
+        raise AgentUpdateError("No matching package found in the agent manifest for requested version: {0} in goal state incarnation: {1}, "
                         "skipping agent update".format(str(version), self._gs_id))
 
     @staticmethod
@@ -245,6 +243,15 @@ def __get_all_agents_on_disk():
         path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME))
         return [GuestAgent.from_installed_agent(path=agent_dir) for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)]
 
+    @staticmethod
+    def __get_daemon_version_for_update():
+        daemon_version = get_daemon_version()
+        if daemon_version != FlexibleVersion(VERSION_0):
+            return daemon_version
+        # We return 0.0.0.0 if daemon version is not specified. In that case,
+        # use the min version as 2.2.53 as we started setting the daemon version starting 2.2.53.
+        return FlexibleVersion("2.2.53")
+
     @staticmethod
     def __log_event(level, msg, success=True):
         if level == LogLevel.INFO:
@@ -291,11 +298,20 @@ def run(self, goal_state):
             if warn_msg != "":
                 self.__log_event(LogLevel.WARNING, warn_msg)
 
-            msg = "Goal state {0} is requesting a new agent version {1}, will update the agent before processing the goal state.".format(
-                self._gs_id, str(requested_version))
-            self.__log_event(LogLevel.INFO, msg)
-
             try:
+                daemon_version = self.__get_daemon_version_for_update()
+                if requested_version < daemon_version:
+                    # Don't process the update if the requested version is less than daemon version,
+                    # as historically we don't support downgrades below daemon versions. So daemon will not pickup that requested version rather start with
+                    # installed latest version again. When that happens agent go into loop of downloading the requested version, exiting and start again with same version.
+                    #
+                    raise AgentUpdateError("The Agent received a request to downgrade to version {0}, but downgrading to a version less than "
+                                           "the Agent installed on the image ({1}) is not supported. Skipping downgrade.".format(requested_version, daemon_version))
+
+                msg = "Goal state {0} is requesting a new agent version {1}, will update the agent before processing the goal state.".format(
+                    self._gs_id, str(requested_version))
+                self.__log_event(LogLevel.INFO, msg)
+
                 agent = self.__download_and_get_agent(goal_state, agent_family, agent_manifest, requested_version)
 
                 if agent.is_blacklisted or not agent.is_downloaded:
@@ -314,9 +330,13 @@ def run(self, goal_state):
         except Exception as err:
             if isinstance(err, AgentUpgradeExitException):
                 raise err
+            elif isinstance(err, AgentUpdateError):
+                error_msg = ustr(err)
+            else:
+                error_msg = "Unable to update Agent: {0}".format(textutil.format_exception(err))
+            self.__log_event(LogLevel.WARNING, error_msg, success=False)
             if "Missing requested version" not in GAUpdateReportState.report_error_msg:
-                GAUpdateReportState.report_error_msg = "Unable to update Agent: {0}".format(textutil.format_exception(err))
-            self.__log_event(LogLevel.WARNING, GAUpdateReportState.report_error_msg, success=False)
+                GAUpdateReportState.report_error_msg = error_msg
 
     def get_vmagent_update_status(self):
         """

tests/data/wire/ga_manifest.xml

@@ -25,6 +25,9 @@
         <Plugin>
             <Version>2.1.0</Version><Uris><Uri>http://mock-goal-state/ga-manifests/OSTCExtensions.WALinuxAgent__2.1.0</Uri></Uris>
         </Plugin>
+        <Plugin>
+            <Version>2.5.0</Version><Uris><Uri>http://mock-goal-state/ga-manifests/OSTCExtensions.WALinuxAgent__2.5.0</Uri></Uris>
+        </Plugin>
         <Plugin>
             <Version>9.9.9.10</Version>
             <Uris>

tests/ga/test_agent_update_handler.py

@@ -80,7 +80,7 @@ def __assert_agent_requested_version_in_goal_state(self, mock_telemetry, inc=1,
 
     def __assert_no_agent_package_telemetry_emitted(self, mock_telemetry, version="9.9.9.10"):
         upgrade_event_msgs = [kwarg['message'] for _, kwarg in mock_telemetry.call_args_list if
-                              'Unable to update Agent: No matching package found in the agent manifest for requested version: {0}'.format(version) in kwarg['message'] and kwarg[
+                              'No matching package found in the agent manifest for requested version: {0}'.format(version) in kwarg['message'] and kwarg[
                                   'op'] == WALAEventOperation.AgentUpgrade]
         self.assertEqual(1, len(upgrade_event_msgs),
                          "Did not find the event indicating that the agent package not found. Got: {0}".format(
@@ -217,7 +217,7 @@ def test_it_should_downgrade_agent_if_requested_version_is_available_less_than_c
         self.prepare_agents()
         self.assertEqual(20, self.agent_count(), "Agent directories not set properly")
 
-        downgraded_version = "1.2.0"
+        downgraded_version = "2.5.0"
 
         with self.__get_agent_update_handler(test_data=data_file) as (agent_update_handler, mock_telemetry):
             agent_update_handler._protocol.mock_wire_data.set_extension_config_requested_version(downgraded_version)
@@ -230,6 +230,28 @@ def test_it_should_downgrade_agent_if_requested_version_is_available_less_than_c
                 versions=[downgraded_version, str(CURRENT_VERSION)])
             self.assertIn("Agent update found, Exiting current process", ustr(context.exception.reason))
 
+    def test_it_should_not_downgrade_below_daemon_version(self):
+        data_file = DATA_FILE.copy()
+        data_file["ext_conf"] = "wire/ext_conf_requested_version.xml"
+
+        # Set the test environment by adding 20 random agents to the agent directory
+        self.prepare_agents()
+        self.assertEqual(20, self.agent_count(), "Agent directories not set properly")
+
+        downgraded_version = "1.2.0"
+
+        with self.__get_agent_update_handler(test_data=data_file) as (agent_update_handler, mock_telemetry):
+            agent_update_handler._protocol.mock_wire_data.set_extension_config_requested_version(downgraded_version)
+            agent_update_handler._protocol.mock_wire_data.set_incarnation(2)
+            agent_update_handler._protocol.client.update_goal_state()
+            agent_update_handler.run(agent_update_handler._protocol.get_goal_state())
+            self.assertFalse(os.path.exists(self.agent_dir(downgraded_version)),
+                             "New agent directory should not be found")
+            self.assertEqual(1, len([kwarg['message'] for _, kwarg in mock_telemetry.call_args_list if
+                                     "The Agent received a request to downgrade to version" in kwarg[
+                                         'message'] and kwarg[
+                                         'op'] == WALAEventOperation.AgentUpgrade]), "We should allow downgrade above daemon version")
+
     def test_handles_if_requested_version_not_found_in_pkgs_to_download(self):
         data_file = DATA_FILE.copy()
         data_file["ext_conf"] = "wire/ext_conf_requested_version.xml"

tests/ga/test_update.py

@@ -1818,7 +1818,7 @@ def test_it_should_mark_current_agent_as_bad_version_on_downgrade(self):
         self.assertTrue(os.path.exists(self.agent_dir(CURRENT_VERSION)))
         self.assertFalse(next(agent for agent in self.agents() if agent.version == CURRENT_VERSION).is_blacklisted,
                          "The current agent should not be blacklisted")
-        downgraded_version = "1.2.0"
+        downgraded_version = "2.5.0"
 
         data_file = mockwiredata.DATA_FILE.copy()
         data_file["ext_conf"] = "wire/ext_conf_requested_version.xml"

tests_e2e/orchestrator/runbook.yml

@@ -51,7 +51,7 @@ variable:
   #
   # The test suites to execute
   - name: test_suites
-    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, agent_update, fips"
+    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips"
   - name: cloud
     value: "AzureCloud"
     is_case_visible: true

tests_e2e/pipeline/pipeline.yml

@@ -43,6 +43,11 @@ parameters:
     - failed
     - no
 
+  - name: collect_lisa_logs
+    displayName: Collect LISA logs
+    type: boolean
+    default: true
+
   - name: keep_environment
     displayName: Keep the test VMs (do not delete them)
     type: string
@@ -117,6 +122,11 @@ jobs:
           TEST_SUITES: ${{ parameters.test_suites }}
           VM_SIZE: ${{ parameters.vm_size }}
 
+      - bash: $(Build.SourcesDirectory)/tests_e2e/pipeline/scripts/collect_artifacts.sh
+        displayName: "Collect test artifacts"
+        env:
+          COLLECT_LISA_LOGS: ${{ parameters.collect_lisa_logs }}
+
       - publish: $(Build.ArtifactStagingDirectory)
         artifact: 'artifacts'
         displayName: 'Publish test artifacts'

tests_e2e/pipeline/scripts/collect_artifacts.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Moves the relevant logs to the staging directory
+#
+set -euxo pipefail
+
+#
+# The execute_test.sh script gives ownership of the log directory to the 'waagent' user in
+# the Docker container; re-take ownership
+#
+sudo find "$LOGS_DIRECTORY" -exec chown "$USER" {} \;
+
+#
+# Move the logs for failed tests to a temporary location
+#
+mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp
+for log in $(grep -l MARKER-LOG-WITH-ERRORS "$LOGS_DIRECTORY"/*.log); do
+  mv "$log" "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp
+done
+
+#
+# Move the environment logs to "environment_logs"
+#
+if ls "$LOGS_DIRECTORY"/env-*.log > /dev/null 2>&1; then
+  mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/environment_logs
+  mv "$LOGS_DIRECTORY"/env-*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"/environment_logs
+fi
+
+#
+# Move the rest of the logs to "test_logs"
+#
+if ls "$LOGS_DIRECTORY"/*.log > /dev/null 2>&1; then
+  mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/test_logs
+  mv "$LOGS_DIRECTORY"/*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"/test_logs
+fi
+
+#
+# Move the logs for failed tests to the main directory
+#
+if ls "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp/*.log > /dev/null 2>&1; then
+  mv "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp/*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"
+fi
+rmdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp
+
+#
+# Move the logs collected from the test VMs to vm_logs
+#
+if ls "$LOGS_DIRECTORY"/*.tgz > /dev/null 2>&1; then
+  mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/vm_logs
+  mv "$LOGS_DIRECTORY"/*.tgz "$BUILD_ARTIFACTSTAGINGDIRECTORY"/vm_logs
+fi
+
+#
+# Move the main LISA log and the JUnit report to "runbook_logs"
+#
+# Note that files created by LISA are under .../lisa/<date>/<unique_name>"
+#
+mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/runbook_logs
+mv "$LOGS_DIRECTORY"/lisa/*/*/lisa-*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"/runbook_logs
+mv "$LOGS_DIRECTORY"/lisa/*/*/agent.junit.xml "$BUILD_ARTIFACTSTAGINGDIRECTORY"/runbook_logs
+
+#
+# Move the rest of the LISA logs to "lisa_logs"
+#
+if [[ ${COLLECT_LISA_LOGS,,} == 'true' ]]; then  # case-insensitive comparison
+  mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/lisa_logs
+  mv "$LOGS_DIRECTORY"/lisa/*/*/* "$BUILD_ARTIFACTSTAGINGDIRECTORY"/lisa_logs
+fi
+