Merge branch 'main' into bring-up

cloudinit/cmd/main.py

@@ -210,6 +210,35 @@ def attempt_cmdline_url(path, network=True, cmdline=None):
             (cmdline_name, url, path))
 
 
+def purge_cache_on_python_version_change(init):
+    """Purge the cache if python version changed on us.
+
+    There could be changes not represented in our cache (obj.pkl) after we
+    upgrade to a new version of python, so at that point clear the cache
+    """
+    current_python_version = '%d.%d' % (
+        sys.version_info.major, sys.version_info.minor
+    )
+    python_version_path = os.path.join(
+        init.paths.get_cpath('data'), 'python-version'
+    )
+    if os.path.exists(python_version_path):
+        cached_python_version = open(python_version_path).read()
+        # The Python version has changed out from under us, anything that was
+        # pickled previously is likely useless due to API changes.
+        if cached_python_version != current_python_version:
+            LOG.debug('Python version change detected. Purging cache')
+            init.purge_cache(True)
+            util.write_file(python_version_path, current_python_version)
+    else:
+        if os.path.exists(init.paths.get_ipath_cur('obj_pkl')):
+            LOG.info(
+                'Writing python-version file. '
+                'Cache compatibility status is currently unknown.'
+            )
+        util.write_file(python_version_path, current_python_version)
+
+
 def main_init(name, args):
     deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]
     if args.local:
@@ -276,6 +305,7 @@ def main_init(name, args):
         util.logexc(LOG, "Failed to initialize, likely bad things to come!")
     # Stage 4
     path_helper = init.paths
+    purge_cache_on_python_version_change(init)
     mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK
 
     if mode == sources.DSMODE_NETWORK:

cloudinit/cmd/tests/test_main.py

@@ -17,6 +17,8 @@
 
 
 class TestMain(FilesystemMockingTestCase):
+    with_logs = True
+    allowed_subp = False
 
     def setUp(self):
         super(TestMain, self).setUp()

tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py

@@ -18,7 +18,7 @@
 no_ssh_fingerprints: true
 """
 
-USER_DATA_SSH_AUTHKEY_ENABLE="""\
+USER_DATA_SSH_AUTHKEY_ENABLE = """\
 #cloud-config
 ssh_genkeytypes:
   - ecdsa

tests/integration_tests/modules/test_version_change.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+
+from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import ASSETS_DIR
+
+
+PICKLE_PATH = Path('/var/lib/cloud/instance/obj.pkl')
+TEST_PICKLE = ASSETS_DIR / 'test_version_change.pkl'
+
+
+def _assert_no_pickle_problems(log):
+    assert 'Failed loading pickled blob' not in log
+    assert 'Traceback' not in log
+    assert 'WARN' not in log
+
+
+def test_reboot_without_version_change(client: IntegrationInstance):
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Python version change detected' not in log
+    assert 'Cache compatibility status is currently unknown.' not in log
+    _assert_no_pickle_problems(log)
+
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Python version change detected' not in log
+    assert 'Could not determine Python version used to write cache' not in log
+    _assert_no_pickle_problems(log)
+
+    # Now ensure that loading a bad pickle gives us problems
+    client.push_file(TEST_PICKLE, PICKLE_PATH)
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Failed loading pickled blob from {}'.format(PICKLE_PATH) in log
+
+
+def test_cache_purged_on_version_change(client: IntegrationInstance):
+    # Start by pushing the invalid pickle so we'll hit an error if the
+    # cache didn't actually get purged
+    client.push_file(TEST_PICKLE, PICKLE_PATH)
+    client.execute("echo '1.0' > /var/lib/cloud/data/python-version")
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Python version change detected. Purging cache' in log
+    _assert_no_pickle_problems(log)
+
+
+def test_log_message_on_missing_version_file(client: IntegrationInstance):
+    # Start by pushing a pickle so we can see the log message
+    client.push_file(TEST_PICKLE, PICKLE_PATH)
+    client.execute("rm /var/lib/cloud/data/python-version")
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert (
+        'Writing python-version file. '
+        'Cache compatibility status is currently unknown.'
+    ) in log

tests/integration_tests/util.py

@@ -3,10 +3,14 @@
 import os
 import time
 from contextlib import contextmanager
+from pathlib import Path
 
 log = logging.getLogger('integration_testing')
 
 
+ASSETS_DIR = Path('tests/integration_tests/assets')
+
+
 def verify_ordered_items_in_text(to_verify: list, text: str):
     """Assert all items in list appear in order in text.