Fix both integration tests and make them handle incompatible upgrades

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

Author: marceloneppel

src/charm.py

@@ -332,6 +332,7 @@ def __init__(self, *args):
 
         if (
             self.refresh is not None
+            and self.refresh.workload_allowed_to_start
             and not self.refresh.next_unit_allowed_to_refresh
             and int(self.unit.name.split("/")[1]) >= self.refresh._get_partition()
         ):

src/refresh.py

@@ -38,7 +38,9 @@ def is_compatible(
             old_workload_version=old_workload_version,
             new_workload_version=new_workload_version,
         ):
-            logger.error("Charm version is not compatible 1")
+            logger.error(
+                f"Charm version is not compatible 1 - old_charm_version: {old_charm_version}, new_charm_version: {new_charm_version}, old_workload_version: {old_workload_version}, new_workload_version: {new_workload_version}"
+            )
             return False
 
         # Check workload version compatibility

tests/integration/ha_tests/test_upgrade.py

@@ -6,6 +6,7 @@
 import shutil
 import zipfile
 from pathlib import Path
+from time import sleep
 
 import pytest
 import tomli
@@ -128,10 +129,6 @@ async def test_upgrade_from_edge(ops_test: OpsTest, charm, continuous_writes) ->
     await application.refresh(path=charm, resources=resources)
 
     logger.info("Wait for upgrade to start")
-
-    # Blocked status is expected due to:
-    # (on PR) compatibility checks (on PR charm revision is '16/1.25.0+dirty...')
-    # (non-PR) the first unit upgraded and paused (pause-after-unit-refresh=first)
     await ops_test.model.block_until(lambda: application.status == "blocked", timeout=60 * 3)
 
     logger.info("Wait for refresh to block as paused or incompatible")
@@ -144,23 +141,34 @@ async def test_upgrade_from_edge(ops_test: OpsTest, charm, continuous_writes) ->
     refresh_order = sorted(
         application.units, key=lambda unit: int(unit.name.split("/")[1]), reverse=True
     )
-    if "Refresh incompatible" in application.status_message:
+
+    if "Refresh incompatible" in refresh_order[0].workload_status_message:
         logger.info("Application refresh is blocked due to incompatibility")
 
         action = await refresh_order[0].run_action(
             "force-refresh-start", **{"check-compatibility": False}
         )
         await action.wait()
 
-    logger.info("Wait for first incompatible unit to upgrade")
-    async with ops_test.fast_forward("60s"):
-        await ops_test.model.wait_for_idle(
-            apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
-        )
+        logger.info("Wait for first incompatible unit to upgrade")
+        async with ops_test.fast_forward("60s"):
+            await ops_test.model.wait_for_idle(
+                apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
+            )
+    else:
+        async with ops_test.fast_forward("60s"):
+            await ops_test.model.block_until(
+                lambda: all(unit.workload_status == "active" for unit in application.units),
+                timeout=60 * 3,
+            )
 
-    logger.info("Run resume-refresh action")
-    action = await refresh_order[1].run_action("resume-refresh")
+    sleep(60)
+
+    leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME)
+    logger.info(f"Run resume-refresh action on {leader_unit.name}")
+    action = await leader_unit.run_action("resume-refresh")
     await action.wait()
+    logger.info(f"Results from the action: {action.results}")
 
     logger.info("Wait for upgrade to complete")
     async with ops_test.fast_forward("60s"):
@@ -227,9 +235,14 @@ async def test_fail_and_rollback(ops_test, charm, continuous_writes) -> None:
 
     logger.info("Wait for upgrade to fail")
 
+    # Highest to lowest unit number
+    refresh_order = sorted(
+        application.units, key=lambda unit: int(unit.name.split("/")[1]), reverse=True
+    )
+
     await ops_test.model.block_until(
         lambda: application.status == "blocked"
-        and "incompatible" in application.status_message.lower(),
+        and "Refresh incompatible" in refresh_order[0].workload_status_message,
         timeout=TIMEOUT,
     )
 
@@ -244,9 +257,24 @@ async def test_fail_and_rollback(ops_test, charm, continuous_writes) -> None:
     await ops_test.model.block_until(lambda: application.status == "blocked", timeout=TIMEOUT)
 
     logger.info("Wait for application to recover")
+    async with ops_test.fast_forward("60s"):
+        await ops_test.model.block_until(
+            lambda: all(unit.workload_status == "active" for unit in application.units),
+            timeout=60 * 3,
+        )
+
+    sleep(60)
+
+    leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME)
+    logger.info(f"Run resume-refresh action on {leader_unit.name}")
+    action = await leader_unit.run_action("resume-refresh")
+    await action.wait()
+    logger.info(f"Results from the action: {action.results}")
+
+    logger.info("Wait for upgrade to complete")
     async with ops_test.fast_forward("60s"):
         await ops_test.model.wait_for_idle(
-            apps=[DATABASE_APP_NAME], status="active", timeout=TIMEOUT
+            apps=[DATABASE_APP_NAME], status="active", idle_period=30, timeout=TIMEOUT
         )
 
     logger.info("Ensure continuous_writes after rollback procedure")

tests/integration/ha_tests/test_upgrade_from_stable.py

@@ -123,17 +123,36 @@ async def test_upgrade_from_stable(ops_test: OpsTest, charm):
     logger.info("Wait for upgrade to start")
     await ops_test.model.block_until(lambda: application.status == "blocked", timeout=60 * 3)
 
-    logger.info("Wait for refresh to block as paused")
+    logger.info("Wait for refresh to block as paused or incompatible")
     async with ops_test.fast_forward("60s"):
         await ops_test.model.wait_for_idle(
             apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
         )
 
-    async with ops_test.fast_forward("60s"):
-        await ops_test.model.block_until(
-            lambda: all(unit.workload_status == "active" for unit in application.units),
-            timeout=60 * 3,
+    # Highest to lowest unit number
+    refresh_order = sorted(
+        application.units, key=lambda unit: int(unit.name.split("/")[1]), reverse=True
+    )
+
+    if "Refresh incompatible" in application.status_message:
+        logger.info("Application refresh is blocked due to incompatibility")
+
+        action = await refresh_order[0].run_action(
+            "force-refresh-start", **{"check-compatibility": False}
         )
+        await action.wait()
+
+        logger.info("Wait for first incompatible unit to upgrade")
+        async with ops_test.fast_forward("60s"):
+            await ops_test.model.wait_for_idle(
+                apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
+            )
+    else:
+        async with ops_test.fast_forward("60s"):
+            await ops_test.model.block_until(
+                lambda: all(unit.workload_status == "active" for unit in application.units),
+                timeout=60 * 3,
+            )
 
     sleep(60)
 

tests/unit/conftest.py

@@ -1,6 +1,6 @@
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
-from unittest.mock import PropertyMock
+from unittest.mock import Mock, PropertyMock, patch
 
 import pytest
 from charms.tempo_coordinator_k8s.v0.charm_tracing import charm_tracing_disabled
@@ -19,20 +19,43 @@ def disable_charm_tracing():
         yield
 
 
-class _MockRefresh:
-    in_progress = False
-    next_unit_allowed_to_refresh = True
-    workload_allowed_to_start = True
-    app_status_higher_priority = None
-    unit_status_higher_priority = None
+# class _MockRefresh:
+#     in_progress = False
+#     next_unit_allowed_to_refresh = True
+#     workload_allowed_to_start = True
+#     app_status_higher_priority = None
+#     unit_status_higher_priority = None
+#
+#     def __init__(self, _, /):
+#         pass
+#
+#     def unit_status_lower_priority(self, *, workload_is_running=True):
+#         return None
 
-    def __init__(self, _, /):
-        pass
 
-    def unit_status_lower_priority(self, *, workload_is_running=True):
-        return None
+# @pytest.fixture(autouse=True)
+# def patch(monkeypatch):
+#     monkeypatch.setattr("charm_refresh.Kubernetes", _MockRefresh)
 
 
 @pytest.fixture(autouse=True)
-def patch(monkeypatch):
-    monkeypatch.setattr("charm_refresh.Kubernetes", _MockRefresh)
+def mock_refresh():
+    """Fixture to shunt refresh logic and events."""
+    refresh_mock = Mock()
+    refresh_mock.in_progress = False
+    refresh_mock.unit_status_higher_priority = None
+    refresh_mock.unit_status_lower_priority.return_value = None
+    refresh_mock.next_unit_allowed_to_refresh = True
+    refresh_mock.workload_allowed_to_start = True
+
+    # Mock the _RefreshVersions class to avoid KeyError when charm key is missing
+    versions_mock = Mock()
+    versions_mock.charm = "v1/4.0.0"
+    versions_mock.workload = "4.0.0"
+
+    with (
+        patch("charm_refresh.Kubernetes", Mock(return_value=refresh_mock)),
+        patch("charm.PostgreSQLRefresh", Mock(return_value=None)),
+        patch("charm_refresh._main._RefreshVersions", Mock(return_value=versions_mock)),
+    ):
+        yield