[DPE-6965] Wait for all storages to detatch in smoke test (#869)

* Separate storage pools

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix charm test

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix async replication

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix replica bootstrap

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix data directory removal on restore

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix backup do microceph test

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix upgrade integration tests

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Add test to check new multiple storages

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Add storages'descriptions

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix test_charm_garbage_ignorance

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix restore cluster test

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Fix storage re-use test

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

* Reduce volume size

* List storage

* Poor man's jubilant

* Cleanup

* Correct charm path

* Fix base

* Debug missing storage

* Wait for storage to be available

---------

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>
Co-authored-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

Author: dragomirp

.github/workflows/integration_test.yaml

@@ -157,7 +157,7 @@ jobs:
       - name: juju status
         timeout-minutes: 1
         if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }}
-        run: sudo juju status --color --relations | tee ~/logs/juju-status.txt
+        run: sudo juju status --color --relations --storage | tee ~/logs/juju-status.txt
       - name: juju debug-log
         timeout-minutes: 3
         if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }}

tests/integration/ha_tests/test_smoke.py

@@ -2,7 +2,6 @@
 # Copyright 2021 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-import asyncio
 import logging
 
 import pytest
@@ -43,10 +42,10 @@ async def test_app_force_removal(ops_test: OpsTest, charm: str):
             num_units=1,
             base=CHARM_BASE,
             storage={
-                "archive": {"pool": "lxd-btrfs", "size": 8046},
-                "data": {"pool": "lxd-btrfs", "size": 8046},
-                "logs": {"pool": "lxd-btrfs", "size": 8046},
-                "temp": {"pool": "lxd-btrfs", "size": 8046},
+                "archive": {"pool": "lxd-btrfs", "size": 2048},
+                "data": {"pool": "lxd-btrfs", "size": 2048},
+                "logs": {"pool": "lxd-btrfs", "size": 2048},
+                "temp": {"pool": "lxd-btrfs", "size": 2048},
             },
             config={"profile": "testing"},
         )
@@ -66,7 +65,7 @@ async def test_app_force_removal(ops_test: OpsTest, charm: str):
         storage_ids = get_storage_ids(ops_test, primary_name)
 
         # Check if storage exists after application deployed
-        logger.info("werifing is storage exists")
+        logger.info("verifying that storage exists")
         for storage_id in storage_ids:
             for attempt in Retrying(
                 stop=stop_after_delay(15 * 3), wait=wait_fixed(3), reraise=True
@@ -89,7 +88,7 @@ async def test_app_force_removal(ops_test: OpsTest, charm: str):
         )
 
         # Storage should remain
-        logger.info("werifing is storage exists")
+        logger.info("verifying that storage exists")
         for storage_id in storage_ids:
             for attempt in Retrying(
                 stop=stop_after_delay(15 * 3), wait=wait_fixed(3), reraise=True
@@ -107,6 +106,8 @@ async def test_charm_garbage_ignorance(ops_test: OpsTest, charm: str):
         for attempt in Retrying(stop=stop_after_delay(30 * 3), wait=wait_fixed(3), reraise=True):
             with attempt:
                 garbage_storages = await get_detached_storages(ops_test)
+                assert len(garbage_storages) == 4
+                logger.info(f"Collected storages: {garbage_storages}")
 
         logger.info("add unit with attached storage")
         await add_unit_with_storage(ops_test, APPLICATION_NAME, garbage_storages)
@@ -146,31 +147,29 @@ async def test_app_resources_conflicts_v3(ops_test: OpsTest, charm: str):
     """Test application deploy in dirty environment with garbage storage from another application."""
     async with ops_test.fast_forward():
         logger.info("checking garbage storage")
-        garbage_storage = None
+        garbage_storages = None
         for attempt in Retrying(stop=stop_after_delay(30 * 3), wait=wait_fixed(3), reraise=True):
             with attempt:
-                garbage_storage = await get_detached_storages(ops_test)
+                garbage_storages = await get_detached_storages(ops_test)
+                assert len(garbage_storages) == 4
+                logger.info(f"Collected storages: {garbage_storages}")
 
         logger.info("deploying duplicate application with attached storage")
         await ops_test.model.deploy(
             charm,
             application_name=DUP_APPLICATION_NAME,
             num_units=1,
             base=CHARM_BASE,
-            attach_storage=[tag.storage(storage) for storage in garbage_storage],
+            attach_storage=[tag.storage(storage) for storage in garbage_storages],
             config={"profile": "testing"},
         )
 
         # Reducing the update status frequency to speed up the triggering of deferred events.
-        await ops_test.model.set_config({"update-status-hook-interval": "10s"})
-
-        logger.info("waiting for duplicate application to be blocked")
-        try:
+        async with ops_test.fast_forward("60s"):
+            logger.info("waiting for duplicate application to be waiting")
             await ops_test.model.wait_for_idle(
-                apps=[DUP_APPLICATION_NAME], timeout=1000, status="blocked"
+                apps=[DUP_APPLICATION_NAME], timeout=1000, status="waiting", idle_period=30
             )
-        except asyncio.TimeoutError:
-            logger.info("Application is not in blocked state. Checking logs...")
 
         # Since application have postgresql db in storage from external application it should not be able to connect due to new password
         logger.info("checking operator password auth")