[DPE-4533] Pause Patroni in the TLS test (#534)

* Increase Patroni's loop_wait

* Allow setting and getting configs using tls

* Wait for loop to clear

* Mover loop_wait restore

* Pause patroni

* Skip on fail

* Increase update status interval

* Further increase the update hook interval

* Different ffwd context

* Switch back ffwding

* Retry grep

* Reraise and log retries

* Union signature

Author: dragomirp

tests/integration/ha_tests/helpers.py

@@ -7,7 +7,7 @@
 import subprocess
 from pathlib import Path
 from tempfile import mkstemp
-from typing import Dict, Optional, Set, Tuple
+from typing import Dict, Optional, Set, Tuple, Union
 
 import psycopg2
 import requests
@@ -137,7 +137,11 @@ async def app_name(
 
 
 async def change_patroni_setting(
-    ops_test: OpsTest, setting: str, value: int, use_random_unit: bool = False
+    ops_test: OpsTest,
+    setting: str,
+    value: Union[int, bool],
+    use_random_unit: bool = False,
+    tls: bool = False,
 ) -> None:
     """Change the value of one of the Patroni settings.
 
@@ -146,8 +150,13 @@ async def change_patroni_setting(
         setting: the name of the setting.
         value: the value to assign to the setting.
         use_random_unit: whether to use a random unit (default is False,
-            so it uses the primary)
+            so it uses the primary).
+        tls: if Patroni is serving using tls.
     """
+    if tls:
+        schema = "https"
+    else:
+        schema = "http"
     for attempt in Retrying(stop=stop_after_delay(30 * 2), wait=wait_fixed(3)):
         with attempt:
             app = await app_name(ops_test)
@@ -158,8 +167,9 @@ async def change_patroni_setting(
                 primary_name = await get_primary(ops_test, app)
                 unit_ip = get_unit_address(ops_test, primary_name)
             requests.patch(
-                f"http://{unit_ip}:8008/config",
+                f"{schema}://{unit_ip}:8008/config",
                 json={setting: value},
+                verify=not tls,
             )
 
 
@@ -399,22 +409,27 @@ async def get_ip_from_inside_the_unit(ops_test: OpsTest, unit_name: str) -> str:
     return stdout.splitlines()[0].strip()
 
 
-async def get_patroni_setting(ops_test: OpsTest, setting: str) -> Optional[int]:
+async def get_patroni_setting(ops_test: OpsTest, setting: str, tls: bool = False) -> Optional[int]:
     """Get the value of one of the integer Patroni settings.
 
     Args:
         ops_test: ops_test instance.
         setting: the name of the setting.
+        tls: if Patroni is serving using tls.
 
     Returns:
         the value of the configuration or None if it's using the default value.
     """
+    if tls:
+        schema = "https"
+    else:
+        schema = "http"
     for attempt in Retrying(stop=stop_after_delay(30 * 2), wait=wait_fixed(3)):
         with attempt:
             app = await app_name(ops_test)
             primary_name = await get_primary(ops_test, app)
             unit_ip = get_unit_address(ops_test, primary_name)
-            configuration_info = requests.get(f"http://{unit_ip}:8008/config")
+            configuration_info = requests.get(f"{schema}://{unit_ip}:8008/config", verify=not tls)
             value = configuration_info.json().get(setting)
             return int(value) if value is not None else None
 

tests/integration/test_tls.py

@@ -6,9 +6,12 @@
 
 import pytest as pytest
 from pytest_operator.plugin import OpsTest
-from tenacity import Retrying, stop_after_attempt, stop_after_delay, wait_exponential
+from tenacity import Retrying, stop_after_attempt, stop_after_delay, wait_exponential, wait_fixed
 
 from . import architecture
+from .ha_tests.helpers import (
+    change_patroni_setting,
+)
 from .helpers import (
     CHARM_SERIES,
     DATABASE_APP_NAME,
@@ -65,6 +68,7 @@ async def test_deploy_active(ops_test: OpsTest):
 
 
 @pytest.mark.group(1)
+@pytest.mark.abort_on_fail
 async def test_tls_enabled(ops_test: OpsTest) -> None:
     """Test that TLS is enabled when relating to the TLS Certificates Operator."""
     async with ops_test.fast_forward():
@@ -106,6 +110,10 @@ async def test_tls_enabled(ops_test: OpsTest) -> None:
         )
         change_primary_start_timeout(ops_test, primary, 0)
 
+        # Pause Patroni so it doesn't wipe the custom changes
+        await change_patroni_setting(ops_test, "pause", True, use_random_unit=True, tls=True)
+
+    async with ops_test.fast_forward("24h"):
         for attempt in Retrying(
             stop=stop_after_delay(60 * 5), wait=wait_exponential(multiplier=1, min=2, max=30)
         ):
@@ -158,12 +166,18 @@ async def test_tls_enabled(ops_test: OpsTest) -> None:
 
         # Check the logs to ensure TLS is being used by pg_rewind.
         primary = await get_primary(ops_test, primary)
-        await run_command_on_unit(
-            ops_test,
-            primary,
-            "grep 'connection authorized: user=rewind database=postgres SSL enabled' /var/snap/charmed-postgresql/common/var/log/postgresql/postgresql-*.log",
-        )
+        for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(5), reraise=True):
+            with attempt:
+                logger.info("Trying to grep for rewind logs.")
+                await run_command_on_unit(
+                    ops_test,
+                    primary,
+                    "grep 'connection authorized: user=rewind database=postgres SSL enabled' /var/snap/charmed-postgresql/common/var/log/postgresql/postgresql-*.log",
+                )
+
+        await change_patroni_setting(ops_test, "pause", False, use_random_unit=True, tls=True)
 
+    async with ops_test.fast_forward():
         # Remove the relation.
         await ops_test.model.applications[DATABASE_APP_NAME].remove_relation(
             f"{DATABASE_APP_NAME}:certificates", f"{tls_certificates_app_name}:certificates"