[MISC] Skip backup tests without creds (#884)

dragomirp · carlcsaposs-canonical · web-flow · commit ebf4756ed4e9 · 2025-03-07T19:00:55.000+02:00
* Bump libs

* Skip backups tests if no creds are set

* Check that the cloud config is set to a value

* Typo

* Update tests/integration/conftest.py

Co-authored-by: Carl Csaposs &lt;carl.csaposs@canonical.com&gt;

* Switch all checks to gets

* Typo

---------

Co-authored-by: Carl Csaposs &lt;carl.csaposs@canonical.com&gt;
diff --git a/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py b/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py
@@ -325,7 +325,10 @@ def _remove_stale_otel_sdk_packages():
     SpanExportResult,
 )
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
-from opentelemetry.trace import INVALID_SPAN, Tracer
+from opentelemetry.trace import (
+    INVALID_SPAN,
+    Tracer,
+)
 from opentelemetry.trace import get_current_span as otlp_get_current_span
 from opentelemetry.trace import (
     get_tracer,
@@ -345,7 +348,7 @@ def _remove_stale_otel_sdk_packages():
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
 
-LIBPATCH = 6
+LIBPATCH = 7
 
 PYDEPS = ["opentelemetry-exporter-otlp-proto-http==1.21.0"]
 
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
@@ -1,8 +1,20 @@
 # Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
+import logging
+import os
+import uuid
+
+import boto3
 import pytest
+from pytest_operator.plugin import OpsTest
 
 from . import architecture
+from .helpers import construct_endpoint
+
+AWS = "AWS"
+GCP = "GCP"
+
+logger = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope="session")
@@ -11,3 +23,74 @@ def charm():
     # juju bundle files expect local charms to begin with `./` or `/` to distinguish them from
     # Charmhub charms.
     return f"./postgresql-k8s_ubuntu@22.04-{architecture.architecture}.charm"
+
+
+def get_cloud_config(cloud: str) -> tuple[dict[str, str], dict[str, str]]:
+    # Define some configurations and credentials.
+    if cloud == AWS:
+        return {
+            "endpoint": "https://s3.amazonaws.com",
+            "bucket": "data-charms-testing",
+            "path": f"/postgresql-k8s/{uuid.uuid1()}",
+            "region": "us-east-1",
+        }, {
+            "access-key": os.environ["AWS_ACCESS_KEY"],
+            "secret-key": os.environ["AWS_SECRET_KEY"],
+        }
+    elif cloud == GCP:
+        return {
+            "endpoint": "https://storage.googleapis.com",
+            "bucket": "data-charms-testing",
+            "path": f"/postgresql-k8s/{uuid.uuid1()}",
+            "region": "",
+        }, {
+            "access-key": os.environ["GCP_ACCESS_KEY"],
+            "secret-key": os.environ["GCP_SECRET_KEY"],
+        }
+
+
+def cleanup_cloud(config: dict[str, str], credentials: dict[str, str]) -> None:
+    # Delete the previously created objects.
+    logger.info("deleting the previously created backups")
+    session = boto3.session.Session(
+        aws_access_key_id=credentials["access-key"],
+        aws_secret_access_key=credentials["secret-key"],
+        region_name=config["region"],
+    )
+    s3 = session.resource(
+        "s3", endpoint_url=construct_endpoint(config["endpoint"], config["region"])
+    )
+    bucket = s3.Bucket(config["bucket"])
+    # GCS doesn't support batch delete operation, so delete the objects one by one.
+    for bucket_object in bucket.objects.filter(Prefix=config["path"].lstrip("/")):
+        bucket_object.delete()
+
+
+@pytest.fixture(scope="module")
+async def aws_cloud_configs(ops_test: OpsTest) -> None:
+    if (
+        not os.environ.get("AWS_ACCESS_KEY", "").strip()
+        or not os.environ.get("AWS_SECRET_KEY", "").strip()
+    ):
+        pytest.skip("AWS configs not set")
+        return
+
+    config, credentials = get_cloud_config(AWS)
+    yield config, credentials
+
+    cleanup_cloud(config, credentials)
+
+
+@pytest.fixture(scope="module")
+async def gcp_cloud_configs(ops_test: OpsTest) -> None:
+    if (
+        not os.environ.get("GCP_ACCESS_KEY", "").strip()
+        or not os.environ.get("GCP_SECRET_KEY", "").strip()
+    ):
+        pytest.skip("GCP configs not set")
+        return
+
+    config, credentials = get_cloud_config(GCP)
+    yield config, credentials
+
+    cleanup_cloud(config, credentials)
diff --git a/tests/integration/test_backups_aws.py b/tests/integration/test_backups_aws.py
@@ -2,19 +2,16 @@
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
 import logging
-import os
-import uuid
 
-import boto3
-import pytest as pytest
+import pytest
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_attempt, wait_exponential
 
 from . import architecture
+from .conftest import AWS
 from .helpers import (
     DATABASE_APP_NAME,
     backup_operations,
-    construct_endpoint,
     db_connect,
     get_password,
     get_primary,
@@ -41,60 +38,12 @@
 
 logger = logging.getLogger(__name__)
 
-AWS = "AWS"
-GCP = "GCP"
-
-
-@pytest.fixture(scope="module")
-async def cloud_configs(ops_test: OpsTest) -> None:
-    # Define some configurations and credentials.
-    configs = {
-        AWS: {
-            "endpoint": "https://s3.amazonaws.com",
-            "bucket": "data-charms-testing",
-            "path": f"/postgresql-k8s/{uuid.uuid1()}",
-            "region": "us-east-1",
-        },
-        GCP: {
-            "endpoint": "https://storage.googleapis.com",
-            "bucket": "data-charms-testing",
-            "path": f"/postgresql-k8s/{uuid.uuid1()}",
-            "region": "",
-        },
-    }
-    credentials = {
-        AWS: {
-            "access-key": os.environ["AWS_ACCESS_KEY"],
-            "secret-key": os.environ["AWS_SECRET_KEY"],
-        },
-        GCP: {
-            "access-key": os.environ["GCP_ACCESS_KEY"],
-            "secret-key": os.environ["GCP_SECRET_KEY"],
-        },
-    }
-    yield configs, credentials
-    # Delete the previously created objects.
-    logger.info("deleting the previously created backups")
-    for cloud, config in configs.items():
-        session = boto3.session.Session(
-            aws_access_key_id=credentials[cloud]["access-key"],
-            aws_secret_access_key=credentials[cloud]["secret-key"],
-            region_name=config["region"],
-        )
-        s3 = session.resource(
-            "s3", endpoint_url=construct_endpoint(config["endpoint"], config["region"])
-        )
-        bucket = s3.Bucket(config["bucket"])
-        # GCS doesn't support batch delete operation, so delete the objects one by one.
-        for bucket_object in bucket.objects.filter(Prefix=config["path"].lstrip("/")):
-            bucket_object.delete()
-
 
 @pytest.mark.abort_on_fail
-async def test_backup_aws(ops_test: OpsTest, charm, cloud_configs: tuple[dict, dict]) -> None:
+async def test_backup_aws(ops_test: OpsTest, charm, aws_cloud_configs: tuple[dict, dict]) -> None:
     """Build and deploy two units of PostgreSQL in AWS and then test the backup and restore actions."""
-    config = cloud_configs[0][AWS]
-    credentials = cloud_configs[1][AWS]
+    config = aws_cloud_configs[0]
+    credentials = aws_cloud_configs[1]
 
     await backup_operations(
         ops_test,
diff --git a/tests/integration/test_backups_gcp.py b/tests/integration/test_backups_gcp.py
@@ -2,23 +2,21 @@
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
 import logging
-import os
 import uuid
 
-import boto3
-import pytest as pytest
+import pytest
 from lightkube.core.client import Client
 from lightkube.resources.core_v1 import Pod
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_attempt, wait_exponential
 
 from . import architecture
+from .conftest import GCP
 from .helpers import (
     DATABASE_APP_NAME,
     backup_operations,
     build_and_deploy,
     cat_file_from_unit,
-    construct_endpoint,
     db_connect,
     get_password,
     get_unit_address,
@@ -43,60 +41,12 @@
 
 logger = logging.getLogger(__name__)
 
-AWS = "AWS"
-GCP = "GCP"
-
-
-@pytest.fixture(scope="module")
-async def cloud_configs(ops_test: OpsTest) -> None:
-    # Define some configurations and credentials.
-    configs = {
-        AWS: {
-            "endpoint": "https://s3.amazonaws.com",
-            "bucket": "data-charms-testing",
-            "path": f"/postgresql-k8s/{uuid.uuid1()}",
-            "region": "us-east-1",
-        },
-        GCP: {
-            "endpoint": "https://storage.googleapis.com",
-            "bucket": "data-charms-testing",
-            "path": f"/postgresql-k8s/{uuid.uuid1()}",
-            "region": "",
-        },
-    }
-    credentials = {
-        AWS: {
-            "access-key": os.environ["AWS_ACCESS_KEY"],
-            "secret-key": os.environ["AWS_SECRET_KEY"],
-        },
-        GCP: {
-            "access-key": os.environ["GCP_ACCESS_KEY"],
-            "secret-key": os.environ["GCP_SECRET_KEY"],
-        },
-    }
-    yield configs, credentials
-    # Delete the previously created objects.
-    logger.info("deleting the previously created backups")
-    for cloud, config in configs.items():
-        session = boto3.session.Session(
-            aws_access_key_id=credentials[cloud]["access-key"],
-            aws_secret_access_key=credentials[cloud]["secret-key"],
-            region_name=config["region"],
-        )
-        s3 = session.resource(
-            "s3", endpoint_url=construct_endpoint(config["endpoint"], config["region"])
-        )
-        bucket = s3.Bucket(config["bucket"])
-        # GCS doesn't support batch delete operation, so delete the objects one by one.
-        for bucket_object in bucket.objects.filter(Prefix=config["path"].lstrip("/")):
-            bucket_object.delete()
-
 
 @pytest.mark.abort_on_fail
-async def test_backup_gcp(ops_test: OpsTest, charm, cloud_configs: tuple[dict, dict]) -> None:
+async def test_backup_gcp(ops_test: OpsTest, charm, gcp_cloud_configs: tuple[dict, dict]) -> None:
     """Build and deploy two units of PostgreSQL in GCP and then test the backup and restore actions."""
-    config = cloud_configs[0][GCP]
-    credentials = cloud_configs[1][GCP]
+    config = gcp_cloud_configs[0]
+    credentials = gcp_cloud_configs[1]
 
     await backup_operations(
         ops_test,
@@ -123,7 +73,9 @@ async def test_backup_gcp(ops_test: OpsTest, charm, cloud_configs: tuple[dict, d
     )
 
 
-async def test_restore_on_new_cluster(ops_test: OpsTest, charm) -> None:
+async def test_restore_on_new_cluster(
+    ops_test: OpsTest, charm, gcp_cloud_configs: tuple[dict, dict]
+) -> None:
     """Test that is possible to restore a backup to another PostgreSQL cluster."""
     previous_database_app_name = f"{DATABASE_APP_NAME}-gcp"
     database_app_name = f"new-{DATABASE_APP_NAME}"
@@ -217,7 +169,7 @@ async def test_restore_on_new_cluster(ops_test: OpsTest, charm) -> None:
 
 
 async def test_invalid_config_and_recovery_after_fixing_it(
-    ops_test: OpsTest, cloud_configs: tuple[dict, dict]
+    ops_test: OpsTest, gcp_cloud_configs: tuple[dict, dict]
 ) -> None:
     """Test that the charm can handle invalid and valid backup configurations."""
     database_app_name = f"new-{DATABASE_APP_NAME}"
@@ -251,10 +203,10 @@ async def test_invalid_config_and_recovery_after_fixing_it(
     logger.info(
         "configuring S3 integrator for a valid cloud, but with the path of another cluster repository"
     )
-    await ops_test.model.applications[S3_INTEGRATOR_APP_NAME].set_config(cloud_configs[0][GCP])
+    await ops_test.model.applications[S3_INTEGRATOR_APP_NAME].set_config(gcp_cloud_configs[0])
     action = await ops_test.model.units.get(f"{S3_INTEGRATOR_APP_NAME}/0").run_action(
         "sync-s3-credentials",
-        **cloud_configs[1][GCP],
+        **gcp_cloud_configs[1],
     )
     await action.wait()
     await wait_for_idle_on_blocked(
@@ -267,7 +219,7 @@ async def test_invalid_config_and_recovery_after_fixing_it(
 
     # Provide valid backup configurations, with another path in the S3 bucket.
     logger.info("configuring S3 integrator for a valid cloud")
-    config = cloud_configs[0][GCP].copy()
+    config = gcp_cloud_configs[0].copy()
     config["path"] = f"/postgresql-k8s/{uuid.uuid1()}"
     await ops_test.model.applications[S3_INTEGRATOR_APP_NAME].set_config(config)
     logger.info("waiting for the database charm to become active")
@@ -276,7 +228,7 @@ async def test_invalid_config_and_recovery_after_fixing_it(
     )
 
 
-async def test_delete_pod(ops_test: OpsTest) -> None:
+async def test_delete_pod(ops_test: OpsTest, gcp_cloud_configs: tuple[dict, dict]) -> None:
     logger.info("Getting original backup config")
     database_app_name = f"new-{DATABASE_APP_NAME}"
     original_pgbackrest_config = await cat_file_from_unit(
diff --git a/tests/integration/test_backups_pitr_aws.py b/tests/integration/test_backups_pitr_aws.py
diff --git a/tests/integration/test_backups_pitr_gcp.py b/tests/integration/test_backups_pitr_gcp.py
