Tests: Add full-roundtrip/integration test for the "snapshot" strategy

pyproject.toml

@@ -107,7 +107,8 @@ test = [
  "pytest<8",
  "pytest-cov<5",
  "pytest-mock<4",
- "testcontainers",
+ "testcontainers<4",
+ "testcontainers-minio==0.0.1rc1",
 ]
 [project.urls]
 changelog = "https://github.com/crate-workbench/cratedb-retention/blob/main/CHANGES.rst"

tests/conftest.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2021-2023, Crate.io Inc.
 # Distributed under the terms of the AGPLv3 license, see LICENSE.
 import pytest
+from testcontainers.minio import MinioContainer
 
 from cratedb_retention.model import DatabaseAddress, JobSettings, RetentionPolicy, RetentionStrategy
 from cratedb_retention.setup.schema import setup_schema
@@ -67,6 +68,15 @@ def cratedb():
  db.finalize()
 
 
+@pytest.fixture(scope="session")
+def minio():
+ """
+ Provide a MinIO service to the test suite.
+ """
+ with MinioContainer(image="docker.io/minio/minio:latest") as minio:
+ yield minio
+
+
 @pytest.fixture()
 def database(cratedb, settings):
  """

tests/test_cli.py

@@ -2,7 +2,7 @@
 # Distributed under the terms of the AGPLv3 license, see LICENSE.
 import pytest
 from click.testing import CliRunner
-from sqlalchemy.exc import OperationalError
+from sqlalchemy.exc import OperationalError, ProgrammingError
 
 from cratedb_retention.cli import cli
 from tests.conftest import TESTDRIVE_DATA_SCHEMA
@@ -245,22 +245,60 @@ def test_run_reallocate(store, database, raw_metrics, raw_metrics_reallocate_pol
  assert database.count_records(f'"{TESTDRIVE_DATA_SCHEMA}"."raw_metrics"') == 6
 
 
-def test_run_snapshot(caplog, store, database, sensor_readings, sensor_readings_snapshot_policy):
+def test_run_snapshot(caplog, store, database, sensor_readings, sensor_readings_snapshot_policy, minio):
  """
  CLI test: Invoke `cratedb-retention run --strategy=snapshot`.
  """
 
- database_url = store.database.dburi
- runner = CliRunner()
+ # Acquire runtime information from MinIO container. In order to let CrateDB talk to
+ # MinIO, we need its Docker-internal IP address (172.17.0.x), not the exposed one.
+ s3_config = minio.get_config()
+ container_ip = minio.get_docker_client().bridge_ip(minio._container.id)
+ s3_url = f"{container_ip}:{minio.port_to_expose}"
+
+ # Prepare a bucket in the S3 storage.
+ minio.get_client().make_bucket("cratedb-cold-storage")
+
+ # TODO: DROP REPOSITORY IF EXISTS
+ try:
+ sql = "DROP REPOSITORY export_cold"
+ database.run_sql(sql)
+ except ProgrammingError as ex:
+ if "RepositoryUnknownException" not in str(ex):
+ raise
+
+ # TODO: CREATE REPOSITORY IF NOT EXISTS
+ sql = f"""
+ CREATE REPOSITORY export_cold TYPE s3
+ WITH (
+ protocol = 'http',
+ endpoint = '{s3_url}',
+ access_key = '{s3_config["access_key"]}',
+ secret_key = '{s3_config["secret_key"]}',
+ bucket = 'cratedb-cold-storage'
+ );
+ """
+ database.run_sql(sql)
+
+ # Check number of records in database.
+ assert database.count_records(f'"{TESTDRIVE_DATA_SCHEMA}"."sensor_readings"') == 9
 
  # Invoke data retention through CLI interface.
- # FIXME: This currently can not be tested, because it needs a snapshot repository.
- # TODO: Provide an embedded MinIO S3 instance to the test suite.
+ database_url = store.database.dburi
+ runner = CliRunner()
  runner.invoke(
  cli,
  args=f'run --cutoff-day=2024-12-31 --strategy=snapshot "{database_url}"',
  catch_exceptions=False,
  )
 
- assert "Data retention SQL statement failed" in caplog.text
- assert "RepositoryUnknownException[Repository 'export_cold' unknown]" in caplog.text
+ # Check number of records in database.
+ assert database.count_records(f'"{TESTDRIVE_DATA_SCHEMA}"."sensor_readings"') == 0
+
+ # Verify that the S3 bucket has been populated correctly, and that the snapshot has the right shape.
+ objs = minio.get_client().list_objects(bucket_name="cratedb-cold-storage")
+ object_names = [obj.object_name for obj in objs]
+ assert "index-1" in object_names
+ assert "index.latest" in object_names
+ assert "indices/" in object_names
+ assert len(object_names) == 7