Parallelize tests

.github/workflows/ci.yaml

@@ -13,6 +13,7 @@ jobs:
         run: python3 -m pip install tox
       - name: Run linters
         run: tox -e lint
+
   unit-test:
     name: Unit tests
     runs-on: ubuntu-latest
@@ -25,8 +26,61 @@ jobs:
         run: tox -e unit
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v2
-  integration-test-lxd:
-    name: Integration tests (lxd)
+
+  integration-test-lxd-charm:
+    name: Integration tests for charm deployment (lxd)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e charm-integration
+
+  integration-test-lxd-database-relation:
+    name: Integration tests for database relation (lxd)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e database-relation-integration
+
+  integration-test-lxd-db-relation:
+    name: Integration tests for db relation (lxd)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e db-relation-integration
+
+  integration-test-lxd-db-admin-relation:
+    name: Integration tests for db-admin relation (lxd)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e db-admin-relation-integration
+
+  integration-test-lxd-password-rotation:
+    name: Integration tests for password rotation (lxd)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -36,4 +90,4 @@ jobs:
         with:
           provider: lxd
       - name: Run integration tests
-        run: tox -e integration
+        run: tox -e password-rotation-integration

.github/workflows/release.yaml

@@ -42,8 +42,8 @@ jobs:
       - name: Run tests
         run: tox -e unit
 
-  integration-test-lxd:
-    name: Integration tests for (lxd)
+  integration-test-charm:
+    name: Integration tests for charm deployment
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -53,15 +53,71 @@ jobs:
         with:
           provider: lxd
       - name: Run integration tests
-        run: tox -e integration
+        run: tox -e charm-integration
+
+  integration-test-database-relation:
+    name: Integration tests for database relation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e database-relation-integration
+
+  integration-test-db-relation:
+    name: Integration tests for db relation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e db-relation-integration
+
+  integration-test-db-admin-relation:
+    name: Integration tests for db-admin relation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e db-admin-relation-integration
+
+  integration-test-password-rotation:
+    name: Integration tests for password rotation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+      - name: Run integration tests
+        run: tox -e password-rotation-integration
 
   release-to-charmhub:
     name: Release to CharmHub
     needs:
       - lib-check
       - lint
       - unit-test
-      - integration-test-lxd
+      - integration-test-charm
+      - integration-test-database-relation
+      - integration-test-db-relation
+      - integration-test-db-admin-relation
+      - integration-test-password-rotation
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

pyproject.toml

@@ -12,6 +12,13 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
+markers = [
+    "charm_tests",
+    "database_relation_tests",
+    "db_relation_tests",
+    "db_admin_relation_tests",
+    "password_rotation_tests",
+]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/new_relations/test_new_relations.py

@@ -31,6 +31,7 @@
 
 
 @pytest.mark.abort_on_fail
+@pytest.mark.database_relation_tests
 async def test_deploy_charms(ops_test: OpsTest, application_charm, database_charm):
     """Deploy both charms (application and database) to use in the tests."""
     # Deploy both charms (multiple units for each application to test that later they correctly
@@ -65,6 +66,7 @@ async def test_deploy_charms(ops_test: OpsTest, application_charm, database_char
         await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", timeout=3000)
 
 
+@pytest.mark.database_relation_tests
 async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest):
     """Test that there is no read-only endpoint in a standalone cluster."""
     async with ops_test.fast_forward():
@@ -91,6 +93,7 @@ async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest):
         )
 
 
+@pytest.mark.database_relation_tests
 async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest):
     """Test that there is read-only endpoint in a scaled up cluster."""
     async with ops_test.fast_forward():
@@ -108,7 +111,7 @@ async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest):
         )
 
 
-@pytest.mark.abort_on_fail
+@pytest.mark.database_relation_tests
 async def test_database_relation_with_charm_libraries(ops_test: OpsTest):
     """Test basic functionality of database relation interface."""
     # Get the connection string to connect to the database using the read/write endpoint.
@@ -156,6 +159,7 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest):
             cursor.execute("DROP TABLE test;")
 
 
+@pytest.mark.database_relation_tests
 async def test_user_with_extra_roles(ops_test: OpsTest):
     """Test superuser actions and the request for more permissions."""
     # Get the connection string to connect to the database.
@@ -176,6 +180,7 @@ async def test_user_with_extra_roles(ops_test: OpsTest):
     connection.close()
 
 
+@pytest.mark.database_relation_tests
 async def test_two_applications_doesnt_share_the_same_relation_data(
     ops_test: OpsTest, application_charm
 ):
@@ -210,6 +215,7 @@ async def test_two_applications_doesnt_share_the_same_relation_data(
     assert application_connection_string != another_application_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_an_application_can_connect_to_multiple_database_clusters(
     ops_test: OpsTest, database_charm
 ):
@@ -242,6 +248,7 @@ async def test_an_application_can_connect_to_multiple_database_clusters(
     assert application_connection_string != another_application_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_an_application_can_connect_to_multiple_aliased_database_clusters(
     ops_test: OpsTest, database_charm
 ):
@@ -277,6 +284,7 @@ async def test_an_application_can_connect_to_multiple_aliased_database_clusters(
     assert application_connection_string != another_application_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_an_application_can_request_multiple_databases(ops_test: OpsTest, application_charm):
     """Test that an application can request additional databases using the same interface."""
     # Relate the charms using another relation and wait for them exchanging some connection data.
@@ -297,6 +305,7 @@ async def test_an_application_can_request_multiple_databases(ops_test: OpsTest,
     assert first_database_connection_string != second_database_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest):
     """Test that relation data, like connection data, is updated correctly when scaling."""
     # Retrieve the list of current database unit names.

tests/integration/test_charm.py

@@ -33,7 +33,9 @@
 
 
 @pytest.mark.abort_on_fail
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("series", SERIES)
+@pytest.mark.skip_if_deployed
 async def test_deploy(ops_test: OpsTest, charm: str, series: str):
     """Deploy the charm-under-test.
 
@@ -58,6 +60,7 @@ async def test_deploy(ops_test: OpsTest, charm: str, series: str):
 
 
 @pytest.mark.abort_on_fail
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("series", SERIES)
 @pytest.mark.parametrize("unit_id", UNIT_IDS)
 async def test_database_is_up(ops_test: OpsTest, series: str, unit_id: int):
@@ -71,6 +74,7 @@ async def test_database_is_up(ops_test: OpsTest, series: str, unit_id: int):
     assert result.status_code == 200
 
 
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("series", SERIES)
 @pytest.mark.parametrize("unit_id", UNIT_IDS)
 async def test_settings_are_correct(ops_test: OpsTest, series: str, unit_id: int):
@@ -120,6 +124,7 @@ async def test_settings_are_correct(ops_test: OpsTest, series: str, unit_id: int
     assert settings["maximum_lag_on_failover"] == 1048576
 
 
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("series", SERIES)
 async def test_scale_down_and_up(ops_test: OpsTest, series: str):
     """Test data is replicated to new units after a scale up."""
@@ -205,6 +210,7 @@ async def test_scale_down_and_up(ops_test: OpsTest, series: str):
     await scale_application(ops_test, application_name, initial_scale)
 
 
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("series", SERIES)
 async def test_persist_data_through_primary_deletion(ops_test: OpsTest, series: str):
     """Test data persists through a primary deletion."""

tests/integration/test_db.py

@@ -24,6 +24,7 @@
 RELATION_NAME = "db"
 
 
+@pytest.mark.db_relation_tests
 async def test_mailman3_core_db(ops_test: OpsTest, charm: str) -> None:
     """Deploy Mailman3 Core to test the 'db' relation."""
     async with ops_test.fast_forward():
@@ -92,6 +93,7 @@ async def test_mailman3_core_db(ops_test: OpsTest, charm: str) -> None:
         assert domain_name not in [domain.mail_host for domain in client.domains]
 
 
+@pytest.mark.db_relation_tests
 async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest):
     """Test that relation data, like connection data, is updated correctly when scaling."""
     # Retrieve the list of current database unit names.

tests/integration/test_db_admin.py

@@ -5,6 +5,7 @@
 import json
 import logging
 
+import pytest as pytest
 from landscape_api.base import run_query
 from pytest_operator.plugin import OpsTest
 
@@ -24,6 +25,7 @@
 DATABASE_UNITS = 3
 
 
+@pytest.mark.db_admin_relation_tests
 async def test_landscape_scalable_bundle_db(ops_test: OpsTest, charm: str) -> None:
     """Deploy Landscape Scalable Bundle to test the 'db-admin' relation."""
     config = {