Replace family and stage tables with enum fields (#238)

* Fix docker image not building

* Insert optional launchpad_api paramter to add_user for testing

* Replace real LaunchpadAPI with FakeLaunchpadAPI for test

* Replace real LaunchpadAPI with FakeLaunchpadAPI for test

* Move FakeLaunchpadAPI into its own file

* Use FakeLaunchpadAPI in test seed script for performance

* Handle warning about trying to test TestEvent class

* Expose family directly from artefact

* Expose family name through artefact instead of family

* Add FakeLaunchpadAPI to git

* Expose stage name from artefact directly

* Expose stage name from artefact directly

* Copy family and stage name fields into artefact table

* Remove stage field from Artefact entity

* Rename Artefact.stage_name to Artefact.stage

* Rename Artefact.family_name to Artefact.family

* Remove stage_id from Artefact

* Remove references to Stage table

* Fix bug with swm tracker

* Drop family and stage tables

* Reorganize artefact fields

* Rename FamilyName to Family

* Match Family name and value to avoid confusion

* Rename Family back to FamilyName

* Make Artefact.family an enum

* Make stage an enum

* Make sure stage name is valid through API

* Fix workflow

.github/workflows/test_backend.yml

@@ -35,7 +35,7 @@ jobs:
           python-version: "3.10"
       - uses: Gr1N/setup-poetry@v8
         with:
-          poetry-version: "1.5.1"
+          poetry-version: "1.8.2"
       - run: poetry install
       - run: poetry run black --check test_observer tests migrations scripts tasks
       - run: poetry run ruff test_observer tests migrations scripts tasks

backend/migrations/versions/2023_05_26_1152-12383cab7248_initial_families_and_stages.py

@@ -12,8 +12,8 @@
 from test_observer.data_access.models_enums import FamilyName
 
 initial_families_and_stages = {
-    FamilyName.SNAP: ["edge", "beta", "candidate", "stable"],
-    FamilyName.DEB: ["proposed", "updates"],
+    FamilyName.snap: ["edge", "beta", "candidate", "stable"],
+    FamilyName.deb: ["proposed", "updates"],
 }
 
 # revision identifiers, used by Alembic.

backend/migrations/versions/2024_12_13_1702-0e09759a33c3_add_charm_family.py

@@ -5,13 +5,13 @@
 Create Date: 2024-12-13 17:02:14.136283+00:00
 
 """
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 
 from test_observer.data_access.models_enums import FamilyName
 
 new_families = {
-    FamilyName.CHARM: ["edge", "beta", "candidate", "stable"],
+    FamilyName.charm: ["edge", "beta", "candidate", "stable"],
 }
 
 

backend/migrations/versions/2025_01_06_1249-7878a1b29384_add_stage_and_family_names_to_artefact.py

@@ -0,0 +1,216 @@
+"""Add stage and family names to artefact
+
+Revision ID: 7878a1b29384
+Revises: 0e09759a33c3
+Create Date: 2025-01-06 12:49:42.039170+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "7878a1b29384"
+down_revision = "0e09759a33c3"
+branch_labels = None
+depends_on = None
+
+
+family_type = sa.Enum("snap", "deb", "charm", name="familyname")
+stage_type = sa.Enum(
+    "proposed", "updates", "edge", "beta", "candidate", "stable", name="stagename"
+)
+
+
+fill_stages_stmt = f"""
+UPDATE artefact
+SET stage = subq.name::{stage_type.name}
+FROM (SELECT id, name FROM stage) as subq
+WHERE subq.id = artefact.stage_id
+"""
+
+fill_families_stmt = f"""
+UPDATE artefact
+SET family = subq.family::{family_type.name}
+FROM (
+    SELECT stage.id stage_id, family.name family
+    FROM stage 
+    JOIN family ON family.id = stage.family_id
+) subq
+WHERE subq.stage_id = artefact.stage_id
+"""
+
+
+def upgrade() -> None:
+    _add_artefact_stage_field()
+    _add_artefact_family_field()
+    _drop_artefact_stage_id_field()
+    _drop_stage_table()
+    _drop_family_table()
+
+
+def _add_artefact_stage_field() -> None:
+    stage_type.create(op.get_bind())
+    op.add_column("artefact", sa.Column("stage", stage_type))
+    op.execute(fill_stages_stmt)
+    op.alter_column("artefact", "stage", nullable=False)
+
+
+def _add_artefact_family_field() -> None:
+    family_type.create(op.get_bind())
+    op.add_column(
+        "artefact",
+        sa.Column("family", family_type),
+    )
+    op.execute(fill_families_stmt)
+    op.alter_column("artefact", "family", nullable=False)
+
+
+def _drop_artefact_stage_id_field() -> None:
+    op.drop_index("artefact_stage_id_ix", table_name="artefact")
+    op.drop_constraint("artefact_stage_id_fkey", "artefact", type_="foreignkey")
+    op.drop_column("artefact", "stage_id")
+
+
+def _drop_family_table() -> None:
+    op.drop_index("family_name_ix", table_name="family")
+    op.drop_table("family")
+
+
+def _drop_stage_table() -> None:
+    op.drop_index("stage_family_id_ix", table_name="stage")
+    op.drop_index("stage_name_ix", table_name="stage")
+    op.drop_table("stage")
+
+
+fill_stage_id_stmt = """
+UPDATE artefact
+SET stage_id = subq.stage_id
+FROM (
+    SELECT stage.id stage_id, stage.name stage_name, family.name family_name
+    FROM stage
+    JOIN family ON family.id = stage.family_id
+) subq
+WHERE subq.stage_name = artefact.stage::text 
+    AND subq.family_name = artefact.family::text
+"""
+
+
+def downgrade() -> None:
+    family_table = _create_family_table()
+    stage_table = _create_stage_table()
+    _fill_tables(family_table, stage_table)
+    _add_artefact_stage_id_field()
+    _drop_family_column()
+    _drop_stage_column()
+
+
+def _create_stage_table() -> sa.Table:
+    result = op.create_table(
+        "stage",
+        sa.Column("name", sa.VARCHAR(length=100), autoincrement=False, nullable=False),
+        sa.Column("position", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("family_id", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), autoincrement=False, nullable=False
+        ),
+        sa.Column(
+            "updated_at", postgresql.TIMESTAMP(), autoincrement=False, nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["family_id"],
+            ["family.id"],
+            name="stage_family_id_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name="stage_pkey"),
+    )
+    op.create_index("stage_name_ix", "stage", ["name"], unique=False)
+    op.create_index("stage_family_id_ix", "stage", ["family_id"], unique=False)
+    return result
+
+
+def _create_family_table() -> sa.Table:
+    result = op.create_table(
+        "family",
+        sa.Column("name", sa.VARCHAR(length=100), autoincrement=False, nullable=False),
+        sa.Column(
+            "id",
+            sa.INTEGER(),
+            server_default=sa.text("nextval('family_id_seq'::regclass)"),
+            autoincrement=True,
+            nullable=False,
+        ),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), autoincrement=False, nullable=False
+        ),
+        sa.Column(
+            "updated_at", postgresql.TIMESTAMP(), autoincrement=False, nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id", name="family_pkey"),
+        postgresql_ignore_search_path=False,
+    )
+    op.create_index("family_name_ix", "family", ["name"], unique=True)
+    return result
+
+
+def _fill_tables(family_table: sa.Table, stage_table: sa.Table) -> None:
+    conn = op.get_bind()
+    families_and_stages = {
+        "snap": ["edge", "beta", "candidate", "stable"],
+        "deb": ["proposed", "updates"],
+        "charm": ["edge", "beta", "candidate", "stable"],
+    }
+
+    for family in families_and_stages:
+        (inserted_family,) = conn.execute(
+            sa.insert(family_table)
+            .values(
+                name=family,
+                created_at=sa.func.now(),
+                updated_at=sa.func.now(),
+            )
+            .returning(family_table.c.id)
+        )
+        stage_position = 10
+        for stage in families_and_stages[family]:
+            conn.execute(
+                sa.insert(stage_table).values(
+                    name=stage,
+                    family_id=inserted_family.id,
+                    position=stage_position,
+                    created_at=sa.func.now(),
+                    updated_at=sa.func.now(),
+                )
+            )
+            stage_position += 10
+
+
+def _add_artefact_stage_id_field() -> None:
+    op.add_column(
+        "artefact",
+        sa.Column("stage_id", sa.INTEGER(), autoincrement=False),
+    )
+
+    op.execute(fill_stage_id_stmt)
+    op.alter_column("artefact", "stage_id", nullable=False)
+    op.create_foreign_key(
+        "artefact_stage_id_fkey",
+        "artefact",
+        "stage",
+        ["stage_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.create_index("artefact_stage_id_ix", "artefact", ["stage_id"], unique=False)
+
+
+def _drop_family_column() -> None:
+    op.drop_column("artefact", "family")
+    op.execute(f"DROP TYPE IF EXISTS {family_type.name}")
+
+
+def _drop_stage_column() -> None:
+    op.drop_column("artefact", "stage")
+    op.execute(f"DROP TYPE IF EXISTS {stage_type.name}")

backend/pyproject.toml

@@ -3,6 +3,7 @@ name = "test-observer"
 version = "0.0.0"
 description = "Observe the status and state of certification tests for various artefacts"
 authors = []
+package-mode = false
 
 [tool.poetry.dependencies]
 python = "^3.10"