Merge pull request #225 from lsst-sqre/tickets/DM-46034

DM-46034: Add Alembic support

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.4
+    rev: v0.6.5
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

Dockerfile

@@ -12,7 +12,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.12.5-slim-bookworm as base-image
+FROM python:3.12.5-slim-bookworm AS base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .
@@ -52,6 +52,13 @@ RUN useradd --create-home appuser
 # Copy the virtualenv
 COPY --from=install-image /opt/venv /opt/venv
 
+# Copy the Alembic configuration and migrations, and set that path as the
+# working directory so that Alembic can be run with a simple entry command
+# and no extra configuration.
+COPY --from=install-image /workdir/alembic.ini /app/alembic.ini
+COPY --from=install-image /workdir/alembic /app/alembic
+WORKDIR /app
+
 # Copy the startup script
 COPY scripts/start-frontend.sh /start-frontend.sh
 

alembic.ini

@@ -0,0 +1,22 @@
+# Alembic configuration for vo-cutouts.
+#
+# This file does not retain the comments that are generated as part of the
+# default template, since they will get out of date with newer versions of
+# Alembic. See the Alembic documentation for details about each setting and
+# for settings that are not used here.
+
+[alembic]
+script_location = %(here)s/alembic
+file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+prepend_sys_path = .
+timezone = UTC
+version_path_separator = os
+
+[post_write_hooks]
+hooks = ruff ruff_format
+ruff.type = exec
+ruff.executable = ruff
+ruff.options = check --fix REVISION_SCRIPT_FILENAME
+ruff_format.type = exec
+ruff_format.executable = ruff
+ruff_format.options = format REVISION_SCRIPT_FILENAME

alembic/README.md

@@ -0,0 +1,14 @@
+# vo-cutouts Alembic configuration
+
+This directory contains the Alembic configuration for managing the vo-cutouts UWS database.
+It is installed into the vo-cutouts Docker image and is used to check whether the schema is up-to-date at startup of any vo-cutouts component.
+It is also used by the Helm hook that updates the vo-cutouts UWS schema if `config.updateSchema` is enabled.
+
+## Generating new migrations
+
+For detailed instructions on how to generate a new Alembic migration, see [the Safir documentation](https://safir.lsst.io/user-guide/database/schema#create-migration).
+
+One of the files in this directory is here only to support creating migrations.
+`docker-compose.yaml` is a [docker-compose](https://docs.docker.com/compose/) configuration file that starts a PostgreSQL instance suitable for generating schema migrations.
+This file is not used at runtime.
+It is used by the tox environment described in the above documentation.

alembic/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3"
+services:
+  postgresql:
+    image: "postgres:latest"
+    hostname: "postgresql"
+    container_name: "postgresql"
+    environment:
+      POSTGRES_PASSWORD: "INSECURE"
+      POSTGRES_USER: "vo-cutouts"
+      POSTGRES_DB: "vo-cutouts"
+    ports:
+      - "5432:5432"

alembic/env.py

@@ -0,0 +1,22 @@
+"""Alembic migration environment."""
+
+from safir.database import run_migrations_offline, run_migrations_online
+from safir.logging import configure_alembic_logging, configure_logging
+from safir.uws import UWSSchemaBase
+
+from alembic import context
+from vocutouts.config import config
+
+# Configure structlog.
+configure_logging(name="vo-cutouts", log_level=config.log_level)
+configure_alembic_logging()
+
+# Run the migrations.
+if context.is_offline_mode():
+    run_migrations_offline(UWSSchemaBase.metadata, config.database_url)
+else:
+    run_migrations_online(
+        UWSSchemaBase.metadata,
+        config.database_url,
+        config.database_password,
+    )

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/20240912_1902_5ab72a20365b_initial_uws_schema.py

@@ -0,0 +1,27 @@
+"""Initial UWS schema
+
+Revision ID: 5ab72a20365b
+Revises:
+Create Date: 2024-09-12 19:02:23.855138+00:00
+
+"""
+
+from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "5ab72a20365b"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

changelog.d/20240913_143738_rra_DM_46034.md

@@ -0,0 +1,3 @@
+### New features
+
+- Use Alembic to manage the schema of the UWS database. When upgrading to this version, set `config.updateSchema` to true in the Helm configuration for the first deployment. This release contains no schema changes, but needs to perform a migration to add the Alembic version information. The vo-cutouts components will now refuse to start if the database schema has changed and the database has not yet been migrated.

pyproject.toml

@@ -26,11 +26,12 @@ dependencies = [
     "python-multipart",
     "uvicorn[standard]",
     # Other dependencies.
+    "alembic[tz]",
     "astropy",
     "click",
     "pydantic>2",
     "pydantic-settings",
-    "safir[uws]>=6.2.0",
+    "safir[db,uws]>=6.4.0",
     "structlog",
 ]
 dynamic = ["version"]

requirements/dev.txt

@@ -229,9 +229,9 @@ httpx==0.27.2 \
     #   -c requirements/main.txt
     #   -r requirements/dev.in
     #   respx
-idna==3.8 \
-    --hash=sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac \
-    --hash=sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603
+idna==3.10 \
+    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
+    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
     # via
     #   -c requirements/main.txt
     #   anyio

requirements/main.txt

@@ -1,5 +1,11 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile --universal --generate-hashes --output-file requirements/main.txt pyproject.toml
+alembic==1.13.2 \
+    --hash=sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef \
+    --hash=sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953
+    # via
+    #   vo-cutouts (pyproject.toml)
+    #   safir
 annotated-types==0.7.0 \
     --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
     --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
@@ -46,9 +52,9 @@ astropy==6.1.3 \
     --hash=sha256:e6bfb987d512bbd70ec0c33ac023598594d35711bdb730cfaf3d8985d338155b \
     --hash=sha256:e81c1fcb54b11190ce84c4106d49af762aff76ec18d50fbb205c556f423c4791
     # via vo-cutouts (pyproject.toml)
-astropy-iers-data==0.2024.9.12.13.29.57 \
-    --hash=sha256:9ae1d147f47f18c335984781c7119e1c12aca47b89653d609980f75c505e7708 \
-    --hash=sha256:ca580347f084a9d9a2e30cbd8cd44665cef52c69e5a1c1d129abb645eb970fea
+astropy-iers-data==0.2024.9.16.0.32.21 \
+    --hash=sha256:2ff6fe868a623e616953a432698b05dd6adac9683d21ac780bfbb94e78f7c344 \
+    --hash=sha256:adf111e1b596470c4437fa44cf767e56f6d4bc2e93068871fd0b30c73476d430
     # via astropy
 asyncpg==0.29.0 \
     --hash=sha256:0009a300cae37b8c525e5b449233d59cd9868fd35431abc470a3e364d2b85cb9 \
@@ -310,9 +316,9 @@ cryptography==43.0.1 \
     # via
     #   pyjwt
     #   safir
-fastapi==0.114.1 \
-    --hash=sha256:1d7bbbeabbaae0acb0c22f0ab0b040f642d3093ca3645f8c876b6f91391861d8 \
-    --hash=sha256:5d4746f6e4b7dff0b4f6b6c6d5445645285f662fe75886e99af7ee2d6b58bb3e
+fastapi==0.114.2 \
+    --hash=sha256:0adb148b62edb09e8c6eeefa3ea934e8f276dabc038c5a82989ea6346050c3da \
+    --hash=sha256:44474a22913057b1acb973ab90f4b671ba5200482e7622816d79105dcece1ac5
     # via
     #   vo-cutouts (pyproject.toml)
     #   safir
@@ -597,9 +603,9 @@ httpx==0.27.2 \
     --hash=sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0 \
     --hash=sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2
     # via safir
-idna==3.8 \
-    --hash=sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac \
-    --hash=sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603
+idna==3.10 \
+    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
+    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
     # via
     #   anyio
     #   httpx
@@ -748,6 +754,10 @@ lxml==5.3.0 \
     --hash=sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945 \
     --hash=sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8
     # via pydantic-xml
+mako==1.3.5 \
+    --hash=sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a \
+    --hash=sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc
+    # via alembic
 markupsafe==2.1.5 \
     --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \
     --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \
@@ -809,7 +819,9 @@ markupsafe==2.1.5 \
     --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \
     --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \
     --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68
-    # via jinja2
+    # via
+    #   jinja2
+    #   mako
 numpy==2.1.1 \
     --hash=sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5 \
     --hash=sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0 \
@@ -1120,75 +1132,29 @@ rsa==4.9 \
     --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \
     --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21
     # via google-auth
-safir==6.3.0 \
-    --hash=sha256:2fcd64bf37dd42eacedd6378341b2487cd06dbaf1f28403301b8d80f60a4fb56 \
-    --hash=sha256:6ad7dad520d87d853628849ef95a348c55dbd0180ad3f15c1cf2f7f8fe32f915
+safir==6.4.0 \
+    --hash=sha256:ba7af071eab0d198e6e15a2117028566f3f4237e02e2278e8bfc2633a7c68228 \
+    --hash=sha256:f38c3f1d7d76d304984b572288826510e5c7a0e1f965b2eabdd7f3bace07c48a
     # via vo-cutouts (pyproject.toml)
-safir-arq==6.3.0 \
-    --hash=sha256:c034e34fa79a7ebb4cfe3c1cb527e4c64c2195b63f919e2b623ae9dd72ac7e1b \
-    --hash=sha256:db110ce3fb0419d1d4a8d83b524ae9a09aa680d0e3f9323115cce836945b10e9
+safir-arq==6.4.0 \
+    --hash=sha256:4db9a1859d42064a702f1878f3da4d972f03140fa4214a420b8168bd5226c276 \
+    --hash=sha256:5017668a1c8f101010b0a96affcb0d2e84596709c9160609841ce35443a80176
     # via safir
-safir-logging==6.3.0 \
-    --hash=sha256:491dfe85de89a3f2daa29c491a22a0551f0961444490418d91ec50c040ae16eb \
-    --hash=sha256:e14754ab0bba6cfa248c3fc4cb5ca28410d97ff3965e831eab6581ed37485e79
+safir-logging==6.4.0 \
+    --hash=sha256:4031a430d738b8fe5bfd29125dce6cbf4e4949879307ba4146648afa3d24cd0a \
+    --hash=sha256:e2dbf0b5d9dabecd70c27bff9bf01629bf0724b05b0f0087a1fe4f45c702215f
     # via safir
 sniffio==1.3.1 \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
     # via
     #   anyio
     #   httpx
-sqlalchemy==2.0.34 \
-    --hash=sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22 \
-    --hash=sha256:13be2cc683b76977a700948411a94c67ad8faf542fa7da2a4b167f2244781cf3 \
-    --hash=sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2 \
-    --hash=sha256:173f5f122d2e1bff8fbd9f7811b7942bead1f5e9f371cdf9e670b327e6703ebd \
-    --hash=sha256:196958cde924a00488e3e83ff917be3b73cd4ed8352bbc0f2989333176d1c54d \
-    --hash=sha256:203d46bddeaa7982f9c3cc693e5bc93db476ab5de9d4b4640d5c99ff219bee8c \
-    --hash=sha256:220574e78ad986aea8e81ac68821e47ea9202b7e44f251b7ed8c66d9ae3f4278 \
-    --hash=sha256:243f92596f4fd4c8bd30ab8e8dd5965afe226363d75cab2468f2c707f64cd83b \
-    --hash=sha256:24af3dc43568f3780b7e1e57c49b41d98b2d940c1fd2e62d65d3928b6f95f021 \
-    --hash=sha256:25691f4adfb9d5e796fd48bf1432272f95f4bbe5f89c475a788f31232ea6afba \
-    --hash=sha256:2e6965346fc1491a566e019a4a1d3dfc081ce7ac1a736536367ca305da6472a8 \
-    --hash=sha256:3166dfff2d16fe9be3241ee60ece6fcb01cf8e74dd7c5e0b64f8e19fab44911b \
-    --hash=sha256:413c85cd0177c23e32dee6898c67a5f49296640041d98fddb2c40888fe4daa2e \
-    --hash=sha256:430093fce0efc7941d911d34f75a70084f12f6ca5c15d19595c18753edb7c33b \
-    --hash=sha256:43f28005141165edd11fbbf1541c920bd29e167b8bbc1fb410d4fe2269c1667a \
-    --hash=sha256:526ce723265643dbc4c7efb54f56648cc30e7abe20f387d763364b3ce7506c82 \
-    --hash=sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e \
-    --hash=sha256:5bc08e75ed11693ecb648b7a0a4ed80da6d10845e44be0c98c03f2f880b68ff4 \
-    --hash=sha256:6831a78bbd3c40f909b3e5233f87341f12d0b34a58f14115c9e94b4cdaf726d3 \
-    --hash=sha256:6a1e03db964e9d32f112bae36f0cc1dcd1988d096cfd75d6a588a3c3def9ab2b \
-    --hash=sha256:6daeb8382d0df526372abd9cb795c992e18eed25ef2c43afe518c73f8cccb721 \
-    --hash=sha256:6e7cde3a2221aa89247944cafb1b26616380e30c63e37ed19ff0bba5e968688d \
-    --hash=sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a \
-    --hash=sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f \
-    --hash=sha256:79cb400c360c7c210097b147c16a9e4c14688a6402445ac848f296ade6283bbc \
-    --hash=sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83 \
-    --hash=sha256:80bd73ea335203b125cf1d8e50fef06be709619eb6ab9e7b891ea34b5baa2287 \
-    --hash=sha256:895184dfef8708e15f7516bd930bda7e50ead069280d2ce09ba11781b630a434 \
-    --hash=sha256:8fddde2368e777ea2a4891a3fb4341e910a056be0bb15303bf1b92f073b80c02 \
-    --hash=sha256:95d0b2cf8791ab5fb9e3aa3d9a79a0d5d51f55b6357eecf532a120ba3b5524db \
-    --hash=sha256:9661268415f450c95f72f0ac1217cc6f10256f860eed85c2ae32e75b60278ad8 \
-    --hash=sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74 \
-    --hash=sha256:9ea54f7300553af0a2a7235e9b85f4204e1fc21848f917a3213b0e0818de9a24 \
-    --hash=sha256:9ebc11c54c6ecdd07bb4efbfa1554538982f5432dfb8456958b6d46b9f834bb7 \
-    --hash=sha256:a17d8fac6df9835d8e2b4c5523666e7051d0897a93756518a1fe101c7f47f2f0 \
-    --hash=sha256:ae92bebca3b1e6bd203494e5ef919a60fb6dfe4d9a47ed2453211d3bd451b9f5 \
-    --hash=sha256:b68094b165a9e930aedef90725a8fcfafe9ef95370cbb54abc0464062dbf808f \
-    --hash=sha256:b75b00083e7fe6621ce13cfce9d4469c4774e55e8e9d38c305b37f13cf1e874c \
-    --hash=sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812 \
-    --hash=sha256:bd90c221ed4e60ac9d476db967f436cfcecbd4ef744537c0f2d5291439848768 \
-    --hash=sha256:c29d03e0adf3cc1a8c3ec62d176824972ae29b67a66cbb18daff3062acc6faa8 \
-    --hash=sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b \
-    --hash=sha256:c7db3db284a0edaebe87f8f6642c2b2c27ed85c3e70064b84d1c9e4ec06d5d84 \
-    --hash=sha256:ce119fc4ce0d64124d37f66a6f2a584fddc3c5001755f8a49f1ca0a177ef9796 \
-    --hash=sha256:dbcdf987f3aceef9763b6d7b1fd3e4ee210ddd26cac421d78b3c206d07b2700b \
-    --hash=sha256:e54ef33ea80d464c3dcfe881eb00ad5921b60f8115ea1a30d781653edc2fd6a2 \
-    --hash=sha256:e60ed6ef0a35c6b76b7640fe452d0e47acc832ccbb8475de549a5cc5f90c2c06 \
-    --hash=sha256:fb1b30f31a36c7f3fee848391ff77eebdd3af5750bf95fbf9b8b5323edfdb4ec \
-    --hash=sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580
-    # via safir
+sqlalchemy==2.0.35 \
+    --hash=sha256:e11d7ea4d24f0a262bccf9a7cd6284c976c5369dac21db237cff59586045ab9f
+    # via
+    #   alembic
+    #   safir
 starlette==0.38.5 \
     --hash=sha256:04a92830a9b6eb1442c766199d62260c3d4dc9c4f9188360626b1e0273cb7077 \
     --hash=sha256:632f420a9d13e3ee2a6f18f437b0a9f1faecb0bc42e1942aa2ea0e379a4c4206
@@ -1207,6 +1173,7 @@ typing-extensions==4.12.2 \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
     # via
+    #   alembic
     #   fastapi
     #   pydantic
     #   pydantic-core