From 954e0995415b6530d4737ec5d7ff26cff85522d3 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Thu, 12 Sep 2024 13:55:30 -0700 Subject: [PATCH] Add Alembic support Add support for Alembic to manage the UWS schema that vo-cutouts obtains from Safir. --- .pre-commit-config.yaml | 2 +- Dockerfile | 9 +- alembic.ini | 22 ++++ alembic/README.md | 14 +++ alembic/docker-compose.yaml | 12 ++ alembic/env.py | 22 ++++ alembic/script.py.mako | 26 +++++ ...12_1902_5ab72a20365b_initial_uws_schema.py | 27 +++++ changelog.d/20240913_143738_rra_DM_46034.md | 3 + pyproject.toml | 3 +- requirements/dev.txt | 6 +- requirements/main.txt | 107 ++++++------------ requirements/tox.txt | 50 ++++---- ruff-shared.toml | 103 ++++++++++------- src/vocutouts/cli.py | 59 +++++++++- src/vocutouts/main.py | 3 +- src/vocutouts/workers/uws.py | 4 +- tests/conftest.py | 2 +- tox.ini | 26 +++++ 19 files changed, 352 insertions(+), 148 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/README.md create mode 100644 alembic/docker-compose.yaml create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/20240912_1902_5ab72a20365b_initial_uws_schema.py create mode 100644 changelog.d/20240913_143738_rra_DM_46034.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5cf7572..80c84fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.6.5 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/Dockerfile b/Dockerfile index d910a9b..54b7935 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # - Runs a non-root user. # - Sets up the entrypoint and port. -FROM python:3.12.5-slim-bookworm as base-image +FROM python:3.12.5-slim-bookworm AS base-image # Update system packages COPY scripts/install-base-packages.sh . @@ -52,6 +52,13 @@ RUN useradd --create-home appuser # Copy the virtualenv COPY --from=install-image /opt/venv /opt/venv +# Copy the Alembic configuration and migrations, and set that path as the +# working directory so that Alembic can be run with a simple entry command +# and no extra configuration. +COPY --from=install-image /workdir/alembic.ini /app/alembic.ini +COPY --from=install-image /workdir/alembic /app/alembic +WORKDIR /app + # Copy the startup script COPY scripts/start-frontend.sh /start-frontend.sh diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..c36f6e9 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,22 @@ +# Alembic configuration for vo-cutouts. +# +# This file does not retain the comments that are generated as part of the +# default template, since they will get out of date with newer versions of +# Alembic. See the Alembic documentation for details about each setting and +# for settings that are not used here. + +[alembic] +script_location = %(here)s/alembic +file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s +prepend_sys_path = . +timezone = UTC +version_path_separator = os + +[post_write_hooks] +hooks = ruff ruff_format +ruff.type = exec +ruff.executable = ruff +ruff.options = check --fix REVISION_SCRIPT_FILENAME +ruff_format.type = exec +ruff_format.executable = ruff +ruff_format.options = format REVISION_SCRIPT_FILENAME diff --git a/alembic/README.md b/alembic/README.md new file mode 100644 index 0000000..8f1b063 --- /dev/null +++ b/alembic/README.md @@ -0,0 +1,14 @@ +# vo-cutouts Alembic configuration + +This directory contains the Alembic configuration for managing the vo-cutouts UWS database. +It is installed into the vo-cutouts Docker image and is used to check whether the schema is up-to-date at startup of any vo-cutouts component. +It is also used by the Helm hook that updates the vo-cutouts UWS schema if `config.updateSchema` is enabled. + +## Generating new migrations + +For detailed instructions on how to generate a new Alembic migration, see [the Safir documentation](https://safir.lsst.io/user-guide/database/schema#create-migration). + +One of the files in this directory is here only to support creating migrations. +`docker-compose.yaml` is a [docker-compose](https://docs.docker.com/compose/) configuration file that starts a PostgreSQL instance suitable for generating schema migrations. +This file is not used at runtime. +It is used by the tox environment described in the above documentation. diff --git a/alembic/docker-compose.yaml b/alembic/docker-compose.yaml new file mode 100644 index 0000000..79c230c --- /dev/null +++ b/alembic/docker-compose.yaml @@ -0,0 +1,12 @@ +version: "3" +services: + postgresql: + image: "postgres:latest" + hostname: "postgresql" + container_name: "postgresql" + environment: + POSTGRES_PASSWORD: "INSECURE" + POSTGRES_USER: "vo-cutouts" + POSTGRES_DB: "vo-cutouts" + ports: + - "5432:5432" diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..846362d --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,22 @@ +"""Alembic migration environment.""" + +from safir.database import run_migrations_offline, run_migrations_online +from safir.logging import configure_alembic_logging, configure_logging +from safir.uws import UWSSchemaBase + +from alembic import context +from vocutouts.config import config + +# Configure structlog. +configure_logging(name="vo-cutouts", log_level=config.log_level) +configure_alembic_logging() + +# Run the migrations. +if context.is_offline_mode(): + run_migrations_offline(UWSSchemaBase.metadata, config.database_url) +else: + run_migrations_online( + UWSSchemaBase.metadata, + config.database_url, + config.database_password, + ) diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/20240912_1902_5ab72a20365b_initial_uws_schema.py b/alembic/versions/20240912_1902_5ab72a20365b_initial_uws_schema.py new file mode 100644 index 0000000..b48a09e --- /dev/null +++ b/alembic/versions/20240912_1902_5ab72a20365b_initial_uws_schema.py @@ -0,0 +1,27 @@ +"""Initial UWS schema + +Revision ID: 5ab72a20365b +Revises: +Create Date: 2024-09-12 19:02:23.855138+00:00 + +""" + +from collections.abc import Sequence + +# revision identifiers, used by Alembic. +revision: str = "5ab72a20365b" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/changelog.d/20240913_143738_rra_DM_46034.md b/changelog.d/20240913_143738_rra_DM_46034.md new file mode 100644 index 0000000..33feb3b --- /dev/null +++ b/changelog.d/20240913_143738_rra_DM_46034.md @@ -0,0 +1,3 @@ +### New features + +- Use Alembic to manage the schema of the UWS database. When upgrading to this version, set `config.updateSchema` to true in the Helm configuration for the first deployment. This release contains no schema changes, but needs to perform a migration to add the Alembic version information. The vo-cutouts components will now refuse to start if the database schema has changed and the database has not yet been migrated. diff --git a/pyproject.toml b/pyproject.toml index 35fab44..8fb1709 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,11 +26,12 @@ dependencies = [ "python-multipart", "uvicorn[standard]", # Other dependencies. + "alembic[tz]", "astropy", "click", "pydantic>2", "pydantic-settings", - "safir[uws]>=6.2.0", + "safir[db,uws]>=6.4.0", "structlog", ] dynamic = ["version"] diff --git a/requirements/dev.txt b/requirements/dev.txt index 42cc46d..49b9466 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -229,9 +229,9 @@ httpx==0.27.2 \ # -c requirements/main.txt # -r requirements/dev.in # respx -idna==3.8 \ - --hash=sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac \ - --hash=sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603 +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 # via # -c requirements/main.txt # anyio diff --git a/requirements/main.txt b/requirements/main.txt index eb0ffd3..92b3b92 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1,5 +1,11 @@ # This file was autogenerated by uv via the following command: # uv pip compile --universal --generate-hashes --output-file requirements/main.txt pyproject.toml +alembic==1.13.2 \ + --hash=sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef \ + --hash=sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953 + # via + # vo-cutouts (pyproject.toml) + # safir annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 @@ -46,9 +52,9 @@ astropy==6.1.3 \ --hash=sha256:e6bfb987d512bbd70ec0c33ac023598594d35711bdb730cfaf3d8985d338155b \ --hash=sha256:e81c1fcb54b11190ce84c4106d49af762aff76ec18d50fbb205c556f423c4791 # via vo-cutouts (pyproject.toml) -astropy-iers-data==0.2024.9.12.13.29.57 \ - --hash=sha256:9ae1d147f47f18c335984781c7119e1c12aca47b89653d609980f75c505e7708 \ - --hash=sha256:ca580347f084a9d9a2e30cbd8cd44665cef52c69e5a1c1d129abb645eb970fea +astropy-iers-data==0.2024.9.16.0.32.21 \ + --hash=sha256:2ff6fe868a623e616953a432698b05dd6adac9683d21ac780bfbb94e78f7c344 \ + --hash=sha256:adf111e1b596470c4437fa44cf767e56f6d4bc2e93068871fd0b30c73476d430 # via astropy asyncpg==0.29.0 \ --hash=sha256:0009a300cae37b8c525e5b449233d59cd9868fd35431abc470a3e364d2b85cb9 \ @@ -310,9 +316,9 @@ cryptography==43.0.1 \ # via # pyjwt # safir -fastapi==0.114.1 \ - --hash=sha256:1d7bbbeabbaae0acb0c22f0ab0b040f642d3093ca3645f8c876b6f91391861d8 \ - --hash=sha256:5d4746f6e4b7dff0b4f6b6c6d5445645285f662fe75886e99af7ee2d6b58bb3e +fastapi==0.114.2 \ + --hash=sha256:0adb148b62edb09e8c6eeefa3ea934e8f276dabc038c5a82989ea6346050c3da \ + --hash=sha256:44474a22913057b1acb973ab90f4b671ba5200482e7622816d79105dcece1ac5 # via # vo-cutouts (pyproject.toml) # safir @@ -597,9 +603,9 @@ httpx==0.27.2 \ --hash=sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0 \ --hash=sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2 # via safir -idna==3.8 \ - --hash=sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac \ - --hash=sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603 +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 # via # anyio # httpx @@ -748,6 +754,10 @@ lxml==5.3.0 \ --hash=sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945 \ --hash=sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8 # via pydantic-xml +mako==1.3.5 \ + --hash=sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a \ + --hash=sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc + # via alembic markupsafe==2.1.5 \ --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ @@ -809,7 +819,9 @@ markupsafe==2.1.5 \ --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 - # via jinja2 + # via + # jinja2 + # mako numpy==2.1.1 \ --hash=sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5 \ --hash=sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0 \ @@ -1120,17 +1132,17 @@ rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 # via google-auth -safir==6.3.0 \ - --hash=sha256:2fcd64bf37dd42eacedd6378341b2487cd06dbaf1f28403301b8d80f60a4fb56 \ - --hash=sha256:6ad7dad520d87d853628849ef95a348c55dbd0180ad3f15c1cf2f7f8fe32f915 +safir==6.4.0 \ + --hash=sha256:ba7af071eab0d198e6e15a2117028566f3f4237e02e2278e8bfc2633a7c68228 \ + --hash=sha256:f38c3f1d7d76d304984b572288826510e5c7a0e1f965b2eabdd7f3bace07c48a # via vo-cutouts (pyproject.toml) -safir-arq==6.3.0 \ - --hash=sha256:c034e34fa79a7ebb4cfe3c1cb527e4c64c2195b63f919e2b623ae9dd72ac7e1b \ - --hash=sha256:db110ce3fb0419d1d4a8d83b524ae9a09aa680d0e3f9323115cce836945b10e9 +safir-arq==6.4.0 \ + --hash=sha256:4db9a1859d42064a702f1878f3da4d972f03140fa4214a420b8168bd5226c276 \ + --hash=sha256:5017668a1c8f101010b0a96affcb0d2e84596709c9160609841ce35443a80176 # via safir -safir-logging==6.3.0 \ - --hash=sha256:491dfe85de89a3f2daa29c491a22a0551f0961444490418d91ec50c040ae16eb \ - --hash=sha256:e14754ab0bba6cfa248c3fc4cb5ca28410d97ff3965e831eab6581ed37485e79 +safir-logging==6.4.0 \ + --hash=sha256:4031a430d738b8fe5bfd29125dce6cbf4e4949879307ba4146648afa3d24cd0a \ + --hash=sha256:e2dbf0b5d9dabecd70c27bff9bf01629bf0724b05b0f0087a1fe4f45c702215f # via safir sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ @@ -1138,57 +1150,11 @@ sniffio==1.3.1 \ # via # anyio # httpx -sqlalchemy==2.0.34 \ - --hash=sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22 \ - --hash=sha256:13be2cc683b76977a700948411a94c67ad8faf542fa7da2a4b167f2244781cf3 \ - --hash=sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2 \ - --hash=sha256:173f5f122d2e1bff8fbd9f7811b7942bead1f5e9f371cdf9e670b327e6703ebd \ - --hash=sha256:196958cde924a00488e3e83ff917be3b73cd4ed8352bbc0f2989333176d1c54d \ - --hash=sha256:203d46bddeaa7982f9c3cc693e5bc93db476ab5de9d4b4640d5c99ff219bee8c \ - --hash=sha256:220574e78ad986aea8e81ac68821e47ea9202b7e44f251b7ed8c66d9ae3f4278 \ - --hash=sha256:243f92596f4fd4c8bd30ab8e8dd5965afe226363d75cab2468f2c707f64cd83b \ - --hash=sha256:24af3dc43568f3780b7e1e57c49b41d98b2d940c1fd2e62d65d3928b6f95f021 \ - --hash=sha256:25691f4adfb9d5e796fd48bf1432272f95f4bbe5f89c475a788f31232ea6afba \ - --hash=sha256:2e6965346fc1491a566e019a4a1d3dfc081ce7ac1a736536367ca305da6472a8 \ - --hash=sha256:3166dfff2d16fe9be3241ee60ece6fcb01cf8e74dd7c5e0b64f8e19fab44911b \ - --hash=sha256:413c85cd0177c23e32dee6898c67a5f49296640041d98fddb2c40888fe4daa2e \ - --hash=sha256:430093fce0efc7941d911d34f75a70084f12f6ca5c15d19595c18753edb7c33b \ - --hash=sha256:43f28005141165edd11fbbf1541c920bd29e167b8bbc1fb410d4fe2269c1667a \ - --hash=sha256:526ce723265643dbc4c7efb54f56648cc30e7abe20f387d763364b3ce7506c82 \ - --hash=sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e \ - --hash=sha256:5bc08e75ed11693ecb648b7a0a4ed80da6d10845e44be0c98c03f2f880b68ff4 \ - --hash=sha256:6831a78bbd3c40f909b3e5233f87341f12d0b34a58f14115c9e94b4cdaf726d3 \ - --hash=sha256:6a1e03db964e9d32f112bae36f0cc1dcd1988d096cfd75d6a588a3c3def9ab2b \ - --hash=sha256:6daeb8382d0df526372abd9cb795c992e18eed25ef2c43afe518c73f8cccb721 \ - --hash=sha256:6e7cde3a2221aa89247944cafb1b26616380e30c63e37ed19ff0bba5e968688d \ - --hash=sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a \ - --hash=sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f \ - --hash=sha256:79cb400c360c7c210097b147c16a9e4c14688a6402445ac848f296ade6283bbc \ - --hash=sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83 \ - --hash=sha256:80bd73ea335203b125cf1d8e50fef06be709619eb6ab9e7b891ea34b5baa2287 \ - --hash=sha256:895184dfef8708e15f7516bd930bda7e50ead069280d2ce09ba11781b630a434 \ - --hash=sha256:8fddde2368e777ea2a4891a3fb4341e910a056be0bb15303bf1b92f073b80c02 \ - --hash=sha256:95d0b2cf8791ab5fb9e3aa3d9a79a0d5d51f55b6357eecf532a120ba3b5524db \ - --hash=sha256:9661268415f450c95f72f0ac1217cc6f10256f860eed85c2ae32e75b60278ad8 \ - --hash=sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74 \ - --hash=sha256:9ea54f7300553af0a2a7235e9b85f4204e1fc21848f917a3213b0e0818de9a24 \ - --hash=sha256:9ebc11c54c6ecdd07bb4efbfa1554538982f5432dfb8456958b6d46b9f834bb7 \ - --hash=sha256:a17d8fac6df9835d8e2b4c5523666e7051d0897a93756518a1fe101c7f47f2f0 \ - --hash=sha256:ae92bebca3b1e6bd203494e5ef919a60fb6dfe4d9a47ed2453211d3bd451b9f5 \ - --hash=sha256:b68094b165a9e930aedef90725a8fcfafe9ef95370cbb54abc0464062dbf808f \ - --hash=sha256:b75b00083e7fe6621ce13cfce9d4469c4774e55e8e9d38c305b37f13cf1e874c \ - --hash=sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812 \ - --hash=sha256:bd90c221ed4e60ac9d476db967f436cfcecbd4ef744537c0f2d5291439848768 \ - --hash=sha256:c29d03e0adf3cc1a8c3ec62d176824972ae29b67a66cbb18daff3062acc6faa8 \ - --hash=sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b \ - --hash=sha256:c7db3db284a0edaebe87f8f6642c2b2c27ed85c3e70064b84d1c9e4ec06d5d84 \ - --hash=sha256:ce119fc4ce0d64124d37f66a6f2a584fddc3c5001755f8a49f1ca0a177ef9796 \ - --hash=sha256:dbcdf987f3aceef9763b6d7b1fd3e4ee210ddd26cac421d78b3c206d07b2700b \ - --hash=sha256:e54ef33ea80d464c3dcfe881eb00ad5921b60f8115ea1a30d781653edc2fd6a2 \ - --hash=sha256:e60ed6ef0a35c6b76b7640fe452d0e47acc832ccbb8475de549a5cc5f90c2c06 \ - --hash=sha256:fb1b30f31a36c7f3fee848391ff77eebdd3af5750bf95fbf9b8b5323edfdb4ec \ - --hash=sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580 - # via safir +sqlalchemy==2.0.35 \ + --hash=sha256:e11d7ea4d24f0a262bccf9a7cd6284c976c5369dac21db237cff59586045ab9f + # via + # alembic + # safir starlette==0.38.5 \ --hash=sha256:04a92830a9b6eb1442c766199d62260c3d4dc9c4f9188360626b1e0273cb7077 \ --hash=sha256:632f420a9d13e3ee2a6f18f437b0a9f1faecb0bc42e1942aa2ea0e379a4c4206 @@ -1207,6 +1173,7 @@ typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via + # alembic # fastapi # pydantic # pydantic-core diff --git a/requirements/tox.txt b/requirements/tox.txt index 6a671d7..10bbc8b 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -133,9 +133,9 @@ filelock==3.16.0 \ # via # tox # virtualenv -idna==3.8 \ - --hash=sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac \ - --hash=sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603 +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 # via # -c requirements/dev.txt # -c requirements/main.txt @@ -149,9 +149,9 @@ packaging==24.1 \ # pyproject-api # tox # tox-uv -platformdirs==4.3.2 \ - --hash=sha256:9e5e27a08aa095dd127b9f2e764d74254f482fef22b0970773bfba79d091ab8c \ - --hash=sha256:eb1c8582560b34ed4ba105009a4badf7f6f85768b30126f351328507b2beb617 +platformdirs==4.3.3 \ + --hash=sha256:50a5450e2e84f44539718293cbb1da0a0885c9d14adf21b77bae4e66fc99d9b5 \ + --hash=sha256:d4e0b7d8ec176b341fb03cb11ca12d0276faa8c485f9cd218f613840463fc2c0 # via # tox # virtualenv @@ -211,25 +211,25 @@ urllib3==2.2.3 \ # -c requirements/main.txt # docker # requests -uv==0.4.9 \ - --hash=sha256:0340d2c7bf9afe0098e3301c1885de10e317232cfa346f0ac16374cee284a4cb \ - --hash=sha256:060af185481ef46ab97008cad330f3cd7a7aa1ce3d219b67d27c5a2a551ac2ea \ - --hash=sha256:1a8acc7abb2174bd3c8f5fc98345f2bb602f31b7558e37f3d23bef99ddd58dec \ - --hash=sha256:34bce9f4892130b01a7605d27bbeb71395e9b031d793123c250b79187ee307ca \ - --hash=sha256:45bf0cead2436b1977f71669e945db19990ca70a7765111fb951545815467bb6 \ - --hash=sha256:52101bc8652b4284b78fac52ed7878f3bae414bc4076c377735962666b309dde \ - --hash=sha256:5422680436f4cebef945bb2e562e01c02a4fa0a95f85d1b8010f2ee868a0b8c1 \ - --hash=sha256:55cf2522262ef663114bda5d80375ddc7f7af0d054df89426372a0d494380875 \ - --hash=sha256:566d4d7a475aacd21dbb4aba053cd4f4f52d65acdef2c83c59bcdff08756701e \ - --hash=sha256:5b66a52cb60a2882a882bc5f13afa6daf3172a54fe9fb998529d19418d5aed18 \ - --hash=sha256:630a6fe215829f734278e618c1633c2bb88ee03dc6a92ae9890fabd98ee810a9 \ - --hash=sha256:69529b6bf5de6ec8fbe8e022f5bcbaef778e76136fc37fae6ec7a8b18b3f9024 \ - --hash=sha256:71e87038fcc9f61b2d6f66c4a92354c6d0abe4baae21bb90241693f161ddeaa1 \ - --hash=sha256:8869637ea6231f66fe643be22f9334874db3496844b3d8bfd8efd4227ded3d44 \ - --hash=sha256:9c9b70f016f28cc05633b564d8690cfdb7ebac4d2210d9158819947841e00347 \ - --hash=sha256:b54a9022e9e1fdbf3ae15ef340a0d1d1847dd739df5023896aa8d97d88af1efe \ - --hash=sha256:bf834f7f360a192372d879eda86f6a1dd94195faf68154dcf7c90247098d2bb2 \ - --hash=sha256:f50cbdfbc8399e1211c580e47f42650a184541ee398af95ad29bf9a2e977baba +uv==0.4.10 \ + --hash=sha256:0784f75093a75390d8d480cc8a444516e78f08849db9a13c21791a5f651df4a1 \ + --hash=sha256:0f8b9ba4ecfbea343a00e46d509669606e55fe233d800752c4c25650473df358 \ + --hash=sha256:1b6b6c6b8cc0c4e54ab25e3b46e49d1e583e26c194572eb42bfeebf71b39cca2 \ + --hash=sha256:1ff5130b6f3af79c4e47f63db03215aed15e78cb4f1f51682af6f9949c2bcf00 \ + --hash=sha256:2ff29a2f55a697e78d787a41ab41d4b26421d200728289b88b6241d3b486c436 \ + --hash=sha256:30d1f8348a2b18e21a35c97ce42528781f242d0303881fc92fbacdcb653c8bca \ + --hash=sha256:3be73788db9ceacb94a521cf67ca5cc08bac512aef71145b904ab62a3acabdae \ + --hash=sha256:444e1cdb36d7ef103e52185f918800527c255dc369c9f90eb1f198dfa3f4d5bc \ + --hash=sha256:6ba1cc3070e5c63ce0a1421fbed28bd1b3ff520671d7badda11a501504c78394 \ + --hash=sha256:8fa510dfbbde4f8ad5cd2769568c7b0c3e867b74deaf4beabcca79e74e7550cc \ + --hash=sha256:97a1187e11a9df70d55bc577721ad4a19441cda56e4d69fb2f38d88c7650d2a0 \ + --hash=sha256:99954a94dd6c4bff8a9a963c05bc3988214ea39e7511a52fda35112e1a478447 \ + --hash=sha256:a9dc1f8fca5c4a2f73054d9f56c7397e9fc6ba43baefc503d6f0128d72ea662f \ + --hash=sha256:b89dfd213359a23797155ff8175e5202ed6b84aadeb20df92132127608d46acf \ + --hash=sha256:bc87d6c581cfed0979e0f5ee93383d46006c6d4a5e4eb9f43ef13bce61b50cc2 \ + --hash=sha256:bc99e6b45303f0881a8dc199f0b7ea8261dd1779e576e8477a7721ceeeaafcc7 \ + --hash=sha256:e99e3f761875962942e0743b868bd666021d5e14c3df494e820ef8f45fb88578 \ + --hash=sha256:ff9046a8c5e836e892ac7741e672ee016e92e55c659fa8195595df65a1f3accf # via tox-uv virtualenv==20.26.4 \ --hash=sha256:48f2695d9809277003f30776d155615ffc11328e6a0a8c1f0ec80188d7874a55 \ diff --git a/ruff-shared.toml b/ruff-shared.toml index 823693a..aff2c6e 100644 --- a/ruff-shared.toml +++ b/ruff-shared.toml @@ -27,42 +27,45 @@ docstring-code-format = true [lint] ignore = [ - "ANN401", # sometimes Any is the right type - "ARG001", # unused function arguments are often legitimate - "ARG002", # unused method arguments are often legitimate - "ARG005", # unused lambda arguments are often legitimate - "BLE001", # we want to catch and report Exception in background tasks - "C414", # nested sorted is how you sort by multiple keys with reverse - "D102", # sometimes we use docstring inheritence - "D104", # don't see the point of documenting every package - "D105", # our style doesn't require docstrings for magic methods - "D106", # Pydantic uses a nested Config class that doesn't warrant docs - "D205", # our documentation style allows a folded first line - "EM101", # justification (duplicate string in traceback) is silly - "EM102", # justification (duplicate string in traceback) is silly - "FBT003", # positional booleans are normal for Pydantic field defaults - "FIX002", # point of a TODO comment is that we're not ready to fix it - "G004", # forbidding logging f-strings is appealing, but not our style - "RET505", # disagree that omitting else always makes code more readable - "PLR0911", # often many returns is clearer and simpler style - "PLR0913", # factory pattern uses constructors with many arguments - "PLR2004", # too aggressive about magic values - "PLW0603", # yes global is discouraged but if needed, it's needed - "S105", # good idea but too many false positives on non-passwords - "S106", # good idea but too many false positives on non-passwords - "S107", # good idea but too many false positives on non-passwords - "S603", # not going to manually mark every subprocess call as reviewed - "S607", # using PATH is not a security vulnerability - "SIM102", # sometimes the formatting of nested if statements is clearer - "SIM117", # sometimes nested with contexts are clearer - "TCH001", # we decided to not maintain separate TYPE_CHECKING blocks - "TCH002", # we decided to not maintain separate TYPE_CHECKING blocks - "TCH003", # we decided to not maintain separate TYPE_CHECKING blocks - "TD003", # we don't require issues be created for TODOs - "TID252", # if we're going to use relative imports, use them always - "TRY003", # good general advice but lint is way too aggressive - "TRY301", # sometimes raising exceptions inside try is the best flow - "UP040", # PEP 695 type aliases not yet supported by mypy + "ANN401", # sometimes Any is the right type + "ARG001", # unused function arguments are often legitimate + "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate + "ARG005", # unused lambda arguments are often legitimate + "ASYNC109", # many async functions use asyncio.timeout internally + "BLE001", # we want to catch and report Exception in background tasks + "C414", # nested sorted is how you sort by multiple keys with reverse + "D102", # sometimes we use docstring inheritence + "D104", # don't see the point of documenting every package + "D105", # our style doesn't require docstrings for magic methods + "D106", # Pydantic uses a nested Config class that doesn't warrant docs + "D205", # our documentation style allows a folded first line + "EM101", # justification (duplicate string in traceback) is silly + "EM102", # justification (duplicate string in traceback) is silly + "FBT003", # positional booleans are normal for Pydantic field defaults + "FIX002", # point of a TODO comment is that we're not ready to fix it + "PD011", # attempts to enforce pandas conventions for all data types + "G004", # forbidding logging f-strings is appealing, but not our style + "RET505", # disagree that omitting else always makes code more readable + "PLR0911", # often many returns is clearer and simpler style + "PLR0913", # factory pattern uses constructors with many arguments + "PLR2004", # too aggressive about magic values + "PLW0603", # yes global is discouraged but if needed, it's needed + "S105", # good idea but too many false positives on non-passwords + "S106", # good idea but too many false positives on non-passwords + "S107", # good idea but too many false positives on non-passwords + "S603", # not going to manually mark every subprocess call as reviewed + "S607", # using PATH is not a security vulnerability + "SIM102", # sometimes the formatting of nested if statements is clearer + "SIM117", # sometimes nested with contexts are clearer + "TCH001", # we decided to not maintain separate TYPE_CHECKING blocks + "TCH002", # we decided to not maintain separate TYPE_CHECKING blocks + "TCH003", # we decided to not maintain separate TYPE_CHECKING blocks + "TD003", # we don't require issues be created for TODOs + "TID252", # if we're going to use relative imports, use them always + "TRY003", # good general advice but lint is way too aggressive + "TRY301", # sometimes raising exceptions inside try is the best flow + "UP040", # PEP 695 type aliases not yet supported by mypy # The following settings should be disabled when using ruff format # per https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules @@ -80,16 +83,24 @@ ignore = [ "COM819", "ISC001", "ISC002", - - # Temporary bug workarounds. - "S113", # https://github.com/astral-sh/ruff/issues/12210 ] select = ["ALL"] [lint.per-file-ignores] +"alembic/**" = [ + "INP001", # Alembic files are magical + "D103", # Alembic methods do not need docstrings + "D400", # Alembic migrations have their own docstring format +] +"noxfile.py" = [ + "T201", # print makes sense as output from nox rules +] "src/*/handlers/**" = [ "D103", # FastAPI handlers should not have docstrings ] +"*/src/*/handlers/**" = [ + "D103", # FastAPI handlers should not have docstrings +] "tests/**" = [ "C901", # tests are allowed to be complex, sometimes that's convenient "D101", # tests don't need docstrings @@ -101,6 +112,17 @@ select = ["ALL"] "S301", # allow tests for whether code can be pickled "SLF001", # tests are allowed to access private members ] +"*/tests/**" = [ + "C901", # tests are allowed to be complex, sometimes that's convenient + "D101", # tests don't need docstrings + "D103", # tests don't need docstrings + "PLR0915", # tests are allowed to be long, sometimes that's convenient + "PT012", # way too aggressive about limiting pytest.raises blocks + "S101", # tests should use assert + "S106", # tests are allowed to hard-code dummy passwords + "S301", # allow tests for whether code can be pickled + "SLF001", # tests are allowed to access private members +] # These are too useful as attributes or methods to allow the conflict with the # built-in to rule out their use. @@ -118,8 +140,5 @@ builtins-ignorelist = [ fixture-parentheses = false mark-parentheses = false -[lint.mccabe] -max-complexity = 11 - [lint.pydocstyle] convention = "numpy" diff --git a/src/vocutouts/cli.py b/src/vocutouts/cli.py index ed09eac..c3f5cec 100644 --- a/src/vocutouts/cli.py +++ b/src/vocutouts/cli.py @@ -2,6 +2,9 @@ from __future__ import annotations +import subprocess +from pathlib import Path + import click import structlog from safir.asyncio import run_with_asyncio @@ -31,11 +34,63 @@ def help(ctx: click.Context, topic: str | None) -> None: @main.command() +@click.option( + "--alembic/--no-alembic", + default=True, + help="Mark the database with the current Alembic version.", +) +@click.option( + "--alembic-config-path", + envvar="CUTOUT_ALEMBIC_CONFIG_PATH", + type=click.Path(path_type=Path), + default=Path("/app/alembic.ini"), + help="Alembic configuration file.", +) @click.option( "--reset", is_flag=True, help="Delete all existing database data." ) @run_with_asyncio -async def init(*, reset: bool) -> None: +async def init( + *, alembic: bool, alembic_config_path: Path, reset: bool +) -> None: """Initialize the database storage.""" logger = structlog.get_logger("vocutouts") - await uws.initialize_uws_database(logger, reset=reset) + await uws.initialize_uws_database( + logger, + reset=reset, + use_alembic=alembic, + alembic_config_path=alembic_config_path, + ) + + +@main.command() +@click.option( + "--alembic-config-path", + envvar="CUTOUT_ALEMBIC_CONFIG_PATH", + type=click.Path(path_type=Path), + default=Path("/app/alembic.ini"), + help="Alembic configuration file.", +) +def update_schema(*, alembic_config_path: Path) -> None: + """Update the schema.""" + subprocess.run( + ["alembic", "upgrade", "head"], + check=True, + cwd=str(alembic_config_path.parent), + ) + + +@main.command() +@click.option( + "--alembic-config-path", + envvar="CUTOUT_ALEMBIC_CONFIG_PATH", + type=click.Path(path_type=Path), + default=Path("/app/alembic.ini"), + help="Alembic configuration file.", +) +@run_with_asyncio +async def validate_schema(*, alembic_config_path: Path) -> None: + """Validate that the database schema is current.""" + logger = structlog.get_logger("vocutouts") + if not uws.is_schema_current(logger, alembic_config_path): + raise click.ClickException("Database schema is not current") diff --git a/src/vocutouts/main.py b/src/vocutouts/main.py index a3d76ee..03f13b2 100644 --- a/src/vocutouts/main.py +++ b/src/vocutouts/main.py @@ -27,7 +27,8 @@ @asynccontextmanager async def _lifespan(app: FastAPI) -> AsyncIterator[None]: """Set up and tear down the application.""" - await uws.initialize_fastapi() + logger = structlog.get_logger("vocutouts") + await uws.initialize_fastapi(logger, check_schema=True) yield await uws.shutdown_fastapi() diff --git a/src/vocutouts/workers/uws.py b/src/vocutouts/workers/uws.py index 7047ae3..a0a19ca 100644 --- a/src/vocutouts/workers/uws.py +++ b/src/vocutouts/workers/uws.py @@ -14,5 +14,7 @@ name="vocutouts", profile=config.profile, log_level=config.log_level ) -WorkerSettings = uws.build_worker(structlog.get_logger("vocutouts")) +WorkerSettings = uws.build_worker( + structlog.get_logger("vocutouts"), check_schema=True +) """arq configuration for the UWS database worker.""" diff --git a/tests/conftest.py b/tests/conftest.py index c2d0799..e0d8a96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,7 @@ async def app(arq_queue: MockArqQueue) -> AsyncIterator[FastAPI]: dropped from a persistent database between test cases. """ logger = structlog.get_logger("vocutouts") - await uws.initialize_uws_database(logger, reset=True) + await uws.initialize_uws_database(logger, reset=True, use_alembic=True) async with LifespanManager(main.app): # Ensure that all the components use the same mock arq queue. # Otherwise, the web application will use the one created in its diff --git a/tox.ini b/tox.ini index 7ef66d1..1be0b5e 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,20 @@ deps = -r{toxinidir}/requirements/main.txt -r{toxinidir}/requirements/dev.txt +[testenv:alembic] +description = Run Alembic against a test database +commands = + alembic {posargs} +deps = + -r{toxinidir}/requirements/main.txt + ruff +setenv = + CUTOUT_ARQ_QUEUE_URL = redis://localhost/0 + CUTOUT_DATABASE_URL = postgresql://vo-cutouts@localhost/vo-cutouts + CUTOUT_DATABASE_PASSWORD = INSECURE + CUTOUT_SERVICE_ACCOUNT = vo-cutouts@example.com + CUTOUT_STORAGE_URL = s3://some-bucket + [testenv:coverage-report] description = Compile coverage from each test run. skip_install = true @@ -71,3 +85,15 @@ commands = pre-commit run --all-files description = Run mypy. commands = mypy src/vocutouts tests + +[testenv:vo-cutouts] +description = Run command-line tool against a test database +commands = + vo-cutouts {posargs} +setenv = + CUTOUT_ALEMBIC_CONFIG_PATH = {toxinidir}/alembic.ini + CUTOUT_ARQ_QUEUE_URL = redis://localhost/0 + CUTOUT_DATABASE_URL = postgresql://vo-cutouts@localhost/vo-cutouts + CUTOUT_DATABASE_PASSWORD = INSECURE + CUTOUT_SERVICE_ACCOUNT = vo-cutouts@example.com + CUTOUT_STORAGE_URL = s3://some-bucket