DM-44606: Convert to arq

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.6
+    rev: v0.4.8
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -1,7 +1,6 @@
 # Change log
 
-vo-cutouts is versioned with [semver](https://semver.org/).
-Dependencies are updated to the latest available version during each release, and aren't noted here.
+vo-cutouts is versioned with [semver](https://semver.org/). Dependencies are updated to the latest available version during each release, and aren't noted here.
 
 Find changes for the upcoming release in the project's [changelog.d directory](https://github.com/lsst-sqre/vo-cutouts/tree/main/changelog.d/).
 

Dockerfile.worker

@@ -1,8 +1,8 @@
-# This Dockerfile constructs the image for cutout workers.  These images
+# This Dockerfile constructs the image for cutout workers. These images
 # are based on stack containers and install any required supporting code
-# for the image cutout backend, Dramatiq, and the backend worker definition.
+# for the image cutout backend, arq, and the backend worker definition.
 
-FROM lsstsqre/centos:7-stack-lsst_distrib-w_2024_15
+FROM lsstsqre/centos:7-stack-lsst_distrib-w_2024_16
 
 # Reset the user to root since we need to do system install tasks.
 USER root
@@ -11,12 +11,9 @@ USER root
 COPY scripts/install-worker-packages.sh .
 RUN ./install-worker-packages.sh && rm ./install-worker-packages.sh
 
-# Install the necessary prerequisites.
-COPY scripts/install-worker.sh .
-RUN ./install-worker.sh && rm ./install-worker.sh
-
-# Install the worker code.
-COPY src/vocutouts/workers.py /
+# Install the necessary prerequisites and the vo-cutouts code.
+COPY . /workdir
+RUN /workdir/scripts/install-worker.sh /workdir && rm -r /workdir
 COPY scripts/start-worker.sh /
 
 # Create a non-root user
@@ -25,6 +22,6 @@ RUN useradd --create-home appuser
 # Switch to the non-root user.
 USER appuser
 
-# Start the Dramatiq worker.
+# Start the arq worker.
 WORKDIR /
 CMD ["/start-worker.sh"]

README.md

@@ -3,6 +3,8 @@
 vo-cutouts is a [FastAPI](https://fastapi.tiangolo.com/) web service and associated backend worker implementation that implements an image cutout service for the Rubin Science Platform.
 The underlying work of generating the cutout is delegated to [lsst.image_cutout_backend](https://github.com/lsst-dm/image_cutout_backend/).
 
+See [CHANGELOG.md](https://github.com/lsst-sqre/vo-cutouts/blob/main/CHANGELOG.md) for the change history of vo-cutouts.
+
 This is the concrete implementation of the architecture in [DMTN-139](https://dmtn-139.lsst.io/) (not yet published) and the design in [DMTN-208](https://dmtn-208.lsst.io/).
 
 vo-cutouts implements the image cutout portion of the IVOA [SODA](https://ivoa.net/documents/SODA/20170517/REC-SODA-1.0.html) specification.

changelog.d/20240606_103703_rra_DM_44606.md

@@ -0,0 +1,12 @@
+### Backwards-incompatible changes
+
+- Change the job queuing system from Dramatiq to [arq](https://arq-docs.helpmanual.io/). This change should be transparent to users when creating new jobs, but any in-progress jobs at the time of the upgrade will be orphaned.
+- Use workload identity for all authentication when deployed on Google Cloud. Separate service account keys are no longer required or used. The `vo-cutouts` Google service account now requires the `storage.legacyBucketWriter` role in addition to `storage.objectViewer`.
+
+### New features
+
+- Add support for `gs` storage URLs in addition to `s3` storage URLs. When a `gs` storage URL is used, the image cutout backend will use the Google Cloud Storage Python API to store the results instead of boto, which will work correctly with workload identity.
+
+### Bug fixes
+
+- Queuing a job for execution in the frontend is now async and will not block the event loop, which may help with performance under load.

pyproject.toml

@@ -16,7 +16,10 @@ classifiers = [
     "Operating System :: POSIX",
     "Typing :: Typed",
 ]
-requires-python = ">=3.12"
+# vo-cutouts really requires Python 3.12, but the backend worker runs on
+# Python 3.11 currently (2024-06-05) due to the stack container and we need to
+# be able to install vo-cutouts there so that we can use supporting code.
+requires-python = ">=3.11"
 # Use requirements/main.in for runtime dependencies instead.
 dependencies = []
 dynamic = ["version"]
@@ -181,7 +184,7 @@ select = ["ALL"]
 "src/vocutouts/handlers/**" = [
     "D103",    # FastAPI handlers should not have docstrings
 ]
-"src/vocutouts/workers.py" = [
+"src/vocutouts/workers/cutout.py" = [
     "S108",    # use of /tmp is safe in this context
 ]
 "tests/**" = [

requirements/dev.txt

@@ -12,7 +12,7 @@ cachetools==5.3.3
     # via
     #   -c requirements/main.txt
     #   tox
-certifi==2024.2.2
+certifi==2024.6.2
     # via
     #   -c requirements/main.txt
     #   httpcore
@@ -93,7 +93,7 @@ mypy==1.10.0
     #   sqlalchemy
 mypy-extensions==1.0.0
     # via mypy
-nodeenv==1.9.0
+nodeenv==1.9.1
     # via pre-commit
 packaging==24.0
     # via
@@ -114,7 +114,7 @@ pre-commit==3.7.1
     # via -r requirements/dev.in
 pyproject-api==1.6.1
     # via tox
-pytest==8.2.1
+pytest==8.2.2
     # via
     #   -r requirements/dev.in
     #   pytest-asyncio
@@ -146,13 +146,13 @@ sqlalchemy==2.0.30
     # via
     #   -c requirements/main.txt
     #   -r requirements/dev.in
-tox==4.15.0
+tox==4.15.1
     # via
     #   -r requirements/dev.in
     #   tox-uv
 tox-uv==1.9.0
     # via -r requirements/dev.in
-typing-extensions==4.12.0
+typing-extensions==4.12.1
     # via
     #   -c requirements/main.txt
     #   mypy
@@ -161,7 +161,7 @@ urllib3==2.2.1
     # via
     #   -c requirements/main.txt
     #   requests
-uv==0.2.5
+uv==0.2.8
     # via tox-uv
 virtualenv==20.26.2
     # via

requirements/main.in

@@ -12,20 +12,20 @@ starlette
 uvicorn[standard]
 
 # Other dependencies.
+arq
 astropy
 asyncpg
 click
-dramatiq[redis]
 google-auth
 google-cloud-storage
 jinja2
 psycopg2
 pydantic
 pydantic-settings
-#safir[db,gcs]>=5.2.1
+#safir[arq,db,gcs]>=5.2.1
 sqlalchemy[asyncio]
 structlog
 
 # Uncomment this, change the branch, comment out safir above, and run make
 # update-deps-no-hashes to test against an unreleased version of Safir.
-safir[db,gcs] @ git+https://github.com/lsst-sqre/safir@main
+safir[arq,db,gcs] @ git+https://github.com/lsst-sqre/safir@main

requirements/main.txt

@@ -7,17 +7,21 @@ anyio==4.4.0
     #   httpx
     #   starlette
     #   watchfiles
+arq==0.26.0
+    # via
+    #   -r requirements/main.in
+    #   safir
 astropy==6.1.0
     # via -r requirements/main.in
-astropy-iers-data==0.2024.5.27.0.30.8
+astropy-iers-data==0.2024.6.3.0.31.14
     # via astropy
 asyncpg==0.29.0
     # via
     #   -r requirements/main.in
     #   safir
 cachetools==5.3.3
     # via google-auth
-certifi==2024.2.2
+certifi==2024.6.2
     # via
     #   httpcore
     #   httpx
@@ -29,17 +33,16 @@ charset-normalizer==3.3.2
 click==8.1.7
     # via
     #   -r requirements/main.in
+    #   arq
     #   safir
     #   typer
     #   uvicorn
-cryptography==42.0.7
+cryptography==42.0.8
     # via
     #   pyjwt
     #   safir
 dnspython==2.6.1
     # via email-validator
-dramatiq==1.17.0
-    # via -r requirements/main.in
 email-validator==2.1.1
     # via fastapi
 fastapi==0.111.0
@@ -73,14 +76,16 @@ google-crc32c==1.5.0
     #   google-resumable-media
 google-resumable-media==2.7.0
     # via google-cloud-storage
-googleapis-common-protos==1.63.0
+googleapis-common-protos==1.63.1
     # via google-api-core
 greenlet==3.0.3
     # via sqlalchemy
 h11==0.14.0
     # via
     #   httpcore
     #   uvicorn
+hiredis==2.3.2
+    # via redis
 httpcore==1.0.5
     # via httpx
 httptools==0.6.1
@@ -113,8 +118,6 @@ orjson==3.10.3
     # via fastapi
 packaging==24.0
     # via astropy
-prometheus-client==0.20.0
-    # via dramatiq
 proto-plus==1.23.0
     # via google-api-core
 protobuf==4.25.3
@@ -132,15 +135,15 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.7.2
+pydantic==2.7.3
     # via
     #   -r requirements/main.in
     #   fastapi
     #   pydantic-settings
     #   safir
-pydantic-core==2.18.3
+pydantic-core==2.18.4
     # via pydantic
-pydantic-settings==2.2.1
+pydantic-settings==2.3.1
     # via -r requirements/main.in
 pyerfa==2.0.1.4
     # via astropy
@@ -160,8 +163,8 @@ pyyaml==6.0.1
     # via
     #   astropy
     #   uvicorn
-redis==5.0.4
-    # via dramatiq
+redis==4.6.0
+    # via arq
 requests==2.32.3
     # via
     #   google-api-core
@@ -170,7 +173,7 @@ rich==13.7.1
     # via typer
 rsa==4.9
     # via google-auth
-safir @ git+https://github.com/lsst-sqre/safir@f98d329edc0f8578c197167a8be8d581c0d2c2c4
+safir @ git+https://github.com/lsst-sqre/safir@9ce8b39d810df41cf028afe1c8befea5613951a8
     # via -r requirements/main.in
 shellingham==1.5.4
     # via typer
@@ -193,7 +196,7 @@ structlog==24.2.0
     #   safir
 typer==0.12.3
     # via fastapi-cli
-typing-extensions==4.12.0
+typing-extensions==4.12.1
     # via
     #   fastapi
     #   pydantic
@@ -206,7 +209,7 @@ uritemplate==4.1.1
     # via gidgethub
 urllib3==2.2.1
     # via requests
-uvicorn==0.30.0
+uvicorn==0.30.1
     # via
     #   -r requirements/main.in
     #   fastapi

requirements/tox.txt

@@ -5,7 +5,7 @@ cachetools==5.3.3
     #   -c requirements/dev.txt
     #   -c requirements/main.txt
     #   tox
-certifi==2024.2.2
+certifi==2024.6.2
     # via
     #   -c requirements/dev.txt
     #   -c requirements/main.txt
@@ -64,7 +64,7 @@ requests==2.32.3
     #   -c requirements/dev.txt
     #   -c requirements/main.txt
     #   docker
-tox==4.15.0
+tox==4.15.1
     # via
     #   -c requirements/dev.txt
     #   -r requirements/tox.in
@@ -82,7 +82,7 @@ urllib3==2.2.1
     #   -c requirements/main.txt
     #   docker
     #   requests
-uv==0.2.5
+uv==0.2.8
     # via
     #   -c requirements/dev.txt
     #   tox-uv

scripts/install-worker.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
 # This script updates and installs the necessary prerequisites for an image
-# cutout backend starting with a stack container.
+# cutout backend starting with a stack container. It takes one parameter, the
+# directory in which to do the installation.
 
 # Bash "strict mode", to help catch problems and bugs in the shell
 # script. Every bash script you write should include this. See
@@ -30,5 +31,8 @@ cd image_cutout_backend
 setup -r .
 scons install declare -t current
 
-# Install Python dependencies.
-pip install --no-cache-dir 'dramatiq[redis]' safir structlog
+# Install Python dependencies and the vo-cutouts code.
+cd "$1"
+pip install --no-cache-dir pydantic-settings google-auth google-cloud-storage
+pip install --no-cache-dir 'safir[arq,db,gcs] @ git+https://github.com/lsst-sqre/safir@main'
+pip install --no-cache-dir --no-deps .

scripts/start-worker.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 # This script is installed in the worker image and starts the cutout backend
-# worker using Dramatiq.  It must run with the stack environment configured
-# and the image cutout backend imported.
+# worker using arq. It must run with the stack environment configured and the
+# image cutout backend imported.
 
 # Bash "strict mode", to help catch problems and bugs in the shell
 # script. Every bash script you write should include this. See
@@ -15,7 +15,5 @@ source /opt/lsst/software/stack/loadLSST.bash
 setup lsst_distrib
 setup image_cutout_backend
 
-# Start Dramatiq with the worker.  Limit workers to one process (we will scale
-# horizontally in Kubernetes by adding more pods) and one thread (Butler is
-# not thread-safe when one instance is reused across multiple threads).
-dramatiq workers -Q cutout -p 1 -t 1
+# Start Dramatiq with the worker.
+arq vocutouts.workers.cutout.WorkerSettings