Use celery for preview tasks

.env.ci

@@ -18,6 +18,8 @@ MINIO_ACCESS_KEY=rainfall-ci-minio-access-key
 MINIO_SECRET_KEY=rainfall-ci-minio-secret-key
 MINIO_BUCKET=rainfall-test
 
+REDIS_URL=redis://localhost:6379
+
 RAINFALL_ENV=test
 MASTODON_REDIRECT_URL=http://localhost:5000/api/v1/mastodon/login
 MASTODON_APP_NAME='Rainfall Dev'

.github/workflows/ci.yml

@@ -21,6 +21,11 @@ jobs:
           --health-timeout 5s
           --health-retries 3
 
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python

Dockerfile

@@ -56,4 +56,4 @@ RUN pipenv install --deploy --ignore-pipfile
 COPY . .
 COPY --from=build /usr/src/app/frontend ./rainfall-frontend
 
-CMD ["pipenv", "run", "gunicorn", "--error-logfile", "-", "--access-logfile", "-", "-b", "0.0.0.0", "rainfall.main:create_app()"]
+ENTRYPOINT ["./entrypoint.sh"]

Pipfile

@@ -19,10 +19,14 @@ alembic = "*"
 authlib = "*"
 pymysql = "*"
 minio = "*"
+celery = {extras = ["redis"], version = "*"}
 
 [dev-packages]
 pytest = "*"
 pytest-cov = "*"
 
 [requires]
 python_version = "3.12"
+
+[scripts]
+worker = "pipenv run python -m celery -A rainfall.main worker -l info"

alembic/versions/20a525107a88_add_task_id_to_sites.py

@@ -0,0 +1,34 @@
+"""Add task id to sites
+
+Revision ID: 20a525107a88
+Revises: 3b83d643e6cb
+Create Date: 2025-01-06 14:57:01.760943
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '20a525107a88'
+down_revision: Union[str, None] = '3b83d643e6cb'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('sites', sa.Column('preview_task_id', sa.String(length=255), nullable=True))
+    op.drop_index('fk_sites_sites_users_id', table_name='sites')
+    op.create_foreign_key(None, 'sites', 'users', ['user_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'sites', type_='foreignkey')
+    op.create_index('fk_sites_sites_users_id', 'sites', ['user_id'], unique=False)
+    op.drop_column('sites', 'preview_task_id')
+    # ### end Alembic commands ###

docker-compose-dev.yml

@@ -22,6 +22,12 @@ services:
       MINIO_ROOT_PASSWORD: rainfall-minio
     command: server --console-address :9001 /data
 
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"
+
+
   # Used for formatting the volume as ext4. Otherwise, some minio operations may fail.
   # Uncomment to format (with docker compose up) and comment afterwards so the volume
   # is not formatted again.

entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [[ $# -gt 0 ]]; then
+    # If we pass a command, run it (for celery workers)
+    exec "$@"
+else
+    # Else default to starting the server
+    exec "pipenv run gunicorn --error-logfile - --access-logfile - -b 0.0.0.0 rainfall.main:create_app()"
+fi

fly.toml

@@ -8,9 +8,14 @@ primary_region = "sjc"
 
 [build]
 
+[processes]
+  app = "pipenv run gunicorn --error-logfile - --access-logfile - -b 0.0.0.0 rainfall.main:create_app()"
+  worker = "pipenv run python -m celery -A rainfall.tasks worker -l info"
+
 [mounts]
   source="rainfall_data"
   destination="/data"
+  processes = ["worker"]
 
 [http_service]
   internal_port = 8000