Added celery

.gitignore

@@ -62,7 +62,8 @@ target/
 # pyenv
 .python-version
 
-
+# celery beat schedule file
+celerybeat-schedule
 
 # Environments
 .venv

Procfile

@@ -1,2 +1,4 @@
 release: python manage.py migrate
 web: gunicorn config.wsgi:application
+worker: REMAP_SIGTERM=SIGQUIT celery -A config.celery_app worker --loglevel=info
+beat: REMAP_SIGTERM=SIGQUIT celery -A config.celery_app beat --loglevel=info

README.md

@@ -45,6 +45,33 @@ To run the tests, check your test coverage, and generate an HTML coverage report
 
 Moved to [Live reloading and SASS compilation](https://cookiecutter-django.readthedocs.io/en/latest/developing-locally.html#sass-compilation-live-reloading).
 
+### Celery
+
+This app comes with Celery.
+
+To run a celery worker:
+
+```bash
+cd flamerelay
+celery -A config.celery_app worker -l info
+```
+
+Please note: For Celery's import magic to work, it is important _where_ the celery commands are run. If you are in the same folder with _manage.py_, you should be right.
+
+To run [periodic tasks](https://docs.celeryq.dev/en/stable/userguide/periodic-tasks.html), you'll need to start the celery beat scheduler service. You can start it as a standalone process:
+
+```bash
+cd flamerelay
+celery -A config.celery_app beat
+```
+
+or you can embed the beat service inside a worker with the `-B` option (not recommended for production use):
+
+```bash
+cd flamerelay
+celery -A config.celery_app worker -B -l info
+```
+
 ### Email Server
 
 In development, it is often nice to be able to see emails that are being sent from your application. For that reason local SMTP server [MailHog](https://github.com/mailhog/MailHog) with a web interface is available as docker container.

compose/local/django/Dockerfile

@@ -60,6 +60,18 @@ RUN sed -i 's/\r$//g' /start
 RUN chmod +x /start
 
 
+COPY ./compose/local/django/celery/worker/start /start-celeryworker
+RUN sed -i 's/\r$//g' /start-celeryworker
+RUN chmod +x /start-celeryworker
+
+COPY ./compose/local/django/celery/beat/start /start-celerybeat
+RUN sed -i 's/\r$//g' /start-celerybeat
+RUN chmod +x /start-celerybeat
+
+COPY ./compose/local/django/celery/flower/start /start-flower
+RUN sed -i 's/\r$//g' /start-flower
+RUN chmod +x /start-flower
+
 
 # copy application code to WORKDIR
 COPY . ${APP_HOME}

compose/local/django/celery/beat/start

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+
+rm -f './celerybeat.pid'
+exec watchfiles celery.__main__.main --args '-A config.celery_app beat -l INFO'

compose/local/django/celery/flower/start

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+exec watchfiles celery.__main__.main \
+    --args \
+    "-A config.celery_app -b \"${CELERY_BROKER_URL}\" flower --basic_auth=\"${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}\""

compose/local/django/celery/worker/start

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+
+exec watchfiles celery.__main__.main --args '-A config.celery_app worker -l INFO'

compose/production/django/Dockerfile

@@ -65,6 +65,19 @@ RUN chmod +x /entrypoint
 COPY --chown=django:django ./compose/production/django/start /start
 RUN sed -i 's/\r$//g' /start
 RUN chmod +x /start
+COPY --chown=django:django ./compose/production/django/celery/worker/start /start-celeryworker
+RUN sed -i 's/\r$//g' /start-celeryworker
+RUN chmod +x /start-celeryworker
+
+
+COPY --chown=django:django ./compose/production/django/celery/beat/start /start-celerybeat
+RUN sed -i 's/\r$//g' /start-celerybeat
+RUN chmod +x /start-celerybeat
+
+
+COPY ./compose/production/django/celery/flower/start /start-flower
+RUN sed -i 's/\r$//g' /start-flower
+RUN chmod +x /start-flower
 
 
 # copy application code to WORKDIR

compose/production/django/celery/beat/start

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+exec celery -A config.celery_app beat -l INFO

compose/production/django/celery/flower/start

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+
+exec celery \
+    -A config.celery_app \
+    -b "${CELERY_BROKER_URL}" \
+    flower \
+    --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}"

compose/production/django/celery/worker/start

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+exec celery -A config.celery_app worker -l INFO

compose/production/django/entrypoint

@@ -6,6 +6,9 @@ set -o nounset
 
 
 
+# N.B. If only .env files supported variable expansion...
+export CELERY_BROKER_URL="${REDIS_URL}"
+
 
 if [ -z "${POSTGRES_USER}" ]; then
     base_postgres_image_default_user='postgres'

compose/production/traefik/Dockerfile

@@ -1,4 +1,4 @@
-FROM traefik:2.10.0
+FROM traefik:2.10.1
 RUN mkdir -p /etc/traefik/acme \
   && touch /etc/traefik/acme/acme.json \
   && chmod 600 /etc/traefik/acme/acme.json

compose/production/traefik/traefik.yml

@@ -15,11 +15,14 @@ entryPoints:
     # https
     address: ':443'
 
+  flower:
+    address: ':5555'
+
 certificatesResolvers:
   letsencrypt:
     # https://docs.traefik.io/master/https/acme/#lets-encrypt
     acme:
-      email: 'jvacek@pm.me'
+      email: 'daniel-roy-greenfeld@example.com'
       storage: /etc/traefik/acme/acme.json
       # https://docs.traefik.io/master/https/acme/#httpchallenge
       httpChallenge:
@@ -28,7 +31,7 @@ certificatesResolvers:
 http:
   routers:
     web-secure-router:
-      rule: 'Host(`flamerelay.org`) || Host(`www.flamerelay.org`)'
+      rule: 'Host(`example.com`) || Host(`www.example.com`)'
       entryPoints:
         - web-secure
       middlewares:
@@ -38,8 +41,17 @@ http:
         # https://docs.traefik.io/master/routing/routers/#certresolver
         certResolver: letsencrypt
 
+    flower-secure-router:
+      rule: 'Host(`example.com`)'
+      entryPoints:
+        - flower
+      service: flower
+      tls:
+        # https://docs.traefik.io/master/routing/routers/#certresolver
+        certResolver: letsencrypt
+
     web-media-router:
-      rule: '(Host(`flamerelay.org`) || Host(`www.flamerelay.org`)) && PathPrefix(`/media/`)'
+      rule: '(Host(`example.com`) || Host(`www.example.com`)) && PathPrefix(`/media/`)'
       entryPoints:
         - web-secure
       middlewares:
@@ -61,6 +73,11 @@ http:
         servers:
           - url: http://django:5000
 
+    flower:
+      loadBalancer:
+        servers:
+          - url: http://flower:5555
+
     django-media:
       loadBalancer:
         servers:

config/__init__.py

@@ -0,0 +1,5 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery_app import app as celery_app
+
+__all__ = ("celery_app",)

config/celery_app.py

@@ -0,0 +1,17 @@
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
+
+app = Celery("flamerelay")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()

config/settings/base.py

@@ -69,6 +69,7 @@
     "allauth",
     "allauth.account",
     "allauth.socialaccount",
+    "django_celery_beat",
     "rest_framework",
     "rest_framework.authtoken",
     "corsheaders",
@@ -252,7 +253,40 @@
     "root": {"level": "INFO", "handlers": ["console"]},
 }
 
-
+# Celery
+# ------------------------------------------------------------------------------
+if USE_TZ:
+    # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-timezone
+    CELERY_TIMEZONE = TIME_ZONE
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-broker_url
+CELERY_BROKER_URL = env("CELERY_BROKER_URL")
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_backend
+CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-extended
+CELERY_RESULT_EXTENDED = True
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-backend-always-retry
+# https://github.com/celery/celery/pull/6122
+CELERY_RESULT_BACKEND_ALWAYS_RETRY = True
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-backend-max-retries
+CELERY_RESULT_BACKEND_MAX_RETRIES = 10
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-accept_content
+CELERY_ACCEPT_CONTENT = ["json"]
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-task_serializer
+CELERY_TASK_SERIALIZER = "json"
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_serializer
+CELERY_RESULT_SERIALIZER = "json"
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-time-limit
+# TODO: set to whatever value is adequate in your circumstances
+CELERY_TASK_TIME_LIMIT = 5 * 60
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-soft-time-limit
+# TODO: set to whatever value is adequate in your circumstances
+CELERY_TASK_SOFT_TIME_LIMIT = 60
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#beat-scheduler
+CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#worker-send-task-events
+CELERY_WORKER_SEND_TASK_EVENTS = True
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_send_sent_event
+CELERY_TASK_SEND_SENT_EVENT = True
 # django-allauth
 # ------------------------------------------------------------------------------
 ACCOUNT_ALLOW_REGISTRATION = env.bool("DJANGO_ACCOUNT_ALLOW_REGISTRATION", True)

config/settings/local.py

@@ -62,6 +62,10 @@
     "django_extensions",
     "django_fastdev",
 ]  # noqa: F405
+# Celery
+# ------------------------------------------------------------------------------
 
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates
+CELERY_TASK_EAGER_PROPAGATES = True
 # Your stuff...
 # ------------------------------------------------------------------------------

config/settings/production.py

@@ -1,6 +1,7 @@
 import logging
 
 import sentry_sdk
+from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
@@ -142,7 +143,12 @@
     level=SENTRY_LOG_LEVEL,  # Capture info and above as breadcrumbs
     event_level=logging.ERROR,  # Send errors as events
 )
-integrations = [sentry_logging, DjangoIntegration(), RedisIntegration()]
+integrations = [
+    sentry_logging,
+    DjangoIntegration(),
+    CeleryIntegration(),
+    RedisIntegration(),
+]
 sentry_sdk.init(
     dsn=SENTRY_DSN,
     integrations=integrations,

flamerelay/users/tasks.py

@@ -0,0 +1,11 @@
+from django.contrib.auth import get_user_model
+
+from config import celery_app
+
+User = get_user_model()
+
+
+@celery_app.task()
+def get_users_count():
+    """A pointless Celery task to demonstrate usage."""
+    return User.objects.count()

flamerelay/users/tests/test_tasks.py

@@ -0,0 +1,16 @@
+import pytest
+from celery.result import EagerResult
+
+from flamerelay.users.tasks import get_users_count
+from flamerelay.users.tests.factories import UserFactory
+
+pytestmark = pytest.mark.django_db
+
+
+def test_user_count(settings):
+    """A basic test to execute the get_users_count Celery task."""
+    UserFactory.create_batch(3)
+    settings.CELERY_TASK_ALWAYS_EAGER = True
+    task_result = get_users_count.delay()
+    assert isinstance(task_result, EagerResult)
+    assert task_result.result == 3