use task processor with server

Flagsmith · Feb 26, 2025 · 121128e · 121128e
1 parent 5521b45
commit 121128e
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 60 deletions.
diff --git a/api/app/settings/common.py b/api/app/settings/common.py
@@ -986,14 +986,17 @@
 # Used to control the size(number of identities) of the project that can be self migrated to edge
 MAX_SELF_MIGRATABLE_IDENTITIES = env.int("MAX_SELF_MIGRATABLE_IDENTITIES", 100000)
 
+# RUN_BY_PROCESSOR is set by the task processor entrypoint
+TASK_PROCESSOR_MODE = env.bool("RUN_BY_PROCESSOR", False)
+
 # Setting to allow asynchronous tasks to be run synchronously for testing purposes
 # or in a separate thread for self-hosted users
 TASK_RUN_METHOD = env.enum(
     "TASK_RUN_METHOD",
     type=TaskRunMethod,
     default=(
         TaskRunMethod.TASK_PROCESSOR.value
-        if env.bool("RUN_BY_PROCESSOR", False)
+        if TASK_PROCESSOR_MODE
         else TaskRunMethod.SEPARATE_THREAD.value
     ),
 )

diff --git a/api/app/urls.py b/api/app/urls.py
@@ -1,4 +1,5 @@
 import importlib
+from typing import TYPE_CHECKING
 
 from django.conf import settings
 from django.contrib import admin
@@ -10,67 +11,74 @@
 from . import views
 
 urlpatterns = [
-    re_path(r"^api/v1/", include("api.urls.deprecated", namespace="api-deprecated")),
-    re_path(r"^api/v1/", include("api.urls.v1", namespace="api-v1")),
-    re_path(r"^api/v2/", include("api.urls.v2", namespace="api-v2")),
-    re_path(r"^admin/", admin.site.urls),
     re_path(r"^health/liveness/?", views.version_info),
     re_path(r"^health/readiness/?", include("health_check.urls")),
-    re_path(r"^health", include("health_check.urls", namespace="health")),
-    # Aptible health checks must be on /healthcheck and cannot redirect
-    # see https://www.aptible.com/docs/core-concepts/apps/connecting-to-apps/app-endpoints/https-endpoints/health-checks
-    path("healthcheck", include("health_check.urls", namespace="aptible")),
-    re_path(r"^version", views.version_info, name="version-info"),
-    re_path(
-        r"^sales-dashboard/",
-        include("sales_dashboard.urls", namespace="sales_dashboard"),
-    ),
-    # this url is used to generate email content for the password reset workflow
-    re_path(
-        r"^password-reset/confirm/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,"
-        r"13}-[0-9A-Za-z]{1,20})/$",
-        password_reset_redirect,
-        name="password_reset_confirm",
-    ),
-    re_path(
-        r"^config/project-overrides",
-        views.project_overrides,
-        name="project_overrides",
-    ),
     path("processor/", include("task_processor.urls")),
-    path(
-        "robots.txt",
-        TemplateView.as_view(template_name="robots.txt", content_type="text/plain"),
-    ),
 ]
 
-if settings.DEBUG:
-    import debug_toolbar  # type: ignore[import-untyped,unused-ignore]
+if not settings.TASK_PROCESSOR_MODE:
+    urlpatterns += [
+        re_path(
+            r"^api/v1/", include("api.urls.deprecated", namespace="api-deprecated")
+        ),
+        re_path(r"^api/v1/", include("api.urls.v1", namespace="api-v1")),
+        re_path(r"^api/v2/", include("api.urls.v2", namespace="api-v2")),
+        re_path(r"^admin/", admin.site.urls),
+        re_path(r"^health", include("health_check.urls", namespace="health")),
+        # Aptible health checks must be on /healthcheck and cannot redirect
+        # see https://www.aptible.com/docs/core-concepts/apps/connecting-to-apps/app-endpoints/https-endpoints/health-checks
+        path("healthcheck", include("health_check.urls", namespace="aptible")),
+        re_path(r"^version", views.version_info, name="version-info"),
+        re_path(
+            r"^sales-dashboard/",
+            include("sales_dashboard.urls", namespace="sales_dashboard"),
+        ),
+        # this url is used to generate email content for the password reset workflow
+        re_path(
+            r"^password-reset/confirm/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,"
+            r"13}-[0-9A-Za-z]{1,20})/$",
+            password_reset_redirect,
+            name="password_reset_confirm",
+        ),
+        re_path(
+            r"^config/project-overrides",
+            views.project_overrides,
+            name="project_overrides",
+        ),
+        path(
+            "robots.txt",
+            TemplateView.as_view(template_name="robots.txt", content_type="text/plain"),
+        ),
+    ]
+
+if settings.DEBUG or TYPE_CHECKING:
+    import debug_toolbar
 
     urlpatterns = [
-        re_path(r"^__debug__/", include(debug_toolbar.urls)),  # type: ignore[attr-defined,unused-ignore]
+        re_path(r"^__debug__/", include(debug_toolbar.urls)),  # type: ignore[attr-defined]
     ] + urlpatterns
 
 if settings.SAML_INSTALLED:
-    urlpatterns.append(path("api/v1/auth/saml/", include("saml.urls")))
+    urlpatterns += [
+        path("api/v1/auth/saml/", include("saml.urls")),
+    ]
 
 if settings.WORKFLOWS_LOGIC_INSTALLED:  # pragma: no cover
     workflow_views = importlib.import_module("workflows_logic.views")
-    urlpatterns.extend(
-        [
-            path("api/v1/features/workflows/", include("workflows_logic.urls")),
-            path(
-                "api/v1/environments/<str:environment_api_key>/create-change-request/",
-                workflow_views.create_change_request,
-                name="create-change-request",
-            ),
-            path(
-                "api/v1/environments/<str:environment_api_key>/list-change-requests/",
-                workflow_views.list_change_requests,
-                name="list-change-requests",
-            ),
-        ]
-    )
+    urlpatterns += [
+        path("api/v1/features/workflows/", include("workflows_logic.urls")),
+        path(
+            "api/v1/environments/<str:environment_api_key>/create-change-request/",
+            workflow_views.create_change_request,
+            name="create-change-request",
+        ),
+        path(
+            "api/v1/environments/<str:environment_api_key>/list-change-requests/",
+            workflow_views.list_change_requests,
+            name="list-change-requests",
+        ),
+    ]
+
 
 if settings.SERVE_FE_ASSETS:  # pragma: no cover
     # add route to serve FE assets for any unrecognised paths

diff --git a/api/poetry.lock b/api/poetry.lock
diff --git a/api/pyproject.toml b/api/pyproject.toml
@@ -146,7 +146,7 @@ pygithub = "2.1.1"
 hubspot-api-client = "^8.2.1"
 djangorestframework-dataclasses = "^1.3.1"
 pyotp = "^2.9.0"
-flagsmith-task-processor = { git = "https://github.com/Flagsmith/flagsmith-task-processor", tag = "v1.2.1" }
+flagsmith-task-processor = { git = "https://github.com/Flagsmith/flagsmith-task-processor", rev = "2f4e95787d15f3c0fc86ecd0fc7036b353fbeba6" }
 flagsmith-common = { git = "https://github.com/Flagsmith/flagsmith-common", tag = "v1.4.2" }
 tzdata = "^2024.1"
 djangorestframework-simplejwt = "^5.3.1"

diff --git a/api/scripts/run-docker.sh b/api/scripts/run-docker.sh
@@ -1,6 +1,10 @@
 #!/bin/sh
 set -e
 
+# common environment variables
+ACCESS_LOG_FORMAT=${ACCESS_LOG_FORMAT:-'%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %({origin}i)s %({access-control-allow-origin}o)s'}
+GUNICORN_LOGGER_CLASS=${GUNICORN_LOGGER_CLASS:-'util.logging.GunicornJsonCapableLogger'}
+
 waitfordb() {
   if [ -z "${SKIP_WAIT_FOR_DB}" ]; then
      python manage.py waitfordb "$@"
@@ -27,8 +31,8 @@ serve() {
              --workers ${GUNICORN_WORKERS:-3} \
              --threads ${GUNICORN_THREADS:-2} \
              --access-logfile $ACCESS_LOG_LOCATION \
-             --logger-class ${GUNICORN_LOGGER_CLASS:-'util.logging.GunicornJsonCapableLogger'} \
-             --access-logformat ${ACCESS_LOG_FORMAT:-'%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %({origin}i)s %({access-control-allow-origin}o)s'} \
+             --logger-class $GUNICORN_LOGGER_CLASS \
+             --access-logformat $ACCESS_LOG_FORMAT \
              --keep-alive ${GUNICORN_KEEP_ALIVE:-2} \
              ${STATSD_HOST:+--statsd-host $STATSD_HOST:$STATSD_PORT} \
              ${STATSD_HOST:+--statsd-prefix $STATSD_PREFIX} \
@@ -39,11 +43,16 @@ run_task_processor() {
     if [ -n "$ANALYTICS_DATABASE_URL" ] || [ -n "$DJANGO_DB_NAME_ANALYTICS" ]; then
         waitfordb --waitfor 30 --migrations --database analytics
     fi
-    RUN_BY_PROCESSOR=1 exec python manage.py runprocessor \
-      --sleepintervalms ${TASK_PROCESSOR_SLEEP_INTERVAL:-500} \
-      --graceperiodms ${TASK_PROCESSOR_GRACE_PERIOD_MS:-20000} \
-      --numthreads ${TASK_PROCESSOR_NUM_THREADS:-5} \
-      --queuepopsize ${TASK_PROCESSOR_QUEUE_POP_SIZE:-10}
+    RUN_BY_PROCESSOR=1 python manage.py runprocessor \
+             --sleepintervalms ${TASK_PROCESSOR_SLEEP_INTERVAL_MS:-${TASK_PROCESSOR_SLEEP_INTERVAL:-500}} \
+             --graceperiodms ${TASK_PROCESSOR_GRACE_PERIOD_MS:-20000} \
+             --numthreads ${TASK_PROCESSOR_NUM_THREADS:-5} \
+             --queuepopsize ${TASK_PROCESSOR_QUEUE_POP_SIZE:-10} \
+             gunicorn \
+             --bind 0.0.0.0:8000 \
+             --access-logfile $ACCESS_LOG_LOCATION \
+             --logger-class $GUNICORN_LOGGER_CLASS \
+             --access-logformat $ACCESS_LOG_FORMAT
 }
 migrate_analytics_db(){
     # if `$ANALYTICS_DATABASE_URL` or DJANGO_DB_NAME_ANALYTICS is set