Nuke Celery (#4521)

* enforce hard limits on non-responsive work horses by workers

* move differences from Worker to helper methods to help make the specialization clearer

* move HardLimitingWorker to redash/tasks

* move schedule.py to /tasks

* explain the motivation for HardLimitingWorker

* pleasing CodeClimate

* pleasing CodeClimate

* port query execution to RQ

* get rid of argsrepr

* avoid star imports

* allow queries to be cancelled in RQ

* return QueryExecutionErrors as job results

* fix TestTaskEnqueue and QueryExecutorTests

* remove Celery monitoring

* get rid of QueryTask and use RQ jobs directly (with a job serializer)

* Revert "remove Celery monitoring"

This reverts commit 37a74ea.

* reduce occurences of the word 'task'

* use Worker, Queue and Job instead of spreading names that share behavior details

* remove locks for failed jobs as well

* did I not commit that colon? oh my

* push the redis connection to RQ's stack on every request to avoid verbose connection setting

* use a connection context for tests

* remove Celery monitoring

* 👋 Celery

* remove Celery from Cypress

* black it up

* some more black

* return all started/queued job ids (for future monitoring

* Restyled by prettier (#4522)

* remove celery.py

* remove some frontend residuals that reappeared after a merge

Co-authored-by: restyled-io[bot] <32688539+restyled-io[bot]@users.noreply.github.com>

.circleci/docker-compose.cypress.yml

@@ -31,25 +31,12 @@ services:
       REDASH_LOG_LEVEL: "INFO"
       REDASH_REDIS_URL: "redis://redis:6379/0"
       REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
-  celery_worker:
-    build: ../
-    command: celery_worker
-    depends_on:
-      - server
-    environment:
-      PYTHONUNBUFFERED: 0
-      REDASH_LOG_LEVEL: "INFO"
-      REDASH_REDIS_URL: "redis://redis:6379/0"
-      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
-      QUEUES: "queries,scheduled_queries"
-      WORKERS_COUNT: 2
   cypress:
     build:
       context: ../
       dockerfile: .circleci/Dockerfile.cypress
     depends_on:
       - server
-      - celery_worker
       - worker
       - scheduler
     environment:

bin/docker-entrypoint

@@ -1,15 +1,6 @@
 #!/bin/bash
 set -e
 
-celery_worker() {
-  WORKERS_COUNT=${WORKERS_COUNT:-2}
-  QUEUES=${QUEUES:-queries,scheduled_queries}
-  WORKER_EXTRA_OPTIONS=${WORKER_EXTRA_OPTIONS:-}
-
-  echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..."
-  exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair $WORKER_EXTRA_OPTIONS
-}
-
 scheduler() {
   echo "Starting RQ scheduler..."
 
@@ -37,15 +28,6 @@ dev_worker() {
   exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq worker $QUEUES
 }
 
-dev_celery_worker() {
-  WORKERS_COUNT=${WORKERS_COUNT:-2}
-  QUEUES=${QUEUES:-queries,scheduled_queries}
-
-  echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..."
-
-  exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair
-}
-
 server() {
   # Recycle gunicorn workers every n-th request. See http://docs.gunicorn.org/en/stable/settings.html#max-requests for more details.
   MAX_REQUESTS=${MAX_REQUESTS:-1000}
@@ -57,10 +39,6 @@ create_db() {
   exec /app/manage.py database create_tables
 }
 
-celery_healthcheck() {
-  exec /usr/local/bin/celery inspect ping --app=redash.worker -d celery@$HOSTNAME
-}
-
 rq_healthcheck() {
   exec /app/manage.py rq healthcheck
 }
@@ -72,13 +50,10 @@ help() {
   echo ""
 
   echo "server -- start Redash server (with gunicorn)"
-  echo "celery_worker -- start Celery worker"
-  echo "dev_celery_worker -- start Celery worker process which picks up code changes and reloads"
   echo "worker -- start a single RQ worker"
   echo "dev_worker -- start a single RQ worker with code reloading"
   echo "scheduler -- start an rq-scheduler instance"
   echo "dev_scheduler -- start an rq-scheduler instance with code reloading"
-  echo "celery_healthcheck -- runs a Celery healthcheck. Useful for Docker's HEALTHCHECK mechanism."
   echo "rq_healthcheck -- runs a RQ healthcheck that verifies that all local workers are active. Useful for Docker's HEALTHCHECK mechanism."
   echo ""
   echo "shell -- open shell"
@@ -117,14 +92,6 @@ case "$1" in
     shift
     dev_scheduler
     ;;
-  celery_worker)
-    shift
-    celery_worker
-    ;;
-  dev_celery_worker)
-    shift
-    dev_celery_worker
-    ;;
   dev_worker)
     shift
     dev_worker
@@ -133,10 +100,6 @@ case "$1" in
     shift
     rq_healthcheck
     ;;
-  celery_healthcheck)
-    shift
-    celery_healthcheck
-    ;;
   dev_server)
     export FLASK_DEBUG=1
     exec /app/manage.py runserver --debugger --reload -h 0.0.0.0

client/app/components/admin/Layout.jsx

@@ -15,9 +15,6 @@ export default function Layout({ activeTab, children }) {
           <Tabs.TabPane key="system_status" tab={<a href="admin/status">System Status</a>}>
             {activeTab === "system_status" ? children : null}
           </Tabs.TabPane>
-          <Tabs.TabPane key="tasks" tab={<a href="admin/queries/tasks">Celery Status</a>}>
-            {activeTab === "tasks" ? children : null}
-          </Tabs.TabPane>
           <Tabs.TabPane key="jobs" tab={<a href="admin/queries/jobs">RQ Status</a>}>
             {activeTab === "jobs" ? children : null}
           </Tabs.TabPane>

client/app/components/admin/RQStatus.jsx

@@ -3,9 +3,34 @@ import React from "react";
 import PropTypes from "prop-types";
 
 import Badge from "antd/lib/badge";
+import Card from "antd/lib/card";
+import Spin from "antd/lib/spin";
 import Table from "antd/lib/table";
 import { Columns } from "@/components/items-list/components/ItemsTable";
 
+// CounterCard
+
+export function CounterCard({ title, value, loading }) {
+  return (
+    <Spin spinning={loading}>
+      <Card>
+        {title}
+        <div className="f-20">{value}</div>
+      </Card>
+    </Spin>
+  );
+}
+
+CounterCard.propTypes = {
+  title: PropTypes.string.isRequired,
+  value: PropTypes.oneOfType([PropTypes.number, PropTypes.string]),
+  loading: PropTypes.bool.isRequired,
+};
+
+CounterCard.defaultProps = {
+  value: "",
+};
+
 // Tables
 
 const otherJobsColumns = [

client/app/pages/admin/Jobs.jsx

@@ -6,8 +6,7 @@ import Alert from "antd/lib/alert";
 import Tabs from "antd/lib/tabs";
 import * as Grid from "antd/lib/grid";
 import Layout from "@/components/admin/Layout";
-import { CounterCard } from "@/components/admin/CeleryStatus";
-import { WorkersTable, QueuesTable, OtherJobsTable } from "@/components/admin/RQStatus";
+import { CounterCard, WorkersTable, QueuesTable, OtherJobsTable } from "@/components/admin/RQStatus";
 
 import { $http, $location, $rootScope } from "@/services/ng";
 import recordEvent from "@/services/recordEvent";