Skip to content

[SE-4304] [SE-4495] feat: add celerybeat configuration #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gabor-boros merged 1 commit into from
Jul 14, 2021
Merged

[SE-4304] [SE-4495] feat: add celerybeat configuration #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions playbooks/roles/edxapp/defaults/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,12 @@ EDXAPP_CELERY_BROKER_VHOST: ""
EDXAPP_CELERY_BROKER_USE_SSL: false
EDXAPP_CELERY_EVENT_QUEUE_TTL: !!null
EDXAPP_CELERY_TIMEZONE: "UTC"
EDXAPP_CELERYBEAT_SCHEDULER: "celery.beat:PersistentScheduler"
EDXAPP_ENABLE_CELERY_BEAT: false
EDXAPP_SINGLE_BEAT_LOCK_TIME: 60
# EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL must be smaller than EDXAPP_SINGLE_BEAT_LOCK_TIME / 2
EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL: 29
EDXAPP_SINGLE_BEAT_VERSION: "0.4.2"
EDXAPP_BRANCH_IO_KEY: ""

EDXAPP_AUTH_USE_OPENID_PROVIDER: true
Expand Down Expand Up @@ -1409,6 +1415,7 @@ generic_env_config: &edxapp_generic_env
LOCATION: "{{ EDXAPP_CACHE_COURSE_STRUCTURE_MEMCACHE }}"
# Default to two hours
TIMEOUT: "7200"
CELERYBEAT_SCHEDULER: "{{ EDXAPP_CELERYBEAT_SCHEDULER }}"
CELERY_BROKER_TRANSPORT: "{{ EDXAPP_CELERY_BROKER_TRANSPORT }}"
CELERY_BROKER_HOSTNAME: "{{ EDXAPP_CELERY_BROKER_HOSTNAME }}"
COMMENTS_SERVICE_URL: "{{ EDXAPP_COMMENTS_SERVICE_URL }}"
Expand Down
13 changes: 13 additions & 0 deletions playbooks/roles/edxapp/tasks/deploy.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,7 @@
with_items:
- "lms.sh"
- "cms.sh"
- "beat_scheduler.sh"
- "worker.sh"
- "reload_lms_config.sh"
- "reload_cms_config.sh"
Expand All @@ -370,6 +371,18 @@
when:
- celery_worker is not defined

- name: install single-beat to run only one celerybeat scheduler
pip:
name: single-beat
version: "{{ EDXAPP_SINGLE_BEAT_VERSION|default(omit) }}"
virtualenv: "{{ edxapp_venv_dir }}"
state: present
become_user: "{{ edxapp_user }}"
when: EDXAPP_ENABLE_CELERY_BEAT
tags:
- install
- install:app-requirements

# creates the supervisor jobs for the
# service variants configured, runs
# gather_assets and db migrations
Expand Down
22 changes: 22 additions & 0 deletions playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash

# {{ ansible_managed }}

{% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
source {{ edxapp_app_dir }}/edxapp_env
{% if COMMON_ENABLE_NEWRELIC_APP %}
{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery beat' %}

export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini"
if command -v ec2metadata >/dev/null 2>&1; then
INSTANCEID=$(ec2metadata --instance-id);
HOSTNAME=$(hostname)
export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
fi
{% else %}
{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery beat' %}
{% endif %}

# We exec so that celery is the child of supervisor and can be managed properly

exec {{ executable }} $@
35 changes: 35 additions & 0 deletions playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,38 @@ autorestart=true

[group:edxapp_worker]
programs={%- for w in edxapp_workers %}{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}{%- if not loop.last %},{%- endif %}{%- endfor %}


{% if EDXAPP_ENABLE_CELERY_BEAT|bool and EDXAPP_CELERY_BROKER_TRANSPORT == "redis" %}
[program:celerybeat_scheduler]
environment=
{% if COMMON_ENABLE_NEWRELIC_APP %}
NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }}-lms,
NEW_RELIC_DISTRIBUTED_TRACING_ENABLED={{ EDXAPP_WORKERS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }},
NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},
{% endif -%}
LANG={{ EDXAPP_LANG }},
PYTHONPATH={{ edxapp_code_dir }},
SERVICE_VARIANT=lms,
BOTO_CONFIG="{{ edxapp_app_dir }}/.boto",
EDX_REST_API_CLIENT_NAME=edx.lms.core.default,
SINGLE_BEAT_LOCK_TIME={{ EDXAPP_SINGLE_BEAT_LOCK_TIME }},
SINGLE_BEAT_HEARTBEAT_INTERVAL={{ EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL }},
SINGLE_BEAT_IDENTIFIER="celerybeat",
SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}",
SINGLE_BEAT_WAIT_MODE="supervised"
user={{ common_web_user }}
directory={{ edxapp_code_dir }}
stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log

command={{ edxapp_app_dir }}/beat_scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid"

killasgroup=true
stopwaitsecs={{ EDXAPP_WORKER_DEFAULT_STOPWAITSECS }}
; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
; are auto restarted even when exiting with code 0.
; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
autorestart=true
{% endif %}