diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aeb2650..b7c00b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -34,8 +34,8 @@ jobs: needs: lint strategy: matrix: - python-version: [ 3.6, 3.7, 3.8, 3.9 ] - django-version: [ "2.2", "3.0", "3.1" ] + python-version: [ 3.8, 3.9, "3.10" ] + django-version: [ "3.2", "4.0" ] env: PYTHON: ${{ matrix.python-version }} DJANGO: ${{ matrix.django-version }} diff --git a/README.md b/README.md index 52e4330..d2c807c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ django-apscheduler is a great choice for quickly and easily adding basic schedul with minimal dependencies and very little additional configuration. The ideal use case probably involves running a handful of tasks on a fixed execution schedule. -The tradeoff of this simplicity is that you need to **be careful to ensure that you only have ***one*** scheduler +The trade-off of this simplicity is that you need to **be careful to ensure that you only have ***one*** scheduler actively running at a particular point in time**. This limitation stems from the fact that APScheduler does not currently have any [interprocess synchronization and @@ -131,60 +131,62 @@ logger = logging.getLogger(__name__) def my_job(): - # Your job processing logic here... - pass + # Your job processing logic here... + pass -# The `close_old_connections` decorator ensures that database connections, that have become unusable or are obsolete, -# are closed before and after our job has run. +# The `close_old_connections` decorator ensures that database connections, that have become +# unusable or are obsolete, are closed before and after our job has run. @util.close_old_connections def delete_old_job_executions(max_age=604_800): - """ - This job deletes APScheduler job execution entries older than `max_age` from the database. It helps to prevent the - database from filling up with old historical records that are no longer useful. - - :param max_age: The maximum length of time to retain historical job execution records. Defaults - to 7 days. - """ - DjangoJobExecution.objects.delete_old_job_executions(max_age) + """ + This job deletes APScheduler job execution entries older than `max_age` from the database. + It helps to prevent the database from filling up with old historical records that are no + longer useful. + + :param max_age: The maximum length of time to retain historical job execution records. + Defaults to 7 days. + """ + DjangoJobExecution.objects.delete_old_job_executions(max_age) class Command(BaseCommand): - help = "Runs APScheduler." - - def handle(self, *args, **options): - scheduler = BlockingScheduler(timezone=settings.TIME_ZONE) - scheduler.add_jobstore(DjangoJobStore(), "default") - - scheduler.add_job( - my_job, - trigger=CronTrigger(second="*/10"), # Every 10 seconds - id="my_job", # The `id` assigned to each job MUST be unique - max_instances=1, - replace_existing=True, - ) - logger.info("Added job 'my_job'.") - - scheduler.add_job( - delete_old_job_executions, - trigger=CronTrigger( - day_of_week="mon", hour="00", minute="00" - ), # Midnight on Monday, before start of the next work week. - id="delete_old_job_executions", - max_instances=1, - replace_existing=True, - ) - logger.info( - "Added weekly job: 'delete_old_job_executions'." - ) - - try: - logger.info("Starting scheduler...") - scheduler.start() - except KeyboardInterrupt: - logger.info("Stopping scheduler...") - scheduler.shutdown() - logger.info("Scheduler shut down successfully!") + help = "Runs APScheduler." + + def handle(self, *args, **options): + scheduler = BlockingScheduler(timezone=settings.TIME_ZONE) + scheduler.add_jobstore(DjangoJobStore(), "default") + + scheduler.add_job( + my_job, + trigger=CronTrigger(second="*/10"), # Every 10 seconds + id="my_job", # The `id` assigned to each job MUST be unique + max_instances=1, + replace_existing=True, + ) + logger.info("Added job 'my_job'.") + + scheduler.add_job( + delete_old_job_executions, + trigger=CronTrigger( + day_of_week="mon", hour="00", minute="00" + ), # Midnight on Monday, before start of the next work week. + id="delete_old_job_executions", + max_instances=1, + replace_existing=True, + ) + logger.info( + "Added weekly job: 'delete_old_job_executions'." + ) + + try: + logger.info("Starting scheduler...") + scheduler.start() + except KeyboardInterrupt: + logger.info("Stopping scheduler...") + scheduler.shutdown() + logger.info("Scheduler shut down successfully!") + ``` - This management command should be invoked via `./manage.py runapscheduler` whenever the web server serving your Django @@ -196,7 +198,7 @@ class Command(BaseCommand): job store, then you will need to include `jobstore='djangojobstore'` in your `scheduler.add_job()` calls. -Advanced Usage +Advanced usage -------------- django-apscheduler assumes that you are already familiar with APScheduler and its proper use. If not, then please head @@ -211,7 +213,7 @@ depending on your environment and use case. If you would prefer running a `Backg order to re-enable threading support. -Supported Databases +Supported databases ------------------- Please take note of the list of databases that @@ -219,7 +221,7 @@ are [officially supported by Django](https://docs.djangoproject.com/en/dev/ref/d probably won't work with unsupported databases like Microsoft SQL Server, MongoDB, and the like. -Database Connections and Timeouts +Database connections and timeouts --------------------------------- django-apscheduler is dependent on the standard Django @@ -245,16 +247,16 @@ Common footguns --------------- Unless you have a very specific set of requirements, and have intimate knowledge of the inner workings of APScheduler, -you shouldn't be using `BackgroundScheduler`. This can lead to all sorts of temptations like: +you shouldn't be using `BackgroundScheduler`. Doing so can lead to all sorts of temptations like: * Firing up a scheduler inside of a Django view. This will most likely cause more than one scheduler to run concurrently and lead to jobs running multiple times (see the above introduction to this README for a more thorough treatment of the subject). -* Bootstrapping a scheduler somewhere else inside of your Django application. It feels like this should solve the - problem mentioned above and guarantee that only one scheduler is running. The downside is that you have just delegated - all of your background task processing to whatever webserver you are using (Gunicorn, uWSGI, etc.). It will probably - kill any long-running threads (your jobs) with extreme prejudice (thinking that they are caused by misbehaving HTTP - requests). +* Bootstrapping a scheduler somewhere else inside your Django application. It feels like this should solve the problem + mentioned above and guarantee that only one scheduler is running. The downside is that you have just delegated the + management of all of your background task processing threads to whatever webserver you are using (Gunicorn, uWSGI, + etc.). The webserver will probably kill any long-running threads (your jobs) with extreme prejudice (thinking that + they are caused by misbehaving HTTP requests). Relying on `BlockingScheduler` forces you to run APScheduler in its own dedicated process that is not handled or monitored by the webserver. The example code provided in `runapscheduler.py` above is a good starting point. diff --git a/django_apscheduler/models.py b/django_apscheduler/models.py index e83cfdc..7a5efb6 100644 --- a/django_apscheduler/models.py +++ b/django_apscheduler/models.py @@ -1,6 +1,7 @@ from datetime import timedelta, datetime from django.db import models, transaction +from django.db.models import UniqueConstraint from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -213,3 +214,8 @@ def __str__(self): class Meta: ordering = ("-run_time",) + constraints = [ + UniqueConstraint( + fields=["job_id", "run_time"], name="unique_job_executions" + ) + ] diff --git a/django_apscheduler/util.py b/django_apscheduler/util.py index dac4600..ab42536 100644 --- a/django_apscheduler/util.py +++ b/django_apscheduler/util.py @@ -55,11 +55,11 @@ def get_apscheduler_datetime(dt: datetime, scheduler: BaseScheduler) -> datetime def retry_on_db_operational_error(func): """ This decorator can be used to wrap a database-related method so that it will be retried when a - django.db.OperationalError is encountered. + django.db.OperationalError or django.db.InterfaceError is encountered. - The rationale is that django.db.OperationalError is usually raised when attempting to use an old database - connection that the database backend has since closed. Closing the Django connection as well, and re-trying with - a fresh connection, is usually sufficient to solve the problem. + The rationale is that these exceptions are usually raised when attempting to use an old database connection that + the database backend has since closed. Closing the Django connection as well, and re-trying with a fresh connection, + is usually sufficient to solve the problem. It is a reluctant workaround for users that persistently have issues with stale database connections (most notably: 2006, 'MySQL server has gone away'). @@ -69,10 +69,9 @@ def retry_on_db_operational_error(func): workaround is probably justified. CAUTION: any method that this decorator is applied to MUST be idempotent (i.e. the method can be retried a second - time without any unwanted side effects). If your method performs any actions before the django.db.OperationalError - is raised then those actions will be repeated. If you don't want that to happen then it would be best to handle the - django.db.OperationalError exception manually and call `db.close_old_connections()` in an appropriate fashion - inside your own method instead. + time without any unwanted side effects). If your method performs any actions before the database exception is + raised then those actions will be repeated. If you don't want that to happen then it would be best to handle the + exception manually and call `db.close_old_connections()` in an appropriate fashion inside your own method instead. The following list of alternative workarounds were also considered: @@ -82,9 +81,9 @@ def retry_on_db_operational_error(func): standard). The database overhead, and associated performance penalty, that this approach would impose seem unreasonable. See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-CONN_MAX_AGE. - 2. Using a custom QuerySet or database backend that handles django.db.OperationalError automatically: this would - be more convenient than having to decorate individual methods, but it would also break when a DB operation needs - to be re-tried as part of an atomic transaction. See: https://github.com/django/django/pull/2740 + 2. Using a custom QuerySet or database backend that handles the relevant database exceptions automatically: this + would be more convenient than having to decorate individual methods, but it would also break when a DB operation + needs to be re-tried as part of an atomic transaction. See: https://github.com/django/django/pull/2740 3. Pinging the database before each operation to see if it is still available: django-apscheduler used to make use of this approach (see: https://github.com/jcass77/django-apscheduler/blob/9ac06b33d19961da6c36d5ac814d4338beb11309/django_apscheduler/models.py#L16-L51). @@ -97,7 +96,7 @@ def retry_on_db_operational_error(func): def func_wrapper(*args, **kwargs): try: result = func(*args, **kwargs) - except db.OperationalError as e: + except (db.OperationalError, db.InterfaceError) as e: logger.warning( f"DB error executing '{func.__name__}' ({e}). Retrying with a new DB connection..." ) diff --git a/docs/changelog.md b/docs/changelog.md index 0f97e86..7c8ab50 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,19 @@ This changelog is used to track all major changes to django-apscheduler. +## v0.6.1 (2022-03-05) + +**Fixes** + +- Also handle `db.InterfaceError` when retrying database-related operations (thanks @zmmfsj-z). +- Add `unique_job_executions` constraint to ensure that only one `DjangoJobExecution` can be created for + each `DjangoJob` for a specific run time (Mitigates [#156](https://github.com/jcass77/django-apscheduler/issues/156)). +- Update CI configuration to test on Django 3.2 and 4.0, and Python 3.10 ( + Resolves [#163](https://github.com/jcass77/django-apscheduler/issues/163)). +- Drop official support for Django<3.2 and Python<3.8. This is slightly ahead of the official dates published in + https://www.djangoproject.com/download/ and https://devguide.python.org/#status-of-python-branches, but makes the test + workflows simpler and easier to maintain. If you are using older releases they **might** still work... + ## v0.6.0 (2021-06-17) **Fixes** @@ -18,8 +31,8 @@ This changelog is used to track all major changes to django-apscheduler. a `django.db.OperationalError` is encountered (Partial resolution of [#145](https://github.com/jcass77/django-apscheduler/issues/145)). - Introduce a `close_old_connections` utility decorator to enforce Django's `CONN_MAX_AGE` setting. (Partial resolution - of [#145](https://github.com/jcass77/django-apscheduler/issues/145)). **This decorator should be applied to all of - your jobs that require access to the database.** + of [#145](https://github.com/jcass77/django-apscheduler/issues/145) - thanks @bluetech). **This decorator should be + applied to all of your jobs that require access to the database.** ## v0.5.2 (2021-01-28) diff --git a/requirements/base.txt b/requirements/base.txt index 73ef2b2..68dcac0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,3 +1,3 @@ # General # ------------------------------------------------------------------------------ -APScheduler>=3.2 # https://github.com/agronholm/apscheduler +APScheduler>=3.2,<4.0 # https://github.com/agronholm/apscheduler diff --git a/requirements/local.txt b/requirements/local.txt index 3842721..fd1375c 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -2,12 +2,12 @@ # Testing # ------------------------------------------------------------------------------ -pytest~=5.4 # https://github.com/pytest-dev/pytest +pytest~=6.2 # https://github.com/pytest-dev/pytest pytest-sugar~=0.9 # https://github.com/Frozenball/pytest-sugar pytest-pythonpath~=0.7 # https://github.com/bigsassy/pytest-pythonpath # Django # ------------------------------------------------------------------------------ django # https://www.djangoproject.com/ -django-coverage-plugin~=1.6 # https://github.com/nedbat/django_coverage_plugin -pytest-django~=3.6 # https://github.com/pytest-dev/pytest-django +django-coverage-plugin~=2.0 # https://github.com/nedbat/django_coverage_plugin +pytest-django~=4.5 # https://github.com/pytest-dev/pytest-django diff --git a/setup.py b/setup.py index 3b3c3be..fc7733d 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name="django-apscheduler", - version="0.6.0", + version="0.6.1", description="APScheduler for Django", long_description=long_description, long_description_content_type="text/markdown", @@ -26,19 +26,17 @@ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Framework :: Django", - "Framework :: Django :: 2.2", - "Framework :: Django :: 3.0", - "Framework :: Django :: 3.1", + "Framework :: Django :: 3.2", + "Framework :: Django :: 4.0", ], keywords="django apscheduler django-apscheduler", packages=find_packages(exclude=("tests",)), install_requires=[ - "django>=2.2", + "django>=3.2", "apscheduler>=3.2,<4.0", ], zip_safe=False,