refactor: use utility from openedex_events to reset application state

CHANGELOG.rst

@@ -16,6 +16,14 @@ Unreleased
 
 *
 
+[0.3.3] - 2023-10-13
+************************************************
+
+Added
+=====
+* Use utility from openedx_events to reset application state before processing
+  event
+
 [0.3.2] - 2023-09-01
 ************************************************
 

edx_event_bus_redis/__init__.py

@@ -5,6 +5,6 @@
 from edx_event_bus_redis.internal.consumer import RedisEventConsumer
 from edx_event_bus_redis.internal.producer import create_producer
 
-__version__ = '0.3.2'
+__version__ = '0.3.3'
 
 default_app_config = 'edx_event_bus_redis.apps.EdxEventBusRedisConfig'  # pylint: disable=invalid-name

edx_event_bus_redis/internal/consumer.py

@@ -6,12 +6,11 @@
 from typing import Optional
 
 from django.conf import settings
-from django.db import connection
 from edx_django_utils.monitoring import record_exception, set_custom_attribute
 from edx_toggles.toggles import SettingToggle
 from openedx_events.event_bus import EventBusConsumer
 from openedx_events.event_bus.avro.deserializer import deserialize_bytes_to_event_data
-from openedx_events.tooling import OpenEdxPublicSignal
+from openedx_events.tooling import OpenEdxPublicSignal, prepare_for_new_work_cycle
 from redis.exceptions import ConnectionError as RedisConnectionError
 from redis.exceptions import ResponseError
 from walrus import Database
@@ -70,21 +69,6 @@ class EventConsumptionException(Exception):
     """
 
 
-def _reconnect_to_db_if_needed():
-    """
-    Reconnects the db connection if needed.
-
-    This is important because Django only does connection validity/age checks as part of
-    its request/response cycle, which isn't in effect for the consume-loop. If we don't
-    force these checks, a broken connection will remain broken indefinitely. For most
-    consumers, this will cause event processing to fail.
-    """
-    has_connection = bool(connection.connection)
-    requires_reconnect = has_connection and not connection.is_usable()
-    if requires_reconnect:
-        connection.connect()
-
-
 class RedisEventConsumer(EventBusConsumer):
     """
     Construct consumer for the given topic and group. The consumer can then
@@ -240,8 +224,9 @@ def _consume_indefinitely(self):
                         if isinstance(redis_raw_msg, list):
                             redis_raw_msg = redis_raw_msg[0]
                         msg = RedisMessage.parse(redis_raw_msg, self.full_topic)
-                        # Before processing, make sure our db connection is still active
-                        _reconnect_to_db_if_needed()
+                        # Before processing, try to make sure our application state is cleaned
+                        # up as would happen at the start of a Django request/response cycle.
+                        prepare_for_new_work_cycle()
                         self.emit_signals_from_message(msg)
                         consecutive_errors = 0
 

edx_event_bus_redis/internal/tests/test_consumer.py

@@ -287,7 +287,7 @@ def test_consecutive_error_limit(self):
         assert mock_emit.call_args_list == [call(self.normal_message)] * exception_count
         assert exc_info.value.args == ("Too many consecutive errors, exiting (4 in a row)",)
 
-    @patch('edx_event_bus_redis.internal.consumer.connection')
+    @patch('openedx_events.tooling.connection')
     @ddt.data(
         (False, False, False),  # no connection, don't reconnect
         (True, False, True),  # connection unusable, reconnect expected

requirements/base.in

@@ -2,8 +2,9 @@
 -c constraints.txt
 
 Django             # Web application framework
-# openedx-events 8.0.0 removes an argument from consumer initialization
-openedx-events>=8.0.0                   # Events API
+# openedx-events 9.0.1 adds utility to reset application state
+# openedx-events>=9.0.1                   # Events API
+git+https://github.com/open-craft/openedx-events@navin/work-cycle-utility
 edx_django_utils
 edx_toggles
 

requirements/base.txt

@@ -39,6 +39,7 @@ edx-django-utils==5.7.0
     # via
     #   -r requirements/base.in
     #   edx-toggles
+    #   openedx-events
 edx-opaque-keys[django]==2.5.1
     # via openedx-events
 edx-toggles==5.1.0
@@ -51,11 +52,11 @@ markupsafe==2.1.3
     # via jinja2
 newrelic==9.1.0
     # via edx-django-utils
-openedx-events==9.0.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/work-cycle-utility
     # via -r requirements/base.in
 pbr==5.11.1
     # via stevedore
-psutil==5.9.5
+psutil==5.9.6
     # via edx-django-utils
 pycparser==2.21
     # via cffi

requirements/dev.txt

@@ -52,10 +52,11 @@ code-annotations==1.5.0
 coverage[toml]==7.3.2
     # via
     #   -r requirements/quality.txt
+    #   coverage
     #   pytest-cov
 ddt==1.6.0
     # via -r requirements/quality.txt
-diff-cover==7.7.0
+diff-cover==8.0.0
     # via -r requirements/dev.in
 dill==0.3.7
     # via
@@ -89,13 +90,15 @@ edx-django-utils==5.7.0
     # via
     #   -r requirements/quality.txt
     #   edx-toggles
+    #   openedx-events
 edx-i18n-tools==1.3.0
     # via -r requirements/dev.in
 edx-lint==5.3.4
     # via -r requirements/quality.txt
 edx-opaque-keys[django]==2.5.1
     # via
     #   -r requirements/quality.txt
+    #   edx-opaque-keys
     #   openedx-events
 edx-toggles==5.1.0
     # via -r requirements/quality.txt
@@ -147,7 +150,7 @@ newrelic==9.1.0
     # via
     #   -r requirements/quality.txt
     #   edx-django-utils
-openedx-events==9.0.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/work-cycle-utility
     # via -r requirements/quality.txt
 packaging==23.2
     # via
@@ -180,15 +183,15 @@ pluggy==1.3.0
     #   tox
 polib==1.2.0
     # via edx-i18n-tools
-psutil==5.9.5
+psutil==5.9.6
     # via
     #   -r requirements/quality.txt
     #   edx-django-utils
 py==1.11.0
     # via
     #   -r requirements/ci.txt
     #   tox
-pycodestyle==2.11.0
+pycodestyle==2.11.1
     # via -r requirements/quality.txt
 pycparser==2.21
     # via

requirements/doc.txt

@@ -49,6 +49,7 @@ code-annotations==1.5.0
 coverage[toml]==7.3.2
     # via
     #   -r requirements/test.txt
+    #   coverage
     #   pytest-cov
 cryptography==41.0.4
     # via secretstorage
@@ -82,9 +83,11 @@ edx-django-utils==5.7.0
     # via
     #   -r requirements/test.txt
     #   edx-toggles
+    #   openedx-events
 edx-opaque-keys[django]==2.5.1
     # via
     #   -r requirements/test.txt
+    #   edx-opaque-keys
     #   openedx-events
 edx-toggles==5.1.0
     # via -r requirements/test.txt
@@ -141,7 +144,7 @@ newrelic==9.1.0
     #   edx-django-utils
 nh3==0.2.14
     # via readme-renderer
-openedx-events==9.0.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/work-cycle-utility
     # via -r requirements/test.txt
 packaging==23.2
     # via
@@ -160,7 +163,7 @@ pluggy==1.3.0
     # via
     #   -r requirements/test.txt
     #   pytest
-psutil==5.9.5
+psutil==5.9.6
     # via
     #   -r requirements/test.txt
     #   edx-django-utils

requirements/pip.txt

@@ -8,7 +8,7 @@ wheel==0.41.2
     # via -r requirements/pip.in
 
 # The following packages are considered to be unsafe in a requirements file:
-pip==23.2.1
+pip==23.3
     # via -r requirements/pip.in
 setuptools==68.2.2
     # via -r requirements/pip.in

requirements/quality.txt

@@ -41,6 +41,7 @@ code-annotations==1.5.0
 coverage[toml]==7.3.2
     # via
     #   -r requirements/test.txt
+    #   coverage
     #   pytest-cov
 ddt==1.6.0
     # via -r requirements/test.txt
@@ -69,11 +70,13 @@ edx-django-utils==5.7.0
     # via
     #   -r requirements/test.txt
     #   edx-toggles
+    #   openedx-events
 edx-lint==5.3.4
     # via -r requirements/quality.in
 edx-opaque-keys[django]==2.5.1
     # via
     #   -r requirements/test.txt
+    #   edx-opaque-keys
     #   openedx-events
 edx-toggles==5.1.0
     # via -r requirements/test.txt
@@ -109,7 +112,7 @@ newrelic==9.1.0
     # via
     #   -r requirements/test.txt
     #   edx-django-utils
-openedx-events==9.0.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/work-cycle-utility
     # via -r requirements/test.txt
 packaging==23.2
     # via
@@ -125,11 +128,11 @@ pluggy==1.3.0
     # via
     #   -r requirements/test.txt
     #   pytest
-psutil==5.9.5
+psutil==5.9.6
     # via
     #   -r requirements/test.txt
     #   edx-django-utils
-pycodestyle==2.11.0
+pycodestyle==2.11.1
     # via -r requirements/quality.in
 pycparser==2.21
     # via

requirements/test.txt

@@ -31,7 +31,9 @@ code-annotations==1.5.0
     #   -r requirements/test.in
     #   edx-toggles
 coverage[toml]==7.3.2
-    # via pytest-cov
+    # via
+    #   coverage
+    #   pytest-cov
 ddt==1.6.0
     # via -r requirements/test.in
     # via
@@ -56,9 +58,11 @@ edx-django-utils==5.7.0
     # via
     #   -r requirements/base.txt
     #   edx-toggles
+    #   openedx-events
 edx-opaque-keys[django]==2.5.1
     # via
     #   -r requirements/base.txt
+    #   edx-opaque-keys
     #   openedx-events
 edx-toggles==5.1.0
     # via -r requirements/base.txt
@@ -82,7 +86,7 @@ newrelic==9.1.0
     # via
     #   -r requirements/base.txt
     #   edx-django-utils
-openedx-events==9.0.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/work-cycle-utility
     # via -r requirements/base.txt
 packaging==23.2
     # via pytest
@@ -92,7 +96,7 @@ pbr==5.11.1
     #   stevedore
 pluggy==1.3.0
     # via pytest
-psutil==5.9.5
+psutil==5.9.6
     # via
     #   -r requirements/base.txt
     #   edx-django-utils