Cleanly shut down on SIGTERM or SIGINT

README.md

@@ -141,7 +141,7 @@ Event types and their expected `message` format are documented below.
 | copy_image    | `{"artwork_id": INT, "old_gid": UUID, "new_gid": UUID, "suffix": TEXT}` | copies an image from one bucket to another (after a release is merged)  |
 | delete_image  | `{"gid": UUID, "artwork_id": INT, "suffix": TEXT}`                      | deletes an image (including after a release is merged or deleted)       |
 | deindex       | `{"gid": UUID}`                                                         | deletes index.json (after a release is deleted)                         |
-| noop          | `{}` or `{"fail": BOOL}`                                                | for debugging (does nothing, or fails if `{"fail": true}` is specified) |
+| noop          | `{}` or `{"fail": BOOL}` or `{"sleep": REAL}`                           | for testing/debugging (does nothing, or optionally fails or sleeps)     |
 
 Failed events (any that encounter an exception during their execution) are
 tried up to 5 times; only after all attempts have been exhausted is an

handlers_base.py

@@ -18,6 +18,7 @@
 
 import logging
 import json
+import time
 
 from psycopg import sql
 from textwrap import dedent
@@ -287,6 +288,8 @@ def noop(self, pg_conn, event):
         message = event['message']
         if message.get('fail'):
             raise Exception('Failure (no-op)')
+        if 'sleep' in message:
+            time.sleep(message['sleep'])
 
 
 class MusicBrainzEventHandler(EventHandler):

indexer.py

@@ -169,6 +169,11 @@ def run_event_handler(pg_conn, event, handler):
         pg_conn.commit()
 
 
+# When set to True, indicates to the `indexer` event loop that it should
+# stop once idle.
+SHUTDOWN_SIGNAL = False
+
+
 def indexer(
     config,
     pg_conn,
@@ -191,7 +196,7 @@ def indexer(
 
     idle_loops = 0
 
-    while True:
+    while not SHUTDOWN_SIGNAL:
         time.sleep(sleep_amount)
 
         event = get_next_event(pg_conn)
@@ -323,6 +328,14 @@ def reload_configuration(signum, frame):
 
     signal.signal(signal.SIGHUP, reload_configuration)
 
+    def shutdown(signum, frame):
+        logging.info(f'Got signal {signum}, shutting down')
+        global SHUTDOWN_SIGNAL
+        SHUTDOWN_SIGNAL = True
+
+    signal.signal(signal.SIGTERM, shutdown)
+    signal.signal(signal.SIGINT, shutdown)
+
     if 'sentry' in config:
         sentry_dsn = config['sentry'].get('dsn')
         if sentry_dsn:

run_tests.sh

@@ -7,5 +7,6 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 ./create_test_db.sh
 
 ./tests/test_concurrency.sh
+./tests/test_signals.sh
 
 exec coverage run -m unittest discover . "test_*.py"

tests/test_signals.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../"
+
+read -r -d '' SQL <<'EOF'
+SET client_min_messages TO WARNING;
+INSERT INTO artwork_indexer.event_queue (id, entity_type, action, message, created)
+     VALUES (1, 'release', 'noop', '{"sleep": 3}', NOW() - interval '1 minute');
+EOF
+
+psql -U musicbrainz -d musicbrainz_test_artwork_indexer -c "$SQL" -q > /dev/null
+
+run_indexer() {
+    exec python indexer.py \
+        --max-wait=1 \
+        --config=config.tests.ini
+}
+
+run_indexer &
+run1_pid=$!
+
+sleep 1.5
+
+kill -TERM "$run1_pid"
+wait "$run1_pid"
+
+read -r -d '' STATE_SQL <<'EOF'
+SELECT state
+  FROM artwork_indexer.event_queue
+ WHERE id = 1;
+EOF
+
+event_state="$(psql -U musicbrainz -d musicbrainz_test_artwork_indexer -c "$STATE_SQL" -tAq)"
+if [[ "$event_state" != 'completed' ]]; then
+    echo 'ERROR: Event was not completed after SIGTERM'
+    exit 1
+fi
+
+read -r -d '' SQL <<'EOF'
+UPDATE artwork_indexer.event_queue
+   SET state = 'queued', attempts = 0
+ WHERE id = 1;
+EOF
+
+psql -U musicbrainz -d musicbrainz_test_artwork_indexer -c "$SQL"
+
+run_indexer &
+run2_pid=$!
+
+sleep 1.5
+
+kill -INT "$run2_pid"
+wait "$run2_pid"
+
+event_state="$(psql -U musicbrainz -d musicbrainz_test_artwork_indexer -c "$STATE_SQL" -tAq)"
+if [[ "$event_state" != 'completed' ]]; then
+    echo 'ERROR: Event was not completed after SIGINT'
+    exit 1
+fi