upgrade to Python 3.12

.readthedocs.yaml

@@ -7,9 +7,9 @@ version: 2
 
 # Set the version of Python and other tools you might need
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.11"
+    python: "3.12"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

CHANGELOG.md

@@ -11,6 +11,7 @@
 - [Docs] Clarify how to reference query_key values in flatline alerts - [#1320](https://github.com/jertel/elastalert2/pull/1320) - @jertel
 - Fix percentiles aggregation type in Spike Metric Aggregation rules - [#1323](https://github.com/jertel/elastalert2/pull/1323) - @jertel
 - [Docs] Extend FAQ / troubleshooting section with information on Elasticsearch RBAC - [#1324](https://github.com/jertel/elastalert2/pull/1324) - @chr-b
+- Upgrade to Python 3.12 - [#1327](https://github.com/jertel/elastalert2/pull/1327) - @jertel
 
 # 2.15.0
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim as builder
+FROM python:3.12-slim as builder
 
 LABEL description="ElastAlert 2 Official Image"
 LABEL maintainer="Jason Ertel"
@@ -10,7 +10,7 @@ RUN mkdir -p /opt/elastalert && \
     pip install setuptools wheel && \
     python setup.py sdist bdist_wheel
 
-FROM python:3.11-slim
+FROM python:3.12-slim
 
 ARG GID=1000
 ARG UID=1000

docs/source/running_elastalert.rst

@@ -206,11 +206,11 @@ Requirements
 
 - Elasticsearch 7.x or 8.x, or OpenSearch 1.x or 2.x
 - ISO8601 or Unix timestamped data
-- Python 3.11. Require OpenSSL 1.1.1 or newer.
+- Python 3.12. Require OpenSSL 1.1.1 or newer.
 - pip
-- Packages on Ubuntu 21.x: build-essential python3-pip python3.11 python3.11-dev libffi-dev libssl-dev
+- Packages on Ubuntu 21.x: build-essential python3-pip python3.12 python3.12-dev libffi-dev libssl-dev
 
-If you want to install python 3.11 on CentOS, please install python 3.11 from the source code after installing 'Development Tools'.
+If you want to install python 3.12 on CentOS, please install python 3.12 from the source code after installing 'Development Tools'.
 
 Downloading and Configuring
 ---------------------------

elastalert/alerters/gelf.py

@@ -65,7 +65,8 @@ def sent_tcp(self, gelf_msg):
 
         try:
             if self.ca_cert:
-                tcp_socket = ssl.wrap_socket(tcp_socket, ca_certs=self.ca_cert)
+                ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+                tcp_socket = ctx.wrap_socket(tcp_socket, ca_certs=self.ca_cert)
                 tcp_socket.sendall(bytes_msg)
             else:
                 tcp_socket.sendall(bytes_msg)

elastalert/elastalert.py

@@ -18,8 +18,6 @@
 from socket import error
 import statsd
 
-
-import dateutil.tz
 import pytz
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.executors.pool import ThreadPoolExecutor
@@ -1130,26 +1128,26 @@ def start(self):
                                name='Internal: Handle Config Change')
         self.scheduler.start()
         while self.running:
-            next_run = datetime.datetime.utcnow() + self.run_every
+            next_run = datetime.datetime.now(tz=datetime.UTC) + self.run_every
 
             # Quit after end_time has been reached
             if self.args.end:
                 endtime = ts_to_dt(self.args.end)
 
-                next_run_dt = next_run.replace(tzinfo=dateutil.tz.tzutc())
+                next_run_dt = next_run.replace(tzinfo=timezone.utc)
                 if next_run_dt > endtime:
                     elastalert_logger.info("End time '%s' falls before the next run time '%s', exiting." % (endtime, next_run_dt))
                     exit(0)
 
-            if next_run < datetime.datetime.utcnow():
+            if next_run < datetime.datetime.now(tz=datetime.UTC):
                 continue
 
             # Show disabled rules
             if self.show_disabled_rules:
                 elastalert_logger.info("Disabled rules are: %s" % (str(self.get_disabled_rules())))
 
             # Wait before querying again
-            sleep_duration = total_seconds(next_run - datetime.datetime.utcnow())
+            sleep_duration = total_seconds(next_run - datetime.datetime.now(tz=datetime.UTC))
             self.sleep_for(sleep_duration)
 
     def wait_until_responsive(self, timeout, clock=timeit.default_timer):
@@ -1209,7 +1207,7 @@ def handle_config_change(self):
 
     def handle_rule_execution(self, rule):
         self.thread_data.alerts_sent = 0
-        next_run = datetime.datetime.utcnow() + rule['run_every']
+        next_run = datetime.datetime.now(tz=datetime.UTC) + rule['run_every']
         # Set endtime based on the rule's delay
         delay = rule.get('query_delay')
         if hasattr(self.args, 'end') and self.args.end:
@@ -1232,7 +1230,7 @@ def handle_rule_execution(self, rule):
             # That means that we need to pause execution after this run
             if endtime_epoch + rule['run_every'].total_seconds() < exec_next - 59:
                 # apscheduler requires pytz tzinfos, so don't use unix_to_dt here!
-                rule['next_starttime'] = datetime.datetime.utcfromtimestamp(exec_next).replace(tzinfo=pytz.utc)
+                rule['next_starttime'] = datetime.datetime.fromtimestamp(exec_next, tz=datetime.UTC).replace(tzinfo=pytz.utc)
                 if rule.get('limit_execution_coverage'):
                     rule['next_min_starttime'] = rule['next_starttime']
                 if not rule['has_run_once']:
@@ -1260,7 +1258,7 @@ def handle_rule_execution(self, rule):
 
             self.thread_data.alerts_sent = 0
 
-            if next_run < datetime.datetime.utcnow():
+            if next_run < datetime.datetime.now(tz=datetime.UTC):
                 # We were processing for longer than our refresh interval
                 # This can happen if --start was specified with a large time period
                 # or if we are running too slow to process events in real time.

elastalert/util.py

@@ -186,7 +186,8 @@ def dt_to_ts_with_format(dt, ts_format):
 
 
 def ts_now():
-    return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
+    now = datetime.datetime.now(tz=datetime.UTC)
+    return now.replace(tzinfo=dateutil.tz.tzutc())
 
 
 def ts_utc_to_tz(ts, tz_name):
@@ -268,16 +269,16 @@ def total_seconds(dt):
 
 
 def dt_to_int(dt):
-    dt = dt.replace(tzinfo=None)
-    return int(total_seconds((dt - datetime.datetime.utcfromtimestamp(0))) * 1000)
+    dt = dt.replace(tzinfo=datetime.UTC)
+    return int(total_seconds((dt - datetime.datetime.fromtimestamp(0, tz=datetime.UTC))) * 1000)
 
 
 def unixms_to_dt(ts):
     return unix_to_dt(float(ts) / 1000)
 
 
 def unix_to_dt(ts):
-    dt = datetime.datetime.utcfromtimestamp(float(ts))
+    dt = datetime.datetime.fromtimestamp(float(ts), tz=datetime.UTC)
     dt = dt.replace(tzinfo=dateutil.tz.tzutc())
     return dt
 

setup.py

@@ -21,7 +21,7 @@
         "Discussion Forum": "https://github.com/jertel/elastalert2/discussions",
     },
     classifiers=[
-        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: OS Independent',
     ],

tests/Dockerfile-test

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.12-slim
 
 RUN apt update && apt upgrade -y
 RUN apt install -y gcc libffi-dev

tests/alerters/command_test.py

@@ -20,7 +20,7 @@ def test_command_getinfo():
              'nested': {'field': 1}}
     with mock.patch("elastalert.alerters.command.subprocess.Popen") as mock_popen:
         alert.alert([match])
-    assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
+    mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
     expected_data = {
         'type': 'command',
         'command': '/bin/test/ --arg foobarbaz'
@@ -39,7 +39,7 @@ def test_command_old_style_string_format1(caplog):
     alert = CommandAlerter(rule)
     with mock.patch("elastalert.alerters.command.subprocess.Popen") as mock_popen:
         alert.alert([match])
-    assert mock_popen.called_with('/bin/test --arg foobarbaz', stdin=subprocess.PIPE, shell=False)
+    mock_popen.assert_called_with(['/bin/test/ --arg foobarbaz'], stdin=subprocess.PIPE, shell=True)
     assert ('elastalert', logging.WARNING, 'Warning! You could be vulnerable to shell injection!') == caplog.record_tuples[0]
     assert ('elastalert', logging.INFO, 'Alert sent to Command') == caplog.record_tuples[1]
 
@@ -53,7 +53,7 @@ def test_command_old_style_string_format2():
     alert = CommandAlerter(rule)
     with mock.patch("elastalert.alerters.command.subprocess.Popen") as mock_popen:
         alert.alert([match])
-    assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True)
+    mock_popen.assert_called_with(['/bin/test/foo.sh'], stdin=subprocess.PIPE, shell=True)
 
 
 def test_command_pipe_match_json():
@@ -67,8 +67,8 @@ def test_command_pipe_match_json():
         mock_popen.return_value = mock_subprocess
         mock_subprocess.communicate.return_value = (None, None)
         alert.alert([match])
-    assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
-    assert mock_subprocess.communicate.called_with(input=json.dumps(match))
+    mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
+    mock_subprocess.communicate.assert_called_with(input=(json.dumps([match]) + '\n').encode())
 
 
 def test_command_pipe_alert_text():
@@ -83,8 +83,8 @@ def test_command_pipe_alert_text():
         mock_popen.return_value = mock_subprocess
         mock_subprocess.communicate.return_value = (None, None)
         alert.alert([match])
-    assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
-    assert mock_subprocess.communicate.called_with(input=alert_text.encode())
+    mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
+    mock_subprocess.communicate.assert_called_with(input=alert_text.encode())
 
 
 def test_command_fail_on_non_zero_exit():
@@ -99,7 +99,7 @@ def test_command_fail_on_non_zero_exit():
             mock_popen.return_value = mock_subprocess
             mock_subprocess.wait.return_value = 1
             alert.alert([match])
-    assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
+    mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)
     assert "Non-zero exit code while running command" in str(exception)
 
 

tests/alerters/gelf_test.py

@@ -211,7 +211,7 @@ def test_gelf_sent_tcp_with_custom_ca(caplog):
     expected_data = json.dumps(expected_data).encode('utf-8') + b'\x00'
 
     with mock.patch('socket.socket') as mock_socket:
-        with mock.patch('ssl.wrap_socket') as mock_ssl_wrap_socket:
+        with mock.patch('ssl.SSLContext.wrap_socket') as mock_ssl_wrap_socket:
             mock_ssl_wrap_socket.return_value = mock_socket
             alert.alert([match])
             mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)

tests/base_test.py

@@ -215,7 +215,7 @@ def test_match(ea):
     with mock.patch('elastalert.elastalert.elasticsearch_client'):
         ea.run_rule(ea.rules[0], END, START)
 
-    ea.rules[0]['alert'][0].alert.called_with({'@timestamp': END_TIMESTAMP})
+    ea.rules[0]['alert'][0].alert.assert_called_with([{'@timestamp': END, 'num_hits': 0, 'num_matches': 1}])
     assert ea.rules[0]['alert'][0].alert.call_count == 1
 
 
@@ -375,7 +375,6 @@ def test_agg_matchtime_timestamp_field(ea):
     with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:
         ea.send_pending_alerts()
         # Assert that current_es was refreshed from the aggregate rules
-        assert mock_es.called_with(host='', port='')
         assert mock_es.call_count == 2
     assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]])
 
@@ -505,7 +504,6 @@ def test_agg_matchtime(ea):
     with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:
         ea.send_pending_alerts()
         # Assert that current_es was refreshed from the aggregate rules
-        assert mock_es.called_with(host='', port='')
         assert mock_es.call_count == 2
     assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]])
 
@@ -607,7 +605,6 @@ def test_agg_with_aggregation_key(ea):
         mock_es.return_value = ea.thread_data.current_es
         ea.send_pending_alerts()
         # Assert that current_es was refreshed from the aggregate rules
-        assert mock_es.called_with(host='', port='')
         assert mock_es.call_count == 2
     assert_alerts(ea, [[hits_timestamps[0], hits_timestamps[2]], [hits_timestamps[1]]])
 
@@ -641,7 +638,7 @@ def test_silence(ea):
     with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
         with mock.patch('elastalert.elastalert.elasticsearch_client'):
             # Converted twice to add tzinfo
-            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
+            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(hours=5)))
             ea.run_rule(ea.rules[0], END, START)
     assert ea.rules[0]['alert'][0].alert.call_count == 1
 
@@ -684,7 +681,7 @@ def test_silence_query_key(ea):
     with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
         with mock.patch('elastalert.elastalert.elasticsearch_client'):
             # Converted twice to add tzinfo
-            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
+            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(hours=5)))
             ea.run_rule(ea.rules[0], END, START)
     assert ea.rules[0]['alert'][0].alert.call_count == 2
 
@@ -711,7 +708,7 @@ def test_realert(ea):
     with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
         with mock.patch('elastalert.elastalert.elasticsearch_client'):
             # mock_ts is converted twice to add tzinfo
-            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(minutes=10)))
+            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(minutes=10)))
             ea.rules[0]['type'].matches = matches
             ea.run_rule(ea.rules[0], END, START)
             assert ea.rules[0]['alert'][0].alert.call_count == 2

tests/tox.ini

@@ -1,6 +1,6 @@
 [tox]
 project = elastalert
-envlist = py311,docs
+envlist = py312,docs
 setupdir = ..
 
 [testenv]

tests/util_test.py

@@ -65,8 +65,8 @@ def test_parse_deadline(spec, expected_deadline):
     # Note: Can't mock ``utcnow`` directly because ``datetime`` is a built-in.
     class MockDatetime(datetime):
         @staticmethod
-        def utcnow():
-            return dt('2017-07-07T10:00:00.000Z')
+        def now(tz=None):
+            return datetime(2017, 7, 7, 10, 0, 0)
 
     with mock.patch('datetime.datetime', MockDatetime):
         assert parse_deadline(spec) == expected_deadline