Fix CVE 2023 37920

.github/workflows/ci.yml

@@ -160,7 +160,7 @@ jobs:
 
       - name: Check semgrep rules
         if: steps.changed-files.outputs.all_changed_files
-        run: semgrep -c .semgrep_rules -c r/python --error -l python --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}
+        run: semgrep -c .semgrep_rules -c r/python --error --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}
 
   containerbuild:
     strategy:

CHANGELOG.md

@@ -9,6 +9,8 @@ It contains the time at which a pre-detection was created by the processor.
 
 ### Bugfix
 
+* fix CVE-2023-37920 Removal of e-Tugra root certificate
+
 ## v6.8.1
 ### Bugfix
 

logprep/processor/pseudonymizer/processor.py

@@ -39,7 +39,7 @@
 import re
 from functools import cached_property
 from logging import Logger
-from typing import Any, List, Optional, Tuple, Union, Pattern
+from typing import Any, List, Optional, Pattern, Tuple, Union
 from urllib.parse import parse_qs
 
 from attr import define, field, validators
@@ -328,7 +328,9 @@ def _parse_url_parts(self, tld_extractor: TLDExtract, url_str: str) -> dict:
         parts["domain"] = url.domain
         parts["subdomain"] = url.subdomain
         parts["suffix"] = url.suffix
-        url_list = ".".join(list(url))
+        url_list = list(url)
+        url_list.pop()
+        url_list = ".".join(url_list)
         parts["path"] = self._find_first(
             rf"(?:^[a-z0-9]+\:\/\/)?{url_list}(?:\:\d+)?([^#^\?]*).*", url_str
         )

requirements.in

@@ -1,5 +1,6 @@
+aiohttp>=3.8.5 # CVE-2023-37276
 attrs
-certifi>=2022.12.07
+certifi>=2023.7.22 # CVE-2023-37920
 ciso8601 # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11
 colorama
 confluent-kafka>2

requirements.txt

@@ -2,66 +2,72 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --resolver=backtracking requirements.in
+#    pip-compile ./requirements.in
 #
-aiohttp==3.8.4
-    # via geoip2
+aiohttp==3.8.5
+    # via
+    #   -r ./requirements.in
+    #   geoip2
 aiosignal==1.3.1
     # via aiohttp
-anyio==3.7.0
-    # via starlette
-async-timeout==4.0.2
+annotated-types==0.5.0
+    # via pydantic
+anyio==3.7.1
+    # via
+    #   fastapi
+    #   starlette
+async-timeout==4.0.3
     # via aiohttp
 attrs==23.1.0
     # via
-    #   -r requirements.in
+    #   -r ./requirements.in
     #   aiohttp
-boto3==1.26.146
-    # via -r requirements.in
-botocore==1.29.146
+boto3==1.28.56
+    # via -r ./requirements.in
+botocore==1.31.56
     # via
     #   boto3
     #   s3transfer
-certifi==2023.5.7
+certifi==2023.7.22
     # via
-    #   -r requirements.in
+    #   -r ./requirements.in
     #   elasticsearch
     #   opensearch-py
     #   requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via
     #   aiohttp
     #   requests
 ciso8601==2.3.0
-    # via -r requirements.in
-click==8.1.3
+    # via -r ./requirements.in
+click==8.1.7
     # via uvicorn
 colorama==0.4.6
-    # via -r requirements.in
-confluent-kafka==2.1.1
-    # via -r requirements.in
-deepdiff==6.3.0
-    # via -r requirements.in
+    # via -r ./requirements.in
+confluent-kafka==2.2.0
+    # via -r ./requirements.in
+deepdiff==6.5.0
+    # via -r ./requirements.in
 elasticsearch==7.17.9
-    # via -r requirements.in
-exceptiongroup==1.1.1
+    # via -r ./requirements.in
+exceptiongroup==1.1.3
     # via anyio
-fastapi==0.96.0
-    # via -r requirements.in
-filelock==3.12.0
+fastapi==0.103.1
+    # via -r ./requirements.in
+filelock==3.12.4
     # via
     #   tldextract
     #   urlextract
-frozenlist==1.3.3
+frozenlist==1.4.0
     # via
     #   aiohttp
     #   aiosignal
 geoip2==4.7.0
-    # via -r requirements.in
+    # via -r ./requirements.in
 h11==0.14.0
     # via uvicorn
-hyperscan==0.4.0 ; sys_platform == "linux" and platform_machine == "x86_64"
-    # via -r requirements.in
+hyperscan==0.5.0 ; sys_platform == "linux" and platform_machine == "x86_64"
+    # via -r ./requirements.in
 idna==3.4
     # via
     #   anyio
@@ -73,83 +79,85 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-joblib==1.2.0
+joblib==1.3.2
     # via
-    #   -r requirements.in
+    #   -r ./requirements.in
     #   scikit-learn
 jsonref==1.1.0
-    # via -r requirements.in
+    # via -r ./requirements.in
 luqum==0.13.0
-    # via -r requirements.in
-maxminddb==2.3.0
+    # via -r ./requirements.in
+maxminddb==2.4.0
     # via geoip2
-msgspec==0.15.1
-    # via -r requirements.in
+msgspec==0.18.2
+    # via -r ./requirements.in
 multidict==6.0.4
     # via
     #   aiohttp
     #   yarl
-mysql-connector-python==8.0.33
-    # via -r requirements.in
-numpy==1.24.3
+mysql-connector-python==8.1.0
+    # via -r ./requirements.in
+numpy==1.26.0
     # via
-    #   -r requirements.in
+    #   -r ./requirements.in
     #   scikit-learn
     #   scipy
-opensearch-py==2.2.0
-    # via -r requirements.in
+opensearch-py==2.3.1
+    # via -r ./requirements.in
 ordered-set==4.1.0
     # via deepdiff
-platformdirs==3.5.1
+platformdirs==3.10.0
     # via urlextract
 ply==3.11
     # via luqum
-prometheus-client==0.17.0
-    # via -r requirements.in
-protobuf==3.20.3
+prometheus-client==0.17.1
+    # via -r ./requirements.in
+protobuf==4.21.12
     # via
-    #   -r requirements.in
+    #   -r ./requirements.in
     #   mysql-connector-python
-pycryptodome==3.18.0
-    # via -r requirements.in
-pydantic==1.10.8
+pycryptodome==3.19.0
+    # via -r ./requirements.in
+pydantic==2.4.2
     # via fastapi
+pydantic-core==2.10.1
+    # via pydantic
 pygrok==1.0.0
-    # via -r requirements.in
-pyparsing==3.0.9
-    # via -r requirements.in
+    # via -r ./requirements.in
+pyparsing==3.1.1
+    # via -r ./requirements.in
 python-dateutil==2.8.2
     # via
     #   botocore
     #   opensearch-py
-pytz==2023.3
-    # via -r requirements.in
-pyyaml==6.0
-    # via -r requirements.in
-regex==2023.6.3
+pytz==2023.3.post1
+    # via -r ./requirements.in
+pyyaml==6.0.1
+    # via -r ./requirements.in
+regex==2023.8.8
     # via pygrok
 requests==2.31.0
     # via
-    #   -r requirements.in
+    #   -r ./requirements.in
     #   geoip2
     #   opensearch-py
     #   requests-file
     #   tldextract
 requests-file==1.5.1
     # via tldextract
-ruamel-yaml==0.17.31
-    # via -r requirements.in
+ruamel-yaml==0.17.33
+    # via -r ./requirements.in
 ruamel-yaml-clib==0.2.7
     # via ruamel-yaml
-s3transfer==0.6.1
+s3transfer==0.7.0
     # via boto3
 schedule==1.2.0
-    # via -r requirements.in
-scikit-learn==1.2.2
-    # via -r requirements.in
-scipy==1.10.1
+    # via -r ./requirements.in
+scikit-learn==1.3.1
+    # via -r ./requirements.in
+scipy==1.11.3
     # via
-    #   -r requirements.in
+    #   -r ./requirements.in
     #   scikit-learn
 six==1.16.0
     # via
@@ -160,27 +168,30 @@ sniffio==1.3.0
     # via anyio
 starlette==0.27.0
     # via fastapi
-threadpoolctl==3.1.0
+threadpoolctl==3.2.0
     # via scikit-learn
-tldextract==3.4.4
-    # via -r requirements.in
-typing-extensions==4.6.3
+tldextract==3.6.0
+    # via -r ./requirements.in
+typing-extensions==4.8.0
     # via
+    #   fastapi
     #   pydantic
+    #   pydantic-core
     #   starlette
-uritools==4.0.1
+    #   uvicorn
+uritools==4.0.2
     # via urlextract
 urlextract==1.8.0
-    # via -r requirements.in
+    # via -r ./requirements.in
 urllib3==1.26.16
     # via
     #   botocore
     #   elasticsearch
     #   opensearch-py
     #   requests
-uvicorn==0.22.0
-    # via -r requirements.in
-wheel==0.40.0
-    # via -r requirements.in
+uvicorn==0.23.2
+    # via -r ./requirements.in
+wheel==0.41.2
+    # via -r ./requirements.in
 yarl==1.9.2
     # via aiohttp