diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efa6a4aff..a3ad84dce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,7 +160,7 @@ jobs: - name: Check semgrep rules if: steps.changed-files.outputs.all_changed_files - run: semgrep -c .semgrep_rules -c r/python --error -l python --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }} + run: semgrep -c .semgrep_rules -c r/python --error --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }} containerbuild: strategy: diff --git a/CHANGELOG.md b/CHANGELOG.md index a8237ad4f..6ee56404d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ It contains the time at which a pre-detection was created by the processor. ### Bugfix +* fix CVE-2023-37920 Removal of e-Tugra root certificate + ## v6.8.1 ### Bugfix diff --git a/logprep/processor/pseudonymizer/processor.py b/logprep/processor/pseudonymizer/processor.py index 8e9accf60..e0ba367ad 100644 --- a/logprep/processor/pseudonymizer/processor.py +++ b/logprep/processor/pseudonymizer/processor.py @@ -39,7 +39,7 @@ import re from functools import cached_property from logging import Logger -from typing import Any, List, Optional, Tuple, Union, Pattern +from typing import Any, List, Optional, Pattern, Tuple, Union from urllib.parse import parse_qs from attr import define, field, validators @@ -328,7 +328,9 @@ def _parse_url_parts(self, tld_extractor: TLDExtract, url_str: str) -> dict: parts["domain"] = url.domain parts["subdomain"] = url.subdomain parts["suffix"] = url.suffix - url_list = ".".join(list(url)) + url_list = list(url) + url_list.pop() + url_list = ".".join(url_list) parts["path"] = self._find_first( rf"(?:^[a-z0-9]+\:\/\/)?{url_list}(?:\:\d+)?([^#^\?]*).*", url_str ) diff --git a/requirements.in b/requirements.in index 4e937c97e..3b93e2c96 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,6 @@ +aiohttp>=3.8.5 # CVE-2023-37276 attrs -certifi>=2022.12.07 +certifi>=2023.7.22 # CVE-2023-37920 ciso8601 # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 colorama confluent-kafka>2 diff --git a/requirements.txt b/requirements.txt index 1f1a8a4bb..391a18fae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,66 +2,72 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --resolver=backtracking requirements.in +# pip-compile ./requirements.in # -aiohttp==3.8.4 - # via geoip2 +aiohttp==3.8.5 + # via + # -r ./requirements.in + # geoip2 aiosignal==1.3.1 # via aiohttp -anyio==3.7.0 - # via starlette -async-timeout==4.0.2 +annotated-types==0.5.0 + # via pydantic +anyio==3.7.1 + # via + # fastapi + # starlette +async-timeout==4.0.3 # via aiohttp attrs==23.1.0 # via - # -r requirements.in + # -r ./requirements.in # aiohttp -boto3==1.26.146 - # via -r requirements.in -botocore==1.29.146 +boto3==1.28.56 + # via -r ./requirements.in +botocore==1.31.56 # via # boto3 # s3transfer -certifi==2023.5.7 +certifi==2023.7.22 # via - # -r requirements.in + # -r ./requirements.in # elasticsearch # opensearch-py # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests ciso8601==2.3.0 - # via -r requirements.in -click==8.1.3 + # via -r ./requirements.in +click==8.1.7 # via uvicorn colorama==0.4.6 - # via -r requirements.in -confluent-kafka==2.1.1 - # via -r requirements.in -deepdiff==6.3.0 - # via -r requirements.in + # via -r ./requirements.in +confluent-kafka==2.2.0 + # via -r ./requirements.in +deepdiff==6.5.0 + # via -r ./requirements.in elasticsearch==7.17.9 - # via -r requirements.in -exceptiongroup==1.1.1 + # via -r ./requirements.in +exceptiongroup==1.1.3 # via anyio -fastapi==0.96.0 - # via -r requirements.in -filelock==3.12.0 +fastapi==0.103.1 + # via -r ./requirements.in +filelock==3.12.4 # via # tldextract # urlextract -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal geoip2==4.7.0 - # via -r requirements.in + # via -r ./requirements.in h11==0.14.0 # via uvicorn -hyperscan==0.4.0 ; sys_platform == "linux" and platform_machine == "x86_64" - # via -r requirements.in +hyperscan==0.5.0 ; sys_platform == "linux" and platform_machine == "x86_64" + # via -r ./requirements.in idna==3.4 # via # anyio @@ -73,83 +79,85 @@ jmespath==1.0.1 # via # boto3 # botocore -joblib==1.2.0 +joblib==1.3.2 # via - # -r requirements.in + # -r ./requirements.in # scikit-learn jsonref==1.1.0 - # via -r requirements.in + # via -r ./requirements.in luqum==0.13.0 - # via -r requirements.in -maxminddb==2.3.0 + # via -r ./requirements.in +maxminddb==2.4.0 # via geoip2 -msgspec==0.15.1 - # via -r requirements.in +msgspec==0.18.2 + # via -r ./requirements.in multidict==6.0.4 # via # aiohttp # yarl -mysql-connector-python==8.0.33 - # via -r requirements.in -numpy==1.24.3 +mysql-connector-python==8.1.0 + # via -r ./requirements.in +numpy==1.26.0 # via - # -r requirements.in + # -r ./requirements.in # scikit-learn # scipy -opensearch-py==2.2.0 - # via -r requirements.in +opensearch-py==2.3.1 + # via -r ./requirements.in ordered-set==4.1.0 # via deepdiff -platformdirs==3.5.1 +platformdirs==3.10.0 # via urlextract ply==3.11 # via luqum -prometheus-client==0.17.0 - # via -r requirements.in -protobuf==3.20.3 +prometheus-client==0.17.1 + # via -r ./requirements.in +protobuf==4.21.12 # via - # -r requirements.in + # -r ./requirements.in # mysql-connector-python -pycryptodome==3.18.0 - # via -r requirements.in -pydantic==1.10.8 +pycryptodome==3.19.0 + # via -r ./requirements.in +pydantic==2.4.2 # via fastapi +pydantic-core==2.10.1 + # via pydantic pygrok==1.0.0 - # via -r requirements.in -pyparsing==3.0.9 - # via -r requirements.in + # via -r ./requirements.in +pyparsing==3.1.1 + # via -r ./requirements.in python-dateutil==2.8.2 # via # botocore # opensearch-py -pytz==2023.3 - # via -r requirements.in -pyyaml==6.0 - # via -r requirements.in -regex==2023.6.3 +pytz==2023.3.post1 + # via -r ./requirements.in +pyyaml==6.0.1 + # via -r ./requirements.in +regex==2023.8.8 # via pygrok requests==2.31.0 # via - # -r requirements.in + # -r ./requirements.in # geoip2 # opensearch-py # requests-file # tldextract requests-file==1.5.1 # via tldextract -ruamel-yaml==0.17.31 - # via -r requirements.in +ruamel-yaml==0.17.33 + # via -r ./requirements.in ruamel-yaml-clib==0.2.7 # via ruamel-yaml -s3transfer==0.6.1 +s3transfer==0.7.0 # via boto3 schedule==1.2.0 - # via -r requirements.in -scikit-learn==1.2.2 - # via -r requirements.in -scipy==1.10.1 + # via -r ./requirements.in +scikit-learn==1.3.1 + # via -r ./requirements.in +scipy==1.11.3 # via - # -r requirements.in + # -r ./requirements.in # scikit-learn six==1.16.0 # via @@ -160,27 +168,30 @@ sniffio==1.3.0 # via anyio starlette==0.27.0 # via fastapi -threadpoolctl==3.1.0 +threadpoolctl==3.2.0 # via scikit-learn -tldextract==3.4.4 - # via -r requirements.in -typing-extensions==4.6.3 +tldextract==3.6.0 + # via -r ./requirements.in +typing-extensions==4.8.0 # via + # fastapi # pydantic + # pydantic-core # starlette -uritools==4.0.1 + # uvicorn +uritools==4.0.2 # via urlextract urlextract==1.8.0 - # via -r requirements.in + # via -r ./requirements.in urllib3==1.26.16 # via # botocore # elasticsearch # opensearch-py # requests -uvicorn==0.22.0 - # via -r requirements.in -wheel==0.40.0 - # via -r requirements.in +uvicorn==0.23.2 + # via -r ./requirements.in +wheel==0.41.2 + # via -r ./requirements.in yarl==1.9.2 # via aiohttp diff --git a/requirements_dev.txt b/requirements_dev.txt index ec250de75..caae21475 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,127 +2,133 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --resolver=backtracking requirements_dev.in +# pip-compile ./requirements_dev.in # -aiohttp==3.8.4 +aiohttp==3.8.5 # via - # -r requirements.txt + # -r ./requirements.txt # geoip2 aiosignal==1.3.1 # via - # -r requirements.txt + # -r ./requirements.txt # aiohttp -anyio==3.7.0 +annotated-types==0.5.0 # via - # -r requirements.txt + # -r ./requirements.txt + # pydantic +anyio==3.7.1 + # via + # -r ./requirements.txt + # fastapi # httpcore # starlette -astroid==2.15.5 +astroid==2.15.8 # via pylint -async-timeout==4.0.2 +async-timeout==4.0.3 # via - # -r requirements.txt + # -r ./requirements.txt # aiohttp attrs==23.1.0 # via - # -r requirements.txt + # -r ./requirements.txt # aiohttp # glom # jsonschema + # referencing # semgrep -black==23.3.0 - # via -r requirements_dev.in +black==23.9.1 + # via -r ./requirements_dev.in boltons==21.0.0 # via # face # glom # semgrep -boto3==1.26.146 - # via -r requirements.txt -botocore==1.29.146 +boto3==1.28.56 + # via -r ./requirements.txt +botocore==1.31.56 # via - # -r requirements.txt + # -r ./requirements.txt # boto3 # s3transfer -bracex==2.3.post1 +bracex==2.4 # via wcmatch -certifi==2023.5.7 +certifi==2023.7.22 # via - # -r requirements.txt + # -r ./requirements.txt # elasticsearch # httpcore # httpx # opensearch-py # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via - # -r requirements.txt + # -r ./requirements.txt # aiohttp # requests ciso8601==2.3.0 - # via -r requirements.txt -click==8.1.3 + # via -r ./requirements.txt +click==8.1.7 # via - # -r requirements.txt + # -r ./requirements.txt # black # click-option-group # semgrep # uvicorn -click-option-group==0.5.5 +click-option-group==0.5.6 # via semgrep colorama==0.4.6 # via - # -r requirements.txt + # -r ./requirements.txt # semgrep -confluent-kafka==2.1.1 - # via -r requirements.txt -coverage[toml]==7.2.7 +confluent-kafka==2.2.0 + # via -r ./requirements.txt +coverage[toml]==7.3.1 # via pytest-cov -deepdiff==6.3.0 - # via -r requirements.txt +deepdiff==6.5.0 + # via -r ./requirements.txt defusedxml==0.7.1 # via semgrep -dill==0.3.6 +dill==0.3.7 # via pylint elasticsearch==7.17.9 - # via -r requirements.txt -exceptiongroup==1.1.1 + # via -r ./requirements.txt +exceptiongroup==1.1.3 # via - # -r requirements.txt + # -r ./requirements.txt # anyio # pytest face==22.0.0 # via glom -fastapi==0.96.0 - # via -r requirements.txt -filelock==3.12.0 +fastapi==0.103.1 + # via -r ./requirements.txt +filelock==3.12.4 # via - # -r requirements.txt + # -r ./requirements.txt # tldextract # urlextract -frozenlist==1.3.3 +frozenlist==1.4.0 # via - # -r requirements.txt + # -r ./requirements.txt # aiohttp # aiosignal geoip2==4.7.0 - # via -r requirements.txt + # via -r ./requirements.txt glom==22.1.0 # via semgrep h11==0.14.0 # via - # -r requirements.txt + # -r ./requirements.txt # httpcore # uvicorn -httpcore==0.17.2 +httpcore==0.18.0 # via httpx -httpx==0.24.1 - # via -r requirements_dev.in -hyperscan==0.4.0 ; sys_platform == "linux" and platform_machine == "x86_64" - # via -r requirements.txt +httpx==0.25.0 + # via -r ./requirements_dev.in +hyperscan==0.5.0 ; sys_platform == "linux" and platform_machine == "x86_64" + # via -r ./requirements.txt idna==3.4 # via - # -r requirements.txt + # -r ./requirements.txt # anyio # httpx # requests @@ -133,130 +139,138 @@ iniconfig==2.0.0 # via pytest isort==5.12.0 # via - # -r requirements_dev.in + # -r ./requirements_dev.in # pylint jinja2==3.1.2 - # via -r requirements_dev.in + # via -r ./requirements_dev.in jmespath==1.0.1 # via - # -r requirements.txt + # -r ./requirements.txt # boto3 # botocore -joblib==1.2.0 +joblib==1.3.2 # via - # -r requirements.txt + # -r ./requirements.txt # scikit-learn jsonref==1.1.0 - # via -r requirements.txt -jsonschema==4.17.3 + # via -r ./requirements.txt +jsonschema==4.19.1 # via semgrep +jsonschema-specifications==2023.7.1 + # via jsonschema lazy-object-proxy==1.9.0 # via astroid luqum==0.13.0 - # via -r requirements.txt -markdown-it-py==2.2.0 + # via -r ./requirements.txt +markdown-it-py==3.0.0 # via rich markupsafe==2.1.3 # via jinja2 -maxminddb==2.3.0 +maxminddb==2.4.0 # via - # -r requirements.txt + # -r ./requirements.txt # geoip2 mccabe==0.7.0 # via pylint mdurl==0.1.2 # via markdown-it-py -msgspec==0.15.1 - # via -r requirements.txt +msgspec==0.18.2 + # via -r ./requirements.txt multidict==6.0.4 # via - # -r requirements.txt + # -r ./requirements.txt # aiohttp # yarl mypy-extensions==1.0.0 # via black -mysql-connector-python==8.0.33 - # via -r requirements.txt -numpy==1.24.3 +mysql-connector-python==8.1.0 + # via -r ./requirements.txt +numpy==1.26.0 # via - # -r requirements.txt + # -r ./requirements.txt # scikit-learn # scipy -opensearch-py==2.2.0 - # via -r requirements.txt +opensearch-py==2.3.1 + # via -r ./requirements.txt ordered-set==4.1.0 # via - # -r requirements.txt + # -r ./requirements.txt # deepdiff packaging==23.1 # via # black # pytest # semgrep -pathspec==0.11.1 +pathspec==0.11.2 # via black -peewee==3.16.2 +peewee==3.16.3 # via semgrep -platformdirs==3.5.1 +platformdirs==3.10.0 # via - # -r requirements.txt + # -r ./requirements.txt # black # pylint # urlextract -pluggy==1.0.0 +pluggy==1.3.0 # via pytest ply==3.11 # via - # -r requirements.txt + # -r ./requirements.txt # luqum -prometheus-client==0.17.0 - # via -r requirements.txt -protobuf==3.20.3 +prometheus-client==0.17.1 + # via -r ./requirements.txt +protobuf==4.21.12 # via - # -r requirements.txt + # -r ./requirements.txt # mysql-connector-python -pycryptodome==3.18.0 - # via -r requirements.txt -pydantic==1.10.8 +pycryptodome==3.19.0 + # via -r ./requirements.txt +pydantic==2.4.2 # via - # -r requirements.txt + # -r ./requirements.txt # fastapi -pygments==2.15.1 +pydantic-core==2.10.1 + # via + # -r ./requirements.txt + # pydantic +pygments==2.16.1 # via rich pygrok==1.0.0 - # via -r requirements.txt -pylint==2.17.4 - # via -r requirements_dev.in -pyparsing==3.0.9 - # via -r requirements.txt -pyrsistent==0.19.3 - # via jsonschema -pytest==7.3.1 - # via - # -r requirements_dev.in + # via -r ./requirements.txt +pylint==2.17.6 + # via -r ./requirements_dev.in +pyparsing==3.1.1 + # via -r ./requirements.txt +pytest==7.4.2 + # via + # -r ./requirements_dev.in # pytest-cov pytest-cov==4.1.0 - # via -r requirements_dev.in + # via -r ./requirements_dev.in python-dateutil==2.8.2 # via - # -r requirements.txt + # -r ./requirements.txt # botocore # opensearch-py python-lsp-jsonrpc==1.0.0 # via semgrep -pytz==2023.3 - # via -r requirements.txt -pyyaml==6.0 +pytz==2023.3.post1 + # via -r ./requirements.txt +pyyaml==6.0.1 # via - # -r requirements.txt + # -r ./requirements.txt # responses -regex==2023.6.3 +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications +regex==2023.8.8 # via - # -r requirements.txt + # -r ./requirements.txt # pygrok requests==2.31.0 # via - # -r requirements.txt + # -r ./requirements.txt # geoip2 # opensearch-py # requests-file @@ -265,56 +279,60 @@ requests==2.31.0 # tldextract requests-file==1.5.1 # via - # -r requirements.txt + # -r ./requirements.txt # tldextract -responses==0.23.1 - # via -r requirements_dev.in -rich==13.4.1 +responses==0.23.3 + # via -r ./requirements_dev.in +rich==13.5.3 # via semgrep -ruamel-yaml==0.17.31 +rpds-py==0.10.3 # via - # -r requirements.txt + # jsonschema + # referencing +ruamel-yaml==0.17.33 + # via + # -r ./requirements.txt # semgrep ruamel-yaml-clib==0.2.7 # via - # -r requirements.txt + # -r ./requirements.txt # ruamel-yaml -s3transfer==0.6.1 +s3transfer==0.7.0 # via - # -r requirements.txt + # -r ./requirements.txt # boto3 schedule==1.2.0 - # via -r requirements.txt -scikit-learn==1.2.2 - # via -r requirements.txt -scipy==1.10.1 + # via -r ./requirements.txt +scikit-learn==1.3.1 + # via -r ./requirements.txt +scipy==1.11.3 # via - # -r requirements.txt + # -r ./requirements.txt # scikit-learn -semgrep==1.24.1 - # via -r requirements_dev.in +semgrep==1.41.0 + # via -r ./requirements_dev.in six==1.16.0 # via - # -r requirements.txt + # -r ./requirements.txt # opensearch-py # python-dateutil # requests-file sniffio==1.3.0 # via - # -r requirements.txt + # -r ./requirements.txt # anyio # httpcore # httpx starlette==0.27.0 # via - # -r requirements.txt + # -r ./requirements.txt # fastapi -threadpoolctl==3.1.0 +threadpoolctl==3.2.0 # via - # -r requirements.txt + # -r ./requirements.txt # scikit-learn -tldextract==3.4.4 - # via -r requirements.txt +tldextract==3.6.0 + # via -r ./requirements.txt tomli==2.0.1 # via # black @@ -322,45 +340,48 @@ tomli==2.0.1 # pylint # pytest # semgrep -tomlkit==0.11.8 +tomlkit==0.12.1 # via pylint -types-pyyaml==6.0.12.10 +types-pyyaml==6.0.12.12 # via responses -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via - # -r requirements.txt + # -r ./requirements.txt # astroid # black + # fastapi # pydantic + # pydantic-core # pylint # semgrep # starlette -ujson==5.7.0 + # uvicorn +ujson==5.8.0 # via python-lsp-jsonrpc -uritools==4.0.1 +uritools==4.0.2 # via - # -r requirements.txt + # -r ./requirements.txt # urlextract urlextract==1.8.0 - # via -r requirements.txt + # via -r ./requirements.txt urllib3==1.26.16 # via - # -r requirements.txt + # -r ./requirements.txt # botocore # elasticsearch # opensearch-py # requests # responses # semgrep -uvicorn==0.22.0 - # via -r requirements.txt -wcmatch==8.4.1 +uvicorn==0.23.2 + # via -r ./requirements.txt +wcmatch==8.5 # via semgrep -wheel==0.40.0 - # via -r requirements.txt +wheel==0.41.2 + # via -r ./requirements.txt wrapt==1.15.0 # via astroid yarl==1.9.2 # via - # -r requirements.txt + # -r ./requirements.txt # aiohttp