diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 86e085ff5..92dfd13f2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/python-3/.devcontainer/base.Dockerfile -# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster -ARG VARIANT="3.10-bullseye" +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon) +ARG VARIANT="3.11-bullseye" FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8d7ed4977..1ef937c23 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,7 @@ "dockerfile": "Dockerfile", "context": "..", "args": { - // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + // Update 'VARIANT' to pick a Python version: 3, ... // Append -bullseye or -buster to pin to an OS version. // Use -bullseye variants on local on arm64/Apple Silicon. "VARIANT": "3.12", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d60e4a0d..e83507f38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,11 +12,11 @@ jobs: uses: ./.github/workflows/testing.yml build-docs: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 @@ -40,11 +40,11 @@ jobs: make clean html code-quality: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 @@ -97,7 +97,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest steps: @@ -126,7 +126,7 @@ jobs: - name: Ensure logprep is available in image run: | docker run --rm ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-${{ github.head_ref }} --version - + # This step will build the image again, but every layer will already be cached, so it is nearly instantaneous. - name: Push image uses: docker/build-push-action@v5 @@ -138,7 +138,7 @@ jobs: PYTHON_VERSION=${{ matrix.python-version }} tags: | ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-${{ github.head_ref }} - + # To avoid the trivy-db becoming outdated, we save the cache for one day - name: Get date id: date @@ -149,9 +149,8 @@ jobs: with: path: cache/db key: trivy-cache-${{ steps.date.outputs.date }} - restore-keys: - trivy-cache- - + restore-keys: trivy-cache- + - name: Scan image using Trivy uses: aquasecurity/trivy-action@0.24.0 env: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 90e144c09..58ee2c703 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,11 +10,11 @@ jobs: uses: ./.github/workflows/testing.yml code-quality: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish-latest-dev-release-to-pypi.yml b/.github/workflows/publish-latest-dev-release-to-pypi.yml index 0c49be251..3689f4cb3 100644 --- a/.github/workflows/publish-latest-dev-release-to-pypi.yml +++ b/.github/workflows/publish-latest-dev-release-to-pypi.yml @@ -49,7 +49,7 @@ jobs: containerbuild: strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/publish-release-to-pypi.yml b/.github/workflows/publish-release-to-pypi.yml index a9a9d46af..9943426bb 100644 --- a/.github/workflows/publish-release-to-pypi.yml +++ b/.github/workflows/publish-release-to-pypi.yml @@ -62,7 +62,7 @@ jobs: containerbuild: strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest needs: publish-latest-release-to-pypi diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 87d7c49e0..2047d7e02 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,16 +5,16 @@ on: jobs: rust: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Perform tests for rust modules run: cargo test python: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] test-type: ["unit", "acceptance"] steps: - uses: actions/checkout@v4 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3a5c7f86a..b5bf628ba 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,11 +7,11 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 apt_packages: - "pandoc" tools: - python: "3.10" + python: "3.11" # You can also specify other tool versions: # nodejs: "16" rust: "latest" diff --git a/Dockerfile b/Dockerfile index cc9153771..7c2394c7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.11 FROM bitnami/python:${PYTHON_VERSION} as build ARG LOGPREP_VERSION=latest diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 34189b618..5878b172b 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -5,7 +5,7 @@ Installation PIP === -Python should be present on the system. Currently, Python 3.10 - 3.12 are supported. +Python should be present on the system. Currently, Python 3.11 - 3.13 are supported. To install Logprep you have following options: **1. Option:** latest stable release diff --git a/logprep/util/grok/grok.py b/logprep/util/grok/grok.py index 39d3d3bfd..25da7b774 100644 --- a/logprep/util/grok/grok.py +++ b/logprep/util/grok/grok.py @@ -27,22 +27,16 @@ import string import sys from hashlib import md5 +from importlib import resources from itertools import chain from pathlib import Path from re import error import numpy as np -from importlib import resources from attrs import define, field, validators from logprep.util.decorators import timeout -if sys.version_info.minor < 11: - # because needed possessive quantifiers and atomic grouping - # added to re module in python 3.11 - import regex as re # pylint: disable=shadowed-import - - DEFAULT_PATTERNS_DIRS = [str(resources.files(__package__) / "patterns/ecs-v1")] LOGSTASH_NOTATION = r"(([^\[\]\{\}\.:]*)?(\[[^\[\]\{\}\.:]*\])*)" GROK = r"%\{" + rf"([A-Z0-9_]*)(:({LOGSTASH_NOTATION}))?(:(int|float))?" + r"\}" diff --git a/logprep/util/time.py b/logprep/util/time.py index 32a209289..f02190611 100644 --- a/logprep/util/time.py +++ b/logprep/util/time.py @@ -4,8 +4,6 @@ from typing import Union from zoneinfo import ZoneInfo -import ciso8601 - from logprep.abc.exceptions import LogprepException UTC = ZoneInfo("UTC") @@ -35,7 +33,7 @@ def from_string(cls, source: str, set_missing_utc: bool = True) -> datetime: datetime object """ try: - time_object = ciso8601.parse_datetime(source) # pylint: disable=c-extension-no-member + time_object = datetime.fromisoformat(source) # pylint: disable=c-extension-no-member if set_missing_utc: time_object = cls._set_utc_if_timezone_is_missing(time_object) return time_object diff --git a/pyproject.toml b/pyproject.toml index 85d38a036..3c998bd25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ binding = "PyO3" # Default value, can be omitted [project] name = "logprep" description = "Logprep allows to collect, process and forward log messages from various data sources." -requires-python = ">=3.10" +requires-python = ">=3.11" readme = "README.md" dynamic = ["version"] license = { file = "LICENSE" } @@ -37,9 +37,9 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: POSIX :: Linux", @@ -56,10 +56,9 @@ keywords = [ "logdata", ] dependencies = [ - "aiohttp>=3.9.2", # CVE-2024-23334 + "aiohttp>=3.9.2", # CVE-2024-23334 "attrs", - "certifi>=2023.7.22", # CVE-2023-37920 - "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 + "certifi>=2023.7.22", # CVE-2023-37920 "colorama", "confluent-kafka>2", "geoip2", @@ -67,7 +66,7 @@ dependencies = [ "jsonref", "luqum", "more-itertools==8.10.0", - "mysql-connector-python>=9.1.0", # CVE-2024-21272 + "mysql-connector-python>=9.1.0", # CVE-2024-21272 "numpy>=1.26.0", "opensearch-py", "prometheus_client", @@ -79,12 +78,11 @@ dependencies = [ "joblib", "pyyaml", "requests>=2.31.0", - "regex", "ruamel.yaml", "schedule", "tldextract", "urlextract", - "urllib3>=1.26.17", # CVE-2023-43804 + "urllib3>=1.26.17", # CVE-2023-43804 "uvicorn", "deepdiff", "msgspec", diff --git a/tests/unit/processor/timestamp_differ/test_timestamp_differ.py b/tests/unit/processor/timestamp_differ/test_timestamp_differ.py index dd082562a..edb0bf6a8 100644 --- a/tests/unit/processor/timestamp_differ/test_timestamp_differ.py +++ b/tests/unit/processor/timestamp_differ/test_timestamp_differ.py @@ -343,7 +343,7 @@ "subfield": {"field2": "2022-12-05 12:00:00"}, "tags": ["_timestamp_differ_failure"], }, - r".*ProcessingWarning.*Invalid character while parsing", + r".*ProcessingWarning.*Invalid isoformat string: 'non-timestamp'", ), ( "diff between two timestamps with partial timestamp format match",