From 4628e397d42dac5b0b57ac84e22d2c9b2fe18009 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Fri, 13 Sep 2024 16:57:19 +0200 Subject: [PATCH 1/4] add support for python 3.13 and drop support for python 3.10 --- .devcontainer/Dockerfile | 4 ++-- .devcontainer/devcontainer.json | 2 +- .github/workflows/ci.yml | 15 +++++++-------- .github/workflows/main.yml | 2 +- .../publish-latest-dev-release-to-pypi.yml | 2 +- .github/workflows/publish-release-to-pypi.yml | 2 +- .github/workflows/testing.yml | 2 +- .readthedocs.yaml | 2 +- Dockerfile | 2 +- doc/source/installation.rst | 2 +- pyproject.toml | 4 ++-- 11 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 86e085ff5..92dfd13f2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/python-3/.devcontainer/base.Dockerfile -# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster -ARG VARIANT="3.10-bullseye" +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon) +ARG VARIANT="3.11-bullseye" FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8d7ed4977..1ef937c23 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,7 @@ "dockerfile": "Dockerfile", "context": "..", "args": { - // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + // Update 'VARIANT' to pick a Python version: 3, ... // Append -bullseye or -buster to pin to an OS version. // Use -bullseye variants on local on arm64/Apple Silicon. "VARIANT": "3.12", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d60e4a0d..7018ca7a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 @@ -44,7 +44,7 @@ jobs: strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 @@ -97,7 +97,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest steps: @@ -126,7 +126,7 @@ jobs: - name: Ensure logprep is available in image run: | docker run --rm ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-${{ github.head_ref }} --version - + # This step will build the image again, but every layer will already be cached, so it is nearly instantaneous. - name: Push image uses: docker/build-push-action@v5 @@ -138,7 +138,7 @@ jobs: PYTHON_VERSION=${{ matrix.python-version }} tags: | ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-${{ github.head_ref }} - + # To avoid the trivy-db becoming outdated, we save the cache for one day - name: Get date id: date @@ -149,9 +149,8 @@ jobs: with: path: cache/db key: trivy-cache-${{ steps.date.outputs.date }} - restore-keys: - trivy-cache- - + restore-keys: trivy-cache- + - name: Scan image using Trivy uses: aquasecurity/trivy-action@0.24.0 env: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 90e144c09..b283950e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish-latest-dev-release-to-pypi.yml b/.github/workflows/publish-latest-dev-release-to-pypi.yml index 0c49be251..3689f4cb3 100644 --- a/.github/workflows/publish-latest-dev-release-to-pypi.yml +++ b/.github/workflows/publish-latest-dev-release-to-pypi.yml @@ -49,7 +49,7 @@ jobs: containerbuild: strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/publish-release-to-pypi.yml b/.github/workflows/publish-release-to-pypi.yml index a9a9d46af..9943426bb 100644 --- a/.github/workflows/publish-release-to-pypi.yml +++ b/.github/workflows/publish-release-to-pypi.yml @@ -62,7 +62,7 @@ jobs: containerbuild: strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest needs: publish-latest-release-to-pypi diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 87d7c49e0..46742890a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] test-type: ["unit", "acceptance"] steps: - uses: actions/checkout@v4 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3a5c7f86a..3012f9cb2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,7 +11,7 @@ build: apt_packages: - "pandoc" tools: - python: "3.10" + python: "3.11" # You can also specify other tool versions: # nodejs: "16" rust: "latest" diff --git a/Dockerfile b/Dockerfile index cc9153771..7c2394c7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.11 FROM bitnami/python:${PYTHON_VERSION} as build ARG LOGPREP_VERSION=latest diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 34189b618..5878b172b 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -5,7 +5,7 @@ Installation PIP === -Python should be present on the system. Currently, Python 3.10 - 3.12 are supported. +Python should be present on the system. Currently, Python 3.11 - 3.13 are supported. To install Logprep you have following options: **1. Option:** latest stable release diff --git a/pyproject.toml b/pyproject.toml index 85d38a036..4aebc6943 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ binding = "PyO3" # Default value, can be omitted [project] name = "logprep" description = "Logprep allows to collect, process and forward log messages from various data sources." -requires-python = ">=3.10" +requires-python = ">=3.11" readme = "README.md" dynamic = ["version"] license = { file = "LICENSE" } @@ -37,9 +37,9 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: POSIX :: Linux", From 4814c144a20624bb2085ca78fd9b61404a1f86b3 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Thu, 3 Oct 2024 16:35:50 +0200 Subject: [PATCH 2/4] upgrade ubuntu version --- .github/workflows/ci.yml | 4 ++-- .github/workflows/main.yml | 2 +- .github/workflows/testing.yml | 4 ++-- .readthedocs.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7018ca7a9..e83507f38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: uses: ./.github/workflows/testing.yml build-docs: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: @@ -40,7 +40,7 @@ jobs: make clean html code-quality: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b283950e8..58ee2c703 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/testing.yml code-quality: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 46742890a..2047d7e02 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,13 +5,13 @@ on: jobs: rust: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Perform tests for rust modules run: cargo test python: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.11", "3.12", "3.13"] diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3012f9cb2..b5bf628ba 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,7 +7,7 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 apt_packages: - "pandoc" tools: From bcd07abcc92d46ebdc1abddd31d3ba03b4a97460 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Wed, 9 Oct 2024 19:03:19 +0200 Subject: [PATCH 3/4] remove regex dependency cause it is only needed for versions lower than python 3.11 --- logprep/util/grok/grok.py | 8 +------- pyproject.toml | 11 +++++------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/logprep/util/grok/grok.py b/logprep/util/grok/grok.py index 39d3d3bfd..25da7b774 100644 --- a/logprep/util/grok/grok.py +++ b/logprep/util/grok/grok.py @@ -27,22 +27,16 @@ import string import sys from hashlib import md5 +from importlib import resources from itertools import chain from pathlib import Path from re import error import numpy as np -from importlib import resources from attrs import define, field, validators from logprep.util.decorators import timeout -if sys.version_info.minor < 11: - # because needed possessive quantifiers and atomic grouping - # added to re module in python 3.11 - import regex as re # pylint: disable=shadowed-import - - DEFAULT_PATTERNS_DIRS = [str(resources.files(__package__) / "patterns/ecs-v1")] LOGSTASH_NOTATION = r"(([^\[\]\{\}\.:]*)?(\[[^\[\]\{\}\.:]*\])*)" GROK = r"%\{" + rf"([A-Z0-9_]*)(:({LOGSTASH_NOTATION}))?(:(int|float))?" + r"\}" diff --git a/pyproject.toml b/pyproject.toml index 4aebc6943..38114f961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,10 @@ keywords = [ "logdata", ] dependencies = [ - "aiohttp>=3.9.2", # CVE-2024-23334 + "aiohttp>=3.9.2", # CVE-2024-23334 "attrs", - "certifi>=2023.7.22", # CVE-2023-37920 - "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 + "certifi>=2023.7.22", # CVE-2023-37920 + "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 "colorama", "confluent-kafka>2", "geoip2", @@ -67,7 +67,7 @@ dependencies = [ "jsonref", "luqum", "more-itertools==8.10.0", - "mysql-connector-python>=9.1.0", # CVE-2024-21272 + "mysql-connector-python>=9.1.0", # CVE-2024-21272 "numpy>=1.26.0", "opensearch-py", "prometheus_client", @@ -79,12 +79,11 @@ dependencies = [ "joblib", "pyyaml", "requests>=2.31.0", - "regex", "ruamel.yaml", "schedule", "tldextract", "urlextract", - "urllib3>=1.26.17", # CVE-2023-43804 + "urllib3>=1.26.17", # CVE-2023-43804 "uvicorn", "deepdiff", "msgspec", From 1c819e2ad0919d4d0de301062e7f44810434ad86 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Sat, 19 Oct 2024 21:10:10 +0200 Subject: [PATCH 4/4] remove ciso8601 --- logprep/util/time.py | 4 +--- pyproject.toml | 1 - .../unit/processor/timestamp_differ/test_timestamp_differ.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/logprep/util/time.py b/logprep/util/time.py index 32a209289..f02190611 100644 --- a/logprep/util/time.py +++ b/logprep/util/time.py @@ -4,8 +4,6 @@ from typing import Union from zoneinfo import ZoneInfo -import ciso8601 - from logprep.abc.exceptions import LogprepException UTC = ZoneInfo("UTC") @@ -35,7 +33,7 @@ def from_string(cls, source: str, set_missing_utc: bool = True) -> datetime: datetime object """ try: - time_object = ciso8601.parse_datetime(source) # pylint: disable=c-extension-no-member + time_object = datetime.fromisoformat(source) # pylint: disable=c-extension-no-member if set_missing_utc: time_object = cls._set_utc_if_timezone_is_missing(time_object) return time_object diff --git a/pyproject.toml b/pyproject.toml index 38114f961..3c998bd25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,6 @@ dependencies = [ "aiohttp>=3.9.2", # CVE-2024-23334 "attrs", "certifi>=2023.7.22", # CVE-2023-37920 - "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 "colorama", "confluent-kafka>2", "geoip2", diff --git a/tests/unit/processor/timestamp_differ/test_timestamp_differ.py b/tests/unit/processor/timestamp_differ/test_timestamp_differ.py index dd082562a..edb0bf6a8 100644 --- a/tests/unit/processor/timestamp_differ/test_timestamp_differ.py +++ b/tests/unit/processor/timestamp_differ/test_timestamp_differ.py @@ -343,7 +343,7 @@ "subfield": {"field2": "2022-12-05 12:00:00"}, "tags": ["_timestamp_differ_failure"], }, - r".*ProcessingWarning.*Invalid character while parsing", + r".*ProcessingWarning.*Invalid isoformat string: 'non-timestamp'", ), ( "diff between two timestamps with partial timestamp format match",