Replace setup.py usage with pyproject.toml using hatch

.automation/build.py

@@ -3286,6 +3286,14 @@ def generate_version():
     )
     print(process.stdout)
     print(process.stderr)
+    # Update python project version:
+    process = subprocess.run(
+        ["hatch", "version", RELEASE_TAG],
+        stdout=subprocess.PIPE,
+        text=True,
+        shell=True,
+        check=False,
+    )
     # Update changelog
     if UPDATE_CHANGELOG is True:
         changelog_file = f"{REPO_HOME}/CHANGELOG.md"

.automation/test/python_mypy/pyproject.toml

@@ -0,0 +1 @@
+[tool.pyright]

.config/make/python.mak

@@ -1,6 +1,5 @@
 # Python default launcher
 python_launcher ?= python3.11
-python_requirements_file ?= .config/python/dev/requirements.txt
 python_requirements_dev_file ?= .config/python/dev/requirements.txt
 
 UV = $(shell command -v uv 2> /dev/null)
@@ -10,7 +9,7 @@ UV = $(shell command -v uv 2> /dev/null)
 python-bootstrap: ## Bootstrap python
 	$(MAKE) python-venv-init
 	$(MAKE) python-venv-upgrade
-	$(MAKE) python-venv-requirements
+	$(MAKE) python-venv-editable-install
 
 .PHONY: python-bootstrap-dev
 python-bootstrap-dev: ## Bootstrap python for dev env
@@ -33,22 +32,22 @@ else
 endif
 
 .PHONY: python-venv-upgrade
-python-venv-upgrade: ## Upgrade venv with pip, setuptools and wheel
+python-venv-upgrade: ## Upgrade venv with pip
 ifeq ($(strip $(UV)),)
 	source .venv/bin/activate
-	pip install --upgrade pip setuptools wheel
+	pip install --upgrade pip
 else
-	$(UV) pip install --upgrade pip setuptools wheel
+	$(UV) pip install --upgrade pip
 endif
 
 
-.PHONY: python-venv-requirements
-python-venv-requirements: ## Install or upgrade from $(python_requirements_file)
+.PHONY: python-venv-editable-install
+python-venv-editable-install: ## Install or upgrade from local project's pyproject.toml
 ifeq ($(strip $(UV)),)
 	source .venv/bin/activate
-	pip install --upgrade --requirement $(python_requirements_file)
+	pip install --upgrade -e .
 else
-	$(UV) pip install --upgrade --requirement $(python_requirements_file)
+	$(UV) pip install --upgrade -e .
 endif
 
 .PHONY: python-venv-requirements-dev

.cspell.json

@@ -715,6 +715,7 @@
         "displaymath",
         "distutils",
         "djlint",
+        "dmypy",
         "dockerfilelint",
         "dockerfilelintrc",
         "dockerignore",
@@ -881,6 +882,7 @@
         "holdy",
         "hprof",
         "hsts",
+        "htmlcov",
         "htmlhint",
         "htmlhintrc",
         "htmlout",
@@ -1104,6 +1106,7 @@
         "norionomura",
         "normalsize",
         "noserver",
+        "nosetests",
         "nostic",
         "nosw",
         "notemp",
@@ -1318,6 +1321,7 @@
         "scolladon",
         "scriptsize",
         "scshape",
+        "sdist",
         "seaneagan",
         "secretlint",
         "secretlintrc",
@@ -1540,4 +1544,4 @@
         "zaach",
         "zricethezav"
     ]
-}
+}

.github/CONTRIBUTING.md

@@ -119,17 +119,15 @@ In order to be able to run locally a server that serves all the documentation an
 Commands to execute (only one time):
 
 ```bash
-mkdir venv
-python -m venv venv/
-source venv/bin/activate
-pip install --upgrade -r .config/python/dev/requirements.txt
+pip install pipx
+pipx install hatch
+hatch shell
 ```
 
-Commands to run every time you want to enter the environment and run the server:
+Commands to run every time you want to build the docs and run the server:
 
 ```bash
-source venv/bin/activate
-mkdocs serve
+hatch run build:serve
 ```
 
 By default it listens on `http://127.0.0.1:8000/`.

.gitignore

@@ -99,6 +99,7 @@ coverage.xml
 !**/salesforce/good/report/*
 !**/salesforce/bad/report/*
 
+# mkdocs documentation
 site/
 
 .sfdx/
@@ -126,4 +127,73 @@ server/server-files/
 .venv
 *megalinter_file_names_cspell.txt
 
-.DS_Store
+.DS_Store
+
+################################
+# Python:
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json

CHANGELOG.md

@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-linter.yml file, or with `oxsecurity/megalinter:beta` docker image
 
 - Core
+  - Replace usage of setup.py with a pyproject.toml package install, by @echoix in [#3893](https://github.com/oxsecurity/megalinter/pull/3893)
 
 - Media
 

Dockerfile

@@ -75,6 +75,20 @@ FROM alpine/terragrunt:${TERRAFORM_TERRAGRUNT_VERSION} AS terragrunt
 # FROM alpine/terragrunt:${TERRAFORM_TERRAGRUNT_VERSION} AS terragrunt
 #FROM__END
 
+##################
+# Build wheel for megalinter python package
+##################
+FROM ghcr.io/astral-sh/uv:0.2.37 AS uv
+FROM python:3.12.5-alpine3.20 AS build-ml-core
+WORKDIR /
+COPY pyproject.toml .
+COPY --from=uv /uv /bin/uv
+RUN --mount=type=cache,target=/root/.cache/uv \
+     uv pip install --system -r pyproject.toml
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system .
+
 ##################
 # Get base image #
 ##################
@@ -792,11 +806,11 @@ RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh |
 ################################
 # Installs python dependencies #
 ################################
-COPY megalinter /megalinter
-RUN PYTHONDONTWRITEBYTECODE=1 python /megalinter/setup.py install \
-    && PYTHONDONTWRITEBYTECODE=1 python /megalinter/setup.py clean --all \
-    && rm -rf /var/cache/apk/* \
-    && find . \( -type f \( -iname \*.pyc -o -iname \*.pyo \) -o -type d -iname __pycache__ \) -delete
+COPY --from=build-ml-core pyproject.toml README.md ./
+COPY --from=build-ml-core megalinter /megalinter/
+RUN --mount=type=cache,target=/root/.cache/uv,from=build-ml-core \
+    --mount=from=uv,source=/uv,target=/bin/uv \
+    uv pip install --system -e .
 
 #######################################
 # Copy scripts and rules to container #

Dockerfile-quick

@@ -9,7 +9,26 @@
 ##################
 ARG GITHUB_TOKEN
 ARG MEGALINTER_BASE_IMAGE=oxsecurity/megalinter:beta
-FROM $MEGALINTER_BASE_IMAGE
+FROM $MEGALINTER_BASE_IMAGE as base
+
+##################
+# Build wheel for megalinter python package
+##################
+FROM ghcr.io/astral-sh/uv:0.2.37 AS uv
+FROM python:3.12.5-alpine3.20 AS build-ml-core
+WORKDIR /
+COPY pyproject.toml .
+COPY --from=uv /uv /bin/uv
+RUN --mount=type=cache,target=/root/.cache/uv \
+     uv pip install --system -r pyproject.toml
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system .
+
+#######################
+# Continue base image #
+#######################
+FROM base
 
 ######################
 # Set the entrypoint #
@@ -21,10 +40,12 @@ ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 ################################
 # Installs python dependencies #
 ################################
-COPY megalinter /megalinter
-RUN python /megalinter/setup.py install \
-    && python /megalinter/setup.py clean --all \
-    && rm -rf /var/cache/apk/*
+COPY --from=build-ml-core pyproject.toml README.md ./
+COPY --from=build-ml-core megalinter /megalinter/
+RUN --mount=type=cache,target=/root/.cache/uv,from=build-ml-core \
+    --mount=from=uv,source=/uv,target=/bin/uv \
+    uv pip install --system -e .
+
 #######################################
 # Copy scripts and rules to container #
 #######################################

flavors/c_cpp/Dockerfile

@@ -52,6 +52,20 @@ FROM jdkato/vale:${SPELL_VALE_VERSION} AS vale
 FROM lycheeverse/lychee:latest-alpine AS lychee
 #FROM__END
 
+##################
+# Build wheel for megalinter python package
+##################
+FROM ghcr.io/astral-sh/uv:0.2.37 AS uv
+FROM python:3.12.5-alpine3.20 AS build-ml-core
+WORKDIR /
+COPY pyproject.toml .
+COPY --from=uv /uv /bin/uv
+RUN --mount=type=cache,target=/root/.cache/uv \
+     uv pip install --system -r pyproject.toml
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system .
+
 ##################
 # Get base image #
 ##################
@@ -321,11 +335,11 @@ RUN curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/rel
 ################################
 # Installs python dependencies #
 ################################
-COPY megalinter /megalinter
-RUN PYTHONDONTWRITEBYTECODE=1 python /megalinter/setup.py install \
-    && PYTHONDONTWRITEBYTECODE=1 python /megalinter/setup.py clean --all \
-    && rm -rf /var/cache/apk/* \
-    && find . \( -type f \( -iname \*.pyc -o -iname \*.pyo \) -o -type d -iname __pycache__ \) -delete
+COPY --from=build-ml-core pyproject.toml README.md ./
+COPY --from=build-ml-core megalinter /megalinter/
+RUN --mount=type=cache,target=/root/.cache/uv,from=build-ml-core \
+    --mount=from=uv,source=/uv,target=/bin/uv \
+    uv pip install --system -e .
 
 #######################################
 # Copy scripts and rules to container #

flavors/ci_light/Dockerfile

@@ -33,6 +33,20 @@ FROM zricethezav/gitleaks:${REPOSITORY_GITLEAKS_VERSION} AS gitleaks
 FROM trufflesecurity/trufflehog:${REPOSITORY_TRUFFLEHOG_VERSION} AS trufflehog
 #FROM__END
 
+##################
+# Build wheel for megalinter python package
+##################
+FROM ghcr.io/astral-sh/uv:0.2.37 AS uv
+FROM python:3.12.5-alpine3.20 AS build-ml-core
+WORKDIR /
+COPY pyproject.toml .
+COPY --from=uv /uv /bin/uv
+RUN --mount=type=cache,target=/root/.cache/uv \
+     uv pip install --system -r pyproject.toml
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system .
+
 ##################
 # Get base image #
 ##################
@@ -215,11 +229,11 @@ RUN curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh |
 ################################
 # Installs python dependencies #
 ################################
-COPY megalinter /megalinter
-RUN PYTHONDONTWRITEBYTECODE=1 python /megalinter/setup.py install \
-    && PYTHONDONTWRITEBYTECODE=1 python /megalinter/setup.py clean --all \
-    && rm -rf /var/cache/apk/* \
-    && find . \( -type f \( -iname \*.pyc -o -iname \*.pyo \) -o -type d -iname __pycache__ \) -delete
+COPY --from=build-ml-core pyproject.toml README.md ./
+COPY --from=build-ml-core megalinter /megalinter/
+RUN --mount=type=cache,target=/root/.cache/uv,from=build-ml-core \
+    --mount=from=uv,source=/uv,target=/bin/uv \
+    uv pip install --system -e .
 
 #######################################
 # Copy scripts and rules to container #