From 60f3f76c4f18b14252474554d04d5f39ec5fe977 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Thu, 6 Jun 2024 16:06:37 +1200 Subject: [PATCH 1/7] add support for dynamic dependencies with setuptools --- python/deptry/dependency_getter/builder.py | 74 +++++++++++++++++++++- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/python/deptry/dependency_getter/builder.py b/python/deptry/dependency_getter/builder.py index 14fb16f4..90166a79 100644 --- a/python/deptry/dependency_getter/builder.py +++ b/python/deptry/dependency_getter/builder.py @@ -45,21 +45,89 @@ def build(self) -> DependencyGetter: return PoetryDependencyGetter(self.config, self.package_module_name_map) if self._project_uses_pdm(pyproject_toml): - return PDMDependencyGetter(self.config, self.package_module_name_map, self.pep621_dev_dependency_groups) + return PDMDependencyGetter( + self.config, + self.package_module_name_map, + self.pep621_dev_dependency_groups, + ) + + if self._project_uses_setuptools(pyproject_toml) and self._project_uses_dynamic_dependencies( + pyproject_toml + ): + requirements_txt_file = self._project_dynamic_requirements_file(pyproject_toml) + return RequirementsTxtDependencyGetter( + self.config, self.package_module_name_map, requirements_txt_file, () + ) if self._project_uses_pep_621(pyproject_toml): return PEP621DependencyGetter( - self.config, self.package_module_name_map, self.pep621_dev_dependency_groups + self.config, + self.package_module_name_map, + self.pep621_dev_dependency_groups, ) check, requirements_files = self._project_uses_requirements_files() if check: return RequirementsTxtDependencyGetter( - self.config, self.package_module_name_map, requirements_files, self.requirements_files_dev + self.config, + self.package_module_name_map, + requirements_files, + self.requirements_files_dev, ) raise DependencySpecificationNotFoundError(self.requirements_files) + def _project_uses_setuptools(self, pyproject_toml: dict[str, Any]) -> bool: + try: + if pyproject_toml["build-system"]["build-backend"] == "setuptools.build_meta": + logging.debug( + "pyproject.toml has the entry" + " build-system.build-backend == 'setuptools.build-meta'" + ", so setuptools is used to manage dependencies." + ) + return True + else: + logging.debug( + "pyproject.toml does not have" + " build-system.build-backend == 'setuptools.build-meta'" + ", so setuptools is not used to manage dependencies." + ) + return False + except KeyError: + logging.debug( + "pyproject.toml does not contain a build-system.build-backend entry, " + "so setuptools is not used to manage dependencies." + ) + return False + + def _project_uses_dynamic_dependencies(self, pyproject_toml: dict[str, Any]) -> bool: + try: + if "dependencies" not in pyproject_toml["project"]["dynamic"]: + logging.debug( + "pyproject.toml does not have" + " dependencies listed as dynamic metadata," + " so dynamic dependencies are not used." + ) + return False + else: + pyproject_toml["tool"]["setuptools"]["dynamic"]["dependencies"]["file"] + logging.debug( + "pyproject.toml has dependencies listed as dynamic metadata" + " and contains a populated tools.setuptools.dynamic.dependencies.file" + " entry, so dynamic dependencies are used." + ) + return True + except KeyError: + logging.debug( + "pyproject.toml either does not contain a project.dynamic entry or " + "tools.setuptools.dynamic.dependencies.file entry, so dynamic " + "dependencies are not used." + ) + return False + + def _project_dynamic_requirements_file(self, pyproject_toml: dict[str, Any]) -> tuple[str, ...]: + return tuple(pyproject_toml["tool"]["setuptools"]["dynamic"]["dependencies"]["file"]) + def _project_contains_pyproject_toml(self) -> bool: if self.config.exists(): logging.debug("pyproject.toml found!") From 489534282d154676d0acafce40d847e1a4efe9d2 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Fri, 7 Jun 2024 12:10:45 +1200 Subject: [PATCH 2/7] pass existing unit-tests --- tests/unit/dependency_getter/test_builder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/dependency_getter/test_builder.py b/tests/unit/dependency_getter/test_builder.py index 5ea7ead4..b3db5c64 100644 --- a/tests/unit/dependency_getter/test_builder.py +++ b/tests/unit/dependency_getter/test_builder.py @@ -130,6 +130,10 @@ def test_dependency_specification_not_found_raises_exception(tmp_path: Path, cap "pyproject.toml does not contain a [tool.pdm.dev-dependencies] section, so PDM is not used to specify the" " project's dependencies." ), + ( + "pyproject.toml does not have build-system.build-backend == 'setuptools.build-meta', so setuptools is" + " not used to manage dependencies." + ), ( "pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's" " dependencies." From c7618edefc9847d7cd8e96d7f5328ee3070b398e Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Fri, 7 Jun 2024 12:31:57 +1200 Subject: [PATCH 3/7] add tests; make logs consistent --- python/deptry/dependency_getter/builder.py | 10 +++++----- tests/unit/dependency_getter/test_builder.py | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/python/deptry/dependency_getter/builder.py b/python/deptry/dependency_getter/builder.py index 90166a79..48fc89e7 100644 --- a/python/deptry/dependency_getter/builder.py +++ b/python/deptry/dependency_getter/builder.py @@ -82,21 +82,21 @@ def _project_uses_setuptools(self, pyproject_toml: dict[str, Any]) -> bool: if pyproject_toml["build-system"]["build-backend"] == "setuptools.build_meta": logging.debug( "pyproject.toml has the entry" - " build-system.build-backend == 'setuptools.build-meta'" - ", so setuptools is used to manage dependencies." + " build-system.build-backend == 'setuptools.build_meta'" + ", so setuptools is used to specify the project's dependencies." ) return True else: logging.debug( "pyproject.toml does not have" - " build-system.build-backend == 'setuptools.build-meta'" - ", so setuptools is not used to manage dependencies." + " build-system.build-backend == 'setuptools.build_meta'" + ", so setuptools is not used to specify the project's dependencies." ) return False except KeyError: logging.debug( "pyproject.toml does not contain a build-system.build-backend entry, " - "so setuptools is not used to manage dependencies." + "so setuptools is not used to specify the project's dependencies." ) return False diff --git a/tests/unit/dependency_getter/test_builder.py b/tests/unit/dependency_getter/test_builder.py index b3db5c64..5518bdaa 100644 --- a/tests/unit/dependency_getter/test_builder.py +++ b/tests/unit/dependency_getter/test_builder.py @@ -83,6 +83,22 @@ def test_both(tmp_path: Path) -> None: spec = DependencyGetterBuilder(pyproject_toml_path).build() assert isinstance(spec, PoetryDependencyGetter) +def test_setuptools_dynamic_dependencies(tmp_path: Path) -> None: + with run_within_dir(tmp_path): + with Path("requirements.txt").open('w') as f: + f.write('foo >= 1.0') + with Path("pyproject.toml").open('w') as f: + f.write(''' + [build-system] + build-backend = "setuptools.build_meta" + [project] + dynamic = ["dependencies"] + [tool.setuptools.dynamic] + dependencies = {file = ["requirements.txt"]} + ''') + + spec = DependencyGetterBuilder(Path("pyproject.toml")).build() + assert isinstance(spec, RequirementsTxtDependencyGetter) def test_requirements_files(tmp_path: Path) -> None: with run_within_dir(tmp_path): @@ -131,8 +147,8 @@ def test_dependency_specification_not_found_raises_exception(tmp_path: Path, cap " project's dependencies." ), ( - "pyproject.toml does not have build-system.build-backend == 'setuptools.build-meta', so setuptools is" - " not used to manage dependencies." + "pyproject.toml does not have build-system.build-backend == 'setuptools.build_meta', so setuptools is" + " not used to specify the project's dependencies." ), ( "pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's" From 1755e03d0a4e0c54a6404a72e0e28e329cd8d2fb Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Fri, 7 Jun 2024 12:42:07 +1200 Subject: [PATCH 4/7] add docs --- docs/usage.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index a1b1d9eb..519ddbe1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -37,10 +37,16 @@ To determine the project's dependencies, _deptry_ will scan the directory it is 2. If a `pyproject.toml` file with a `[tool.pdm.dev-dependencies]` section is found, _deptry_ will assume it uses PDM and extract: - dependencies from `[project.dependencies]` and `[project.optional-dependencies]` sections - development dependencies from `[tool.pdm.dev-dependencies]` section and from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. -3. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract: +3. If a `pyproject.toml` file containing the following is found: + 1. a `[build-system]` section containing `build-backend = "setuptools.build_meta"`, + 2. a `[project]` section a list of `dynamic` values containing `dependencies`, + 3. a `[tools.setuptools.dynamic]` section containing `dependencies = { file = some_requirements_file }`, + then _deptry_ will assume it uses setuptools project with dynamic dependencies and extract dependencies + from the file specified in the `[tools.setuptools.dynamic]` section. +4. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract: - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`. - development dependencies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. -4. If a `requirements.in` or `requirements.txt` file is found, _deptry_ will: +5. If a `requirements.in` or `requirements.txt` file is found, _deptry_ will: - extract dependencies from that file. - extract development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist From 0504722676dcbc0875a1cec911c027bbc347894d Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Mon, 10 Jun 2024 11:00:34 +1200 Subject: [PATCH 5/7] add functional test --- .../pyproject.toml | 9 ++ .../requirements.txt | 5 + .../src/main.py | 6 + .../src/notebook.ipynb | 37 ++++++ .../cli/test_cli_dynamic_dependencies.py | 105 ++++++++++++++++++ tests/functional/utils.py | 1 + 6 files changed, 163 insertions(+) create mode 100644 tests/data/project_with_dynamic_dependencies/pyproject.toml create mode 100644 tests/data/project_with_dynamic_dependencies/requirements.txt create mode 100644 tests/data/project_with_dynamic_dependencies/src/main.py create mode 100644 tests/data/project_with_dynamic_dependencies/src/notebook.ipynb create mode 100644 tests/functional/cli/test_cli_dynamic_dependencies.py diff --git a/tests/data/project_with_dynamic_dependencies/pyproject.toml b/tests/data/project_with_dynamic_dependencies/pyproject.toml new file mode 100644 index 00000000..ebe16813 --- /dev/null +++ b/tests/data/project_with_dynamic_dependencies/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +dynamic = ["dependencies"] + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements.txt"] } diff --git a/tests/data/project_with_dynamic_dependencies/requirements.txt b/tests/data/project_with_dynamic_dependencies/requirements.txt new file mode 100644 index 00000000..d5780b03 --- /dev/null +++ b/tests/data/project_with_dynamic_dependencies/requirements.txt @@ -0,0 +1,5 @@ +click==8.1.3 +isort==5.10.1 +pkginfo==1.8.3 +requests==2.28.1 +toml==0.10.2 diff --git a/tests/data/project_with_dynamic_dependencies/src/main.py b/tests/data/project_with_dynamic_dependencies/src/main.py new file mode 100644 index 00000000..3fb31814 --- /dev/null +++ b/tests/data/project_with_dynamic_dependencies/src/main.py @@ -0,0 +1,6 @@ +from os import chdir, walk +from pathlib import Path + +import click +import white as w +from urllib3 import contrib diff --git a/tests/data/project_with_dynamic_dependencies/src/notebook.ipynb b/tests/data/project_with_dynamic_dependencies/src/notebook.ipynb new file mode 100644 index 00000000..a51bdb9d --- /dev/null +++ b/tests/data/project_with_dynamic_dependencies/src/notebook.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "9f4924ec-2200-4801-9d49-d4833651cbc4", + "metadata": {}, + "outputs": [], + "source": [ + "import click\n", + "from urllib3 import contrib\n", + "import toml" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/functional/cli/test_cli_dynamic_dependencies.py b/tests/functional/cli/test_cli_dynamic_dependencies.py new file mode 100644 index 00000000..9a210d67 --- /dev/null +++ b/tests/functional/cli/test_cli_dynamic_dependencies.py @@ -0,0 +1,105 @@ + +from __future__ import annotations + +import uuid +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from tests.functional.utils import Project +from tests.utils import get_issues_report + +if TYPE_CHECKING: + from tests.utils import PipVenvFactory + + +@pytest.mark.xdist_group(name=Project.DYNAMIC_DEPENDENCIES) +def test_cli_single_requirements_files(pip_venv_factory: PipVenvFactory) -> None: + with pip_venv_factory( + Project.DYNAMIC_DEPENDENCIES, + install_command=( + "pip install -r requirements.txt" + ), + ) as virtual_env: + issue_report = f"{uuid.uuid4()}.json" + result = virtual_env.run( + f"deptry . -o {issue_report}" + ) + + assert result.returncode == 1 + assert get_issues_report(Path(issue_report)) == [ + { + "error": { + "code": "DEP002", + "message": "'isort' defined as a dependency but not used in the codebase", + }, + "module": "isort", + "location": { + "file": str(Path("requirements.txt")), + "line": None, + "column": None, + }, + }, + { + "error": { + "code": "DEP002", + "message": "'pkginfo' defined as a dependency but not used in the codebase", + }, + "module": "pkginfo", + "location": { + "file": str(Path("requirements.txt")), + "line": None, + "column": None, + }, + }, + { + "error": { + "code": "DEP002", + "message": "'requests' defined as a dependency but not used in the codebase", + }, + "module": "requests", + "location": { + "file": str(Path("requirements.txt")), + "line": None, + "column": None, + }, + }, + { + "error": { + "code": "DEP001", + "message": "'white' imported but missing from the dependency definitions", + }, + "module": "white", + "location": { + "file": str(Path("src/main.py")), + "line": 5, + "column": 8, + }, + }, + { + "error": { + "code": "DEP003", + "message": "'urllib3' imported but it is a transitive dependency", + }, + "module": "urllib3", + "location": { + "file": str(Path("src/main.py")), + "line": 6, + "column": 1, + }, + }, + { + "error": { + "code": "DEP003", + "message": "'urllib3' imported but it is a transitive dependency", + }, + "module": "urllib3", + "location": { + "file": str(Path("src/notebook.ipynb")), + "line": 2, + "column": 1, + }, + }, + ] + diff --git a/tests/functional/utils.py b/tests/functional/utils.py index 08853829..5ed12331 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -17,6 +17,7 @@ class Project(str, Enum): REQUIREMENTS_TXT = "project_with_requirements_txt" REQUIREMENTS_IN = "project_with_requirements_in" SRC_DIRECTORY = "project_with_src_directory" + DYNAMIC_DEPENDENCIES = 'project_with_dynamic_dependencies' def __str__(self) -> str: return self.value From 883055abe25e73ca5aa218358944d4cb4d66e04e Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Mon, 10 Jun 2024 11:08:00 +1200 Subject: [PATCH 6/7] fix docs and logging --- docs/usage.md | 12 +++++++----- python/deptry/dependency_getter/builder.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 519ddbe1..705a1b6e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -38,11 +38,13 @@ To determine the project's dependencies, _deptry_ will scan the directory it is - dependencies from `[project.dependencies]` and `[project.optional-dependencies]` sections - development dependencies from `[tool.pdm.dev-dependencies]` section and from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. 3. If a `pyproject.toml` file containing the following is found: - 1. a `[build-system]` section containing `build-backend = "setuptools.build_meta"`, - 2. a `[project]` section a list of `dynamic` values containing `dependencies`, - 3. a `[tools.setuptools.dynamic]` section containing `dependencies = { file = some_requirements_file }`, - then _deptry_ will assume it uses setuptools project with dynamic dependencies and extract dependencies - from the file specified in the `[tools.setuptools.dynamic]` section. + 1. a `[build-system]` section containing `build-backend = "setuptools.build_meta"` and, + 2. a `[project]` section with a `dynamic` key, where the corresponding list includes `"dependencies"` and, + 3. a `[tool.setuptools.dynamic]` section containing `dependencies = { file = [some_requirements_file] }`, + + then _deptry_ will assume the project uses setuptools with dynamic dependencies. It will extract dependencies + from the file specified in the `[tool.setuptools.dynamic]` section. + 4. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract: - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`. - development dependencies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. diff --git a/python/deptry/dependency_getter/builder.py b/python/deptry/dependency_getter/builder.py index 48fc89e7..be4f774e 100644 --- a/python/deptry/dependency_getter/builder.py +++ b/python/deptry/dependency_getter/builder.py @@ -113,14 +113,14 @@ def _project_uses_dynamic_dependencies(self, pyproject_toml: dict[str, Any]) -> pyproject_toml["tool"]["setuptools"]["dynamic"]["dependencies"]["file"] logging.debug( "pyproject.toml has dependencies listed as dynamic metadata" - " and contains a populated tools.setuptools.dynamic.dependencies.file" + " and contains a populated tool.setuptools.dynamic.dependencies.file" " entry, so dynamic dependencies are used." ) return True except KeyError: logging.debug( "pyproject.toml either does not contain a project.dynamic entry or " - "tools.setuptools.dynamic.dependencies.file entry, so dynamic " + "tool.setuptools.dynamic.dependencies.file entry, so dynamic " "dependencies are not used." ) return False From dcd71ab4adb3fc7b2e93e2ca26a0d48c3e2358a6 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Mon, 10 Jun 2024 11:11:51 +1200 Subject: [PATCH 7/7] reformat files --- .../functional/cli/test_cli_dynamic_dependencies.py | 10 ++-------- tests/functional/utils.py | 2 +- tests/unit/dependency_getter/test_builder.py | 12 +++++++----- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/functional/cli/test_cli_dynamic_dependencies.py b/tests/functional/cli/test_cli_dynamic_dependencies.py index 9a210d67..22b1b925 100644 --- a/tests/functional/cli/test_cli_dynamic_dependencies.py +++ b/tests/functional/cli/test_cli_dynamic_dependencies.py @@ -1,4 +1,3 @@ - from __future__ import annotations import uuid @@ -18,14 +17,10 @@ def test_cli_single_requirements_files(pip_venv_factory: PipVenvFactory) -> None: with pip_venv_factory( Project.DYNAMIC_DEPENDENCIES, - install_command=( - "pip install -r requirements.txt" - ), + install_command=("pip install -r requirements.txt"), ) as virtual_env: issue_report = f"{uuid.uuid4()}.json" - result = virtual_env.run( - f"deptry . -o {issue_report}" - ) + result = virtual_env.run(f"deptry . -o {issue_report}") assert result.returncode == 1 assert get_issues_report(Path(issue_report)) == [ @@ -102,4 +97,3 @@ def test_cli_single_requirements_files(pip_venv_factory: PipVenvFactory) -> None }, }, ] - diff --git a/tests/functional/utils.py b/tests/functional/utils.py index 5ed12331..e6b077cd 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -17,7 +17,7 @@ class Project(str, Enum): REQUIREMENTS_TXT = "project_with_requirements_txt" REQUIREMENTS_IN = "project_with_requirements_in" SRC_DIRECTORY = "project_with_src_directory" - DYNAMIC_DEPENDENCIES = 'project_with_dynamic_dependencies' + DYNAMIC_DEPENDENCIES = "project_with_dynamic_dependencies" def __str__(self) -> str: return self.value diff --git a/tests/unit/dependency_getter/test_builder.py b/tests/unit/dependency_getter/test_builder.py index 5518bdaa..7c8a22fe 100644 --- a/tests/unit/dependency_getter/test_builder.py +++ b/tests/unit/dependency_getter/test_builder.py @@ -83,23 +83,25 @@ def test_both(tmp_path: Path) -> None: spec = DependencyGetterBuilder(pyproject_toml_path).build() assert isinstance(spec, PoetryDependencyGetter) + def test_setuptools_dynamic_dependencies(tmp_path: Path) -> None: with run_within_dir(tmp_path): - with Path("requirements.txt").open('w') as f: - f.write('foo >= 1.0') - with Path("pyproject.toml").open('w') as f: - f.write(''' + with Path("requirements.txt").open("w") as f: + f.write("foo >= 1.0") + with Path("pyproject.toml").open("w") as f: + f.write(""" [build-system] build-backend = "setuptools.build_meta" [project] dynamic = ["dependencies"] [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} - ''') + """) spec = DependencyGetterBuilder(Path("pyproject.toml")).build() assert isinstance(spec, RequirementsTxtDependencyGetter) + def test_requirements_files(tmp_path: Path) -> None: with run_within_dir(tmp_path): with Path("requirements.txt").open("w") as f: