From b3a86d69d113e4fc765ee301e12c16a9071ee6d6 Mon Sep 17 00:00:00 2001 From: Johan Herland Date: Wed, 15 Mar 2023 11:49:00 +0100 Subject: [PATCH] Refactor `extract_declared_dependencies` parser APIs in preparation for #227 (#229) * test_extract_declared_dependencies_succes: Harmless test refactoring Move the dependency_factory() call into the test case instead of in the parametrized test data. Also remove unnecessary "__" prefix from test ids. * extract_declared_dependencies: Refactor API of parser functions Take a single Path argument instead of separate contents + source. This allows the parser function itself to choose how to open the file and pass it to the underlying parser. This is also necessary for the upcoming rewrite of our requirements.txt parser (#227). * test_extract_declared_dependencies_pyproject_toml: Remove unnecessary dedent() * extract_declared_dependencies: Rename parser functions The rename reflects that these now parse file paths, and not extracted file contents. --------- Co-authored-by: Vince Reuter --- fawltydeps/extract_declared_dependencies.py | 53 +-- noxfile.py | 2 +- ...st_extract_declared_dependencies_errors.py | 39 +- ...ct_declared_dependencies_pyproject_toml.py | 355 ++++++++---------- ...t_extract_declared_dependencies_success.py | 239 ++++++------ 5 files changed, 323 insertions(+), 365 deletions(-) diff --git a/fawltydeps/extract_declared_dependencies.py b/fawltydeps/extract_declared_dependencies.py index 31ca0d04..e8567dae 100644 --- a/fawltydeps/extract_declared_dependencies.py +++ b/fawltydeps/extract_declared_dependencies.py @@ -5,9 +5,11 @@ import logging import re import sys +from dataclasses import replace from functools import partial from itertools import takewhile from pathlib import Path +from tempfile import NamedTemporaryFile from typing import Callable, Iterable, Iterator, NamedTuple, Optional, Set, Tuple from pkg_resources import Requirement @@ -56,17 +58,16 @@ def parse_one_req(req_text: str, source: Location) -> DeclaredDependency: return DeclaredDependency(req_name, source) -def parse_requirements_contents( - text: str, source: Location -) -> Iterator[DeclaredDependency]: +def parse_requirements_txt(path: Path) -> Iterator[DeclaredDependency]: """Extract dependencies (packages names) from a requirements file. This is usually a requirements.txt file or any other file following the Requirements File Format as documented here: https://pip.pypa.io/en/stable/reference/requirements-file-format/. """ + source = Location(path) parse_one = partial(parse_one_req, source=source) - for line in text.splitlines(): + for line in path.read_text().splitlines(): cleaned = line.lstrip() if ( not cleaned # skip empty lines @@ -91,7 +92,7 @@ def parse_requirements_contents( logger.warning(f"Could not parse {source} line {line!r}: {exc}") -def parse_setup_contents(text: str, source: Location) -> Iterator[DeclaredDependency]: +def parse_setup_py(path: Path) -> Iterator[DeclaredDependency]: """Extract dependencies (package names) from setup.py. This file can contain arbitrary Python code, and simply executing it has @@ -100,7 +101,7 @@ def parse_setup_contents(text: str, source: Location) -> Iterator[DeclaredDepend the `install_requires` and `extras_require` keyword args from that function call. """ - + source = Location(path) # Attempt to keep track of simple variable assignments (name -> value) # declared in the setup.py prior to the setup() call, so that we can # resolve any variable references in the arguments to the setup() call. @@ -139,7 +140,7 @@ def _is_setup_function_call(node: ast.AST) -> bool: and node.value.func.id == "setup" ) - setup_contents = ast.parse(text, filename=str(source.path)) + setup_contents = ast.parse(path.read_text(), filename=str(source.path)) for node in ast.walk(setup_contents): tracked_vars.evaluate(node) if _is_setup_function_call(node): @@ -149,9 +150,7 @@ def _is_setup_function_call(node: ast.AST) -> bool: break -def parse_setup_cfg_contents( - text: str, source: Location -) -> Iterator[DeclaredDependency]: +def parse_setup_cfg(path: Path) -> Iterator[DeclaredDependency]: """Extract dependencies (package names) from setup.cfg. `ConfigParser` basic building blocks are "sections" @@ -163,16 +162,22 @@ def parse_setup_cfg_contents( The declaration uses `section` + `option` syntax where section may be [options] or [options.{requirements_type}]. """ + source = Location(path) parser = configparser.ConfigParser() try: - parser.read_string(text) + parser.read([path]) except configparser.Error as exc: logger.debug(exc) logger.error("Could not parse contents of `%s`", source) return def parse_value(value: str) -> Iterator[DeclaredDependency]: - yield from parse_requirements_contents(value, source=source) + # Ugly hack since parse_requirements_txt() accepts only a path: + with NamedTemporaryFile(mode="wt") as tmp: + tmp.write(value) + tmp.flush() + for dep in parse_requirements_txt(Path(tmp.name)): + yield replace(dep, source=source) def extract_section(section: str) -> Iterator[DeclaredDependency]: if section in parser: @@ -288,9 +293,7 @@ def parse_pyproject_elements( ) -def parse_pyproject_contents( - text: str, source: Location -) -> Iterator[DeclaredDependency]: +def parse_pyproject_toml(path: Path) -> Iterator[DeclaredDependency]: """Extract dependencies (package names) from pyproject.toml. There are multiple ways to declare dependencies inside a pyproject.toml. @@ -298,7 +301,9 @@ def parse_pyproject_contents( - PEP 621 core metadata fields - Poetry-specific metadata in `tool.poetry` sections. """ - parsed_contents = tomllib.loads(text) + source = Location(path) + with path.open("rb") as tomlfile: + parsed_contents = tomllib.load(tomlfile) yield from parse_pep621_pyproject_contents(parsed_contents, source) @@ -314,7 +319,7 @@ class ParsingStrategy(NamedTuple): """Named pairing of an applicability criterion and a dependency parser""" applies_to_path: Callable[[Path], bool] - execute: Callable[[str, Location], Iterator[DeclaredDependency]] + execute: Callable[[Path], Iterator[DeclaredDependency]] def first_applicable_parser( @@ -333,18 +338,18 @@ def first_applicable_parser( PARSER_CHOICES = { ParserChoice.PYPROJECT_TOML: ParsingStrategy( - lambda path: path.name == "pyproject.toml", parse_pyproject_contents + lambda path: path.name == "pyproject.toml", parse_pyproject_toml ), ParserChoice.REQUIREMENTS_TXT: ParsingStrategy( lambda path: re.compile(r".*\brequirements\b.*\.(txt|in)").match(path.name) is not None, - parse_requirements_contents, + parse_requirements_txt, ), ParserChoice.SETUP_CFG: ParsingStrategy( - lambda path: path.name == "setup.cfg", parse_setup_cfg_contents + lambda path: path.name == "setup.cfg", parse_setup_cfg ), ParserChoice.SETUP_PY: ParsingStrategy( - lambda path: path.name == "setup.py", parse_setup_contents + lambda path: path.name == "setup.py", parse_setup_py ), } @@ -375,7 +380,7 @@ def extract_declared_dependencies_from_path( ctx="Parsing given dependencies path isn't supported", path=path ) parser = choice_and_parser[1] - yield from parser.execute(path.read_text(), Location(path)) + yield from parser.execute(path) elif path.is_dir(): logger.debug("Extracting dependencies from files under %s", path) for file in walk_dir(path): @@ -384,9 +389,7 @@ def extract_declared_dependencies_from_path( continue if parser_choice is None or choice_and_parser[0] == parser_choice: logger.debug(f"Extracting dependencies: {file}") - yield from choice_and_parser[1].execute( - file.read_text(), Location(file) - ) + yield from choice_and_parser[1].execute(file) else: raise UnparseablePathException( ctx="Dependencies declaration path is neither dir nor file", path=path diff --git a/noxfile.py b/noxfile.py index 8e0d3dfe..cc8203db 100644 --- a/noxfile.py +++ b/noxfile.py @@ -81,7 +81,7 @@ def lint(session): session.run("pylint", "fawltydeps") session.run( "pylint", - "--disable=missing-function-docstring,invalid-name,redefined-outer-name,too-many-lines", + "--disable=missing-function-docstring,invalid-name,redefined-outer-name,too-many-lines,too-many-arguments", "tests", ) diff --git a/tests/test_extract_declared_dependencies_errors.py b/tests/test_extract_declared_dependencies_errors.py index 3b68a8af..4df3c98a 100644 --- a/tests/test_extract_declared_dependencies_errors.py +++ b/tests/test_extract_declared_dependencies_errors.py @@ -1,15 +1,13 @@ """Test unhappy path, where parsing of dependencies fails""" import logging -from pathlib import Path -from textwrap import dedent import pytest from fawltydeps.extract_declared_dependencies import ( extract_declared_dependencies, - parse_setup_cfg_contents, - parse_setup_contents, + parse_setup_cfg, + parse_setup_py, ) from fawltydeps.types import Location, UnparseablePathException @@ -25,20 +23,22 @@ def test_extract_declared_dependencies__unsupported_file__raises_error( ) -def test_parse_setup_cfg_contents__malformed__logs_error(caplog): - setup_contents = dedent( - """\ - [options - install_requires = - pandas - """ +def test_parse_setup_cfg__malformed__logs_error(write_tmp_files, caplog): + tmp_path = write_tmp_files( + { + "setup.cfg": """\ + [options + install_requires = + pandas + """, + } ) expected = [] caplog.set_level(logging.ERROR) - source = Location(Path("setup.cfg")) - result = list(parse_setup_cfg_contents(setup_contents, source)) - assert f"Could not parse contents of `{source}`" in caplog.text + path = tmp_path / "setup.cfg" + result = list(parse_setup_cfg(path)) + assert f"Could not parse contents of `{Location(path)}`" in caplog.text assert expected == result @@ -124,11 +124,14 @@ def test_parse_setup_cfg_contents__malformed__logs_error(caplog): ), ], ) -def test_parse_setup_contents__cannot_parse__logs_warning( - caplog, code, expect, fail_arg +def test_parse_setup_py__cannot_parse__logs_warning( + write_tmp_files, caplog, code, expect, fail_arg ): + tmp_path = write_tmp_files({"setup.py": code}) + path = tmp_path / "setup.py" + caplog.set_level(logging.WARNING) - result = list(parse_setup_contents(dedent(code), Location(Path("setup.py")))) + result = list(parse_setup_py(path)) assert f"Could not parse contents of `{fail_arg}`" in caplog.text - assert "setup.py" in caplog.text + assert str(path) in caplog.text assert expect == result diff --git a/tests/test_extract_declared_dependencies_pyproject_toml.py b/tests/test_extract_declared_dependencies_pyproject_toml.py index 0ef32d96..b7bcaf96 100644 --- a/tests/test_extract_declared_dependencies_pyproject_toml.py +++ b/tests/test_extract_declared_dependencies_pyproject_toml.py @@ -1,11 +1,9 @@ """Test extracting dependencies from pyproject.toml""" import logging -from pathlib import Path -from textwrap import dedent import pytest -from fawltydeps.extract_declared_dependencies import parse_pyproject_contents +from fawltydeps.extract_declared_dependencies import parse_pyproject_toml from fawltydeps.types import DeclaredDependency, Location @@ -13,70 +11,62 @@ "pyproject_toml,expected_deps", [ pytest.param( - dedent( - """\ - [tool.poetry] - - [tool.poetry.dependencies] - python = "^3.8" - isort = "^5.10" - tomli = "^2.0.1" - """ - ), + """\ + [tool.poetry] + + [tool.poetry.dependencies] + python = "^3.8" + isort = "^5.10" + tomli = "^2.0.1" + """, ["isort", "tomli"], id="poetry_main_dependencies", ), pytest.param( - dedent( - """\ - [tool.poetry] - - [tool.poetry.group.dev.dependencies] - black = "^22" - mypy = "^0.991" - - [tool.poetry.group.test.dependencies] - pytest = "^5.0.0" - """ - ), + """\ + [tool.poetry] + + [tool.poetry.group.dev.dependencies] + black = "^22" + mypy = "^0.991" + + [tool.poetry.group.test.dependencies] + pytest = "^5.0.0" + """, ["black", "mypy", "pytest"], id="poetry_group_dependencies", ), pytest.param( - dedent( - """\ - [tool.poetry] - - [tool.poetry.extras] - test = ["pytest < 5.0.0", "pytest-cov[all]"] - dev = ["pylint >= 2.15.8"] - """ - ), + """\ + [tool.poetry] + + [tool.poetry.extras] + test = ["pytest < 5.0.0", "pytest-cov[all]"] + dev = ["pylint >= 2.15.8"] + """, ["pytest", "pytest-cov", "pylint"], id="poetry_extra_dependencies", ), pytest.param( - dedent( - """\ - [tool.poetry] - - [tool.poetry.dependencies] - python = "^3.8" - isort = "^5.10" - tomli = "^2.0.1" - - [tool.poetry.group.dev.dependencies] - black = "^22" - mypy = "^0.991" - - [tool.poetry.group.experimental.dependencies] - django = "^2.1" - - [tool.poetry.extras] - test = ["pytest < 5.0.0", "pytest-cov[all]"] - dev = ["pylint >= 2.15.8"] - """ - ), + """\ + [tool.poetry] + + [tool.poetry.dependencies] + python = "^3.8" + isort = "^5.10" + tomli = "^2.0.1" + + [tool.poetry.group.dev.dependencies] + black = "^22" + mypy = "^0.991" + + [tool.poetry.group.experimental.dependencies] + django = "^2.1" + + [tool.poetry.extras] + test = ["pytest < 5.0.0", "pytest-cov[all]"] + dev = ["pylint >= 2.15.8"] + """, [ "isort", "tomli", @@ -90,75 +80,67 @@ id="poetry_main_group_and_extra_dependencies", ), pytest.param( - dedent( - """\ - [project] - name = "fawltydeps" - - dependencies = ["isort", "django>2.1; os_name != 'nt'"] - """ - ), + """\ + [project] + name = "fawltydeps" + + dependencies = ["isort", "django>2.1; os_name != 'nt'"] + """, ["isort", "django"], id="pep621_main_dependencies", ), pytest.param( - dedent( - """\ - [project] - - [project.optional-dependencies] - test = ["pytest < 5.0.0", "pytest-cov[all]"] - dev = ["pylint >= 2.15.8"] - """ - ), + """\ + [project] + + [project.optional-dependencies] + test = ["pytest < 5.0.0", "pytest-cov[all]"] + dev = ["pylint >= 2.15.8"] + """, ["pytest", "pytest-cov", "pylint"], id="pep621_optional_dependencies", ), pytest.param( - dedent( - """\ - [project] - name = "fawltydeps" - - dependencies = ["isort", "django>2.1; os_name != 'nt'"] - - [project.optional-dependencies] - test = ["pytest < 5.0.0", "pytest-cov[all]"] - dev = ["pylint >= 2.15.8"] - """ - ), + """\ + [project] + name = "fawltydeps" + + dependencies = ["isort", "django>2.1; os_name != 'nt'"] + + [project.optional-dependencies] + test = ["pytest < 5.0.0", "pytest-cov[all]"] + dev = ["pylint >= 2.15.8"] + """, ["isort", "django", "pytest", "pytest-cov", "pylint"], id="pep_621_main_and_optional_dependencies", ), pytest.param( - dedent( - """\ - [project] - name = "fawltydeps" - - dependencies = ["pandas", "pydantic>1.10.4"] - - [project.optional-dependencies] - dev = ["pylint >= 2.15.8"] - - [tool.poetry] - [tool.poetry.dependencies] - python = "^3.8" - isort = "^5.10" - tomli = "^2.0.1" - - [tool.poetry.group.dev.dependencies] - black = "^22" - mypy = "^0.991" - - [tool.poetry.group.experimental.dependencies] - django = "^2.1" - - [tool.poetry.extras] - alpha = ["pytorch < 1.12.1", "numpy >= 1.17.2"] - dev = ["flake >= 5.0.1"] - """ - ), + """\ + [project] + name = "fawltydeps" + + dependencies = ["pandas", "pydantic>1.10.4"] + + [project.optional-dependencies] + dev = ["pylint >= 2.15.8"] + + [tool.poetry] + [tool.poetry.dependencies] + python = "^3.8" + isort = "^5.10" + tomli = "^2.0.1" + + [tool.poetry.group.dev.dependencies] + black = "^22" + mypy = "^0.991" + + [tool.poetry.group.experimental.dependencies] + django = "^2.1" + + [tool.poetry.extras] + alpha = ["pytorch < 1.12.1", "numpy >= 1.17.2"] + dev = ["flake >= 5.0.1"] + """, [ "pandas", "pydantic", @@ -176,12 +158,14 @@ ), ], ) -def test_parse_pyproject_content__pep621_or_poetry_dependencies__yields_dependencies( - pyproject_toml, expected_deps +def test_parse_pyproject_toml__pep621_or_poetry_dependencies__yields_dependencies( + write_tmp_files, pyproject_toml, expected_deps ): - source = Location(Path("pyproject.toml")) - result = list(parse_pyproject_contents(pyproject_toml, source)) - expected = [DeclaredDependency(dep, source) for dep in expected_deps] + tmp_path = write_tmp_files({"pyproject.toml": pyproject_toml}) + path = tmp_path / "pyproject.toml" + + result = list(parse_pyproject_toml(path)) + expected = [DeclaredDependency(dep, Location(path)) for dep in expected_deps] assert result == expected @@ -189,105 +173,89 @@ def test_parse_pyproject_content__pep621_or_poetry_dependencies__yields_dependen "pyproject_toml,expected,metadata_standard,field_types", [ pytest.param( - dedent( - """\ - [tool.poetry] - dependencies = ["pylint"] - """ - ), + """\ + [tool.poetry] + dependencies = ["pylint"] + """, [], "Poetry", ["main"], id="poetry_dependencies_as_one_element_list", ), pytest.param( - dedent( - """\ - [tool.poetry] - dependencies = "pylint" - """ - ), + """\ + [tool.poetry] + dependencies = "pylint" + """, [], "Poetry", ["main"], id="poetry_dependencies_as_str", ), pytest.param( - dedent( - """\ - [tool.poetry] - [tool.poetry.group.dev] - dependencies = ["black > 22", "mypy"] - """ - ), + """\ + [tool.poetry] + [tool.poetry.group.dev] + dependencies = ["black > 22", "mypy"] + """, [], "Poetry", ["group"], id="poetry_dependencies_as_list", ), pytest.param( - dedent( - """\ - [tool.poetry] - [tool.poetry.extras] - test = "pytest" - """ - ), + """\ + [tool.poetry] + [tool.poetry.extras] + test = "pytest" + """, [], "Poetry", ["extra"], id="poetry_extra_requirements_as_str_instead_of_list", ), pytest.param( - dedent( - """\ - [tool.poetry] - extras = ["pytest"] - """ - ), + """\ + [tool.poetry] + extras = ["pytest"] + """, [], "Poetry", ["extra"], id="poetry_extra_requirements_as_list_instead_of_dict", ), pytest.param( - dedent( - """\ - [tool.poetry] + """\ + [tool.poetry] - dependencies = ["pylint"] + dependencies = ["pylint"] - [tool.poetry.group.dev] - dependencies = ["black > 22", "mypy"] + [tool.poetry.group.dev] + dependencies = ["black > 22", "mypy"] - [tool.poetry.extras] - black = "^22" - """ - ), + [tool.poetry.extras] + black = "^22" + """, [], "Poetry", ["main", "group", "extra"], id="poetry_all_dependencies_malformatted", ), pytest.param( - dedent( - """\ - [project.dependencies] - pylint = "" - """ - ), + """\ + [project.dependencies] + pylint = "" + """, [], "PEP621", ["main"], id="pep621_dependencies_as_dict_instead_of_list", ), pytest.param( - dedent( - """\ - [project] - optional-dependencies = ["pylint"] - """ - ), + """\ + [project] + optional-dependencies = ["pylint"] + """, [], "PEP621", ["optional"], @@ -296,15 +264,17 @@ def test_parse_pyproject_content__pep621_or_poetry_dependencies__yields_dependen ], ) def test_parse_pyproject_content__malformatted_poetry_dependencies__yields_no_dependencies( - caplog, pyproject_toml, expected, metadata_standard, field_types + write_tmp_files, caplog, pyproject_toml, expected, metadata_standard, field_types ): - source = Location(Path("pyproject.toml")) + tmp_path = write_tmp_files({"pyproject.toml": pyproject_toml}) + path = tmp_path / "pyproject.toml" + caplog.set_level(logging.ERROR) - result = list(parse_pyproject_contents(pyproject_toml, source)) + result = list(parse_pyproject_toml(path)) assert result == expected for field_type in field_types: assert ( - f"Failed to parse {metadata_standard} {field_type} dependencies in {source.path}" + f"Failed to parse {metadata_standard} {field_type} dependencies in {path}" in caplog.text ) @@ -313,23 +283,19 @@ def test_parse_pyproject_content__malformatted_poetry_dependencies__yields_no_de "pyproject_toml,expected,expected_logs", [ pytest.param( - dedent( - """\ - [project] - name = "fawltydeps" - """ - ), + """\ + [project] + name = "fawltydeps" + """, [], [("PEP621", "main"), ("PEP621", "optional")], id="missing_pep621_fields", ), pytest.param( - dedent( - """\ - [tool.poetry] - name = "fawltydeps" - """ - ), + """\ + [tool.poetry] + name = "fawltydeps" + """, [], [ ("PEP621", "main"), @@ -342,18 +308,17 @@ def test_parse_pyproject_content__malformatted_poetry_dependencies__yields_no_de ), ], ) -def test_parse_pyproject_contents__missing_dependencies__logs_debug_message( - caplog, tmp_path, pyproject_toml, expected, expected_logs +def test_parse_pyproject_toml__missing_dependencies__logs_debug_message( + write_tmp_files, caplog, tmp_path, pyproject_toml, expected, expected_logs ): - caplog.set_level(logging.DEBUG) - path_hint = tmp_path / "pyproject.toml" - - result = list(parse_pyproject_contents(pyproject_toml, path_hint)) + tmp_path = write_tmp_files({"pyproject.toml": pyproject_toml}) + path = tmp_path / "pyproject.toml" + caplog.set_level(logging.DEBUG) + result = list(parse_pyproject_toml(path)) assert expected == result - for metadata_standard, field_type in expected_logs: assert ( - f"Failed to find {metadata_standard} {field_type} dependencies in {path_hint}" + f"Failed to find {metadata_standard} {field_type} dependencies in {path}" in caplog.text ) diff --git a/tests/test_extract_declared_dependencies_success.py b/tests/test_extract_declared_dependencies_success.py index d8957a8e..6c110441 100644 --- a/tests/test_extract_declared_dependencies_success.py +++ b/tests/test_extract_declared_dependencies_success.py @@ -7,9 +7,9 @@ from fawltydeps.extract_declared_dependencies import ( extract_declared_dependencies, - parse_requirements_contents, - parse_setup_cfg_contents, - parse_setup_contents, + parse_requirements_txt, + parse_setup_cfg, + parse_setup_py, ) from fawltydeps.types import DeclaredDependency, Location @@ -21,18 +21,15 @@ def dependency_factory(data: List[str], path: str) -> List[DeclaredDependency]: @pytest.mark.parametrize( - "file_content,expected", + "file_content,expect_deps", [ pytest.param( """\ pandas click """, - dependency_factory( - ["pandas", "click"], - "requirements.txt", - ), - id="__simple_requirements_success", + ["pandas", "click"], + id="simple_requirements_success", ), pytest.param( """\ @@ -40,73 +37,53 @@ def dependency_factory(data: List[str], path: str) -> List[DeclaredDependency]: click >=1.2 """, - dependency_factory( - ["pandas", "click"], - "requirements.txt", - ), - id="__requirements_with_versions__yields_names", + ["pandas", "click"], + id="requirements_with_versions__yields_names", ), pytest.param( - dedent( - """\ - requests [security] @ https://github.com/psf/requests/archive/refs/heads/main.zip - """ - ), - dependency_factory( - ["requests"], - "requirements.txt", - ), - id="__requirements_with_url_based_specifier__yields_names", + """\ + requests [security] @ https://github.com/psf/requests/archive/refs/heads/main.zip + """, + ["requests"], + id="requirements_with_url_based_specifier__yields_names", ), pytest.param( - dedent( - """\ - # this is a comment - click >=1.2 - """ - ), - dependency_factory( - ["click"], - "requirements.txt", - ), - id="__requirements_with_comment__ignores_comment", + """\ + # this is a comment + click >=1.2 + """, + ["click"], + id="requirements_with_comment__ignores_comment", ), pytest.param( - dedent( - """\ - -e . - click >=1.2 - """ - ), - dependency_factory( - ["click"], - "requirements.txt", - ), - id="__requirements_with_option__ignores_option", + """\ + -e . + click >=1.2 + """, + ["click"], + id="requirements_with_option__ignores_option", ), pytest.param( - dedent( - """\ - . # for running tests - click >=1.2 - """ - ), - dependency_factory( - ["click"], - "requirements.txt", - ), - id="__requirements_with_option__ignores_option_Issue200", + """\ + . # for running tests + click >=1.2 + """, + ["click"], + id="requirements_with_option__ignores_option_Issue200", ), ], ) -def test_parse_requirements_contents(file_content, expected): - source = Location(Path("requirements.txt")) - result = list(parse_requirements_contents(dedent(file_content), source)) +def test_parse_requirements_txt(write_tmp_files, file_content, expect_deps): + tmp_path = write_tmp_files({"requirements.txt": file_content}) + path = tmp_path / "requirements.txt" + + expected = dependency_factory(expect_deps, path) + result = list(parse_requirements_txt(path)) assert_unordered_equivalence(result, expected) @pytest.mark.parametrize( - "file_content,expected", + "file_content,expect_deps", [ pytest.param( """\ @@ -117,8 +94,8 @@ def test_parse_requirements_contents(file_content, expected): install_requires=["pandas", "click"] ) """, - dependency_factory(["pandas", "click"], "setup.py"), - id="__simple_requirements_in_setup_py__succeeds", + ["pandas", "click"], + id="simple_requirements_in_setup_py__succeeds", ), pytest.param( """\ @@ -129,8 +106,8 @@ def test_parse_requirements_contents(file_content, expected): install_requires=["pandas", "click>=1.2"] ) """, - dependency_factory(["pandas", "click"], "setup.py"), - id="__requirements_with_versions__yields_names", + ["pandas", "click"], + id="requirements_with_versions__yields_names", ), pytest.param( """\ @@ -141,7 +118,7 @@ def test_parse_requirements_contents(file_content, expected): ) """, [], - id="__no_requirements__yields_nothing", + id="no_requirements__yields_nothing", ), pytest.param( """\ @@ -156,8 +133,8 @@ def random_version(): install_requires=["pandas", "click>=1.2"] ) """, - dependency_factory(["pandas", "click"], "setup.py"), - id="__handles_nested_functions__yields_names", + ["pandas", "click"], + id="handles_nested_functions__yields_names", ), pytest.param( """\ @@ -177,8 +154,8 @@ def setup(**kwargs): install_requires=["pandas", "click>=1.2"] ) """, - dependency_factory(["pandas", "click"], "setup.py"), - id="__two_setup_calls__uses_only_top_level", + ["pandas", "click"], + id="two_setup_calls__uses_only_top_level", ), pytest.param( """\ @@ -192,8 +169,8 @@ def setup(**kwargs): } ) """, - dependency_factory(["annoy", "jieba"], "setup.py"), - id="__extras_present__yields_names", + ["annoy", "jieba"], + id="extras_present__yields_names", ), pytest.param( """\ @@ -208,8 +185,8 @@ def setup(**kwargs): } ) """, - dependency_factory(["pandas", "click", "annoy", "jieba"], "setup.py"), - id="__extras_and_regular_dependencies__yields_all_names", + ["pandas", "click", "annoy", "jieba"], + id="extras_and_regular_dependencies__yields_all_names", ), pytest.param( """\ @@ -222,8 +199,8 @@ def setup(**kwargs): install_requires=my_deps, ) """, - dependency_factory(["pandas", "click"], "setup.py"), - id="__direct_list_variable_reference__succeeds", + ["pandas", "click"], + id="direct_list_variable_reference__succeeds", ), pytest.param( """\ @@ -239,8 +216,8 @@ def setup(**kwargs): extras_require=my_extra_deps, ) """, - dependency_factory(["annoy", "jieba"], "setup.py"), - id="__direct_dict_variable_reference__succeeds", + ["annoy", "jieba"], + id="direct_dict_variable_reference__succeeds", ), pytest.param( """\ @@ -254,8 +231,8 @@ def setup(**kwargs): install_requires=[pandas, click], ) """, - dependency_factory(["pandas", "click"], "setup.py"), - id="__variable_reference_inside_list__succeeds", + ["pandas", "click"], + id="variable_reference_inside_list__succeeds", ), pytest.param( """\ @@ -273,8 +250,8 @@ def setup(**kwargs): }, ) """, - dependency_factory(["annoy", "foobar"], "setup.py"), - id="__variable_reference_inside_dict__succeeds", + ["annoy", "foobar"], + id="variable_reference_inside_dict__succeeds", ), pytest.param( """\ @@ -296,19 +273,22 @@ def setup(**kwargs): extras_require=my_extra_deps, ) """, - dependency_factory(["pandas", "click", "annoy", "foobar"], "setup.py"), - id="__nested_variable_reference__succeeds", + ["pandas", "click", "annoy", "foobar"], + id="nested_variable_reference__succeeds", ), ], ) -def test_parse_setup_contents(file_content, expected): - source = Location(Path("setup.py")) - result = list(parse_setup_contents(dedent(file_content), source)) +def test_parse_setup_py(write_tmp_files, file_content, expect_deps): + tmp_path = write_tmp_files({"setup.py": file_content}) + path = tmp_path / "setup.py" + + expected = dependency_factory(expect_deps, path) + result = list(parse_setup_py(path)) assert_unordered_equivalence(result, expected) @pytest.mark.parametrize( - "file_content,expected", + "file_content,expect_deps", [ pytest.param( """\ @@ -317,8 +297,8 @@ def test_parse_setup_contents(file_content, expected): pandas click """, - dependency_factory(["pandas", "click"], "setup.cfg"), - id="__simple_requirements_in_setup_cfg__succeeds", + ["pandas", "click"], + id="simple_requirements_in_setup_cfg__succeeds", ), pytest.param( """\ @@ -326,23 +306,23 @@ def test_parse_setup_contents(file_content, expected): license_files = LICENSE """, [], - id="__no_requirements_in_setup_cfg__returns_none", + id="no_requirements_in_setup_cfg__returns_none", ), pytest.param( """\ [options.extras_require] test = pytest """, - [DeclaredDependency("pytest", Location(Path("setup.cfg")))], - id="__extra_requirements_section_in_setup_cfg__succeeds", + ["pytest"], + id="extra_requirements_section_in_setup_cfg__succeeds", ), pytest.param( """\ [options.tests_require] test = pytest """, - [DeclaredDependency("pytest", Location(Path("setup.cfg")))], - id="__tests_requirements_section_in_setup_cfg__succeeds", + ["pytest"], + id="tests_requirements_section_in_setup_cfg__succeeds", ), pytest.param( """\ @@ -351,8 +331,8 @@ def test_parse_setup_contents(file_content, expected): hypothesis tox """, - dependency_factory(["hypothesis", "tox"], "setup.cfg"), - id="__tests_requirements_in_setup_cfg__succeeds", + ["hypothesis", "tox"], + id="tests_requirements_in_setup_cfg__succeeds", ), pytest.param( """\ @@ -361,8 +341,8 @@ def test_parse_setup_contents(file_content, expected): hypothesis tox """, - dependency_factory(["hypothesis", "tox"], "setup.cfg"), - id="__extras_requirements_in_setup_cfg__succeeds", + ["hypothesis", "tox"], + id="extras_requirements_in_setup_cfg__succeeds", ), pytest.param( """\ @@ -381,38 +361,45 @@ def test_parse_setup_contents(file_content, expected): [options.tests_require] test = hypothesis """, - dependency_factory( - ["pandas", "click", "tox", "scipy", "pytest", "hypothesis"], "setup.cfg" - ), - id="__all_requirements_types_in_setup_cfg__succeeds", + ["pandas", "click", "tox", "scipy", "pytest", "hypothesis"], + id="all_requirements_types_in_setup_cfg__succeeds", ), ], ) -def test_parse_setup_cfg_contents(file_content, expected): - source = Location(Path("setup.cfg")) - result = list(parse_setup_cfg_contents(dedent(file_content), source)) +def test_parse_setup_cfg(write_tmp_files, file_content, expect_deps): + tmp_path = write_tmp_files({"setup.cfg": file_content}) + path = tmp_path / "setup.cfg" + + expected = dependency_factory(expect_deps, path) + result = list(parse_setup_cfg(path)) assert_unordered_equivalence(result, expected) -def test_parse_setup_contents__multiple_entries_in_extras_require__returns_list(): - setup_contents = dedent( - """\ - from setuptools import setup - - setup( - name="MyLib", - extras_require={ - "simple_parsing":["abc"], - "bert": [ - "bert-serving-server>=1.8.6", - "bert-serving-client>=1.8.6", - "pytorch-transformer", - "flair" - ], - } - ) - """ +def test_parse_setup_py__multiple_entries_in_extras_require__returns_list( + write_tmp_files, +): + tmp_path = write_tmp_files( + { + "setup.py": """\ + from setuptools import setup + + setup( + name="MyLib", + extras_require={ + "simple_parsing":["abc"], + "bert": [ + "bert-serving-server>=1.8.6", + "bert-serving-client>=1.8.6", + "pytorch-transformer", + "flair" + ], + } + ) + """, + } ) + path = tmp_path / "setup.py" + expected = dependency_factory( [ "abc", @@ -421,9 +408,9 @@ def test_parse_setup_contents__multiple_entries_in_extras_require__returns_list( "pytorch-transformer", "flair", ], - Path(""), + path, ) - result = list(parse_setup_contents(setup_contents, Location(Path("")))) + result = list(parse_setup_py(path)) assert_unordered_equivalence(result, expected)