From b246d69889280a19f69fb1d00c0fa2df7490c633 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Fri, 30 Jun 2023 22:36:13 -0400 Subject: [PATCH 1/3] Fix for pyproject.toml dependencies not being included. --- pipenv/utils/resolver.py | 1 + .../requirementslib/models/setup_info.py | 16 +++--- pipenv/vendor/requirementslib/models/utils.py | 57 +++++++++---------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 8fdbd91fbb..653bc265d0 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -363,6 +363,7 @@ def get_deps_from_req( and is_installable_dir(parsed_line.path) ) ): + setup_info.run_pyproject() setup_info.run_setup() requirements = [v for v in getattr(setup_info, "requires", {}).values()] if req.extras: diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 69306d3735..8e37360fa0 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1018,9 +1018,8 @@ def get_metadata(path, pkg_name=None, metadata_type=None): wheel_allowed = metadata_type == "wheel" or metadata_type is None egg_allowed = metadata_type == "egg" or metadata_type is None dist = None # type: Optional[Union[DistInfoDistribution, EggInfoDistribution]] - if wheel_allowed: - dist = get_distinfo_dist(path, pkg_name=pkg_name) - if egg_allowed and dist is None: + dist = get_distinfo_dist(path, pkg_name=pkg_name) + if dist is None: dist = get_egginfo_dist(path, pkg_name=pkg_name) if dist is not None: return get_metadata_from_dist(dist) @@ -1590,17 +1589,18 @@ def run_pyproject(self) -> "SetupInfo": if self.pyproject and self.pyproject.exists(): result = get_pyproject(self.pyproject.parent) if result is not None: - requires, backend = result if self.build_requires is None: self.build_requires = () - if backend: - self.build_backend = backend + if result.get("build_backend"): + self.build_backend = result.get("build_backend") else: self.build_backend = get_default_pyproject_backend() - if requires: - self.build_requires = tuple(set(requires) | set(self.build_requires)) + if result.get("build_requires"): + self.build_requires = tuple(set(result.get("build_requires", [])) | set(self.build_requires)) else: self.build_requires = ("setuptools", "wheel") + if result.get("dependencies"): + self._requirements += make_base_requirements(tuple(set(result.get("dependencies", [])))) return self def get_initial_info(self) -> Dict[str, Any]: diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 111935ca03..319b27b241 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -441,47 +441,46 @@ def get_default_pyproject_backend(): return "setuptools.build_meta" -def get_pyproject(path): - # type: (Union[STRING_TYPE, Path]) -> Optional[Tuple[List[STRING_TYPE], STRING_TYPE]] - """Given a base path, look for the corresponding ``pyproject.toml`` file +def get_pyproject(path: Union[AnyStr, Path]) -> Optional[Dict[str, Union[List[AnyStr], AnyStr]]]: + """ + Given a base path, look for the corresponding ``pyproject.toml`` file and return its build_requires and build_backend. - :param AnyStr path: The root path of the project, should be a directory (will be truncated) - :return: A 2 tuple of build requirements and the build backend - :rtype: Optional[Tuple[List[AnyStr], AnyStr]] + :param path: The root path of the project, should be a directory (will be truncated) + :return: A dictionary with build requirements, build backend, and dependencies """ if not path: return if not isinstance(path, Path): path = Path(path) + if not path.is_dir(): path = path.parent - pp_toml = path.joinpath("pyproject.toml") - setup_py = path.joinpath("setup.py") - if not pp_toml.exists(): - if not setup_py.exists(): - return None - requires = ["setuptools>=40.8", "wheel"] - backend = get_default_pyproject_backend() - else: - pyproject_data = {} - with open(pp_toml.as_posix(), encoding="utf-8") as fh: + + pp_toml = path / "pyproject.toml" + + # Default values + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() + dependencies = [] + + if pp_toml.exists(): + with open(pp_toml, encoding="utf-8") as fh: pyproject_data = tomlkit.loads(fh.read()) + + # Extracting build system information build_system = pyproject_data.get("build-system", None) - if build_system is None: - if setup_py.exists(): - requires = ["setuptools>=40.8", "wheel"] - backend = get_default_pyproject_backend() - else: - requires = ["setuptools>=40.8", "wheel"] - backend = get_default_pyproject_backend() - build_system = {"requires": requires, "build-backend": backend} - pyproject_data["build_system"] = build_system - else: - requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) - backend = build_system.get("build-backend", get_default_pyproject_backend()) - return requires, backend + if build_system is not None: + requires = build_system.get("requires", requires) + backend = build_system.get("build-backend", backend) + + # Extracting project dependencies + project_data = pyproject_data.get("project", None) + if project_data is not None: + dependencies = project_data.get("dependencies", []) + + return {"build_requires": requires, "build_backend": backend, "dependencies": dependencies} def split_markers_from_line(line): From 7787050bc9f28a2257eb0414ee54b8d81578dabc Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Fri, 30 Jun 2023 23:05:12 -0400 Subject: [PATCH 2/3] refactor other usages of get_pyproject --- .../requirementslib/models/requirements.py | 51 ++++++++----------- .../requirementslib/models/setup_info.py | 6 +-- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 1f91923687..8617a68189 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -511,34 +511,29 @@ def pyproject_requires(self): # type: () -> Optional[Tuple[str, ...]] if self._pyproject_requires is None and self.pyproject_toml is not None: if self.path is not None: - pyproject_requires, pyproject_backend = None, None - pyproject_results = get_pyproject(self.path) # type: ignore - if pyproject_results: - pyproject_requires, pyproject_backend = pyproject_results - if pyproject_requires: - self._pyproject_requires = tuple(pyproject_requires) - self._pyproject_backend = pyproject_backend + results = get_pyproject(self.path) # type: ignore + if results and results.get("build_requires"): + self._pyproject_requires = results.get("build_requires") return self._pyproject_requires @property def pyproject_backend(self): # type: () -> Optional[str] if self._pyproject_requires is None and self.pyproject_toml is not None: - pyproject_requires = None # type: Optional[Sequence[str]] - pyproject_backend = None # type: Optional[str] - pyproject_results = get_pyproject(self.path) # type: ignore - if pyproject_results: - pyproject_requires, pyproject_backend = pyproject_results - if not pyproject_backend and self.setup_cfg is not None: - setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) - pyproject_backend = get_default_pyproject_backend() - pyproject_requires = setup_dict.get( - "build_requires", ["setuptools", "wheel"] - ) # type: ignore - if pyproject_requires: - self._pyproject_requires = tuple(pyproject_requires) - if pyproject_backend: - self._pyproject_backend = pyproject_backend + results = get_pyproject(self.path) # type: ignore + if results: + if not results.get("build_backend") and self.setup_cfg is not None: + setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) + self._pyproject_backend = get_default_pyproject_backend() + else: + self._pyproject_backend = results.get("build_backend") + if results.get("build_requires"): + self._pyproject_requires = tuple(results.get("build_requires")) + else: + self._pyproject_requires = setup_dict.get( + "build_requires", ["setuptools", "wheel"] + ) # type: ignore + return self._pyproject_backend def parse_hashes(self): @@ -2052,21 +2047,19 @@ def get_vcs_repo(self, src_dir=None, checkout_dir=None): ) if not self.is_local: vcsrepo.obtain() - pyproject_info = None if self.subdirectory: self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") self.pyproject_path = os.path.join( checkout_dir, self.subdirectory, "pyproject.toml" ) - pyproject_info = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) + result = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) else: self.setup_path = os.path.join(checkout_dir, "setup.py") self.pyproject_path = os.path.join(checkout_dir, "pyproject.toml") - pyproject_info = get_pyproject(checkout_dir) - if pyproject_info is not None: - pyproject_requires, pyproject_backend = pyproject_info - self.pyproject_requires = tuple(pyproject_requires) - self.pyproject_backend = pyproject_backend + result = get_pyproject(checkout_dir) + if result is not None: + self.pyproject_requires = tuple(result.get("build_requires", [])) + self.pyproject_backend = result.get("build_backend") return vcsrepo @cached_property diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 8e37360fa0..d6106655e0 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -812,8 +812,8 @@ def parse_setup_cfg(path: str) -> "Dict[str, Any]": def build_pep517(source_dir, build_dir, config_settings=None, dist_type="wheel"): if config_settings is None: config_settings = {} - requires, backend = get_pyproject(source_dir) - hookcaller = HookCaller(source_dir, backend) + result = get_pyproject(source_dir) or {} + hookcaller = HookCaller(source_dir, result.get("build_backend", get_default_pyproject_backend())) if dist_type == "sdist": get_requires_fn = hookcaller.get_requires_for_build_sdist build_fn = hookcaller.build_sdist @@ -822,7 +822,7 @@ def build_pep517(source_dir, build_dir, config_settings=None, dist_type="wheel") build_fn = hookcaller.build_wheel with BuildEnv() as env: - env.pip_install(requires) + env.pip_install(result.get("build_requires", [])) reqs = get_requires_fn(config_settings) env.pip_install(reqs) return build_fn(build_dir, config_settings) From 9a98977997fd984cc2e0f89481e70b6058f63eed Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Fri, 30 Jun 2023 23:02:17 -0400 Subject: [PATCH 3/3] Add test and news fragment. --- news/5766.bugfix.rst | 1 + tests/integration/test_lock.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 news/5766.bugfix.rst diff --git a/news/5766.bugfix.rst b/news/5766.bugfix.rst new file mode 100644 index 0000000000..71aa5dfae9 --- /dev/null +++ b/news/5766.bugfix.rst @@ -0,0 +1 @@ +Fix issue in requirementslib 3.0.0 where dependencies defined in pyproject.toml were not being included in the lock file. diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 16d5bdb076..665a6f96df 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -25,6 +25,46 @@ def test_lock_handle_eggs(pipenv_instance_private_pypi): assert p.lockfile['default']['randomwords']['version'] == '==0.2.1' +@pytest.mark.lock +@pytest.mark.requirements +def test_lock_gathers_pyproject_dependencies(pipenv_instance_pypi): + """Ensure that running `pipenv install` doesn't install dev packages""" + with pipenv_instance_pypi(chdir=True) as p: + with open(p.pipfile_path, "w") as f: + contents = """ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pipenvtest = { editable = true, path = "." } + """.strip() + f.write(contents) + + # Write the pyproject.toml + pyproject_toml_path = os.path.join(os.path.dirname(p.pipfile_path), "pyproject.toml") + with open(pyproject_toml_path, "w") as f: + contents = """ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "pipenvtest" +version = "0.0.1" +requires-python = ">=3.8" +dependencies = [ + "six" +] + """.strip() + f.write(contents) + c = p.pipenv("lock") + assert c.returncode == 0 + assert "six" in p.lockfile["default"] + + + @pytest.mark.lock @pytest.mark.requirements def test_lock_requirements_file(pipenv_instance_private_pypi):