From d729880f139e7eea5f46cdeda7165789019bc545 Mon Sep 17 00:00:00 2001 From: khanhdq Date: Sun, 31 Jul 2022 01:07:22 +0900 Subject: [PATCH 01/10] Don't create .json files when getting paths --- pipenv/environment.py | 52 +++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 50e0c9fedf..e5cadbade6 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -342,9 +342,8 @@ def build_command( pylib_lines = [] pyinc_lines = [] py_command = ( - "import sysconfig, distutils.sysconfig, io, json, sys; paths = {{" - "%s }}; value = u'{{0}}'.format(json.dumps(paths));" - "fh = io.open('{0}', 'w'); fh.write(value); fh.close()" + "import sysconfig, distutils.sysconfig, io, json, sys; paths = {" + "%s }; value = u'{0}'.format(json.dumps(paths)); print(value)" ) distutils_line = "distutils.sysconfig.get_python_{0}(plat_specific={1})" sysconfig_line = "sysconfig.get_path('{0}')" @@ -354,24 +353,24 @@ def build_command( # XXX: We need to get 'stdlib' or 'platstdlib' sys_prefix = "{}stdlib".format("" if key == "pure" else key) pylib_lines.append( - f"u'{dist_prefix}': u'{{{{0}}}}'.format({distutils_line.format(var, val)})" + f"u'{dist_prefix}': u'{{0}}'.format({distutils_line.format(var, val)})" ) pylib_lines.append( - f"u'{sys_prefix}': u'{{{{0}}}}'.format({sysconfig_line.format(sys_prefix)})" + f"u'{sys_prefix}': u'{{0}}'.format({sysconfig_line.format(sys_prefix)})" ) if python_inc: for key, var, val in (("include", "inc", "0"), ("platinclude", "inc", "1")): pylib_lines.append( - f"u'{key}': u'{{{{0}}}}'.format({distutils_line.format(var, val)})" + f"u'{key}': u'{{0}}'.format({distutils_line.format(var, val)})" ) lines = pylib_lines + pyinc_lines if scripts: lines.append( - "u'scripts': u'{{0}}'.format(%s)" % sysconfig_line.format("scripts") + "u'scripts': u'{0}'.format(%s)" % sysconfig_line.format("scripts") ) if py_version: lines.append( - "u'py_version_short': u'{{0}}'.format(distutils.sysconfig.get_python_version())," + "u'py_version_short': u'{0}'.format(distutils.sysconfig.get_python_version())," ) lines_as_str = ",".join(lines) py_command = py_command % lines_as_str @@ -384,18 +383,13 @@ def get_paths(self) -> Optional[Dict[str, str]]: :return: The python paths for the environment :rtype: Dict[str, str] """ - tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") - tmpfile.close() - tmpfile_path = make_posix(tmpfile.name) py_command = self.build_command( python_lib=True, python_inc=True, scripts=True, py_version=True ) - command = [self.python, "-c", py_command.format(tmpfile_path)] + command = [self.python, "-c", py_command] c = subprocess_run(command) if c.returncode == 0: - paths = {} - with open(tmpfile_path, "r", encoding="utf-8") as fh: - paths = json.load(fh) + paths = json.loads(c.stdout) if "purelib" in paths: paths["libdir"] = paths["purelib"] = make_posix(paths["purelib"]) for key in ( @@ -420,17 +414,12 @@ def get_lib_paths(self) -> Dict[str, str]: :return: The python include path for the environment :rtype: Dict[str, str] """ - tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") - tmpfile.close() - tmpfile_path = make_posix(tmpfile.name) py_command = self.build_command(python_lib=True) - command = [self.python, "-c", py_command.format(tmpfile_path)] + command = [self.python, "-c", py_command] c = subprocess_run(command) paths = None if c.returncode == 0: - paths = {} - with open(tmpfile_path, "r", encoding="utf-8") as fh: - paths = json.load(fh) + paths = json.loads(c.stdout) if "purelib" in paths: paths["libdir"] = paths["purelib"] = make_posix(paths["purelib"]) for key in ("platlib", "platstdlib", "stdlib"): @@ -476,22 +465,17 @@ def get_include_path(self) -> Optional[Dict[str, str]]: :return: The python include path for the environment :rtype: Dict[str, str] """ - tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") - tmpfile.close() - tmpfile_path = make_posix(tmpfile.name) py_command = ( - "import distutils.sysconfig, io, json, sys; paths = {{u'include': " - "u'{{0}}'.format(distutils.sysconfig.get_python_inc(plat_specific=0)), " - "u'platinclude': u'{{0}}'.format(distutils.sysconfig.get_python_inc(" - "plat_specific=1)) }}; value = u'{{0}}'.format(json.dumps(paths));" - "fh = io.open('{0}', 'w'); fh.write(value); fh.close()" + "import distutils.sysconfig, io, json, sys; paths = {u'include': " + "u'{0}'.format(distutils.sysconfig.get_python_inc(plat_specific=0)), " + "u'platinclude': u'{0}'.format(distutils.sysconfig.get_python_inc(" + "plat_specific=1)) }; value = u'{0}'.format(json.dumps(paths));" + "print(value)" ) - command = [self.python, "-c", py_command.format(tmpfile_path)] + command = [self.python, "-c", py_command] c = subprocess_run(command) if c.returncode == 0: - paths = [] - with open(tmpfile_path, "r", encoding="utf-8") as fh: - paths = json.load(fh) + paths = json.loads(c.stdout) for key in ("include", "platinclude"): if key in paths: paths[key] = make_posix(paths[key]) From 837292c1505387055183face46073cb952850e25 Mon Sep 17 00:00:00 2001 From: khanhdq Date: Mon, 1 Aug 2022 04:40:41 +0900 Subject: [PATCH 02/10] Use sysconfig instead of distutils.sysconfig --- pipenv/environment.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 938e7b0edc..36016bbb85 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -342,26 +342,19 @@ def build_command( pylib_lines = [] pyinc_lines = [] py_command = ( - "import sysconfig, distutils.sysconfig, io, json, sys; paths = {" - "%s }; value = u'{0}'.format(json.dumps(paths)); print(value)" + "import sysconfig, json; paths = {%s};" + "value = u'{0}'.format(json.dumps(paths)); print(value)" ) - distutils_line = "distutils.sysconfig.get_python_{0}(plat_specific={1})" sysconfig_line = "sysconfig.get_path('{0}')" if python_lib: - for key, var, val in (("pure", "lib", "0"), ("plat", "lib", "1")): - dist_prefix = f"{key}lib" - # XXX: We need to get 'stdlib' or 'platstdlib' - sys_prefix = "{}stdlib".format("" if key == "pure" else key) + for key in ("purelib", "platlib", "stdlib", "platstdlib"): pylib_lines.append( - f"u'{dist_prefix}': u'{{0}}'.format({distutils_line.format(var, val)})" - ) - pylib_lines.append( - f"u'{sys_prefix}': u'{{0}}'.format({sysconfig_line.format(sys_prefix)})" + f"u'{key}': u'{{0}}'.format({sysconfig_line.format(key)})" ) if python_inc: - for key, var, val in (("include", "inc", "0"), ("platinclude", "inc", "1")): - pylib_lines.append( - f"u'{key}': u'{{0}}'.format({distutils_line.format(var, val)})" + for key in ("include", "platinclude"): + pyinc_lines.append( + f"u'{key}': u'{{0}}'.format({sysconfig_line.format(key)})" ) lines = pylib_lines + pyinc_lines if scripts: @@ -370,7 +363,7 @@ def build_command( ) if py_version: lines.append( - "u'py_version_short': u'{0}'.format(distutils.sysconfig.get_python_version())," + "u'py_version_short': u'{0}'.format(sysconfig.get_python_version())," ) lines_as_str = ",".join(lines) py_command = py_command % lines_as_str @@ -465,13 +458,7 @@ def get_include_path(self) -> Optional[Dict[str, str]]: :return: The python include path for the environment :rtype: Dict[str, str] """ - py_command = ( - "import distutils.sysconfig, io, json, sys; paths = {u'include': " - "u'{0}'.format(distutils.sysconfig.get_python_inc(plat_specific=0)), " - "u'platinclude': u'{0}'.format(distutils.sysconfig.get_python_inc(" - "plat_specific=1)) }; value = u'{0}'.format(json.dumps(paths));" - "print(value)" - ) + py_command = self.build_command(python_inc=True) command = [self.python, "-c", py_command] c = subprocess_run(command) if c.returncode == 0: From 61ee6e79eb4c5b3791dd441f7c51e8c9ace778a3 Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 11:22:19 +0900 Subject: [PATCH 03/10] Add news fragment. --- news/5210.bugfix.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/5210.bugfix.rst diff --git a/news/5210.bugfix.rst b/news/5210.bugfix.rst new file mode 100644 index 0000000000..89710c2ea4 --- /dev/null +++ b/news/5210.bugfix.rst @@ -0,0 +1,2 @@ +Write output from ``subprocess_run`` directly to ``stdout`` instead of creating temporary file. +Remove deprecated ``distutils.sysconfig``, use ``sysconfig`` \ No newline at end of file From 228672f41bc6f45948b01c5e27e2972d99c618f6 Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 11:32:11 +0900 Subject: [PATCH 04/10] Fix linting --- news/5210.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/5210.bugfix.rst b/news/5210.bugfix.rst index 89710c2ea4..e4f8caef4e 100644 --- a/news/5210.bugfix.rst +++ b/news/5210.bugfix.rst @@ -1,2 +1,2 @@ Write output from ``subprocess_run`` directly to ``stdout`` instead of creating temporary file. -Remove deprecated ``distutils.sysconfig``, use ``sysconfig`` \ No newline at end of file +Remove deprecated ``distutils.sysconfig``, use ``sysconfig``. From 48ab50d989be45ce1596df02aab98ea85997ff19 Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 11:33:24 +0900 Subject: [PATCH 05/10] Add newline --- news/5210.bugfix.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/news/5210.bugfix.rst b/news/5210.bugfix.rst index e4f8caef4e..3510b9b92c 100644 --- a/news/5210.bugfix.rst +++ b/news/5210.bugfix.rst @@ -1,2 +1,3 @@ Write output from ``subprocess_run`` directly to ``stdout`` instead of creating temporary file. Remove deprecated ``distutils.sysconfig``, use ``sysconfig``. + From 3a05e37eaa0ee1f6f5bad76e935e5fa58852a2aa Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 11:54:35 +0900 Subject: [PATCH 06/10] Remove newline, add dot. --- news/5210.bugfix.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/news/5210.bugfix.rst b/news/5210.bugfix.rst index 3510b9b92c..e4f8caef4e 100644 --- a/news/5210.bugfix.rst +++ b/news/5210.bugfix.rst @@ -1,3 +1,2 @@ Write output from ``subprocess_run`` directly to ``stdout`` instead of creating temporary file. Remove deprecated ``distutils.sysconfig``, use ``sysconfig``. - From 3ca7c96107bf589634f9e2e2e400effcae1b437b Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 11:55:27 +0900 Subject: [PATCH 07/10] Remove unnecessary exception handling --- pipenv/environment.py | 38 ++------------------------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 36016bbb85..b4e44a1dce 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -9,7 +9,7 @@ import site import sys from pathlib import Path -from sysconfig import get_paths, get_python_version, get_scheme_names +from sysconfig import get_paths, get_scheme_names import pkg_resources @@ -159,17 +159,6 @@ def python_info(self) -> Dict[str, str]: return {"py_version_short": py_version_short, "abiflags": abiflags} return {} - def _replace_parent_version(self, path: str, replace_version: str) -> str: - if not os.path.exists(path): - base, leaf = os.path.split(path) - base, parent = os.path.split(base) - leaf = os.path.join(parent, leaf).replace( - replace_version, - self.python_info.get("py_version_short", get_python_version()), - ) - return os.path.join(base, leaf) - return path - @cached_property def install_scheme(self): if "venv" in get_scheme_names(): @@ -214,30 +203,7 @@ def base_paths(self) -> Dict[str, str]: if self._base_paths: paths = self._base_paths.copy() else: - try: - paths = self.get_paths() - except Exception: - paths = get_paths( - self.install_scheme, - vars={ - "base": prefix, - "platbase": prefix, - }, - ) - current_version = get_python_version() - try: - for k in list(paths.keys()): - if not os.path.exists(paths[k]): - paths[k] = self._replace_parent_version( - paths[k], current_version - ) - except OSError: - # Sometimes virtualenvs are made using virtualenv interpreters and there is no - # include directory, which will cause this approach to fail. This failsafe - # will make sure we fall back to the shell execution to find the real include path - paths = self.get_include_path() - paths.update(self.get_lib_paths()) - paths["scripts"] = self.script_basedir + paths = self.get_paths() if not paths: paths = get_paths( self.install_scheme, From 029d187a2e241b8586373a11daaf3694a43af497 Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 16:22:21 +0900 Subject: [PATCH 08/10] Revert "Remove newline, add dot." This reverts commit 3a05e37eaa0ee1f6f5bad76e935e5fa58852a2aa. --- news/5210.bugfix.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/news/5210.bugfix.rst b/news/5210.bugfix.rst index e4f8caef4e..3510b9b92c 100644 --- a/news/5210.bugfix.rst +++ b/news/5210.bugfix.rst @@ -1,2 +1,3 @@ Write output from ``subprocess_run`` directly to ``stdout`` instead of creating temporary file. Remove deprecated ``distutils.sysconfig``, use ``sysconfig``. + From 04386771800cf6bf703e4a26874ca247544a32c1 Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 16:25:06 +0900 Subject: [PATCH 09/10] Revert "Remove unnecessary exception handling" This reverts commit 3ca7c96107bf589634f9e2e2e400effcae1b437b. --- pipenv/environment.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index b4e44a1dce..36016bbb85 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -9,7 +9,7 @@ import site import sys from pathlib import Path -from sysconfig import get_paths, get_scheme_names +from sysconfig import get_paths, get_python_version, get_scheme_names import pkg_resources @@ -159,6 +159,17 @@ def python_info(self) -> Dict[str, str]: return {"py_version_short": py_version_short, "abiflags": abiflags} return {} + def _replace_parent_version(self, path: str, replace_version: str) -> str: + if not os.path.exists(path): + base, leaf = os.path.split(path) + base, parent = os.path.split(base) + leaf = os.path.join(parent, leaf).replace( + replace_version, + self.python_info.get("py_version_short", get_python_version()), + ) + return os.path.join(base, leaf) + return path + @cached_property def install_scheme(self): if "venv" in get_scheme_names(): @@ -203,7 +214,30 @@ def base_paths(self) -> Dict[str, str]: if self._base_paths: paths = self._base_paths.copy() else: - paths = self.get_paths() + try: + paths = self.get_paths() + except Exception: + paths = get_paths( + self.install_scheme, + vars={ + "base": prefix, + "platbase": prefix, + }, + ) + current_version = get_python_version() + try: + for k in list(paths.keys()): + if not os.path.exists(paths[k]): + paths[k] = self._replace_parent_version( + paths[k], current_version + ) + except OSError: + # Sometimes virtualenvs are made using virtualenv interpreters and there is no + # include directory, which will cause this approach to fail. This failsafe + # will make sure we fall back to the shell execution to find the real include path + paths = self.get_include_path() + paths.update(self.get_lib_paths()) + paths["scripts"] = self.script_basedir if not paths: paths = get_paths( self.install_scheme, From d99f18e85b5b021d0da5aebe800aef1b4d9b1e1b Mon Sep 17 00:00:00 2001 From: khanhdq Date: Tue, 2 Aug 2022 16:25:27 +0900 Subject: [PATCH 10/10] Revert "Revert "Remove newline, add dot."" This reverts commit 029d187a2e241b8586373a11daaf3694a43af497. --- news/5210.bugfix.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/news/5210.bugfix.rst b/news/5210.bugfix.rst index 3510b9b92c..e4f8caef4e 100644 --- a/news/5210.bugfix.rst +++ b/news/5210.bugfix.rst @@ -1,3 +1,2 @@ Write output from ``subprocess_run`` directly to ``stdout`` instead of creating temporary file. Remove deprecated ``distutils.sysconfig``, use ``sysconfig``. -