From bb7f36a65437abfcb274829daee3c0fa7a136d25 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 16 Jan 2023 16:05:39 +0000 Subject: [PATCH] gh-100320: Fix path calculations on Windows when python.exe is moved outside of the normal location (GH-100947) --- Lib/test/test_embed.py | 16 ++++--- Lib/test/test_getpath.py | 20 ++++---- ...-01-11-16-28-09.gh-issue-100320.2DU2it.rst | 3 ++ Modules/getpath.py | 48 ++++++++++++------- 4 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index be7980eebe15df..2affa17e800c07 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -709,8 +709,10 @@ def check_config(self, configs, expected): if MS_WINDOWS: value = config.get(key := 'program_name') if value and isinstance(value, str): - ext = '_d.exe' if debug_build(sys.executable) else '.exe' - config[key] = value[:len(value.lower().removesuffix(ext))] + value = value[:len(value.lower().removesuffix('.exe'))] + if debug_build(sys.executable): + value = value[:len(value.lower().removesuffix('_d'))] + config[key] = value for key, value in list(expected.items()): if value is self.IGNORE_CONFIG: config.pop(key, None) @@ -1283,7 +1285,7 @@ def test_init_setpythonhome(self): stdlib = os.path.join(home, "Lib") # Because we are specifying 'home', module search paths # are fairly static - expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] + expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' stdlib = os.path.join(home, sys.platlibdir, f'python{version}') @@ -1324,7 +1326,7 @@ def test_init_is_python_build_with_home(self): stdlib = os.path.join(home, "Lib") # Because we are specifying 'home', module search paths # are fairly static - expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] + expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' stdlib = os.path.join(home, sys.platlibdir, f'python{version}') @@ -1352,7 +1354,7 @@ def test_init_is_python_build_with_home(self): config['_is_python_build'] = 1 exedir = os.path.dirname(sys.executable) with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: - expected_paths[2] = os.path.normpath( + expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath( os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0])) if not MS_WINDOWS: # PREFIX (default) is set when running in build directory @@ -1429,8 +1431,8 @@ def test_init_pybuilddir_win32(self): module_search_paths = self.module_search_paths() module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3])) - module_search_paths[-2] = stdlibdir - module_search_paths[-1] = tmpdir + module_search_paths[-2] = tmpdir + module_search_paths[-1] = stdlibdir executable = self.test_exe config = { diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 87932b891f808a..8d5b4265f269ee 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -38,8 +38,9 @@ def test_normal_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -64,8 +65,8 @@ def test_buildtree_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\CPython\PCbuild\amd64\python98.zip", - r"C:\CPython\Lib", r"C:\CPython\PCbuild\amd64", + r"C:\CPython\Lib", ], ) actual = getpath(ns, expected) @@ -134,8 +135,9 @@ def test_registry_win32(self): r"C:\Python\python98.zip", "path1-dir", # should not contain not-subdirs - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -148,8 +150,9 @@ def test_registry_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -174,8 +177,9 @@ def test_symlink_normal_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python\DLLs", + r"C:\Python\Lib", + r"C:\Python", ], ) actual = getpath(ns, expected) @@ -202,8 +206,8 @@ def test_symlink_buildtree_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\CPython\PCbuild\amd64\python98.zip", - r"C:\CPython\Lib", r"C:\CPython\PCbuild\amd64", + r"C:\CPython\Lib", ], ) actual = getpath(ns, expected) @@ -232,8 +236,8 @@ def test_buildtree_pythonhome_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Out\python98.zip", - r"C:\CPython\Lib", r"C:\Out", + r"C:\CPython\Lib", ], ) actual = getpath(ns, expected) @@ -255,8 +259,8 @@ def test_no_dlls_win32(self): module_search_paths_set=1, module_search_paths=[ r"C:\Python\python98.zip", - r"C:\Python\Lib", r"C:\Python", + r"C:\Python\Lib", ], ) actual = getpath(ns, expected) diff --git a/Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst b/Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst new file mode 100644 index 00000000000000..c206fc8520a5d9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst @@ -0,0 +1,3 @@ +Ensures the ``PythonPath`` registry key from an install is used when +launching from a different copy of Python that relies on an existing install +to provide a copy of its modules and standard library. diff --git a/Modules/getpath.py b/Modules/getpath.py index fc533a8f5f5eb4..e847141707329a 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -597,25 +597,27 @@ def search_up(prefix, *landmarks, test=isfile): # Detect exec_prefix by searching from executable for the platstdlib_dir if PLATSTDLIB_LANDMARK and not exec_prefix: - if executable_dir: - if os_name == 'nt': - # QUIRK: For compatibility and security, do not search for DLLs - # directory. The fallback below will cover it - exec_prefix = executable_dir - else: - exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) + if os_name == 'nt': + # QUIRK: Windows always assumed these were the same + # gh-100320: Our PYDs are assumed to be relative to the Lib directory + # (that is, prefix) rather than the executable (that is, executable_dir) + exec_prefix = prefix + if not exec_prefix and executable_dir: + exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): if os_name == 'nt': # QUIRK: If DLLs is missing on Windows, don't warn, just assume - # that it's all the same as prefix. - # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into - # sys.path when it doesn't exist, which would give site-packages - # precedence over executable_dir, which is *probably* where our PYDs - # live. Ideally, whoever changes our layout will tell us what the - # layout is, but in the past this worked, so it should keep working. - platstdlib_dir = exec_prefix = prefix + # that they're in exec_prefix + if not platstdlib_dir: + # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into + # sys.path when it doesn't exist in the platstdlib place, which + # would give Lib packages precedence over executable_dir where our + # PYDs *probably* live. Ideally, whoever changes our layout will tell + # us what the layout is, but in the past this worked, so it should + # keep working. + platstdlib_dir = exec_prefix else: warn('Could not find platform dependent libraries ') @@ -701,8 +703,14 @@ def search_up(prefix, *landmarks, test=isfile): except OSError: break if isinstance(v, str): - pythonpath.append(v) + pythonpath.extend(v.split(DELIM)) i += 1 + # Paths from the core key get appended last, but only + # when home was not set and we aren't in a build dir + if not home_was_set and not venv_prefix and not build_prefix: + v = winreg.QueryValue(key, None) + if isinstance(v, str): + pythonpath.extend(v.split(DELIM)) finally: winreg.CloseKey(key) except OSError: @@ -714,13 +722,17 @@ def search_up(prefix, *landmarks, test=isfile): pythonpath.append(joinpath(prefix, p)) # Then add stdlib_dir and platstdlib_dir - if os_name == 'nt' and venv_prefix: - # QUIRK: Windows generates paths differently in a venv + if os_name == 'nt': + # QUIRK: Windows generates paths differently if platstdlib_dir: pythonpath.append(platstdlib_dir) if stdlib_dir: pythonpath.append(stdlib_dir) - if executable_dir not in pythonpath: + if executable_dir and executable_dir not in pythonpath: + # QUIRK: the executable directory is on sys.path + # We keep it low priority, so that properly installed modules are + # found first. It may be earlier in the order if we found some + # reason to put it there. pythonpath.append(executable_dir) else: if stdlib_dir: