Skip to content

[3.11] gh-100320: Fix path calculations on Windows when python.exe is moved outside of the normal location (GH-100947) #101082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 16, 2023
Merged

[3.11] gh-100320: Fix path calculations on Windows when python.exe is moved outside of the normal location (GH-100947) #101082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions Lib/test/test_embed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -709,8 +709,10 @@ def check_config(self, configs, expected):
if MS_WINDOWS:
value = config.get(key := 'program_name')
if value and isinstance(value, str):
ext = '_d.exe' if debug_build(sys.executable) else '.exe'
config[key] = value[:len(value.lower().removesuffix(ext))]
value = value[:len(value.lower().removesuffix('.exe'))]
if debug_build(sys.executable):
value = value[:len(value.lower().removesuffix('_d'))]
config[key] = value
for key, value in list(expected.items()):
if value is self.IGNORE_CONFIG:
config.pop(key, None)
Expand Down Expand Up @@ -1283,7 +1285,7 @@ def test_init_setpythonhome(self):
stdlib = os.path.join(home, "Lib")
# Because we are specifying 'home', module search paths
# are fairly static
expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')]
expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
else:
version = f'{sys.version_info.major}.{sys.version_info.minor}'
stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
Expand Down Expand Up @@ -1324,7 +1326,7 @@ def test_init_is_python_build_with_home(self):
stdlib = os.path.join(home, "Lib")
# Because we are specifying 'home', module search paths
# are fairly static
expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')]
expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
else:
version = f'{sys.version_info.major}.{sys.version_info.minor}'
stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
Expand Down Expand Up @@ -1352,7 +1354,7 @@ def test_init_is_python_build_with_home(self):
config['_is_python_build'] = 1
exedir = os.path.dirname(sys.executable)
with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f:
expected_paths[2] = os.path.normpath(
expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath(
os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0]))
if not MS_WINDOWS:
# PREFIX (default) is set when running in build directory
Expand Down Expand Up @@ -1429,8 +1431,8 @@ def test_init_pybuilddir_win32(self):

module_search_paths = self.module_search_paths()
module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3]))
module_search_paths[-2] = stdlibdir
module_search_paths[-1] = tmpdir
module_search_paths[-2] = tmpdir
module_search_paths[-1] = stdlibdir

executable = self.test_exe
config = {
Expand Down
20 changes: 12 additions & 8 deletions Lib/test/test_getpath.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ def test_normal_win32(self):
module_search_paths_set=1,
module_search_paths=[
r"C:\Python\python98.zip",
r"C:\Python\Lib",
r"C:\Python\DLLs",
r"C:\Python\Lib",
r"C:\Python",
],
)
actual = getpath(ns, expected)
Expand All @@ -64,8 +65,8 @@ def test_buildtree_win32(self):
module_search_paths_set=1,
module_search_paths=[
r"C:\CPython\PCbuild\amd64\python98.zip",
r"C:\CPython\Lib",
r"C:\CPython\PCbuild\amd64",
r"C:\CPython\Lib",
],
)
actual = getpath(ns, expected)
Expand Down Expand Up @@ -134,8 +135,9 @@ def test_registry_win32(self):
r"C:\Python\python98.zip",
"path1-dir",
# should not contain not-subdirs
r"C:\Python\Lib",
r"C:\Python\DLLs",
r"C:\Python\Lib",
r"C:\Python",
],
)
actual = getpath(ns, expected)
Expand All @@ -148,8 +150,9 @@ def test_registry_win32(self):
module_search_paths_set=1,
module_search_paths=[
r"C:\Python\python98.zip",
r"C:\Python\Lib",
r"C:\Python\DLLs",
r"C:\Python\Lib",
r"C:\Python",
],
)
actual = getpath(ns, expected)
Expand All @@ -174,8 +177,9 @@ def test_symlink_normal_win32(self):
module_search_paths_set=1,
module_search_paths=[
r"C:\Python\python98.zip",
r"C:\Python\Lib",
r"C:\Python\DLLs",
r"C:\Python\Lib",
r"C:\Python",
],
)
actual = getpath(ns, expected)
Expand All @@ -202,8 +206,8 @@ def test_symlink_buildtree_win32(self):
module_search_paths_set=1,
module_search_paths=[
r"C:\CPython\PCbuild\amd64\python98.zip",
r"C:\CPython\Lib",
r"C:\CPython\PCbuild\amd64",
r"C:\CPython\Lib",
],
)
actual = getpath(ns, expected)
Expand Down Expand Up @@ -232,8 +236,8 @@ def test_buildtree_pythonhome_win32(self):
module_search_paths_set=1,
module_search_paths=[
r"C:\Out\python98.zip",
r"C:\CPython\Lib",
r"C:\Out",
r"C:\CPython\Lib",
],
)
actual = getpath(ns, expected)
Expand All @@ -255,8 +259,8 @@ def test_no_dlls_win32(self):
module_search_paths_set=1,
module_search_paths=[
r"C:\Python\python98.zip",
r"C:\Python\Lib",
r"C:\Python",
r"C:\Python\Lib",
],
)
actual = getpath(ns, expected)
Expand Down
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Ensures the ``PythonPath`` registry key from an install is used when
launching from a different copy of Python that relies on an existing install
to provide a copy of its modules and standard library.
48 changes: 30 additions & 18 deletions Modules/getpath.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -597,25 +597,27 @@ def search_up(prefix, *landmarks, test=isfile):

# Detect exec_prefix by searching from executable for the platstdlib_dir
if PLATSTDLIB_LANDMARK and not exec_prefix:
if executable_dir:
if os_name == 'nt':
# QUIRK: For compatibility and security, do not search for DLLs
# directory. The fallback below will cover it
exec_prefix = executable_dir
else:
exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
if os_name == 'nt':
# QUIRK: Windows always assumed these were the same
# gh-100320: Our PYDs are assumed to be relative to the Lib directory
# (that is, prefix) rather than the executable (that is, executable_dir)
exec_prefix = prefix
if not exec_prefix and executable_dir:
exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
if not exec_prefix and EXEC_PREFIX:
exec_prefix = EXEC_PREFIX
if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)):
if os_name == 'nt':
# QUIRK: If DLLs is missing on Windows, don't warn, just assume
# that it's all the same as prefix.
# gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into
# sys.path when it doesn't exist, which would give site-packages
# precedence over executable_dir, which is *probably* where our PYDs
# live. Ideally, whoever changes our layout will tell us what the
# layout is, but in the past this worked, so it should keep working.
platstdlib_dir = exec_prefix = prefix
# that they're in exec_prefix
if not platstdlib_dir:
# gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into
# sys.path when it doesn't exist in the platstdlib place, which
# would give Lib packages precedence over executable_dir where our
# PYDs *probably* live. Ideally, whoever changes our layout will tell
# us what the layout is, but in the past this worked, so it should
# keep working.
platstdlib_dir = exec_prefix
else:
warn('Could not find platform dependent libraries <exec_prefix>')

Expand Down Expand Up @@ -701,8 +703,14 @@ def search_up(prefix, *landmarks, test=isfile):
except OSError:
break
if isinstance(v, str):
pythonpath.append(v)
pythonpath.extend(v.split(DELIM))
i += 1
# Paths from the core key get appended last, but only
# when home was not set and we aren't in a build dir
if not home_was_set and not venv_prefix and not build_prefix:
v = winreg.QueryValue(key, None)
if isinstance(v, str):
pythonpath.extend(v.split(DELIM))
finally:
winreg.CloseKey(key)
except OSError:
Expand All @@ -714,13 +722,17 @@ def search_up(prefix, *landmarks, test=isfile):
pythonpath.append(joinpath(prefix, p))

# Then add stdlib_dir and platstdlib_dir
if os_name == 'nt' and venv_prefix:
# QUIRK: Windows generates paths differently in a venv
if os_name == 'nt':
# QUIRK: Windows generates paths differently
if platstdlib_dir:
pythonpath.append(platstdlib_dir)
if stdlib_dir:
pythonpath.append(stdlib_dir)
if executable_dir not in pythonpath:
if executable_dir and executable_dir not in pythonpath:
# QUIRK: the executable directory is on sys.path
# We keep it low priority, so that properly installed modules are
# found first. It may be earlier in the order if we found some
# reason to put it there.
pythonpath.append(executable_dir)
else:
if stdlib_dir:
Expand Down