feat: remove usages of __file__ to support pdm in a zipapp (#1567)

news/1567bugfix.md

@@ -0,0 +1 @@
+Replace the `__file__` usages with `importlib.resources`, to make PDM usable in a zipapp.

pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "virtualenv>=20",
     "pyproject-hooks",
     "requests-toolbelt",
-    "unearth>=0.6.3",
+    "unearth>=0.7",
     "findpython>=0.2.2",
     "tomlkit>=0.11.1,<1",
     "shellingham>=1.3.2",

src/pdm/__version__.py

@@ -1,17 +1,11 @@
-from __future__ import annotations
-
-from packaging.version import Version, parse
-
 from pdm.compat import importlib_metadata
 
 
 def read_version() -> str:
-    return importlib_metadata.version(__package__)
+    return importlib_metadata.version(__package__ or "pdm")
 
 
 try:
     __version__ = read_version()
-    parsed_version: Version | None = parse(__version__)
 except importlib_metadata.PackageNotFoundError:
-    __version__ = "UNKNOWN"
-    parsed_version = None
+    __version__ = "0.0.0+local"

src/pdm/cli/actions.py

@@ -26,6 +26,7 @@
     find_importable_files,
     format_lockfile,
     format_resolution_impossible,
+    get_pep582_path,
     merge_dictionary,
     save_version_specifiers,
     set_env_in_reg,
@@ -45,8 +46,6 @@
 from pdm.resolver import resolve
 from pdm.utils import cd, normalize_name
 
-PEP582_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "pep582")
-
 
 def do_lock(
     project: Project,
@@ -752,13 +751,16 @@ def ask_for_import(project: Project) -> None:
     do_import(project, str(filepath), key)
 
 
-def print_pep582_command(ui: termui.UI, shell: str = "AUTO") -> None:
+def print_pep582_command(project: Project, shell: str = "AUTO") -> None:
     """Print the export PYTHONPATH line to be evaluated by the shell."""
     import shellingham
 
+    pep582_path = get_pep582_path(project)
+    ui = project.core.ui
+
     if os.name == "nt":
         try:
-            set_env_in_reg("PYTHONPATH", PEP582_PATH)
+            set_env_in_reg("PYTHONPATH", pep582_path)
         except PermissionError:
             ui.echo(
                 "Permission denied, please run the terminal as administrator.",
@@ -771,7 +773,7 @@ def print_pep582_command(ui: termui.UI, shell: str = "AUTO") -> None:
             style="success",
         )
         return
-    lib_path = PEP582_PATH.replace("'", "\\'")
+    lib_path = pep582_path.replace("'", "\\'")
     if shell == "AUTO":
         shell = shellingham.detect_shell()[0]
     shell = shell.lower()

src/pdm/cli/commands/run.py

@@ -12,11 +12,10 @@
 from typing import Any, Callable, Iterator, Mapping, NamedTuple, Sequence, cast
 
 from pdm import signals, termui
-from pdm.cli.actions import PEP582_PATH
 from pdm.cli.commands.base import BaseCommand
 from pdm.cli.hooks import KNOWN_HOOKS, HookManager
 from pdm.cli.options import skip_option
-from pdm.cli.utils import check_project_file
+from pdm.cli.utils import check_project_file, get_pep582_path
 from pdm.compat import TypedDict
 from pdm.exceptions import PdmUsageError
 from pdm.project import Project
@@ -165,7 +164,7 @@ def _run_process(
             else:
                 process_env = {**dotenv_env, **process_env}
         pythonpath = process_env.get("PYTHONPATH", "").split(os.pathsep)
-        pythonpath = [PEP582_PATH] + [
+        pythonpath = [get_pep582_path(project)] + [
             p for p in pythonpath if "pdm/pep582" not in p.replace("\\", "/")
         ]
         project_env = project.environment

src/pdm/cli/commands/self_cmd.py

@@ -241,18 +241,16 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
         )
 
     def handle(self, project: Project, options: argparse.Namespace) -> None:
-        from pdm.__version__ import parsed_version, read_version
+        from pdm.__version__ import __version__, read_version
 
         if options.head:
             package = f"pdm @ git+{PDM_REPO}@main"
             version: str | None = "HEAD"
         else:
             version = get_latest_pdm_version_from_pypi(project, options.pre)
             assert version is not None, "No version found"
-            if parsed_version and parsed_version >= parse(version):
-                project.core.ui.echo(
-                    f"Already up-to-date: [primary]{parsed_version}[/]"
-                )
+            if parse(__version__) >= parse(version):
+                project.core.ui.echo(f"Already up-to-date: [primary]{__version__}[/]")
                 return
             package = f"pdm=={version}"
         pip_args = ["install", "--upgrade"] + shlex.split(options.pip_args) + [package]

src/pdm/cli/utils.py

@@ -724,3 +724,15 @@ def get_dist_location(dist: Distribution) -> str:
         editable = direct_url_data.get("dir_info", {}).get("editable", False)
         return f"{'-e ' if editable else ''}{path}"
     return ""
+
+
+def get_pep582_path(project: Project) -> str:
+    import importlib.resources
+
+    script_dir = project.global_config.config_file.parent / "pep582"
+    if script_dir.joinpath("sitecustomize.py").exists():
+        return str(script_dir)
+    script_dir.mkdir(parents=True, exist_ok=True)
+    with importlib.resources.open_binary("pdm.pep582", "sitecustomize.py") as f:
+        script_dir.joinpath("sitecustomize.py").write_bytes(f.read())
+    return str(script_dir)

src/pdm/core.py

@@ -169,7 +169,7 @@ def main(
             os.environ["PDM_IGNORE_SAVED_PYTHON"] = "1"
 
         if options.pep582:
-            print_pep582_command(self.ui, options.pep582)
+            print_pep582_command(project, options.pep582)
             sys.exit(0)
 
         if root_script and root_script not in project.scripts:

src/pdm/models/in_process/__init__.py

@@ -3,20 +3,25 @@
 """
 from __future__ import annotations
 
+import contextlib
 import functools
+import importlib.resources
 import json
 import os
 import subprocess
-from pathlib import Path
-from typing import Any
+from typing import Any, Generator
 
-FOLDER_PATH = Path(__file__).parent
+
+@contextlib.contextmanager
+def _in_process_script(name: str) -> Generator[str, None, None]:
+    with importlib.resources.path(__name__, name) as script:
+        yield str(script)
 
 
 @functools.lru_cache()
 def get_python_abi_tag(executable: str) -> str:
-    script = str(FOLDER_PATH / "get_abi_tag.py")
-    return json.loads(subprocess.check_output(args=[executable, "-Es", script]))
+    with _in_process_script("get_abi_tag.py") as script:
+        return json.loads(subprocess.check_output(args=[executable, "-Es", script]))
 
 
 def get_sys_config_paths(
@@ -27,19 +32,21 @@ def get_sys_config_paths(
     env.pop("__PYVENV_LAUNCHER__", None)
     if vars is not None:
         env["_SYSCONFIG_VARS"] = json.dumps(vars)
-    cmd = [executable, "-Es", str(FOLDER_PATH / "sysconfig_get_paths.py"), kind]
 
-    return json.loads(subprocess.check_output(cmd, env=env))
+    with _in_process_script("sysconfig_get_paths.py") as script:
+        cmd = [executable, "-Es", script, kind]
+        return json.loads(subprocess.check_output(cmd, env=env))
 
 
 def get_pep508_environment(executable: str) -> dict[str, str]:
     """Get PEP 508 environment markers dict."""
-    script = str(FOLDER_PATH / "pep508.py")
-    args = [executable, "-Es", script]
-    return json.loads(subprocess.check_output(args))
+    with _in_process_script("pep508.py") as script:
+        args = [executable, "-Es", script]
+        return json.loads(subprocess.check_output(args))
 
 
 def parse_setup_py(executable: str, path: str) -> dict[str, Any]:
     """Parse setup.py and return the kwargs"""
-    cmd = [executable, "-Es", str(FOLDER_PATH / "parse_setup.py"), path]
-    return json.loads(subprocess.check_output(cmd))
+    with _in_process_script("parse_setup.py") as script:
+        cmd = [executable, "-Es", script, path]
+        return json.loads(subprocess.check_output(cmd))

src/pdm/models/specifiers.py

@@ -3,15 +3,19 @@
 import json
 from functools import lru_cache
 from operator import attrgetter
-from pathlib import Path
 from typing import Any, Iterable, cast
 
 from packaging.specifiers import InvalidSpecifier, SpecifierSet
 
 from pdm.exceptions import InvalidPyVersion
 from pdm.models.versions import Version
 
-MAX_VERSIONS_FILE = Path(__file__).with_name("python_max_versions.json")
+
+def _read_max_versions() -> dict[Version, int]:
+    from importlib.resources import open_binary
+
+    with open_binary("pdm.models", "python_max_versions.json") as fp:
+        return {Version(k): v for k, v in json.load(fp).items()}
 
 
 @lru_cache()
@@ -51,10 +55,7 @@ def _normalize_op_specifier(op: str, version_str: str) -> tuple[str, Version]:
 class PySpecSet(SpecifierSet):
     """A custom SpecifierSet that supports merging with logic operators (&, |)."""
 
-    PY_MAX_MINOR_VERSION = {
-        Version(key): value
-        for key, value in json.loads(MAX_VERSIONS_FILE.read_text()).items()
-    }
+    PY_MAX_MINOR_VERSION = _read_max_versions()
     MAX_MAJOR_VERSION = max(PY_MAX_MINOR_VERSION)[:1].bump()
 
     def __init__(self, version_str: str = "", analyze: bool = True) -> None:

src/pdm/utils.py

@@ -405,10 +405,10 @@ def deprecation_warning(
     """Show a deprecation warning with the given message and raise an error
     after a specified version.
     """
-    from pdm.__version__ import parsed_version
+    from pdm.__version__ import __version__
 
-    if raise_since is not None and parsed_version:
-        if parsed_version >= Version(raise_since):
+    if raise_since is not None:
+        if Version(__version__) >= Version(raise_since):
             raise DeprecationWarning(message)
     warnings.warn(message, DeprecationWarning, stacklevel=stacklevel + 1)
 

tests/cli/test_run.py

@@ -9,7 +9,7 @@
 
 from pdm import termui
 from pdm.cli import actions
-from pdm.cli.actions import PEP582_PATH
+from pdm.cli.utils import get_pep582_path
 from pdm.utils import cd
 
 
@@ -35,7 +35,7 @@ def test_pep582_launcher_for_python_interpreter(project, local_finder, invoke):
     result = invoke(["add", "first"], obj=project)
     assert result.exit_code == 0, result.stderr
     env = os.environ.copy()
-    env.update({"PYTHONPATH": PEP582_PATH})
+    env.update({"PYTHONPATH": get_pep582_path(project)})
     output = subprocess.check_output(
         [str(project.python.executable), str(project.root.joinpath("main.py"))],
         env=env,
@@ -45,7 +45,7 @@ def test_pep582_launcher_for_python_interpreter(project, local_finder, invoke):
 
 def test_auto_isolate_site_packages(project, invoke):
     env = os.environ.copy()
-    env.update({"PYTHONPATH": PEP582_PATH})
+    env.update({"PYTHONPATH": get_pep582_path(project)})
     proc = subprocess.run(
         [str(project.python.executable), "-c", "import sys;print(sys.path, sep='\\n')"],
         env=env,