From a8710697695b78d8a5b43ca61420b4cc31cdd199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 28 Jul 2021 17:01:59 +0100 Subject: [PATCH 01/26] main: implement CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- pyproject.toml | 7 ++ src/installer/__main__.py | 211 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 src/installer/__main__.py diff --git a/pyproject.toml b/pyproject.toml index 5594e27b..93309d83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,3 +10,10 @@ home-page = "https://github.com/pradyunsg/installer" description-file = "README.md" classifiers = ["License :: OSI Approved :: MIT License"] requires-python = ">=3.7" +requires = [ + "build >= 0.2.0", # not a hard runtime requirement -- we can softfail + "packaging", # not a hard runtime requirement -- we can softfail +] + +[tool.flit.scripts] +python-installer = "installer.__main__:entrypoint" diff --git a/src/installer/__main__.py b/src/installer/__main__.py new file mode 100644 index 00000000..f3363eb9 --- /dev/null +++ b/src/installer/__main__.py @@ -0,0 +1,211 @@ +"""Installer CLI.""" + +import argparse +import compileall +import distutils.dist +import pathlib +import platform +import sys +import sysconfig +import warnings +from email.message import Message +from typing import TYPE_CHECKING, Collection, Dict, Iterable, Optional, Sequence, Tuple + +import installer +import installer.destinations +import installer.sources +import installer.utils +from installer.records import RecordEntry +from installer.utils import Scheme + +if TYPE_CHECKING: + from installer.scripts import LauncherKind + + +class InstallerCompatibilityError(Exception): + """Error raised when the install target is not compatible with the environment.""" + + +class _MainDestination(installer.destinations.SchemeDictionaryDestination): + destdir: Optional[pathlib.Path] + + def __init__( + self, + scheme_dict: Dict[str, str], + interpreter: str, + script_kind: "LauncherKind", + hash_algorithm: str = "sha256", + optimization_levels: Collection[int] = (0, 1), + destdir: Optional[str] = None, + ) -> None: + if destdir: + self.destdir = pathlib.Path(destdir).absolute() + self.destdir.mkdir(exist_ok=True, parents=True) + scheme_dict = { + name: self._destdir_path(value) for name, value in scheme_dict.items() + } + else: + self.destdir = None + super().__init__(scheme_dict, interpreter, script_kind, hash_algorithm) + self.optimization_levels = optimization_levels + + def _destdir_path(self, file: str) -> str: + assert self.destdir + file_path = pathlib.Path(file) + rel_path = file_path.relative_to(file_path.anchor) + return str(self.destdir.joinpath(*rel_path.parts)) + + def _compile_record(self, scheme: Scheme, record: RecordEntry) -> None: + if scheme not in ("purelib", "platlib"): + return + for level in self.optimization_levels: + target_path = pathlib.Path(self.scheme_dict[scheme], record.path) + if sys.version_info < (3, 9): + compileall.compile_file(target_path, optimize=level) + else: + compileall.compile_file( + target_path, + optimize=level, + stripdir=str(self.destdir), + ) + + def finalize_installation( + self, + scheme: Scheme, + record_file_path: str, + records: Iterable[Tuple[Scheme, RecordEntry]], + ) -> None: + record_list = list(records) + super().finalize_installation(scheme, record_file_path, record_list) + for scheme, record in record_list: + self._compile_record(scheme, record) + + +def main_parser() -> argparse.ArgumentParser: + """Construct the main parser.""" + parser = argparse.ArgumentParser() + parser.add_argument("wheel", type=str, help="wheel file to install") + parser.add_argument( + "--destdir", + "-d", + metavar="/", + type=str, + default="/", + help="destination directory (prefix to prepend to each file)", + ) + parser.add_argument( + "--optimize", + "-o", + nargs="*", + metavar="level", + type=int, + default=(0, 1), + help="generate bytecode for the specified optimization level(s) (default=0, 1)", + ) + parser.add_argument( + "--skip-dependency-check", + "-s", + action="store_true", + help="don't check if the wheel dependencies are met", + ) + return parser + + +def get_scheme_dict(distribution_name: str) -> Dict[str, str]: + """Calculate the scheme disctionary for the current Python environment.""" + scheme_dict = sysconfig.get_paths() + + # calculate 'headers' path, sysconfig does not have an equivalent + # see https://bugs.python.org/issue44445 + dist_dict = { + "name": distribution_name, + } + distribution = distutils.dist.Distribution(dist_dict) + install_cmd = distribution.get_command_obj("install") + assert install_cmd + install_cmd.finalize_options() + # install_cmd.install_headers is not type hinted + scheme_dict["headers"] = install_cmd.install_headers # type: ignore + + return scheme_dict + + +def check_python_version(metadata: Message) -> None: + """Check if the project support the current interpreter.""" + try: + import packaging.specifiers + except ImportError: + warnings.warn( + "'packaging' module not available, " + "skipping python version compatibility check" + ) + return + + requirement = metadata["Requires-Python"] + if not requirement: + return + + versions = requirement.split(",") + for version in versions: + specifier = packaging.specifiers.Specifier(version) + if platform.python_version() not in specifier: + raise InstallerCompatibilityError( + "Incompatible python version, needed: {}".format(version) + ) + + +def check_dependencies(metadata: Message) -> None: + """Check if the project dependencies are met.""" + try: + import packaging # noqa: F401 + + import build + except ModuleNotFoundError as e: + warnings.warn(f"'{e.name}' module not available, skipping dependency check") + return + + missing = { + unmet + for requirement in metadata.get_all("Requires-Dist") or [] + for unmet_list in build.check_dependency(requirement) + for unmet in unmet_list + } + if missing: + missing_list = ", ".join(missing) + raise InstallerCompatibilityError( + "Missing requirements: {}".format(missing_list) + ) + + +def main(cli_args: Sequence[str], program: Optional[str] = None) -> None: + """Process arguments and perform the install.""" + parser = main_parser() + if program: + parser.prog = program + args = parser.parse_args(cli_args) + + with installer.sources.WheelFile.open(args.wheel) as source: + # compability checks + metadata_contents = source.read_dist_info("METADATA") + metadata = installer.utils.parse_metadata_file(metadata_contents) + check_python_version(metadata) + if not args.skip_dependency_check: + check_dependencies(metadata) + + destination = _MainDestination( + get_scheme_dict(source.distribution), + sys.executable, + installer.utils.get_launcher_kind(), + optimization_levels=args.optimize, + destdir=args.destdir, + ) + installer.install(source, destination, {}) + + +def entrypoint() -> None: + """CLI entrypoint.""" + main(sys.argv[1:]) + + +if __name__ == "__main__": + main(sys.argv[1:], "python -m installer") From ec29913235b2e4dd5ee6a75e38e6302355df8128 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 24 Oct 2021 01:11:50 +0000 Subject: [PATCH 02/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/installer/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index f3363eb9..8bdd29e3 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -157,9 +157,8 @@ def check_python_version(metadata: Message) -> None: def check_dependencies(metadata: Message) -> None: """Check if the project dependencies are met.""" try: - import packaging # noqa: F401 - import build + import packaging # noqa: F401 except ModuleNotFoundError as e: warnings.warn(f"'{e.name}' module not available, skipping dependency check") return From 514d71f99f72a80aa167137561135f46d15ea1cf Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 5 Jan 2022 11:36:04 +0000 Subject: [PATCH 03/26] Remove script, use python -m installer for CLI --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 93309d83..3295ebce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,3 @@ requires = [ "build >= 0.2.0", # not a hard runtime requirement -- we can softfail "packaging", # not a hard runtime requirement -- we can softfail ] - -[tool.flit.scripts] -python-installer = "installer.__main__:entrypoint" From c600a6eafab0e1295955ddb5fe8af19279a0da98 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 5 Jan 2022 11:36:34 +0000 Subject: [PATCH 04/26] Remove Python version & dependency checks --- pyproject.toml | 4 --- src/installer/__main__.py | 66 --------------------------------------- 2 files changed, 70 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3295ebce..5594e27b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,3 @@ home-page = "https://github.com/pradyunsg/installer" description-file = "README.md" classifiers = ["License :: OSI Approved :: MIT License"] requires-python = ">=3.7" -requires = [ - "build >= 0.2.0", # not a hard runtime requirement -- we can softfail - "packaging", # not a hard runtime requirement -- we can softfail -] diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 8bdd29e3..dee64bcd 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -4,11 +4,8 @@ import compileall import distutils.dist import pathlib -import platform import sys import sysconfig -import warnings -from email.message import Message from typing import TYPE_CHECKING, Collection, Dict, Iterable, Optional, Sequence, Tuple import installer @@ -22,10 +19,6 @@ from installer.scripts import LauncherKind -class InstallerCompatibilityError(Exception): - """Error raised when the install target is not compatible with the environment.""" - - class _MainDestination(installer.destinations.SchemeDictionaryDestination): destdir: Optional[pathlib.Path] @@ -102,12 +95,6 @@ def main_parser() -> argparse.ArgumentParser: default=(0, 1), help="generate bytecode for the specified optimization level(s) (default=0, 1)", ) - parser.add_argument( - "--skip-dependency-check", - "-s", - action="store_true", - help="don't check if the wheel dependencies are met", - ) return parser @@ -130,52 +117,6 @@ def get_scheme_dict(distribution_name: str) -> Dict[str, str]: return scheme_dict -def check_python_version(metadata: Message) -> None: - """Check if the project support the current interpreter.""" - try: - import packaging.specifiers - except ImportError: - warnings.warn( - "'packaging' module not available, " - "skipping python version compatibility check" - ) - return - - requirement = metadata["Requires-Python"] - if not requirement: - return - - versions = requirement.split(",") - for version in versions: - specifier = packaging.specifiers.Specifier(version) - if platform.python_version() not in specifier: - raise InstallerCompatibilityError( - "Incompatible python version, needed: {}".format(version) - ) - - -def check_dependencies(metadata: Message) -> None: - """Check if the project dependencies are met.""" - try: - import build - import packaging # noqa: F401 - except ModuleNotFoundError as e: - warnings.warn(f"'{e.name}' module not available, skipping dependency check") - return - - missing = { - unmet - for requirement in metadata.get_all("Requires-Dist") or [] - for unmet_list in build.check_dependency(requirement) - for unmet in unmet_list - } - if missing: - missing_list = ", ".join(missing) - raise InstallerCompatibilityError( - "Missing requirements: {}".format(missing_list) - ) - - def main(cli_args: Sequence[str], program: Optional[str] = None) -> None: """Process arguments and perform the install.""" parser = main_parser() @@ -184,13 +125,6 @@ def main(cli_args: Sequence[str], program: Optional[str] = None) -> None: args = parser.parse_args(cli_args) with installer.sources.WheelFile.open(args.wheel) as source: - # compability checks - metadata_contents = source.read_dist_info("METADATA") - metadata = installer.utils.parse_metadata_file(metadata_contents) - check_python_version(metadata) - if not args.skip_dependency_check: - check_dependencies(metadata) - destination = _MainDestination( get_scheme_dict(source.distribution), sys.executable, From 0037f482663f3104797e36809a14276800efa454 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 5 Jan 2022 12:27:15 +0000 Subject: [PATCH 05/26] Fold destdir support, bytecode compilation into Python API --- src/installer/__main__.py | 68 ++--------------------------------- src/installer/destinations.py | 57 ++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 69 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index dee64bcd..74e16dab 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -1,77 +1,15 @@ """Installer CLI.""" import argparse -import compileall import distutils.dist -import pathlib import sys import sysconfig -from typing import TYPE_CHECKING, Collection, Dict, Iterable, Optional, Sequence, Tuple +from typing import Dict, Optional, Sequence import installer import installer.destinations import installer.sources import installer.utils -from installer.records import RecordEntry -from installer.utils import Scheme - -if TYPE_CHECKING: - from installer.scripts import LauncherKind - - -class _MainDestination(installer.destinations.SchemeDictionaryDestination): - destdir: Optional[pathlib.Path] - - def __init__( - self, - scheme_dict: Dict[str, str], - interpreter: str, - script_kind: "LauncherKind", - hash_algorithm: str = "sha256", - optimization_levels: Collection[int] = (0, 1), - destdir: Optional[str] = None, - ) -> None: - if destdir: - self.destdir = pathlib.Path(destdir).absolute() - self.destdir.mkdir(exist_ok=True, parents=True) - scheme_dict = { - name: self._destdir_path(value) for name, value in scheme_dict.items() - } - else: - self.destdir = None - super().__init__(scheme_dict, interpreter, script_kind, hash_algorithm) - self.optimization_levels = optimization_levels - - def _destdir_path(self, file: str) -> str: - assert self.destdir - file_path = pathlib.Path(file) - rel_path = file_path.relative_to(file_path.anchor) - return str(self.destdir.joinpath(*rel_path.parts)) - - def _compile_record(self, scheme: Scheme, record: RecordEntry) -> None: - if scheme not in ("purelib", "platlib"): - return - for level in self.optimization_levels: - target_path = pathlib.Path(self.scheme_dict[scheme], record.path) - if sys.version_info < (3, 9): - compileall.compile_file(target_path, optimize=level) - else: - compileall.compile_file( - target_path, - optimize=level, - stripdir=str(self.destdir), - ) - - def finalize_installation( - self, - scheme: Scheme, - record_file_path: str, - records: Iterable[Tuple[Scheme, RecordEntry]], - ) -> None: - record_list = list(records) - super().finalize_installation(scheme, record_file_path, record_list) - for scheme, record in record_list: - self._compile_record(scheme, record) def main_parser() -> argparse.ArgumentParser: @@ -125,11 +63,11 @@ def main(cli_args: Sequence[str], program: Optional[str] = None) -> None: args = parser.parse_args(cli_args) with installer.sources.WheelFile.open(args.wheel) as source: - destination = _MainDestination( + destination = installer.destinations.SchemeDictionaryDestination( get_scheme_dict(source.distribution), sys.executable, installer.utils.get_launcher_kind(), - optimization_levels=args.optimize, + bytecode_optimization_levels=args.optimize, destdir=args.destdir, ) installer.install(source, destination, {}) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index 77bf5ecb..3b505bd1 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -1,8 +1,20 @@ """Handles all file writing and post-installation processing.""" +import compileall import io import os -from typing import TYPE_CHECKING, BinaryIO, Dict, Iterable, Tuple, Union +import sys +from pathlib import Path +from typing import ( + TYPE_CHECKING, + BinaryIO, + Collection, + Dict, + Iterable, + Optional, + Tuple, + Union, +) from installer.records import Hash, RecordEntry from installer.scripts import Script @@ -93,6 +105,8 @@ def __init__( interpreter: str, script_kind: "LauncherKind", hash_algorithm: str = "sha256", + bytecode_optimization_levels: Collection[int] = (0, 1), + destdir: Optional[str] = None, ) -> None: """Construct a ``SchemeDictionaryDestination`` object. @@ -102,11 +116,26 @@ def __init__( :param hash_algorithm: the hashing algorithm to use, which is a member of :any:`hashlib.algorithms_available` (ideally from :any:`hashlib.algorithms_guaranteed`). + :param bytecode_optimization_levels: Compile cached bytecode for + installed .py files with these optimization levels. + :param destdir: A staging directory in which to write all files. This + is expected to be the filesystem root at runtime, so embedded paths + will be written as though this was the root. """ self.scheme_dict = scheme_dict self.interpreter = interpreter self.script_kind = script_kind self.hash_algorithm = hash_algorithm + self.bytecode_optimization_levels = bytecode_optimization_levels + self.destdir = destdir + + def _destdir_path(self, scheme: Scheme, path: str) -> str: + file = os.path.join(self.scheme_dict[scheme], path) + if self.destdir is not None: + file_path = Path(file) + rel_path = file_path.relative_to(file_path.anchor) + return os.path.join(self.destdir, rel_path) + return file def write_to_fs(self, scheme: Scheme, path: str, stream: BinaryIO) -> RecordEntry: """Write contents of ``stream`` to the correct location on the filesystem. @@ -118,7 +147,7 @@ def write_to_fs(self, scheme: Scheme, path: str, stream: BinaryIO) -> RecordEntr - Ensures that an existing file is not being overwritten. - Hashes the written content, to determine the entry in the ``RECORD`` file. """ - target_path = os.path.join(self.scheme_dict[scheme], path) + target_path = self._destdir_path(scheme, path) if os.path.exists(target_path): message = f"File already exists: {target_path}" raise FileExistsError(message) @@ -176,24 +205,44 @@ def write_script( with io.BytesIO(data) as stream: entry = self.write_to_fs(Scheme("scripts"), script_name, stream) - path = os.path.join(self.scheme_dict[Scheme("scripts")], script_name) + path = self._destdir_path(Scheme("scripts"), script_name) mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 os.chmod(path, mode) return entry + def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: + """Compile bytecode for a single .py file""" + if scheme not in ("purelib", "platlib"): + return + + target_path = self._destdir_path(scheme, record.path) + for level in self.bytecode_optimization_levels: + if sys.version_info < (3, 9): + compileall.compile_file(target_path, optimize=level) + else: + compileall.compile_file( + target_path, + optimize=level, + stripdir=str(self.destdir), + ) + def finalize_installation( self, scheme: Scheme, record_file_path: str, records: Iterable[Tuple[Scheme, RecordEntry]], ) -> None: - """Finalize installation, by writing the ``RECORD`` file. + """Finalize installation, by writing the ``RECORD`` file & compiling bytecode. :param scheme: scheme to write the ``RECORD`` file in :param record_file_path: path of the ``RECORD`` file with that scheme :param records: entries to write to the ``RECORD`` file """ + record_list = list(records) with construct_record_file(records) as record_stream: self.write_to_fs(scheme, record_file_path, record_stream) + + for scheme, record in record_list: + self._compile_bytecode(scheme, record) From d78df158a9b0b51d95930d94b10fc8fdf6c3ca05 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 6 Jan 2022 11:02:31 +0000 Subject: [PATCH 06/26] Default destdir argument to None --- src/installer/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 74e16dab..4f829f1a 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -21,7 +21,6 @@ def main_parser() -> argparse.ArgumentParser: "-d", metavar="/", type=str, - default="/", help="destination directory (prefix to prepend to each file)", ) parser.add_argument( From b04487ef5c88da855346c2e289a345c2129d1bf4 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 6 Jan 2022 11:03:31 +0000 Subject: [PATCH 07/26] Full stop in docstring --- src/installer/destinations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index ec0654a8..a4809e49 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -213,7 +213,7 @@ def write_script( return entry def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: - """Compile bytecode for a single .py file""" + """Compile bytecode for a single .py file.""" if scheme not in ("purelib", "platlib"): return From d9dd7d217d5833c6edc57ff7cedb141e22daab52 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 6 Jan 2022 13:14:34 +0000 Subject: [PATCH 08/26] Rework command-line options for controlling bytecode generation --- src/installer/__main__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 4f829f1a..b96a416b 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -24,14 +24,18 @@ def main_parser() -> argparse.ArgumentParser: help="destination directory (prefix to prepend to each file)", ) parser.add_argument( - "--optimize", - "-o", - nargs="*", + "--compile-bytecode", + action="append", metavar="level", type=int, - default=(0, 1), + choices=[0, 1, 2], help="generate bytecode for the specified optimization level(s) (default=0, 1)", ) + parser.add_argument( + "--no-compile-bytecode", + action="store_true", + help="don't generate bytecode for installed modules", + ) return parser @@ -61,12 +65,18 @@ def main(cli_args: Sequence[str], program: Optional[str] = None) -> None: parser.prog = program args = parser.parse_args(cli_args) + bytecode_levels = args.compile_bytecode + if args.no_compile_bytecode: + bytecode_levels = [] + elif not bytecode_levels: + bytecode_levels = [0, 1] + with installer.sources.WheelFile.open(args.wheel) as source: destination = installer.destinations.SchemeDictionaryDestination( get_scheme_dict(source.distribution), sys.executable, installer.utils.get_launcher_kind(), - bytecode_optimization_levels=args.optimize, + bytecode_optimization_levels=bytecode_levels, destdir=args.destdir, ) installer.install(source, destination, {}) From 15c992b27e62d5cf00803f9c7004db6f1a68f075 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 12 Jan 2022 10:57:30 +0000 Subject: [PATCH 09/26] Add some tests for CLI interface --- tests/conftest.py | 75 ++++++++++++++++++++++++++++++++++++++++++ tests/test_main.py | 32 ++++++++++++++++++ tests/test_sources.py | 76 ------------------------------------------- 3 files changed, 107 insertions(+), 76 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_main.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..4aa820f9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,75 @@ +import sys +import textwrap +import zipfile + +import pytest + +@pytest.fixture +def fancy_wheel(tmp_path): + path = tmp_path / "fancy-1.0.0-py2.py3-none-any.whl" + files = { + "fancy/": b"""""", + "fancy/__init__.py": b"""\ + def main(): + print("I'm fancy.") + """, + "fancy/__main__.py": b"""\ + if __name__ == "__main__": + from . import main + main() + """, + "fancy-1.0.0.data/data/fancy/": b"""""", + "fancy-1.0.0.data/data/fancy/data.py": b"""\ + # put me in data + """, + "fancy-1.0.0.dist-info/": b"""""", + "fancy-1.0.0.dist-info/top_level.txt": b"""\ + fancy + """, + "fancy-1.0.0.dist-info/entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "fancy-1.0.0.dist-info/WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "fancy-1.0.0.dist-info/METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + # The RECORD file is indirectly validated by the WheelFile, since it only + # provides the items that are a part of the wheel. + "fancy-1.0.0.dist-info/RECORD": b"""\ + fancy/__init__.py,, + fancy/__main__.py,, + fancy-1.0.0.data/data/fancy/data.py,, + fancy-1.0.0.dist-info/top_level.txt,, + fancy-1.0.0.dist-info/entry_points.txt,, + fancy-1.0.0.dist-info/WHEEL,, + fancy-1.0.0.dist-info/METADATA,, + fancy-1.0.0.dist-info/RECORD,, + """, + } + + with zipfile.ZipFile(path, "w") as archive: + for name, indented_content in files.items(): + archive.writestr( + name, + textwrap.dedent(indented_content.decode("utf-8")).encode("utf-8"), + ) + + return path diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 00000000..a1a28f3a --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,32 @@ +from installer.__main__ import get_scheme_dict, main + +def test_get_scheme_dict(): + d = get_scheme_dict(distribution_name='foo') + assert set(d.keys()) >= {'purelib', 'platlib', 'headers', 'scripts', 'data'} + + +def test_main(fancy_wheel, tmp_path): + destdir = tmp_path / 'dest' + + main([str(fancy_wheel), '-d', str(destdir)], "python -m installer") + + installed_py_files = destdir.rglob('*.py') + + assert {f.stem for f in installed_py_files} == {'__init__', '__main__', 'data'} + + installed_pyc_files = destdir.rglob('*.pyc') + assert {f.name.split('.')[0] for f in installed_pyc_files} == { + '__init__', '__main__' + } + +def test_main_no_pyc(fancy_wheel, tmp_path): + destdir = tmp_path / 'dest' + + main([str(fancy_wheel), '-d', str(destdir), '--no-compile-bytecode'], "python -m installer") + + installed_py_files = destdir.rglob('*.py') + + assert {f.stem for f in installed_py_files} == {'__init__', '__main__', 'data'} + + installed_pyc_files = destdir.rglob('*.pyc') + assert set(installed_pyc_files) == set() diff --git a/tests/test_sources.py b/tests/test_sources.py index 72fd6182..23a2d1aa 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1,6 +1,4 @@ import posixpath -import sys -import textwrap import zipfile import pytest @@ -9,80 +7,6 @@ from installer.sources import WheelFile, WheelSource -@pytest.fixture -def fancy_wheel(tmp_path): - path = tmp_path / "fancy-1.0.0-py2.py3-none-any.whl" - files = { - "fancy/": b"""""", - "fancy/__init__.py": b"""\ - def main(): - print("I'm fancy.") - """, - "fancy/__main__.py": b"""\ - if __name__ == "__main__": - from . import main - main() - """, - "fancy-1.0.0.data/data/fancy/": b"""""", - "fancy-1.0.0.data/data/fancy/data.py": b"""\ - # put me in data - """, - "fancy-1.0.0.dist-info/": b"""""", - "fancy-1.0.0.dist-info/top_level.txt": b"""\ - fancy - """, - "fancy-1.0.0.dist-info/entry_points.txt": b"""\ - [console_scripts] - fancy = fancy:main - - [gui_scripts] - fancy-gui = fancy:main - """, - "fancy-1.0.0.dist-info/WHEEL": b"""\ - Wheel-Version: 1.0 - Generator: magic (1.0.0) - Root-Is-Purelib: true - Tag: py3-none-any - """, - "fancy-1.0.0.dist-info/METADATA": b"""\ - Metadata-Version: 2.1 - Name: fancy - Version: 1.0.0 - Summary: A fancy package - Author: Agendaless Consulting - Author-email: nobody@example.com - License: MIT - Keywords: fancy amazing - Platform: UNKNOWN - Classifier: Intended Audience :: Developers - """, - # The RECORD file is indirectly validated by the WheelFile, since it only - # provides the items that are a part of the wheel. - "fancy-1.0.0.dist-info/RECORD": b"""\ - fancy/__init__.py,, - fancy/__main__.py,, - fancy-1.0.0.data/data/fancy/data.py,, - fancy-1.0.0.dist-info/top_level.txt,, - fancy-1.0.0.dist-info/entry_points.txt,, - fancy-1.0.0.dist-info/WHEEL,, - fancy-1.0.0.dist-info/METADATA,, - fancy-1.0.0.dist-info/RECORD,, - """, - } - - if sys.version_info <= (3, 6): - path = str(path) - - with zipfile.ZipFile(path, "w") as archive: - for name, indented_content in files.items(): - archive.writestr( - name, - textwrap.dedent(indented_content.decode("utf-8")).encode("utf-8"), - ) - - return path - - class TestWheelSource: def test_takes_two_arguments(self): WheelSource("distribution", "version") From fdfd28eac247fc8babb896c9e435db67bd3a52f6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jan 2022 10:57:53 +0000 Subject: [PATCH 10/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/conftest.py | 1 + tests/test_main.py | 34 ++++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4aa820f9..237ed768 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import pytest + @pytest.fixture def fancy_wheel(tmp_path): path = tmp_path / "fancy-1.0.0-py2.py3-none-any.whl" diff --git a/tests/test_main.py b/tests/test_main.py index a1a28f3a..ca228920 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,32 +1,38 @@ from installer.__main__ import get_scheme_dict, main + def test_get_scheme_dict(): - d = get_scheme_dict(distribution_name='foo') - assert set(d.keys()) >= {'purelib', 'platlib', 'headers', 'scripts', 'data'} + d = get_scheme_dict(distribution_name="foo") + assert set(d.keys()) >= {"purelib", "platlib", "headers", "scripts", "data"} def test_main(fancy_wheel, tmp_path): - destdir = tmp_path / 'dest' + destdir = tmp_path / "dest" - main([str(fancy_wheel), '-d', str(destdir)], "python -m installer") + main([str(fancy_wheel), "-d", str(destdir)], "python -m installer") - installed_py_files = destdir.rglob('*.py') + installed_py_files = destdir.rglob("*.py") - assert {f.stem for f in installed_py_files} == {'__init__', '__main__', 'data'} + assert {f.stem for f in installed_py_files} == {"__init__", "__main__", "data"} - installed_pyc_files = destdir.rglob('*.pyc') - assert {f.name.split('.')[0] for f in installed_pyc_files} == { - '__init__', '__main__' + installed_pyc_files = destdir.rglob("*.pyc") + assert {f.name.split(".")[0] for f in installed_pyc_files} == { + "__init__", + "__main__", } + def test_main_no_pyc(fancy_wheel, tmp_path): - destdir = tmp_path / 'dest' + destdir = tmp_path / "dest" - main([str(fancy_wheel), '-d', str(destdir), '--no-compile-bytecode'], "python -m installer") + main( + [str(fancy_wheel), "-d", str(destdir), "--no-compile-bytecode"], + "python -m installer", + ) - installed_py_files = destdir.rglob('*.py') + installed_py_files = destdir.rglob("*.py") - assert {f.stem for f in installed_py_files} == {'__init__', '__main__', 'data'} + assert {f.stem for f in installed_py_files} == {"__init__", "__main__", "data"} - installed_pyc_files = destdir.rglob('*.pyc') + installed_pyc_files = destdir.rglob("*.pyc") assert set(installed_pyc_files) == set() From 62b632d0fd9af883100d9cc13c560ed38e8e1ed0 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 12 Jan 2022 10:59:40 +0000 Subject: [PATCH 11/26] Remove unused entrypoint function --- src/installer/__main__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index b96a416b..a03a156f 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -82,10 +82,5 @@ def main(cli_args: Sequence[str], program: Optional[str] = None) -> None: installer.install(source, destination, {}) -def entrypoint() -> None: - """CLI entrypoint.""" - main(sys.argv[1:]) - - if __name__ == "__main__": main(sys.argv[1:], "python -m installer") From ac2d7297211d44daefed37d15910da516c699b00 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 12 Jan 2022 11:00:57 +0000 Subject: [PATCH 12/26] Remove unused import --- tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 237ed768..029cb8f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import sys import textwrap import zipfile From c8a43904b9ba0f3715a6dbb3f7d6fc89b79077db Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 12 Jan 2022 11:09:30 +0000 Subject: [PATCH 13/26] Mark some code which can't be consistently covered by tests --- src/installer/__main__.py | 2 +- src/installer/destinations.py | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index a03a156f..5140c47f 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -82,5 +82,5 @@ def main(cli_args: Sequence[str], program: Optional[str] = None) -> None: installer.install(source, destination, {}) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover main(sys.argv[1:], "python -m installer") diff --git a/src/installer/destinations.py b/src/installer/destinations.py index a4809e49..0c4411e1 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -219,14 +219,10 @@ def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: target_path = self._destdir_path(scheme, record.path) for level in self.bytecode_optimization_levels: - if sys.version_info < (3, 9): - compileall.compile_file(target_path, optimize=level) - else: - compileall.compile_file( - target_path, - optimize=level, - stripdir=str(self.destdir), - ) + kwargs = {} + if sys.version_info >= (3, 9): # pragma: no cover + kwargs['stripdir'] = str(self.destdir) + compileall.compile_file(target_path, optimize=level, **kwargs) def finalize_installation( self, From 86ff36e1d368ed1acfc5341b361d35db9783f676 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jan 2022 11:09:48 +0000 Subject: [PATCH 14/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/installer/destinations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index 0c4411e1..64b74597 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -221,7 +221,7 @@ def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: for level in self.bytecode_optimization_levels: kwargs = {} if sys.version_info >= (3, 9): # pragma: no cover - kwargs['stripdir'] = str(self.destdir) + kwargs["stripdir"] = str(self.destdir) compileall.compile_file(target_path, optimize=level, **kwargs) def finalize_installation( From 679106cfc2dcf5e94581542666d78ce57172e148 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 12 Jan 2022 11:13:41 +0000 Subject: [PATCH 15/26] Add type hint for kwargs dict --- src/installer/destinations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index 64b74597..96cc176d 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import ( TYPE_CHECKING, + Any, BinaryIO, Collection, Dict, @@ -219,7 +220,7 @@ def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: target_path = self._destdir_path(scheme, record.path) for level in self.bytecode_optimization_levels: - kwargs = {} + kwargs: Dict[str, Any] = {} if sys.version_info >= (3, 9): # pragma: no cover kwargs["stripdir"] = str(self.destdir) compileall.compile_file(target_path, optimize=level, **kwargs) From 73f4faefd797a4146de84c43a90ed2c6905f9169 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 13 Jan 2022 22:14:54 +0000 Subject: [PATCH 16/26] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Górny --- src/installer/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 5140c47f..85e0ca20 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -40,7 +40,7 @@ def main_parser() -> argparse.ArgumentParser: def get_scheme_dict(distribution_name: str) -> Dict[str, str]: - """Calculate the scheme disctionary for the current Python environment.""" + """Calculate the scheme dictionary for the current Python environment.""" scheme_dict = sysconfig.get_paths() # calculate 'headers' path, sysconfig does not have an equivalent From 464644bf7ef3ec43e1c57414487ab953c7ff1e1d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 13 Jan 2022 22:16:40 +0000 Subject: [PATCH 17/26] Clearer metavar for destdir argument --- src/installer/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 85e0ca20..59c13df0 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -19,7 +19,7 @@ def main_parser() -> argparse.ArgumentParser: parser.add_argument( "--destdir", "-d", - metavar="/", + metavar="path", type=str, help="destination directory (prefix to prepend to each file)", ) From d150eddd8514a641af8a46bb2027fd6f50a8d731 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 14 Jan 2022 10:21:09 +0000 Subject: [PATCH 18/26] Don't generate bytecode by default in Python API --- src/installer/destinations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index 96cc176d..3974dac5 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -106,7 +106,7 @@ def __init__( interpreter: str, script_kind: "LauncherKind", hash_algorithm: str = "sha256", - bytecode_optimization_levels: Collection[int] = (0, 1), + bytecode_optimization_levels: Collection[int] = (), destdir: Optional[str] = None, ) -> None: """Construct a ``SchemeDictionaryDestination`` object. @@ -118,7 +118,9 @@ def __init__( of :any:`hashlib.algorithms_available` (ideally from :any:`hashlib.algorithms_guaranteed`). :param bytecode_optimization_levels: Compile cached bytecode for - installed .py files with these optimization levels. + installed .py files with these optimization levels. The bytecode + is specific to the minor version of Python (e.g. 3.10) used to + generate it. :param destdir: A staging directory in which to write all files. This is expected to be the filesystem root at runtime, so embedded paths will be written as though this was the root. From 973ac95ea291b509190b4466e61f03a1317507d8 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 14 Jan 2022 10:27:53 +0000 Subject: [PATCH 19/26] Don't use distutils to get headers path --- src/installer/__main__.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 59c13df0..d12ec7e5 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -1,7 +1,7 @@ """Installer CLI.""" import argparse -import distutils.dist +import os.path import sys import sysconfig from typing import Dict, Optional, Sequence @@ -43,17 +43,9 @@ def get_scheme_dict(distribution_name: str) -> Dict[str, str]: """Calculate the scheme dictionary for the current Python environment.""" scheme_dict = sysconfig.get_paths() - # calculate 'headers' path, sysconfig does not have an equivalent - # see https://bugs.python.org/issue44445 - dist_dict = { - "name": distribution_name, - } - distribution = distutils.dist.Distribution(dist_dict) - install_cmd = distribution.get_command_obj("install") - assert install_cmd - install_cmd.finalize_options() - # install_cmd.install_headers is not type hinted - scheme_dict["headers"] = install_cmd.install_headers # type: ignore + # calculate 'headers' path, not currently in sysconfig - + # see https://bugs.python.org/issue44445. This copies what distutils does. + scheme_dict["headers"] = os.path.join(scheme_dict["include"], distribution_name) return scheme_dict From 8263f769a04a58ec470d94da33e42bab3834f47b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 14 Jan 2022 10:36:49 +0000 Subject: [PATCH 20/26] Normalise distribution name to make headers directory --- src/installer/__main__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index d12ec7e5..59e10257 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -2,6 +2,7 @@ import argparse import os.path +import re import sys import sysconfig from typing import Dict, Optional, Sequence @@ -43,9 +44,13 @@ def get_scheme_dict(distribution_name: str) -> Dict[str, str]: """Calculate the scheme dictionary for the current Python environment.""" scheme_dict = sysconfig.get_paths() - # calculate 'headers' path, not currently in sysconfig - - # see https://bugs.python.org/issue44445. This copies what distutils does. - scheme_dict["headers"] = os.path.join(scheme_dict["include"], distribution_name) + # calculate 'headers' path, not currently in sysconfig - see + # https://bugs.python.org/issue44445. This is based on what distutils does. + # For new wheels, the name we get from the wheel filename should already + # be normalised (based on PEP 503, but with _ instead of -), but this is + # not the case for many existing wheels, so normalise it here. + normed_name = re.sub(r"[-_.]+", "_", distribution_name).lower() + scheme_dict["headers"] = os.path.join(scheme_dict["include"], normed_name) return scheme_dict From e29818e959e24953fb0b74c4fa35f2a0c9aebaaf Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 14 Jan 2022 11:41:51 +0000 Subject: [PATCH 21/26] Remove name normalisation for header install path for now --- src/installer/__main__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 59e10257..45579eea 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -46,11 +46,8 @@ def get_scheme_dict(distribution_name: str) -> Dict[str, str]: # calculate 'headers' path, not currently in sysconfig - see # https://bugs.python.org/issue44445. This is based on what distutils does. - # For new wheels, the name we get from the wheel filename should already - # be normalised (based on PEP 503, but with _ instead of -), but this is - # not the case for many existing wheels, so normalise it here. - normed_name = re.sub(r"[-_.]+", "_", distribution_name).lower() - scheme_dict["headers"] = os.path.join(scheme_dict["include"], normed_name) + # TODO: figure out original vs normalised names + scheme_dict["headers"] = os.path.join(scheme_dict["include"], distribution_name) return scheme_dict From 275e73daf89de4fde54fcfc2ec187ecbdda22cd4 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 14 Jan 2022 12:50:50 +0000 Subject: [PATCH 22/26] Fix headers destination directory in a venv --- src/installer/__main__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 45579eea..74143dd1 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -46,8 +46,13 @@ def get_scheme_dict(distribution_name: str) -> Dict[str, str]: # calculate 'headers' path, not currently in sysconfig - see # https://bugs.python.org/issue44445. This is based on what distutils does. - # TODO: figure out original vs normalised names - scheme_dict["headers"] = os.path.join(scheme_dict["include"], distribution_name) + # TODO: figure out original vs normalised distribution names + scheme_dict["headers"] = os.path.join( + sysconfig.get_path( + "include", vars={"installed_base": sysconfig.get_config_var("base")} + ), + distribution_name, + ) return scheme_dict From cf6ba418611df9cc483397fc47a48930a43e7f22 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 14 Jan 2022 12:53:14 +0000 Subject: [PATCH 23/26] Remove unused import --- src/installer/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/installer/__main__.py b/src/installer/__main__.py index 74143dd1..6695d4b9 100644 --- a/src/installer/__main__.py +++ b/src/installer/__main__.py @@ -2,7 +2,6 @@ import argparse import os.path -import re import sys import sysconfig from typing import Dict, Optional, Sequence From 5264e37eefb530ad9c9bb8bff285d1b9e2c7d592 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sat, 15 Jan 2022 13:02:52 +0000 Subject: [PATCH 24/26] Rename private method --- src/installer/destinations.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index 3974dac5..6796a661 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -132,7 +132,7 @@ def __init__( self.bytecode_optimization_levels = bytecode_optimization_levels self.destdir = destdir - def _destdir_path(self, scheme: Scheme, path: str) -> str: + def _path_with_destdir(self, scheme: Scheme, path: str) -> str: file = os.path.join(self.scheme_dict[scheme], path) if self.destdir is not None: file_path = Path(file) @@ -150,7 +150,7 @@ def write_to_fs(self, scheme: Scheme, path: str, stream: BinaryIO) -> RecordEntr - Ensures that an existing file is not being overwritten. - Hashes the written content, to determine the entry in the ``RECORD`` file. """ - target_path = self._destdir_path(scheme, path) + target_path = self._path_with_destdir(scheme, path) if os.path.exists(target_path): message = f"File already exists: {target_path}" raise FileExistsError(message) @@ -208,7 +208,7 @@ def write_script( with io.BytesIO(data) as stream: entry = self.write_to_fs(Scheme("scripts"), script_name, stream) - path = self._destdir_path(Scheme("scripts"), script_name) + path = self._path_with_destdir(Scheme("scripts"), script_name) mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 os.chmod(path, mode) @@ -220,7 +220,7 @@ def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: if scheme not in ("purelib", "platlib"): return - target_path = self._destdir_path(scheme, record.path) + target_path = self._path_with_destdir(scheme, record.path) for level in self.bytecode_optimization_levels: kwargs: Dict[str, Any] = {} if sys.version_info >= (3, 9): # pragma: no cover From 00810eb64ff09a42c8d5d3901f255806c9ac7583 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 19 Jan 2022 11:19:10 +0000 Subject: [PATCH 25/26] Pass quiet=1 to compileall --- src/installer/destinations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index 6796a661..ad0feb36 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -225,7 +225,7 @@ def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: kwargs: Dict[str, Any] = {} if sys.version_info >= (3, 9): # pragma: no cover kwargs["stripdir"] = str(self.destdir) - compileall.compile_file(target_path, optimize=level, **kwargs) + compileall.compile_file(target_path, optimize=level, quiet=1, **kwargs) def finalize_installation( self, From 546fd2d473882acffb078e46492b210d43e2bf1e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 19 Jan 2022 11:34:56 +0000 Subject: [PATCH 26/26] Try to fix path embedded in bytecode files --- src/installer/destinations.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/installer/destinations.py b/src/installer/destinations.py index ad0feb36..f1305f8c 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -3,11 +3,9 @@ import compileall import io import os -import sys from pathlib import Path from typing import ( TYPE_CHECKING, - Any, BinaryIO, Collection, Dict, @@ -221,11 +219,13 @@ def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: return target_path = self._path_with_destdir(scheme, record.path) + dir_path_to_embed = os.path.dirname( # Without destdir + os.path.join(self.scheme_dict[scheme], record.path) + ) for level in self.bytecode_optimization_levels: - kwargs: Dict[str, Any] = {} - if sys.version_info >= (3, 9): # pragma: no cover - kwargs["stripdir"] = str(self.destdir) - compileall.compile_file(target_path, optimize=level, quiet=1, **kwargs) + compileall.compile_file( + target_path, optimize=level, quiet=1, ddir=dir_path_to_embed + ) def finalize_installation( self,