diff --git a/setup.cfg b/setup.cfg index b7f762315..1737e240d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,6 +68,7 @@ virtualenv.create = cpython3-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix cpython3-win = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows cpython2-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Posix + cpython2-mac-framework = virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython2macOsFramework cpython2-win = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Windows pypy2-posix = virtualenv.create.via_global_ref.builtin.pypy.pypy2:PyPy2Posix pypy2-win = virtualenv.create.via_global_ref.builtin.pypy.pypy2:Pypy2Windows diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py index a9bd52c6c..887263eb5 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py @@ -50,8 +50,17 @@ def ensure_directories(self): return dirs +def is_mac_os_framework(interpreter): + framework = bool(interpreter.sysconfig_vars["PYTHONFRAMEWORK"]) + return framework and interpreter.platform == "darwin" + + class CPython2Posix(CPython2, CPythonPosix): - """CPython 2 on POSIX""" + """CPython 2 on POSIX (excluding macOs framework builds)""" + + @classmethod + def can_describe(cls, interpreter): + return is_mac_os_framework(interpreter) is False and super(CPython2Posix, cls).can_describe(interpreter) @classmethod def sources(cls, interpreter): diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py new file mode 100644 index 000000000..529a5935f --- /dev/null +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +"""The Apple Framework builds require their own customization""" +import logging +import os +import struct +import subprocess + +from virtualenv.create.via_global_ref.builtin.cpython.common import CPythonPosix +from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest +from virtualenv.util.path import Path + +from .cpython2 import CPython2, is_mac_os_framework + + +class CPython2macOsFramework(CPython2, CPythonPosix): + @classmethod + def can_describe(cls, interpreter): + return is_mac_os_framework(interpreter) and super(CPython2macOsFramework, cls).can_describe(interpreter) + + def create(self): + super(CPython2macOsFramework, self).create() + + # change the install_name of the copied python executable + current = os.path.join(self.interpreter.prefix, "Python") + fix_mach_o(str(self.exe), current, "@executable_path/../.Python", self.interpreter.max_size) + + @classmethod + def sources(cls, interpreter): + for src in super(CPython2macOsFramework, cls).sources(interpreter): + yield src + + # landmark for exec_prefix + name = "lib-dynload" + yield PathRefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib) + + # this must symlink to the host prefix Python + marker = Path(interpreter.prefix) / "Python" + ref = PathRefToDest(marker, dest=lambda self, _: self.dest / ".Python", must_symlink=True) + yield ref + + @classmethod + def _executables(cls, interpreter): + for _, targets in super(CPython2macOsFramework, cls)._executables(interpreter): + # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the + # stub executable in ${sys.prefix}/bin. + # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951 + fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python" + yield fixed_host_exe, targets + + +def fix_mach_o(exe, current, new, max_size): + """ + https://en.wikipedia.org/wiki/Mach-O + + Mach-O, short for Mach object file format, is a file format for executables, object code, shared libraries, + dynamically-loaded code, and core dumps. A replacement for the a.out format, Mach-O offers more extensibility and + faster access to information in the symbol table. + + Each Mach-O file is made up of one Mach-O header, followed by a series of load commands, followed by one or more + segments, each of which contains between 0 and 255 sections. Mach-O uses the REL relocation format to handle + references to symbols. When looking up symbols Mach-O uses a two-level namespace that encodes each symbol into an + 'object/symbol name' pair that is then linearly searched for by first the object and then the symbol name. + + The basic structure—a list of variable-length "load commands" that reference pages of data elsewhere in the file—was + also used in the executable file format for Accent. The Accent file format was in turn, based on an idea from Spice + Lisp. + + With the introduction of Mac OS X 10.6 platform the Mach-O file underwent a significant modification that causes + binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac + OS X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions, + does not understand. Another significant change to the Mach-O format is the change in how the Link Edit tables + (found in the __LINKEDIT section) function. In 10.6 these new Link Edit tables are compressed by removing unused and + unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format. + """ + try: + logging.debug("change Mach-O for %s from %s to %s", exe, current, new) + _builtin_change_mach_o(max_size)(exe, current, new) + except Exception as e: + logging.warning("Could not call _builtin_change_mac_o: %s. " "Trying to call install_name_tool instead.", e) + try: + cmd = ["install_name_tool", "-change", current, new, exe] + subprocess.check_call(cmd) + except Exception: + logging.fatal("Could not call install_name_tool -- you must " "have Apple's development tools installed") + raise + + +def _builtin_change_mach_o(maxint): + MH_MAGIC = 0xFEEDFACE + MH_CIGAM = 0xCEFAEDFE + MH_MAGIC_64 = 0xFEEDFACF + MH_CIGAM_64 = 0xCFFAEDFE + FAT_MAGIC = 0xCAFEBABE + BIG_ENDIAN = ">" + LITTLE_ENDIAN = "<" + LC_LOAD_DYLIB = 0xC + + class FileView(object): + """A proxy for file-like objects that exposes a given view of a file. Modified from macholib.""" + + def __init__(self, file_obj, start=0, size=maxint): + if isinstance(file_obj, FileView): + self._file_obj = file_obj._file_obj + else: + self._file_obj = file_obj + self._start = start + self._end = start + size + self._pos = 0 + + def __repr__(self): + return "".format(self._start, self._end, self._file_obj) + + def tell(self): + return self._pos + + def _checkwindow(self, seek_to, op): + if not (self._start <= seek_to <= self._end): + msg = "{} to offset {:d} is outside window [{:d}, {:d}]".format(op, seek_to, self._start, self._end) + raise IOError(msg) + + def seek(self, offset, whence=0): + seek_to = offset + if whence == os.SEEK_SET: + seek_to += self._start + elif whence == os.SEEK_CUR: + seek_to += self._start + self._pos + elif whence == os.SEEK_END: + seek_to += self._end + else: + raise IOError("Invalid whence argument to seek: {!r}".format(whence)) + self._checkwindow(seek_to, "seek") + self._file_obj.seek(seek_to) + self._pos = seek_to - self._start + + def write(self, content): + here = self._start + self._pos + self._checkwindow(here, "write") + self._checkwindow(here + len(content), "write") + self._file_obj.seek(here, os.SEEK_SET) + self._file_obj.write(content) + self._pos += len(content) + + def read(self, size=maxint): + assert size >= 0 + here = self._start + self._pos + self._checkwindow(here, "read") + size = min(size, self._end - here) + self._file_obj.seek(here, os.SEEK_SET) + read_bytes = self._file_obj.read(size) + self._pos += len(read_bytes) + return read_bytes + + def read_data(file, endian, num=1): + """Read a given number of 32-bits unsigned integers from the given file with the given endianness.""" + res = struct.unpack(endian + "L" * num, file.read(num * 4)) + if len(res) == 1: + return res[0] + return res + + def mach_o_change(at_path, what, value): + """Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value), + provided it's shorter.""" + + def do_macho(file, bits, endian): + # Read Mach-O header (the magic number is assumed read by the caller) + cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6) + # 64-bits header has one more field. + if bits == 64: + read_data(file, endian) + # The header is followed by n commands + for _ in range(n_commands): + where = file.tell() + # Read command header + cmd, cmd_size = read_data(file, endian, 2) + if cmd == LC_LOAD_DYLIB: + # The first data field in LC_LOAD_DYLIB commands is the offset of the name, starting from the + # beginning of the command. + name_offset = read_data(file, endian) + file.seek(where + name_offset, os.SEEK_SET) + # Read the NUL terminated string + load = file.read(cmd_size - name_offset).decode() + load = load[: load.index("\0")] + # If the string is what is being replaced, overwrite it. + if load == what: + file.seek(where + name_offset, os.SEEK_SET) + file.write(value.encode() + b"\0") + # Seek to the next command + file.seek(where + cmd_size, os.SEEK_SET) + + def do_file(file, offset=0, size=maxint): + file = FileView(file, offset, size) + # Read magic number + magic = read_data(file, BIG_ENDIAN) + if magic == FAT_MAGIC: + # Fat binaries contain nfat_arch Mach-O binaries + n_fat_arch = read_data(file, BIG_ENDIAN) + for _ in range(n_fat_arch): + # Read arch header + cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5) + do_file(file, offset, size) + elif magic == MH_MAGIC: + do_macho(file, 32, BIG_ENDIAN) + elif magic == MH_CIGAM: + do_macho(file, 32, LITTLE_ENDIAN) + elif magic == MH_MAGIC_64: + do_macho(file, 64, BIG_ENDIAN) + elif magic == MH_CIGAM_64: + do_macho(file, 64, LITTLE_ENDIAN) + + assert len(what) >= len(value) + + with open(at_path, "r+b") as f: + do_file(f) + + return mach_o_change diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py index e917ca58c..1f57aef67 100644 --- a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py +++ b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py @@ -32,7 +32,8 @@ def create(self): else: custom_site_text = custom_site.read_text() expected = json.dumps([os.path.relpath(ensure_text(str(i)), ensure_text(str(site_py))) for i in self.libs]) - site_py.write_text(custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected)) + custom_site_text = custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected) + site_py.write_text(custom_site_text) @classmethod def sources(cls, interpreter): diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/site.py b/src/virtualenv/create/via_global_ref/builtin/python2/site.py index 6ec015226..7e7851a62 100644 --- a/src/virtualenv/create/via_global_ref/builtin/python2/site.py +++ b/src/virtualenv/create/via_global_ref/builtin/python2/site.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ A simple shim module to fix up things on Python 2 only. @@ -37,8 +38,7 @@ def load_host_site(): here = __file__ # the distutils.install patterns will be injected relative to this site.py, save it here - with PatchForAppleFrameworkBuilds(): - reload(sys.modules["site"]) # noqa + reload(sys.modules["site"]) # noqa # call system site.py to setup import libraries # and then if the distutils site packages are not on the sys.path we add them via add_site_dir; note we must add # them by invoking add_site_dir to trigger the processing of pth files @@ -56,28 +56,12 @@ def load_host_site(): add_site_dir(full_path) -class PatchForAppleFrameworkBuilds(object): - """Apple Framework builds unconditionally add the global site-package, escape this behaviour""" - - framework = None - - def __enter__(self): - if sys.platform == "darwin": - from sysconfig import get_config_var - - self.framework = get_config_var("PYTHONFRAMEWORK") - if self.framework: - sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = None - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.framework: - sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = self.framework +sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version def read_pyvenv(): """read pyvenv.cfg""" - os_sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version - config_file = "{}{}pyvenv.cfg".format(sys.prefix, os_sep) + config_file = "{}{}pyvenv.cfg".format(sys.prefix, sep) with open(config_file) as file_handler: lines = file_handler.readlines() config = {} @@ -93,23 +77,41 @@ def read_pyvenv(): def rewrite_standard_library_sys_path(): """Once this site file is loaded the standard library paths have already been set, fix them up""" - sep = "\\" if sys.platform == "win32" else "/" - exe_dir = sys.executable[: sys.executable.rfind(sep)] + exe = abs_path(sys.executable) + exe_dir = exe[: exe.rfind(sep)] + prefix, exec_prefix = abs_path(sys.prefix), abs_path(sys.exec_prefix) + base_prefix, base_exec_prefix = abs_path(sys.base_prefix), abs_path(sys.base_exec_prefix) + base_executable = abs_path(sys.base_executable) for at, value in enumerate(sys.path): + value = abs_path(value) # replace old sys prefix path starts with new if value == exe_dir: pass # don't fix the current executable location, notably on Windows this gets added elif value.startswith(exe_dir): # content inside the exe folder needs to remap to original executables folder - orig_exe_folder = sys.base_executable[: sys.base_executable.rfind(sep)] + orig_exe_folder = base_executable[: base_executable.rfind(sep)] value = "{}{}".format(orig_exe_folder, value[len(exe_dir) :]) - elif value.startswith(sys.prefix): - value = "{}{}".format(sys.base_prefix, value[len(sys.prefix) :]) - elif value.startswith(sys.exec_prefix): - value = "{}{}".format(sys.base_exec_prefix, value[len(sys.exec_prefix) :]) + elif value.startswith(prefix): + value = "{}{}".format(base_prefix, value[len(prefix) :]) + elif value.startswith(exec_prefix): + value = "{}{}".format(base_exec_prefix, value[len(exec_prefix) :]) sys.path[at] = value +def abs_path(value): + keep = [] + values = value.split(sep) + i = len(values) - 1 + while i >= 0: + if values[i] == "..": + i -= 1 + else: + keep.append(values[i]) + i -= 1 + value = sep.join(keep[::-1]) + return value + + def disable_user_site_package(): """Flip the switch on enable user site package""" # sys.flags is a c-extension type, so we cannot monkeypatch it, replace it with a python class to flip it diff --git a/src/virtualenv/create/via_global_ref/builtin/ref.py b/src/virtualenv/create/via_global_ref/builtin/ref.py index a579c4b37..15a644aa7 100644 --- a/src/virtualenv/create/via_global_ref/builtin/ref.py +++ b/src/virtualenv/create/via_global_ref/builtin/ref.py @@ -1,3 +1,8 @@ +""" +Virtual environments in the traditional sense are built as reference to the host python. This file allows declarative +references to elements on the file system, allowing our system to automatically detect what modes it can support given +the constraints: e.g. can the file system symlink, can the files be read, executed, etc. +""" from __future__ import absolute_import, unicode_literals import os @@ -14,15 +19,21 @@ @add_metaclass(ABCMeta) class PathRef(object): + """Base class that checks if a file reference can be symlink/copied""" + FS_SUPPORTS_SYMLINK = fs_supports_symlink() FS_CASE_SENSITIVE = fs_is_case_sensitive() - def __init__(self, src): + def __init__(self, src, must_symlink, must_copy): + self.must_symlink = must_symlink + self.must_copy = must_copy self.src = src self.exists = src.exists() self._can_read = None if self.exists else False self._can_copy = None if self.exists else False self._can_symlink = None if self.exists else False + if self.must_copy is True and self.must_symlink is True: + raise ValueError("can copy and symlink at the same time") def __repr__(self): return "{}(src={})".format(self.__class__.__name__, self.src) @@ -43,24 +54,39 @@ def can_read(self): @property def can_copy(self): if self._can_copy is None: - self._can_copy = self.can_read + if self.must_symlink: + self._can_copy = self.can_symlink + else: + self._can_copy = self.can_read return self._can_copy @property def can_symlink(self): if self._can_symlink is None: - self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read + if self.must_copy: + self._can_symlink = self.can_copy + else: + self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read return self._can_symlink @abstractmethod def run(self, creator, symlinks): raise NotImplementedError + def method(self, symlinks): + if self.must_symlink: + return symlink + if self.must_copy: + return copy + return symlink if symlinks else copy + @add_metaclass(ABCMeta) class ExePathRef(PathRef): - def __init__(self, src): - super(ExePathRef, self).__init__(src) + """Base class that checks if a executable can be references via symlink/copy""" + + def __init__(self, src, must_symlink, must_copy): + super(ExePathRef, self).__init__(src, must_symlink, must_copy) self._can_run = None @property @@ -83,22 +109,26 @@ def can_run(self): class PathRefToDest(PathRef): - def __init__(self, src, dest): - super(PathRefToDest, self).__init__(src) + """Link a path on the file system""" + + def __init__(self, src, dest, must_symlink=False, must_copy=False): + super(PathRefToDest, self).__init__(src, must_symlink, must_copy) self.dest = dest def run(self, creator, symlinks): dest = self.dest(creator, self.src) - method = symlink if symlinks else copy + method = self.method(symlinks) dest_iterable = dest if isinstance(dest, list) else (dest,) for dst in dest_iterable: method(self.src, dst) class ExePathRefToDest(PathRefToDest, ExePathRef): - def __init__(self, src, targets, dest, must_copy=False): - ExePathRef.__init__(self, src) - PathRefToDest.__init__(self, src, dest) + """Link a exe path on the file system""" + + def __init__(self, src, targets, dest, must_symlink=False, must_copy=False): + ExePathRef.__init__(self, src, must_symlink, must_copy) + PathRefToDest.__init__(self, src, dest, must_symlink, must_copy) if not self.FS_CASE_SENSITIVE: targets = list(OrderedDict((i.lower(), None) for i in targets).keys()) self.base = targets[0] @@ -108,8 +138,8 @@ def __init__(self, src, targets, dest, must_copy=False): def run(self, creator, symlinks): bin_dir = self.dest(creator, self.src).parent - method = symlink if self.must_copy is False and symlinks else copy dest = bin_dir / self.base + method = self.method(symlinks) method(self.src, dest) make_exe(dest) for extra in self.aliases: diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 61e0927bf..3ffe57e6b 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -79,16 +79,18 @@ def abs_path(v): for element in self.sysconfig_paths.values(): for k in _CONF_VAR_RE.findall(element): config_var_keys.add(u(k[1:-1])) + config_var_keys.add("PYTHONFRAMEWORK") - self.sysconfig_vars = {u(i): u(sysconfig.get_config_var(i)) for i in config_var_keys} + self.sysconfig_vars = {u(i): u(sysconfig.get_config_var(i) or "") for i in config_var_keys} if self.implementation == "PyPy" and sys.version_info.major == 2: - self.sysconfig_vars7[u"implementation_lower"] = u"python" + self.sysconfig_vars[u"implementation_lower"] = u"python" self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()} self.system_stdlib = self.sysconfig_path( "stdlib", {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()}, ) + self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) self._creators = None def _fast_get_system_executable(self): diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index 6d9b4aa9d..753eeb25b 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -80,14 +80,15 @@ def add_parser_arguments(cls, parser, interpreter): def __unicode__(self): result = self.__class__.__name__ + result += "(" if self.extra_search_dir: - result += " extra search dirs = {}".format(", ".join(ensure_text(str(i)) for i in self.extra_search_dir)) - result += " download={}".format(self.download) + result += "extra_search_dir={},".format(", ".join(ensure_text(str(i)) for i in self.extra_search_dir)) + result += "download={},".format(self.download) for package in self.packages: - result += " {}{}".format( + result += " {}{},".format( package, "={}".format(getattr(self, "{}_version".format(package), None) or "latest") ) - return result + return result[:-1] + ")" def __repr__(self): return ensure_str(self.__unicode__()) diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py index 43268ab04..8493c2004 100644 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/via_app_data/via_app_data.py @@ -105,6 +105,9 @@ def installer_class(self, pip_version): return CopyPipInstall def __unicode__(self): - return super(FromAppData, self).__unicode__() + " via={} app_data_dir={}".format( - "symlink" if self.symlinks else "copy", self.app_data_dir.path + base = super(FromAppData, self).__unicode__() + return ( + base[:-1] + + ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.app_data_dir.path) + + base[-1] ) diff --git a/tox.ini b/tox.ini index b7945bdcf..db3db9620 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,8 @@ passenv = https_proxy http_proxy no_proxy HOME PYTEST_* PIP_* CI_RUN TERM extras = testing install_command = python -m pip install {opts} {packages} --disable-pip-version-check commands = + python -m site + python -m pip list python -c 'from os.path import sep; file = open(r"{envsitepackagesdir}\{\}coverage-virtualenv.pth".format(sep), "w"); file.write("import coverage; coverage.process_startup()"); file.close()' python -m coverage erase