start adding macos support

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

setup.cfg

@@ -68,6 +68,7 @@ virtualenv.create =
  cpython3-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix
  cpython3-win = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows
  cpython2-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Posix
+ cpython2-mac-framework = virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython2macOsFramework
  cpython2-win = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Windows
  pypy2-posix = virtualenv.create.via_global_ref.builtin.pypy.pypy2:PyPy2Posix
  pypy2-win = virtualenv.create.via_global_ref.builtin.pypy.pypy2:Pypy2Windows

src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py

@@ -50,8 +50,17 @@ def ensure_directories(self):
  return dirs
 
 
+def is_mac_os_framework(interpreter):
+ framework = bool(interpreter.sysconfig_vars["PYTHONFRAMEWORK"])
+ return framework and interpreter.platform == "darwin"
+
+
 class CPython2Posix(CPython2, CPythonPosix):
- """CPython 2 on POSIX"""
+ """CPython 2 on POSIX (excluding macOs framework builds)"""
+
+ @classmethod
+ def can_describe(cls, interpreter):
+ return is_mac_os_framework(interpreter) is False and super(CPython2Posix, cls).can_describe(interpreter)
 
  @classmethod
  def sources(cls, interpreter):

src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+"""The Apple Framework builds require their own customization"""
+import logging
+import os
+import struct
+import subprocess
+
+from virtualenv.create.via_global_ref.builtin.cpython.common import CPythonPosix
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.util.path import Path
+
+from .cpython2 import CPython2, is_mac_os_framework
+
+
+class CPython2macOsFramework(CPython2, CPythonPosix):
+ @classmethod
+ def can_describe(cls, interpreter):
+ return is_mac_os_framework(interpreter) and super(CPython2macOsFramework, cls).can_describe(interpreter)
+
+ def create(self):
+ super(CPython2macOsFramework, self).create()
+
+ # change the install_name of the copied python executable
+ current = os.path.join(self.interpreter.prefix, "Python")
+ fix_mach_o(str(self.exe), current, "@executable_path/../.Python", self.interpreter.max_size)
+
+ @classmethod
+ def sources(cls, interpreter):
+ for src in super(CPython2macOsFramework, cls).sources(interpreter):
+ yield src
+
+ # landmark for exec_prefix
+ name = "lib-dynload"
+ yield PathRefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib)
+
+ # this must symlink to the host prefix Python
+ marker = Path(interpreter.prefix) / "Python"
+ ref = PathRefToDest(marker, dest=lambda self, _: self.dest / ".Python", must_symlink=True)
+ yield ref
+
+ @classmethod
+ def _executables(cls, interpreter):
+ for _, targets in super(CPython2macOsFramework, cls)._executables(interpreter):
+ # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the
+ # stub executable in ${sys.prefix}/bin.
+ # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951
+ fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python"
+ yield fixed_host_exe, targets
+
+
+def fix_mach_o(exe, current, new, max_size):
+ """
+ https://en.wikipedia.org/wiki/Mach-O
+
+ Mach-O, short for Mach object file format, is a file format for executables, object code, shared libraries,
+ dynamically-loaded code, and core dumps. A replacement for the a.out format, Mach-O offers more extensibility and
+ faster access to information in the symbol table.
+
+ Each Mach-O file is made up of one Mach-O header, followed by a series of load commands, followed by one or more
+ segments, each of which contains between 0 and 255 sections. Mach-O uses the REL relocation format to handle
+ references to symbols. When looking up symbols Mach-O uses a two-level namespace that encodes each symbol into an
+ 'object/symbol name' pair that is then linearly searched for by first the object and then the symbol name.
+
+ The basic structure—a list of variable-length "load commands" that reference pages of data elsewhere in the file—was
+ also used in the executable file format for Accent. The Accent file format was in turn, based on an idea from Spice
+ Lisp.
+
+ With the introduction of Mac OS X 10.6 platform the Mach-O file underwent a significant modification that causes
+ binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac
+ OS X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions,
+ does not understand. Another significant change to the Mach-O format is the change in how the Link Edit tables
+ (found in the __LINKEDIT section) function. In 10.6 these new Link Edit tables are compressed by removing unused and
+ unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format.
+ """
+ try:
+ logging.debug("change Mach-O for %s from %s to %s", exe, current, new)
+ _builtin_change_mach_o(max_size)(exe, current, new)
+ except Exception as e:
+ logging.warning("Could not call _builtin_change_mac_o: %s. " "Trying to call install_name_tool instead.", e)
+ try:
+ cmd = ["install_name_tool", "-change", current, new, exe]
+ subprocess.check_call(cmd)
+ except Exception:
+ logging.fatal("Could not call install_name_tool -- you must " "have Apple's development tools installed")
+ raise
+
+
+def _builtin_change_mach_o(maxint):
+ MH_MAGIC = 0xFEEDFACE
+ MH_CIGAM = 0xCEFAEDFE
+ MH_MAGIC_64 = 0xFEEDFACF
+ MH_CIGAM_64 = 0xCFFAEDFE
+ FAT_MAGIC = 0xCAFEBABE
+ BIG_ENDIAN = ">"
+ LITTLE_ENDIAN = "<"
+ LC_LOAD_DYLIB = 0xC
+
+ class FileView(object):
+ """A proxy for file-like objects that exposes a given view of a file. Modified from macholib."""
+
+ def __init__(self, file_obj, start=0, size=maxint):
+ if isinstance(file_obj, FileView):
+ self._file_obj = file_obj._file_obj
+ else:
+ self._file_obj = file_obj
+ self._start = start
+ self._end = start + size
+ self._pos = 0
+
+ def __repr__(self):
+ return "<fileview [{:d}, {:d}] {!r}>".format(self._start, self._end, self._file_obj)
+
+ def tell(self):
+ return self._pos
+
+ def _checkwindow(self, seek_to, op):
+ if not (self._start <= seek_to <= self._end):
+ msg = "{} to offset {:d} is outside window [{:d}, {:d}]".format(op, seek_to, self._start, self._end)
+ raise IOError(msg)
+
+ def seek(self, offset, whence=0):
+ seek_to = offset
+ if whence == os.SEEK_SET:
+ seek_to += self._start
+ elif whence == os.SEEK_CUR:
+ seek_to += self._start + self._pos
+ elif whence == os.SEEK_END:
+ seek_to += self._end
+ else:
+ raise IOError("Invalid whence argument to seek: {!r}".format(whence))
+ self._checkwindow(seek_to, "seek")
+ self._file_obj.seek(seek_to)
+ self._pos = seek_to - self._start
+
+ def write(self, content):
+ here = self._start + self._pos
+ self._checkwindow(here, "write")
+ self._checkwindow(here + len(content), "write")
+ self._file_obj.seek(here, os.SEEK_SET)
+ self._file_obj.write(content)
+ self._pos += len(content)
+
+ def read(self, size=maxint):
+ assert size >= 0
+ here = self._start + self._pos
+ self._checkwindow(here, "read")
+ size = min(size, self._end - here)
+ self._file_obj.seek(here, os.SEEK_SET)
+ read_bytes = self._file_obj.read(size)
+ self._pos += len(read_bytes)
+ return read_bytes
+
+ def read_data(file, endian, num=1):
+ """Read a given number of 32-bits unsigned integers from the given file with the given endianness."""
+ res = struct.unpack(endian + "L" * num, file.read(num * 4))
+ if len(res) == 1:
+ return res[0]
+ return res
+
+ def mach_o_change(at_path, what, value):
+ """Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value),
+ provided it's shorter."""
+
+ def do_macho(file, bits, endian):
+ # Read Mach-O header (the magic number is assumed read by the caller)
+ cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6)
+ # 64-bits header has one more field.
+ if bits == 64:
+ read_data(file, endian)
+ # The header is followed by n commands
+ for _ in range(n_commands):
+ where = file.tell()
+ # Read command header
+ cmd, cmd_size = read_data(file, endian, 2)
+ if cmd == LC_LOAD_DYLIB:
+ # The first data field in LC_LOAD_DYLIB commands is the offset of the name, starting from the
+ # beginning of the command.
+ name_offset = read_data(file, endian)
+ file.seek(where + name_offset, os.SEEK_SET)
+ # Read the NUL terminated string
+ load = file.read(cmd_size - name_offset).decode()
+ load = load[: load.index("\0")]
+ # If the string is what is being replaced, overwrite it.
+ if load == what:
+ file.seek(where + name_offset, os.SEEK_SET)
+ file.write(value.encode() + b"\0")
+ # Seek to the next command
+ file.seek(where + cmd_size, os.SEEK_SET)
+
+ def do_file(file, offset=0, size=maxint):
+ file = FileView(file, offset, size)
+ # Read magic number
+ magic = read_data(file, BIG_ENDIAN)
+ if magic == FAT_MAGIC:
+ # Fat binaries contain nfat_arch Mach-O binaries
+ n_fat_arch = read_data(file, BIG_ENDIAN)
+ for _ in range(n_fat_arch):
+ # Read arch header
+ cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5)
+ do_file(file, offset, size)
+ elif magic == MH_MAGIC:
+ do_macho(file, 32, BIG_ENDIAN)
+ elif magic == MH_CIGAM:
+ do_macho(file, 32, LITTLE_ENDIAN)
+ elif magic == MH_MAGIC_64:
+ do_macho(file, 64, BIG_ENDIAN)
+ elif magic == MH_CIGAM_64:
+ do_macho(file, 64, LITTLE_ENDIAN)
+
+ assert len(what) >= len(value)
+
+ with open(at_path, "r+b") as f:
+ do_file(f)
+
+ return mach_o_change

src/virtualenv/create/via_global_ref/builtin/python2/python2.py

@@ -32,7 +32,8 @@ def create(self):
  else:
  custom_site_text = custom_site.read_text()
  expected = json.dumps([os.path.relpath(ensure_text(str(i)), ensure_text(str(site_py))) for i in self.libs])
- site_py.write_text(custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected))
+ custom_site_text = custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected)
+ site_py.write_text(custom_site_text)
 
  @classmethod
  def sources(cls, interpreter):

src/virtualenv/create/via_global_ref/builtin/python2/site.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 A simple shim module to fix up things on Python 2 only.
 
@@ -37,8 +38,7 @@ def load_host_site():
 
  here = __file__ # the distutils.install patterns will be injected relative to this site.py, save it here
 
- with PatchForAppleFrameworkBuilds():
- reload(sys.modules["site"]) # noqa
+ reload(sys.modules["site"]) # noqa # call system site.py to setup import libraries
 
  # and then if the distutils site packages are not on the sys.path we add them via add_site_dir; note we must add
  # them by invoking add_site_dir to trigger the processing of pth files
@@ -56,28 +56,12 @@ def load_host_site():
  add_site_dir(full_path)
 
 
-class PatchForAppleFrameworkBuilds(object):
- """Apple Framework builds unconditionally add the global site-package, escape this behaviour"""
-
- framework = None
-
- def __enter__(self):
- if sys.platform == "darwin":
- from sysconfig import get_config_var
-
- self.framework = get_config_var("PYTHONFRAMEWORK")
- if self.framework:
- sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = None
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- if self.framework:
- sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = self.framework
+sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version
 
 
 def read_pyvenv():
  """read pyvenv.cfg"""
- os_sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version
- config_file = "{}{}pyvenv.cfg".format(sys.prefix, os_sep)
+ config_file = "{}{}pyvenv.cfg".format(sys.prefix, sep)
  with open(config_file) as file_handler:
  lines = file_handler.readlines()
  config = {}
@@ -93,23 +77,41 @@ def read_pyvenv():
 
 def rewrite_standard_library_sys_path():
  """Once this site file is loaded the standard library paths have already been set, fix them up"""
- sep = "\\" if sys.platform == "win32" else "/"
- exe_dir = sys.executable[: sys.executable.rfind(sep)]
+ exe = abs_path(sys.executable)
+ exe_dir = exe[: exe.rfind(sep)]
+ prefix, exec_prefix = abs_path(sys.prefix), abs_path(sys.exec_prefix)
+ base_prefix, base_exec_prefix = abs_path(sys.base_prefix), abs_path(sys.base_exec_prefix)
+ base_executable = abs_path(sys.base_executable)
  for at, value in enumerate(sys.path):
+ value = abs_path(value)
  # replace old sys prefix path starts with new
  if value == exe_dir:
  pass # don't fix the current executable location, notably on Windows this gets added
  elif value.startswith(exe_dir):
  # content inside the exe folder needs to remap to original executables folder
- orig_exe_folder = sys.base_executable[: sys.base_executable.rfind(sep)]
+ orig_exe_folder = base_executable[: base_executable.rfind(sep)]
  value = "{}{}".format(orig_exe_folder, value[len(exe_dir) :])
- elif value.startswith(sys.prefix):
- value = "{}{}".format(sys.base_prefix, value[len(sys.prefix) :])
- elif value.startswith(sys.exec_prefix):
- value = "{}{}".format(sys.base_exec_prefix, value[len(sys.exec_prefix) :])
+ elif value.startswith(prefix):
+ value = "{}{}".format(base_prefix, value[len(prefix) :])
+ elif value.startswith(exec_prefix):
+ value = "{}{}".format(base_exec_prefix, value[len(exec_prefix) :])
  sys.path[at] = value
 
 
+def abs_path(value):
+ keep = []
+ values = value.split(sep)
+ i = len(values) - 1
+ while i >= 0:
+ if values[i] == "..":
+ i -= 1
+ else:
+ keep.append(values[i])
+ i -= 1
+ value = sep.join(keep[::-1])
+ return value
+
+
 def disable_user_site_package():
  """Flip the switch on enable user site package"""
  # sys.flags is a c-extension type, so we cannot monkeypatch it, replace it with a python class to flip it