Add support for cross-compiling on iOS

crossenv/__init__.py

@@ -8,11 +8,8 @@
 import subprocess
 import logging
 import importlib
-import types
-from configparser import ConfigParser
 import random
 import shlex
-import platform
 import pprint
 import re
 
@@ -23,6 +20,9 @@
 
 logger = logging.getLogger(__name__)
 
+APPLE_MOBILE_PLATFORMS = {"ios", "tvos", "watchos"}
+
+
 class CrossEnvBuilder(venv.EnvBuilder):
  """
  A class to build a cross-compiling virtual environment useful for
@@ -358,10 +358,20 @@ def find_host_python(self, host):
  # It was probably natively compiled, but not necessarily for this
  # architecture. Guess from HOST_GNU_TYPE.
  host = self.host_gnu_type.split('-')
- if len(host) == 4: # i.e., aarch64-unknown-linux-gnu
- self.host_platform = '-'.join([host[2], host[0]])
- elif len(host) == 3: # i.e., aarch64-linux-gnu, unlikely.
- self.host_platform = '-'.join([host[1], host[0]])
+ if len(host) == 4:
+ if host[1] == "apple":
+ machine = self.host_sysconfigdata.build_time_vars['MULTIARCH']
+ os_name, release = self._split_apple_os_version(host[2])
+ self.host_platform = '-'.join([os_name, release, machine])
+ else: # i.e., aarch64-unknown-linux-gnu
+ self.host_platform = '-'.join([host[2], host[0]])
+ elif len(host) == 3:
+ if host[1] == "apple":
+ machine = self.host_sysconfigdata.build_time_vars['MULTIARCH']
+ os_name, release = self._split_apple_os_version(host[2])
+ self.host_platform = '-'.join([os_name, release, machine])
+ else: # i.e., aarch64-linux-gnu, unlikely.
+ self.host_platform = '-'.join([host[1], host[0]])
  else:
  logger.warning("Cannot determine platform. Using build.")
  self.host_platform = sysconfig.get_platform()
@@ -419,36 +429,58 @@ def run_compiler(arg):
  found_triple = res.stdout.strip()
  if res.returncode == 0 and found_triple:
  expected = self.host_sysconfigdata.build_time_vars['HOST_GNU_TYPE']
- if not self._compare_triples(found_triple, expected):
+ clean_found = self._clean_triple(found_triple)
+ clean_expected = self._clean_triple(expected)
+ if (
+ clean_found is not None
+ and clean_expected is not None
+ and clean_found != clean_expected
+ ):
  logger.warning("The cross-compiler (%r) does not appear to be "
  "for the correct architecture (got %s, expected "
  "%s). Use --cc to correct, if necessary.",
  ' '.join(self.host_cc),
  found_triple,
  expected)
 
- def _compare_triples(self, x, y):
- # They are in the form cpu-vendor-kernel-system or cpu-kernel-system.
- # So we'll get something like: x86_64-linux-gnu or x86_64-pc-linux-gnu.
- # We won't overcomplicate this, since it's just to generate a warning.
+ def _split_apple_os_version(self, value):
+ # Split the Apple OS version from the OS name prefix.
+ if value.startswith("ios"):
+ offset = 3
+ elif value.startswith("tvos"):
+ offset = 4
+ elif value.startswith("watchos"):
+ offset = 7
+ else:
+ raise ValueError("Unknown Apple compiler triple.")
+ return value[:offset], value[offset:]
+
+ def _clean_triple(self, triple):
+ # Triples are in the form cpu-vendor-kernel-system or cpu-kernel-system. So we'll
+ # get something like: x86_64-linux-gnu or x86_64-pc-linux-gnu. We won't
+ # overcomplicate this, since it's just to generate a warning.
  #
- # We return True if we can't make sense of anything and wish to skip
- # the warning.
+ # Apple builds accept use "arm64-apple-ios12.0-simulator" for an iOS simulator.
+ # The host GNU type returned by clang won't include the version number; we also
+ # need to map "arm64" to "aarch64", and allow for the "-simulator" suffix.
 
- parts_x = x.split('-')
- if len(parts_x) == 4:
- del parts_x[1]
- elif len(parts_x) != 3:
- return True # Some other form? Bail out.
+ parts = triple.split('-')
+ if parts[1] == "apple":
+ # Normalize Apple's CPU architecture descriptor to the GNU form.
+  if parts[0] == "arm64":
+  parts[0] = "aarch64"
 
- parts_y = y.split('-')
- if len(parts_y) == 4:
- del parts_y[1]
- elif len(parts_y) != 3:
- return True # Some other form? Bail out.
+ # Remove any version identifier from the OS (e.g., ios13.0 -> ios)
+ parts[2], _ = self._split_apple_os_version(parts[2])
 
- return parts_x == parts_y
+ if len(parts) == 4 and parts[1] != "apple":
+ # 4-part "triples" are expected for Apple simulators;
+ # on any other platform, drop the 4th part.
+ del parts[1]
+ elif len(parts) != 3:
+ return None # Some other form? Bail out.
 
+ return parts
 
  def create(self, env_dir):
  """
@@ -496,15 +528,17 @@ def get_uname_info(self):
  """
  What should uname() return?
  """
-
- # host_platform is _probably_ something like linux-x86_64, but it can
- # vary.
+ # host_platform is _probably_ something like linux-x86_64, but it can vary.
+ # On iOS/tvOS/watchOS, it will be of the form ios-13.0-arm64-iphonesimulator,
+ # telling you sys.platform, the minimum supported OS version, the device ABI,
+ # and the architecture.
  host_info = self.host_platform.split('-')
  if not host_info:
- self.host_sysname = sys.platform
+ self.host_sys_platform = sys.platform
  elif len(host_info) >= 1:
- self.host_sysname = host_info[0]
+ self.host_sys_platform = host_info[0]
 
+ self.host_is_simulator = None
  if self.host_machine is None:
  platform2uname = {
  # On uname.machine=ppc64, _PYTHON_HOST_PLATFORM is linux-powerpc64
@@ -515,10 +549,15 @@ def get_uname_info(self):
  if len(host_info) > 1 and host_info[-1] in platform2uname:
  # Test that this is still a special case when we can.
  self.host_machine = platform2uname[host_info[-1]]
+ elif self.host_sys_platform in APPLE_MOBILE_PLATFORMS:
+ # iOS/tvOS/watchOS return the machine type as the last part of
+ # the host info. The device is a simulator if the last part ends
+ # with "simulator" (e.g., "iphoneos" vs "iphonesimulator")
+ self.host_machine = host_info[-2]
+ self.host_is_simulator = host_info[-1].endswith("simulator")
  else:
  self.host_machine = self.host_gnu_type.split('-')[0]
 
- self.host_release = ''
  if self.macosx_deployment_target:
  try:
  major, minor = self.macosx_deployment_target.split(".")
@@ -533,17 +572,38 @@ def get_uname_info(self):
  else:
  raise ValueError("Unexpected major version %s for MACOSX_DEPLOYMENT_TARGET" %
  major)
+ elif self.host_sys_platform in APPLE_MOBILE_PLATFORMS:
+ self.host_release = host_info[1]
+ else:
+ self.host_release = ''
 
- if self.host_sysname == "darwin":
+ if self.host_sys_platform == "darwin":
  self.sysconfig_platform = "macosx-%s-%s" % (self.macosx_deployment_target,
  self.host_machine)
- elif self.host_sysname == "linux":
+ elif self.host_sys_platform == "linux":
  # Use self.host_machine here as powerpc64le gets converted
  # to ppc64le in self.host_machine
  self.sysconfig_platform = "linux-%s" % (self.host_machine)
  else:
  self.sysconfig_platform = self.host_platform
 
+ self.sysconfig_ext_suffix = self.host_sysconfigdata.build_time_vars['EXT_SUFFIX']
+
+ # Normalize case of the host system/sysname.
+ self.host_system = {
+ "ios": "iOS",
+ "tvos": "tvOS",
+ "watchos": "watchOS",
+ }.get(self.host_sys_platform, self.host_sys_platform.title())
+
+ # on iOS/tvOS/watchOS, platform.uname() reports the actual OS name;
+ # os.uname() reports the *kernel* name.
+ self.host_sysname = {
+ "ios": "Darwin",
+ "tvos": "Darwin",
+ "watchos": "Darwin",
+ }.get(self.host_sys_platform, self.host_sys_platform.title())
+
  def expand_platform_tags(self):
  """
  Convert legacy manylinux tags to PEP600, because pip only looks for one
@@ -676,14 +736,16 @@ def make_cross_python(self, context):
  if self.cross_prefix:
  context.cross_env_dir = self.cross_prefix
  else:
- context.cross_env_dir = os.path.join(context.env_dir, 'cross')
- clear_cross = self.clear in ('default', 'cross-only', 'both')
+ context.cross_env_dir = os.path.join(context.env_dir, "cross")
+ cross_env_name = os.path.split(context.env_dir)[-1]
+
  env = venv.EnvBuilder(
  system_site_packages=False,
  clear=self.clear_cross,
  symlinks=True,
  upgrade=False,
- with_pip=False)
+ with_pip=False,
+ prompt=cross_env_name)
  env.create(context.cross_env_dir)
  context.cross_bin_path = os.path.join(context.cross_env_dir, 'bin')
  context.cross_lib_path = os.path.join(context.cross_env_dir, 'lib')
@@ -769,7 +831,7 @@ def make_cross_python(self, context):
  host_build_time_vars = self.host_sysconfigdata.build_time_vars
  sysconfig_name = self.host_sysconfigdata_name
 
- 
+
  # Install patches to environment
  self.copy_and_patch_sysconfigdata(context)
 
@@ -787,7 +849,9 @@ def make_cross_python(self, context):
  'importlib-metadata-patch.py',
  'platform-patch.py',
  'sysconfig-patch.py',
+ 'subprocess-patch.py',
  'distutils-sysconfig-patch.py',
+ 'pip-_vendor-distlib-scripts-patch.py',
  'pkg_resources-patch.py',
  'packaging-tags-patch.py',
  ]
@@ -839,12 +903,16 @@ def copy_and_patch_sysconfigdata(self, context):
  host_cc = self.real_host_cc[0]
  host_cxx = self.real_host_cxx[0]
  host_ar = self.real_host_ar[0]
+ host_prefix = self.host_sysconfigdata.build_time_vars['prefix']
+
  repl_cc = self.host_cc[0]
  repl_cxx = self.host_cxx[0]
  repl_ar = self.host_ar[0]
+
  find_cc = re.compile(r'(?:^|(?<=\s))%s(?=\s|$)' % re.escape(host_cc))
  find_cxx = re.compile(r'(?:^|(?<=\s))%s(?=\s|$)' % re.escape(host_cxx))
  find_ar = re.compile(r'(?:^|(?<=\s))%s(?=\s|$)' % re.escape(host_ar))
+ find_prefix = re.compile(r'(?:^|(?<=\s))%s' % re.escape(host_prefix))
 
  cross_sysconfig_data = {}
  for key, value in self.host_sysconfigdata.__dict__.items():
@@ -858,6 +926,7 @@ def copy_and_patch_sysconfigdata(self, context):
  value = find_ar.sub(repl_ar, value)
  value = find_cxx.sub(repl_cxx, value)
  value = find_cc.sub(repl_cc, value)
+ value = find_prefix.sub(self.host_home, value)
 
  build_time_vars[key] = value
 

crossenv/scripts/importlib-machinery-patch.py.tmpl

@@ -25,3 +25,5 @@ def _PathFinder_find_spec(cls, fullname, path=None, target=None):
 
  return _original_find_spec(fullname, path, target)
 PathFinder.find_spec = _PathFinder_find_spec
+
+EXTENSION_SUFFIXES = [{{repr(self.sysconfig_ext_suffix)}}, ".abi3.so", ".so"]

crossenv/scripts/os-patch.py.tmpl

@@ -4,7 +4,7 @@ from collections import namedtuple
 uname_result_type = namedtuple('uname_result',
  'sysname nodename release version machine')
 _uname_result = uname_result_type(
- {{repr(self.host_sysname.title())}},
+ {{repr(self.host_sysname)}},
  'build',
  {{repr(self.host_release)}},
  '',

crossenv/scripts/pip-_vendor-distlib-scripts-patch.py.tmpl

@@ -0,0 +1,17 @@
+def _build_shebang(self, executable, post_interp):
+ """
+ Build a shebang line. The default pip behavior will use a "simple" shim
+ if the path to the wrapped Python binary is < 127 chars long, and doesn't
+ contain a space. However, the host python binary isn't actually a binary -
+ it's a shell script that does additional environment modifications - and
+ os.execv() raises "OSError [Errno 8] Exec format error" if the shebang
+ of a script isn't a literal binary.
+
+ So - patch the script writer so that it *always* uses a shim.
+ """
+ result = b'#!/bin/sh\n'
+ result += b"'''exec' " + executable + post_interp + b' "$0" "$@"\n'
+ result += b"' '''"
+ return result
+
+ScriptMaker._build_shebang = _build_shebang

crossenv/scripts/platform-patch.py.tmpl

@@ -3,7 +3,7 @@ from collections import namedtuple
 platform_uname_result_type = namedtuple('uname_result',
  'system node release version machine processor')
 _uname_result = platform_uname_result_type(
- {{repr(self.host_sysname.title())}},
+ {{repr(self.host_system)}},
  'build',
  {{repr(self.host_release)}},
  '',
@@ -22,6 +22,21 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''):
  machine = _uname_result.machine
  return release, versioninfo, machine
 
+IOSVersionInfo = collections.namedtuple(
+ "IOSVersionInfo",
+ ["system", "release", "model", "is_simulator"]
+)
+
+def ios_ver(system="", release="", model="", is_simulator=False):
+ if system == "":
+ system = {{repr(self.host_system)}}
+ if release == "":
+ release = {{repr(self.host_release)}}
+ if model == "":
+ model = {{repr("iPhone" if self.host_is_simulator else "iPhone13,2")}}
+
+ return IOSVersionInfo(system, release, model, {{repr(self.host_is_simulator)}})
+
 # Old, deprecated functions, but we support back to 3.5
 if '_linux_distribution' in globals():
  def _linux_distribution(distname, version, id, supported_dists,

crossenv/scripts/site.py.tmpl

@@ -95,7 +95,7 @@ class CrossenvPatchLegacyLoader(importlib.abc.Loader):
 
 class CrossenvFinder(importlib.abc.MetaPathFinder):
  """Mucks with import machinery in two ways:
- 
+
  1) loads sysconfigdata from our hard-coded path, regardless of sys.path
  2) intercepts and patches modules as they are loaded
  """
@@ -105,11 +105,13 @@ class CrossenvFinder(importlib.abc.MetaPathFinder):
  'importlib.metadata': '{{context.lib_path}}/importlib-metadata-patch.py',
  'sys': '{{context.lib_path}}/sys-patch.py',
  'os': '{{context.lib_path}}/os-patch.py',
+ 'subprocess': '{{context.lib_path}}/subprocess-patch.py',
  'sysconfig': '{{context.lib_path}}/sysconfig-patch.py',
  'distutils.sysconfig': '{{context.lib_path}}/distutils-sysconfig-patch.py',
  'distutils.sysconfig_pypy': '{{context.lib_path}}/distutils-sysconfig-patch.py',
  'platform': '{{context.lib_path}}/platform-patch.py',
  'pkg_resources': '{{context.lib_path}}/pkg_resources-patch.py',
+ 'pip._vendor.distlib.scripts': '{{context.lib_path}}/pip-_vendor-distlib-scripts-patch.py',
  'pip._vendor.pkg_resources': '{{context.lib_path}}/pkg_resources-patch.py',
  'packaging.tags': '{{context.lib_path}}/packaging-tags-patch.py',
  }

crossenv/scripts/subprocess-patch.py.tmpl

@@ -0,0 +1,16 @@
+# We must be on a platform that supports fork and exec
+_can_fork_exec = True
+
+# Set the module-level properties that are dependent on _can_fork_exec
+from _posixsubprocess import fork_exec as _fork_exec
+
+import os
+try:
+ _del_safe.waitpid = os.waitpid
+ _del_safe.waitstatus_to_exitcode = os.waitstatus_to_exitcode
+ _del_safe.WIFSTOPPED = os.WIFSTOPPED
+ _del_safe.WSTOPSIG = os.WSTOPSIG
+ _del_safe.WNOHANG = os.WNOHANG
+except NameError:
+ # Pre Python 3.11 doesn't have the _del_safe helper.
+ pass

crossenv/scripts/sys-patch.py.tmpl

@@ -1,6 +1,6 @@
 cross_compiling = True
 build_path = {{repr(context.build_sys_path)}}
-
+platform = {{repr(self.host_sys_platform)}}
 abiflags = {{repr(host_build_time_vars.get('ABIFLAGS'))}}
 if abiflags is None:
  del abiflags
@@ -12,3 +12,7 @@ if implementation._multiarch is None:
 # Remove cross-python from sys.path. It's not needed after startup.
 path.remove({{repr(context.lib_path)}})
 path.remove({{repr(stdlib)}})
+
+# If a process started by cross-python tries to start a subprocess with sys.executable,
+# make sure that it points at cross-python.
+executable = {{repr(context.cross_env_exe)}}