Merge pull request #482 from asfaltboy/support-quemu-on-github

Support qemu in GitHub Actions

.github/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest]
         python_version: ['3.7']
     timeout-minutes: 180
     steps:
@@ -55,3 +55,24 @@ jobs:
     - name: Test cibuildwheel
       run: |
         python ./bin/run_tests.py
+
+  test-emulated:
+    name: Test emulated cibuildwheel using qemu
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - name: Install dependencies
+      run: |
+        python -m pip install -r requirements-dev.txt
+
+    - name: Set up QEMU
+      id: qemu
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: all
+
+    - name: Run the emulation tests
+      run: |
+        pytest --run-emulation test/test_emulation.py

bin/run_example_ci_configs.py

@@ -8,6 +8,7 @@
 import textwrap
 import time
 import click
+from glob import glob
 from collections import namedtuple
 from urllib.parse import quote
 
@@ -30,49 +31,70 @@ def generate_basic_project(path):
     project.generate(path)
 
 
-CIService = namedtuple('CIService', 'name src_config_path dst_config_path badge_md')
+CIService = namedtuple('CIService', 'name dst_config_path badge_md')
 services = [
     CIService(
         name='appveyor',
-        src_config_path='examples/appveyor-minimal.yml',
         dst_config_path='appveyor.yml',
         badge_md='[![Build status](https://ci.appveyor.com/api/projects/status/wbsgxshp05tt1tif/branch/{branch}?svg=true)](https://ci.appveyor.com/project/joerick/cibuildwheel/branch/{branch})',
     ),
     CIService(
         name='azure-pipelines',
-        src_config_path='examples/azure-pipelines-minimal.yml',
         dst_config_path='azure-pipelines.yml',
         badge_md='[![Build Status](https://dev.azure.com/joerick0429/cibuildwheel/_apis/build/status/joerick.cibuildwheel?branchName={branch})](https://dev.azure.com/joerick0429/cibuildwheel/_build/latest?definitionId=2&branchName={branch})',
     ),
     CIService(
-        name='circle-ci',
-        src_config_path='examples/circleci-minimal.yml',
+        name='circleci',
         dst_config_path='.circleci/config.yml',
         badge_md='[![CircleCI](https://circleci.com/gh/joerick/cibuildwheel/tree/{branch_escaped}.svg?style=svg)](https://circleci.com/gh/joerick/cibuildwheel/tree/{branch})',
     ),
     CIService(
         name='github',
-        src_config_path='examples/github-minimal.yml',
         dst_config_path='.github/workflows/example.yml',
         badge_md='[![Build](https://github.com/joerick/cibuildwheel/workflows/Build/badge.svg?branch={branch})](https://github.com/joerick/cibuildwheel/actions)',
     ),
     CIService(
         name='travis-ci',
-        src_config_path='examples/travis-ci-minimal.yml',
         dst_config_path='.travis.yml',
         badge_md='[![Build Status](https://travis-ci.org/joerick/cibuildwheel.svg?branch={branch})](https://travis-ci.org/joerick/cibuildwheel)',
     ),
     CIService(
         name='gitlab',
-        src_config_path='examples/gitlab-minimal.yml',
         dst_config_path='.gitlab-ci.yml',
         badge_md='[![Gitlab](https://gitlab.com/joerick/cibuildwheel/badges/{branch}/pipeline.svg)](https://gitlab.com/joerick/cibuildwheel/-/commits/{branch})'
     ),
 ]
 
 
+def ci_service_for_config_file(config_file):
+    service_name = Path(config_file).name.rsplit('-', 1)[0]
+
+    for service in services:
+        if service.name == service_name:
+            return service
+
+    raise ValueError(f'unknown ci service for config file {config_file}')
+
+
 @click.command()
-def run_example_ci_configs():
+@click.argument('config_files', nargs=-1, type=click.Path())
+def run_example_ci_configs(config_files=None):
+    '''
+        Test the example configs. If no files are specified, will test
+        examples/*-minimal.yml
+    '''
+
+    if len(config_files) == 0:
+        config_files = glob('examples/*-minimal.yml')
+
+    # check each CI service has at most 1 config file
+    configs_by_service = {}
+    for config_file in config_files:
+        service = ci_service_for_config_file(config_file)
+        if service.name in configs_by_service:
+            raise Exception('You cannot specify more than one config per CI service')
+        configs_by_service[service.name] = config_file
+
     if git_repo_has_changes():
         print('Your git repo has uncommitted changes. Commit or stash before continuing.')
         exit(1)
@@ -91,18 +113,19 @@ def run_example_ci_configs():
         example_project = Path('example_root')
         generate_basic_project(example_project)
 
-        for service in services:
-            src_config_file = Path(service.src_config_path)
+        for config_file in config_files:
+            service = ci_service_for_config_file(config_file)
+            src_config_file = Path(config_file)
             dst_config_file = example_project / service.dst_config_path
 
             dst_config_file.parent.mkdir(parents=True, exist_ok=True)
             shutil.copyfile(src_config_file, dst_config_file)
 
         run(['git', 'add', example_project], check=True)
-        run(['git', 'commit', '-m', textwrap.dedent(f'''
+        run(['git', 'commit', '--no-verify', '-m', textwrap.dedent(f'''
             Test example minimal configs
 
-            Testing files: {[s.src_config_path for s in services]}
+            Testing files: {config_files}
             Generated from branch: {previous_branch}
             Time: {timestamp}
         ''')], check=True)
@@ -116,9 +139,10 @@ def run_example_ci_configs():
         print('> ')
         print('> | Service | Config | Status |')
         print('> |---|---|---|')
-        for service in services:
+        for config_file in config_files:
+            service = ci_service_for_config_file(config_file)
             badge = service.badge_md.format(branch=branch_name, branch_escaped=quote(branch_name, safe=''))
-            print(f'> | {service.name} | `{service.src_config_path}` | {badge} |')
+            print(f'> | {service.name} | `{config_file}` | {badge} |')
         print('> ')
         print('> Generated by `bin/run_example_ci_config.py`')
         print()

cibuildwheel/__main__.py

@@ -17,6 +17,7 @@
     parse_environment,
 )
 from cibuildwheel.util import (
+    Architecture,
     BuildOptions,
     BuildSelector,
     DependencyConstraints,
@@ -67,6 +68,17 @@ def main() -> None:
                             run in Windows, and it will build and test for all versions of
                             Python. Default: auto.
                         ''')
+
+    parser.add_argument('--archs',
+                        default=None,
+                        help='''
+                            Comma-separated list of CPU architectures to build for.
+                            When set to 'auto', builds the architectures natively supported
+                            on this machine. Set this option to build an architecture
+                            via emulation, for example, using binfmt_misc and QEMU.
+                            Default: auto.
+                            Choices: auto, {}
+                        '''.format(", ".join(a.name for a in Architecture)))
     parser.add_argument('--output-dir',
                         default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'),
                         help='Destination folder for the wheels.')
@@ -168,8 +180,14 @@ def main() -> None:
         print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr)
         exit(2)
 
+    if args.archs is not None:
+        archs_config_str = args.archs
+    else:
+        archs_config_str = get_option_from_environment('CIBW_ARCHS', platform=platform, default='auto')
+    archs = Architecture.parse_config(archs_config_str, platform=platform)
+
     if args.print_build_identifiers:
-        print_build_identifiers(platform, build_selector)
+        print_build_identifiers(platform, build_selector, archs)
         exit(0)
 
     manylinux_images: Optional[Dict[str, str]] = None
@@ -202,6 +220,7 @@ def main() -> None:
             manylinux_images[build_platform] = image
 
     build_options = BuildOptions(
+        architectures=archs,
         package_dir=package_dir,
         output_dir=output_dir,
         test_command=test_command,
@@ -284,10 +303,12 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None:
     print('\nHere we go!\n')
 
 
-def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None:
+def print_build_identifiers(
+    platform: str, build_selector: BuildSelector, architectures: List[Architecture]
+) -> None:
     python_configurations: List[Any] = []
     if platform == 'linux':
-        python_configurations = cibuildwheel.linux.get_python_configurations(build_selector)
+        python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures)
     elif platform == 'windows':
         python_configurations = cibuildwheel.windows.get_python_configurations(build_selector)
     elif platform == 'macos':

cibuildwheel/linux.py

@@ -1,4 +1,3 @@
-import platform
 import subprocess
 import sys
 import textwrap
@@ -8,29 +7,10 @@
 
 from .docker_container import DockerContainer
 from .logger import log
-from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
-                   get_build_verbosity_extra_flags, prepare_command)
-
-
-def matches_platform(identifier: str) -> bool:
-    pm = platform.machine()
-    if pm == "x86_64":
-        # x86_64 machines can run i686 docker containers
-        if identifier.endswith('x86_64') or identifier.endswith('i686'):
-            return True
-    elif pm == "i686":
-        if identifier.endswith('i686'):
-            return True
-    elif pm == "aarch64":
-        if identifier.endswith('aarch64'):
-            return True
-    elif pm == "ppc64le":
-        if identifier.endswith('ppc64le'):
-            return True
-    elif pm == "s390x":
-        if identifier.endswith('s390x'):
-            return True
-    return False
+from .util import (
+    Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
+    get_build_verbosity_extra_flags, prepare_command,
+)
 
 
 class PythonConfiguration(NamedTuple):
@@ -43,7 +23,9 @@ def path(self):
         return PurePath(self.path_str)
 
 
-def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]:
+def get_python_configurations(
+    build_selector: BuildSelector, architectures: List[Architecture]
+) -> List[PythonConfiguration]:
     python_configurations = [
         PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'),
         PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'),
@@ -78,8 +60,14 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi
         PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'),
         PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'),
     ]
-    # skip builds as required
-    return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)]
+
+    # return all configurations whose arch is in our `architectures` list,
+    # and match the build/skip rules
+    return [
+        c for c in python_configurations
+        if any(c.identifier.endswith(arch.value) for arch in architectures)
+        and build_selector(c.identifier)
+    ]
 
 
 def build(options: BuildOptions) -> None:
@@ -93,7 +81,7 @@ def build(options: BuildOptions) -> None:
         exit(2)
 
     assert options.manylinux_images is not None
-    python_configurations = get_python_configurations(options.build_selector)
+    python_configurations = get_python_configurations(options.build_selector, options.architectures)
     platforms = [
         ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']),
         ('cp', 'manylinux_i686', options.manylinux_images['i686']),

cibuildwheel/macos.py

@@ -4,13 +4,14 @@
 import subprocess
 import sys
 import tempfile
+import textwrap
 from os import PathLike
 from pathlib import Path
 from typing import Dict, List, NamedTuple, Optional, Sequence, Union
 
 from .environment import ParsedEnvironment
 from .logger import log
-from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
+from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
                    download, get_build_verbosity_extra_flags, get_pip_script,
                    install_certifi_script, prepare_command)
 
@@ -186,6 +187,12 @@ def setup_python(python_configuration: PythonConfiguration,
 
 
 def build(options: BuildOptions) -> None:
+    if options.architectures != [Architecture.x86_64]:
+        raise ValueError(textwrap.dedent(f'''
+            Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment.
+            If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead.
+        '''))
+
     temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel'))
     built_wheel_dir = temp_dir / 'built_wheel'
     repaired_wheel_dir = temp_dir / 'repaired_wheel'

cibuildwheel/util.py

@@ -1,4 +1,6 @@
 import os
+import platform as platform_module
+import re
 import ssl
 import textwrap
 import urllib.request
@@ -121,10 +123,45 @@ def __repr__(self):
         return f'{self.__class__.__name__}{self.base_file_path!r})'
 
 
+class Architecture(Enum):
+    # mac/linux archs
+    x86_64 = 'x86_64'
+    i686 = 'i686'
+    aarch64 = 'aarch64'
+    ppc64le = 'ppc64le'
+    s390x = 's390x'
+
+    # windows archs
+    x86 = 'x86'
+    AMD64 = 'AMD64'
+
+    @staticmethod
+    def parse_config(config: str, platform: str) -> 'List[Architecture]':
+        result = []
+        for arch_str in re.split(r'[\s,]+', config):
+            if arch_str == 'auto':
+                result += Architecture.auto_archs(platform=platform)
+            else:
+                result.append(Architecture(arch_str))
+        return result
+
+    @staticmethod
+    def auto_archs(platform: str) -> 'List[Architecture]':
+        native_architecture = Architecture(platform_module.machine())
+        result = [native_architecture]
+        if platform == 'linux' and native_architecture == Architecture.x86_64:
+            # x86_64 machines can run i686 docker containers
+            result.append(Architecture.i686)
+        if platform == 'windows' and native_architecture == Architecture.AMD64:
+            result.append(Architecture.x86)
+        return result
+
+
 class BuildOptions(NamedTuple):
     package_dir: Path
     output_dir: Path
     build_selector: BuildSelector
+    architectures: List[Architecture]
     environment: ParsedEnvironment
     before_all: str
     before_build: Optional[str]
@@ -164,7 +201,7 @@ def strtobool(val: str) -> bool:
     return False
 
 
-class CIProvider(str, Enum):
+class CIProvider(Enum):
     travis_ci = 'travis'
     appveyor = 'appveyor'
     circle_ci = 'circle_ci'