From add032e76755ae1ae4420ec72e63ec7659403434 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 2 Feb 2025 09:46:41 +0100 Subject: [PATCH 1/2] feat: add armv7l in auto_archs when running on aarch64 This depends on aarch32 EL0 support and thus is done conditionally. --- cibuildwheel/architecture.py | 24 +++++++++++++++++++++--- examples/github-with-qemu.yml | 5 +---- test/test_manylinuxXXXX_only.py | 7 +++++++ test/test_musllinux_X_Y_only.py | 3 ++- test/test_testing.py | 2 +- test/utils.py | 15 ++++++++++++--- unit_test/architecture_test.py | 22 ++++++++++++++++++++-- unit_test/main_tests/conftest.py | 4 ++-- 8 files changed, 66 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 444d37017..43fa4bdb5 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -3,6 +3,8 @@ import functools import platform as platform_module import re +import shutil +import subprocess import sys from collections.abc import Set from enum import Enum @@ -24,6 +26,19 @@ ] +def _check_aarch32_el0() -> bool: + """Check if running armv7l natively on aarch64 is supported""" + if not sys.platform.startswith("linux"): + return False + if platform_module.machine() != "aarch64": + return False + executable = shutil.which("linux32") + if executable is None: + return False + check = subprocess.run([executable, "uname", "-m"], check=False, capture_output=True, text=True) + return check.returncode == 0 and check.stdout.startswith("armv") + + @functools.total_ordering class Architecture(Enum): value: str @@ -114,9 +129,12 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: return set() # can't build anything on this platform result = {native_arch} - if platform == "linux" and Architecture.x86_64 in result: - # x86_64 machines can run i686 containers - result.add(Architecture.i686) + if platform == "linux": + if Architecture.x86_64 in result: + # x86_64 machines can run i686 containers + result.add(Architecture.i686) + elif Architecture.aarch64 in result and _check_aarch32_el0(): + result.add(Architecture.armv7l) if platform == "windows" and Architecture.AMD64 in result: result.add(Architecture.x86) diff --git a/examples/github-with-qemu.yml b/examples/github-with-qemu.yml index ed4836a9c..0a1469299 100644 --- a/examples/github-with-qemu.yml +++ b/examples/github-with-qemu.yml @@ -26,10 +26,7 @@ jobs: # configure cibuildwheel on Linux to build native archs ('auto'), # and to split the remaining architectures between the x86_64 and # ARM runners - # armv7l can be built without QEMU on GitHub Actions ARM runners but that's - # not the case on all ARM64 hardware hence 'auto armv7l' for native archs - # on the GHA ARM64 runner - CIBW_ARCHS_LINUX: ${{ runner.arch == 'X64' && 'auto ppc64le s390x' || 'auto armv7l' }} + CIBW_ARCHS_LINUX: ${{ runner.arch == 'X64' && 'auto ppc64le s390x' || 'auto' }} - uses: actions/upload-artifact@v4 with: diff --git a/test/test_manylinuxXXXX_only.py b/test/test_manylinuxXXXX_only.py index 030ec88f9..b59f599b1 100644 --- a/test/test_manylinuxXXXX_only.py +++ b/test/test_manylinuxXXXX_only.py @@ -108,6 +108,9 @@ def test(manylinux_image, tmp_path): if manylinux_image in {"manylinux_2_28", "manylinux_2_34"} and platform.machine() == "x86_64": # We don't have a manylinux_2_28+ image for i686 add_env["CIBW_ARCHS"] = "x86_64" + if platform.machine() == "aarch64": + # We just have a manylinux_2_31 image for armv7l + add_env["CIBW_ARCHS"] = "aarch64" actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) @@ -150,4 +153,8 @@ def test(manylinux_image, tmp_path): # We don't have a manylinux_2_28+ image for i686 expected_wheels = [w for w in expected_wheels if "i686" not in w] + if platform.machine() == "aarch64": + # We just have a manylinux_2_31 image for armv7l + expected_wheels = [w for w in expected_wheels if "armv7l" not in w] + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_musllinux_X_Y_only.py b/test/test_musllinux_X_Y_only.py index 55ceb2163..0ddd3d2b9 100644 --- a/test/test_musllinux_X_Y_only.py +++ b/test/test_musllinux_X_Y_only.py @@ -38,7 +38,7 @@ def test(musllinux_image, tmp_path): # build the wheels add_env = { - "CIBW_SKIP": "*-manylinux*", + "CIBW_SKIP": "*-manylinux* *_armv7l", "CIBW_MUSLLINUX_X86_64_IMAGE": musllinux_image, "CIBW_MUSLLINUX_I686_IMAGE": musllinux_image, "CIBW_MUSLLINUX_AARCH64_IMAGE": musllinux_image, @@ -54,4 +54,5 @@ def test(musllinux_image, tmp_path): musllinux_versions=[musllinux_image], single_python=True, ) + expected_wheels = [w for w in expected_wheels if "armv7l" not in w] assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_testing.py b/test/test_testing.py index f052a13e2..486afe6d9 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -68,7 +68,7 @@ def test_uname(self): # See #336 for more info. bits = struct.calcsize("P") * 8 if bits == 32: - self.assertIn(platform.machine(), ["i686", "wasm32"]) + self.assertIn(platform.machine(), ["i686", "armv7l","armv8l", "wasm32"]) ''' diff --git a/test/utils.py b/test/utils.py index f956b5876..3357c681f 100644 --- a/test/utils.py +++ b/test/utils.py @@ -173,7 +173,7 @@ def expected_wheels( machine_arch = "aarch64" if manylinux_versions is None: - if machine_arch == "armv7l": + if machine_arch in ("armv7l", "aarch64"): manylinux_versions = ["manylinux_2_17", "manylinux2014", "manylinux_2_31"] elif machine_arch == "x86_64": manylinux_versions = [ @@ -250,14 +250,23 @@ def expected_wheels( if platform == "linux": architectures = [arch_name_for_linux(machine_arch)] - if machine_arch == "x86_64" and not single_arch: - architectures.append("i686") + if not single_arch: + if machine_arch == "x86_64": + architectures.append("i686") + elif ( + machine_arch == "aarch64" + and sys.platform.startswith("linux") + and not python_abi_tag.startswith("pp") + ): + # we assume all CI providers are able to run aarch32 EL0 on aarch64 + architectures.append("armv7l") if len(manylinux_versions) > 0: platform_tags = [ ".".join( f"{manylinux_version}_{architecture}" for manylinux_version in manylinux_versions + if (manylinux_version, architecture) != ("manylinux_2_31", "aarch64") ) for architecture in architectures ] diff --git a/unit_test/architecture_test.py b/unit_test/architecture_test.py index 1c7efd2ee..c1b23a5fd 100644 --- a/unit_test/architecture_test.py +++ b/unit_test/architecture_test.py @@ -1,10 +1,12 @@ from __future__ import annotations import platform as platform_module +import shutil import sys import pytest +import cibuildwheel.architecture from cibuildwheel.architecture import Architecture @@ -24,6 +26,7 @@ def platform_machine(request, monkeypatch): platform_name, platform_value, machine_value, machine_name = request.param monkeypatch.setattr(sys, "platform", platform_value) monkeypatch.setattr(platform_module, "machine", lambda: machine_value) + monkeypatch.setattr(cibuildwheel.architecture, "_check_aarch32_el0", lambda: True) return platform_name, machine_name @@ -34,7 +37,7 @@ def test_arch_auto(platform_machine): expected = { "32": {Architecture.i686}, "64": {Architecture.x86_64, Architecture.i686}, - "arm": {Architecture.aarch64}, + "arm": {Architecture.aarch64, Architecture.armv7l}, } assert arch_set == expected[machine_name] @@ -71,7 +74,7 @@ def test_arch_auto32(platform_machine): platform_name, machine_name = platform_machine arch_set = Architecture.parse_config("auto32", "linux") - expected = {"32": {Architecture.i686}, "64": {Architecture.i686}, "arm": set()} + expected = {"32": {Architecture.i686}, "64": {Architecture.i686}, "arm": {Architecture.armv7l}} assert arch_set == expected[machine_name] arch_set = Architecture.parse_config("auto32", "macos") @@ -80,3 +83,18 @@ def test_arch_auto32(platform_machine): arch_set = Architecture.parse_config("auto32", "windows") expected = {"32": {Architecture.x86}, "64": {Architecture.x86}, "arm": set()} assert arch_set == expected[machine_name] + + +def test_arch_auto_no_aarch32(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(platform_module, "machine", lambda: "aarch64") + monkeypatch.setattr(shutil, "which", lambda *args, **kwargs: None) + + arch_set = Architecture.parse_config("auto", "linux") + assert arch_set == {Architecture.aarch64} + + arch_set = Architecture.parse_config("auto64", "linux") + assert arch_set == {Architecture.aarch64} + + arch_set = Architecture.parse_config("auto32", "linux") + assert len(arch_set) == 0 diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 2235973e4..e93a4b742 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -8,7 +8,7 @@ import pytest -from cibuildwheel import __main__, linux, macos, pyodide, windows +from cibuildwheel import __main__, architecture, linux, macos, pyodide, windows from cibuildwheel.util import file @@ -44,8 +44,8 @@ def ignore_call(*args, **kwargs): monkeypatch.setattr(linux, "build", fail_on_call) monkeypatch.setattr(macos, "build", fail_on_call) monkeypatch.setattr(pyodide, "build", fail_on_call) - monkeypatch.setattr(Path, "mkdir", ignore_call) + monkeypatch.setattr(architecture, "_check_aarch32_el0", lambda: True) @pytest.fixture(autouse=True) From 19f08458f665c9fa4146f144bc7dac97542336f3 Mon Sep 17 00:00:00 2001 From: mayeut Date: Wed, 5 Feb 2025 17:43:36 +0100 Subject: [PATCH 2/2] ci(travis): move to Ubuntu 22.04 / cp312 --- .travis.yml | 36 ++++++++++---------------- CI.md | 12 ++++----- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 2 +- test/utils.py | 8 +++++- unit_test/oci_container_test.py | 4 +-- 7 files changed, 31 insertions(+), 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index 808459794..f3989233a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ os: linux -dist: focal +dist: jammy language: python branches: @@ -8,29 +8,19 @@ branches: jobs: include: - - name: Linux | x86_64 + i686 | Python 3.11 - python: 3.11 + - name: Linux | x86_64 + i686 | Python 3.12 + python: 3.12 services: docker env: PYTHON=python - - name: Linux | arm64 | Python 3.11 - python: 3.11 + - name: Linux | arm64 | Python 3.12 + python: 3.12 services: docker - arch: arm64-graviton2 - group: edge - virt: vm + arch: arm64 env: PYTHON=python - # docker is outdated in the arm64-graviton2 vm focal image (19.x) - # we need to upgrade to get >= 24.0 - addons: - apt: - sources: - - sourceline: 'deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable' - packages: - - docker-ce docker-ce-cli containerd.io - - name: Linux | ppc64le | Python 3.11 - python: 3.11 + - name: Linux | ppc64le | Python 3.12 + python: 3.12 services: docker arch: ppc64le allow_failure: True @@ -40,16 +30,16 @@ jobs: # c.f. https://travis-ci.community/t/running-out-of-disk-space-quota-when-using-docker-on-ppc64le/11634 - PYTEST_ADDOPTS='-k "not test_manylinuxXXXX_only"' - - name: Windows | x86_64 | Python 3.11 + - name: Windows | x86_64 | Python 3.12 os: windows language: shell before_install: - - choco upgrade python3 -y --version 3.11.9 --limit-output --params "/InstallDir:C:\\Python311" + - choco upgrade python3 -y --version 3.12.8 --limit-output --params "/InstallDir:C:\\Python312" env: - - PYTHON=C:\\Python311\\python + - PYTHON=C:\\Python312\\python - - name: Linux | s390x | Python 3.11 - python: 3.11 + - name: Linux | s390x | Python 3.12 + python: 3.12 services: docker arch: s390x allow_failure: True diff --git a/CI.md b/CI.md index 15de33c53..d1282bc36 100644 --- a/CI.md +++ b/CI.md @@ -1,11 +1,11 @@ This is a summary of the host Python versions and platforms covered by the different CI platforms: -| | 3.11 | 3.12 | 3.13 | -|---------|----------------------------------------------|---------------------------------------------|----------------| -| Linux | Azure Pipelines / GitHub Actions / Travis CI | AppVeyor¹ / CircleCI¹ / Cirrus CI / GitLab¹ | GitHub Actions | -| macOS | Azure Pipelines / GitLab¹ | AppVeyor¹ / CircleCI¹ / Cirrus CI / GitLab¹ | GitHub Actions | -| Windows | Azure Pipelines / Travis CI | AppVeyor¹ / Cirrus CI / GitLab¹ | GitHub Actions | +| | 3.11 | 3.12 | 3.13 | +|---------|----------------------------------|---------------------------------------------------------|----------------| +| Linux | Azure Pipelines / GitHub Actions | AppVeyor¹ / CircleCI¹ / Cirrus CI / GitLab¹ / Travis CI | GitHub Actions | +| macOS | Azure Pipelines | AppVeyor¹ / CircleCI¹ / Cirrus CI / GitLab¹ | GitHub Actions | +| Windows | Azure Pipelines | AppVeyor¹ / Cirrus CI / GitLab¹ / Travis CI | GitHub Actions | > ¹ Runs a reduced set of tests to reduce CI load -Non-x86 architectures are covered on Travis CI using Python 3.11. +Non-x86 architectures are covered on Travis CI using Python 3.12. diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index 14457de76..549efdf33 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -2,7 +2,7 @@ # commit, but will only push to PyPI on tagged commits. os: linux -dist: focal +dist: jammy language: python python: "3.12" diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index 5dcd28d61..8a810833c 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -1,5 +1,5 @@ os: linux -dist: focal +dist: jammy language: python python: "3.12" diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index 6e8d181a4..f13ca80ed 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -6,7 +6,7 @@ # distribution is also created. os: linux -dist: focal +dist: jammy language: python python: "3.12" diff --git a/test/utils.py b/test/utils.py index 3357c681f..467bee14f 100644 --- a/test/utils.py +++ b/test/utils.py @@ -18,6 +18,7 @@ import pytest from cibuildwheel.architecture import Architecture +from cibuildwheel.ci import CIProvider, detect_ci_provider from cibuildwheel.util.file import CIBW_CACHE_PATH EMULATED_ARCHS: Final[list[str]] = sorted( @@ -25,6 +26,11 @@ ) SINGLE_PYTHON_VERSION: Final[tuple[int, int]] = (3, 12) +_AARCH64_CAN_RUN_ARMV7: Final[bool] = Architecture.aarch64.value not in EMULATED_ARCHS and { + None: Architecture.armv7l.value not in EMULATED_ARCHS, + CIProvider.travis_ci: False, +}.get(detect_ci_provider(), True) + platform = os.environ.get("CIBW_PLATFORM", "") if platform: pass @@ -257,8 +263,8 @@ def expected_wheels( machine_arch == "aarch64" and sys.platform.startswith("linux") and not python_abi_tag.startswith("pp") + and _AARCH64_CAN_RUN_ARMV7 ): - # we assume all CI providers are able to run aarch32 EL0 on aarch64 architectures.append("armv7l") if len(manylinux_versions) > 0: diff --git a/unit_test/oci_container_test.py b/unit_test/oci_container_test.py index bc3985177..95afddd1a 100644 --- a/unit_test/oci_container_test.py +++ b/unit_test/oci_container_test.py @@ -555,7 +555,7 @@ def test_local_image( ) -> None: if ( detect_ci_provider() in {CIProvider.travis_ci} - and pm in {"s390x", "ppc64le"} + and pm != "x86_64" and platform != DEFAULT_OCI_PLATFORM ): pytest.skip("Skipping test because docker on this platform does not support QEMU") @@ -585,7 +585,7 @@ def test_local_image( def test_multiarch_image(container_engine, platform): if ( detect_ci_provider() in {CIProvider.travis_ci} - and pm in {"s390x", "ppc64le"} + and pm != "x86_64" and platform != DEFAULT_OCI_PLATFORM ): pytest.skip("Skipping test because docker on this platform does not support QEMU")