From 9e7d9019b64c9101b3788294f922fef0f389c975 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 27 May 2024 00:36:47 -0400 Subject: [PATCH 01/14] feat: support uv Signed-off-by: Henry Schreiner --- .github/workflows/test.yml | 9 +- .pre-commit-config.yaml | 1 + cibuildwheel/linux.py | 55 +++++--- cibuildwheel/macos.py | 122 +++++++++++------ cibuildwheel/pyodide.py | 2 +- .../resources/constraints-python310.txt | 2 + .../resources/constraints-python311.txt | 2 + .../resources/constraints-python312.txt | 2 + .../resources/constraints-python313.txt | 2 + .../resources/constraints-python38.txt | 2 + .../resources/constraints-python39.txt | 2 + cibuildwheel/resources/constraints.in | 1 + cibuildwheel/resources/constraints.txt | 2 + cibuildwheel/util.py | 123 +++++++++++------- cibuildwheel/windows.py | 105 ++++++++++----- pyproject.toml | 1 + 16 files changed, 295 insertions(+), 138 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 196ebd7aa..b7c501cab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,9 @@ concurrency: group: test-${{ github.ref }} cancel-in-progress: true +env: + CIBW_BUILD_FRONTEND: "build[uv]" + jobs: lint: name: Linters (mypy, flake8, etc.) @@ -49,6 +52,8 @@ jobs: with: python-version: ${{ matrix.python_version }} + - uses: yezz123/setup-uv@v4 + # Install podman on this CI instance for podman tests on linux # Snippet from: https://github.com/redhat-actions/podman-login/blob/main/.github/workflows/example.yml - name: Install latest podman @@ -69,7 +74,7 @@ jobs: - name: Install dependencies run: | - python -m pip install ".[test]" + uv pip install --system ".[test]" - name: Generate a sample project run: | @@ -159,7 +164,7 @@ jobs: with: python-version: "3.x" - name: Install dependencies - run: python -m pip install ".[test]" + run: python -m pip install ".[test,uv]" - name: Set up QEMU id: qemu diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb3ac4731..38899cef9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,6 +40,7 @@ repos: - types-jinja2 - types-pyyaml - types-requests + - uv - validate-pyproject - id: mypy name: mypy 3.12 diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index c33c0de4f..8e885cbce 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -196,6 +196,8 @@ def build_in_container( log.build_start(config.identifier) build_options = options.build_options(config.identifier) build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") + use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version("3.8") + pip = ["uv", "pip"] if use_uv else ["pip"] dependency_constraint_flags: list[PathOrStr] = [] @@ -229,13 +231,22 @@ def build_in_container( ) sys.exit(1) - which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() - if PurePosixPath(which_pip) != python_bin / "pip": - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) + if use_uv: + which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() + if not which_uv: + print( + "cibuildwheel: uv not found on PATH. You must use a supported manylinux or musllinux environment with uv.", + file=sys.stderr, + ) + sys.exit(1) + else: + which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() + if PurePosixPath(which_pip) != python_bin / "pip": + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -279,10 +290,12 @@ def build_in_container( ], env=env, ) - elif build_frontend.name == "build": + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) + if use_uv: + extra_flags += ["--installer=uv"] container.call( [ "python", @@ -327,26 +340,32 @@ def build_in_container( # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - container.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) + if not use_uv: + container.call( + ["pip", "install", "virtualenv", *dependency_constraint_flags], env=env + ) testing_temp_dir = PurePosixPath( container.call(["mktemp", "-d"], capture_output=True).strip() ) venv_dir = testing_temp_dir / "venv" - # Use embedded dependencies from virtualenv to ensure determinism - venv_args = ["--no-periodic-update", "--pip=embed"] - # In Python<3.12, setuptools & wheel are installed as well - if Version(config.version) < Version("3.12"): - venv_args.extend(("--setuptools=embed", "--wheel=embed")) - container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) + if use_uv: + container.call(["uv", "venv", venv_dir], env=env) + else: + # Use embedded dependencies from virtualenv to ensure determinism + venv_args = ["--no-periodic-update", "--pip=embed"] + # In Python<3.12, setuptools & wheel are installed as well + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) # TODO remove me once virtualenv provides pip>=24.1b1 - if config.version == "3.13": + if config.version == "3.13" and not use_uv: container.call(["pip", "install", "pip>=24.1b1"], env=virtualenv_env) if build_options.before_test: @@ -365,13 +384,13 @@ def build_in_container( # Let's just pick the first one. wheel_to_test = repaired_wheels[0] container.call( - ["pip", "install", str(wheel_to_test) + build_options.test_extras], + [*pip, "install", str(wheel_to_test) + build_options.test_extras], env=virtualenv_env, ) # Install any requirements to run the tests if build_options.test_requires: - container.call(["pip", "install", *build_options.test_requires], env=virtualenv_env) + container.call([*pip, "install", *build_options.test_requires], env=virtualenv_env) # Run the tests from a different directory test_command_prepared = prepare_command( diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 813531b09..f742baac7 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -33,6 +33,7 @@ detect_ci_provider, download, find_compatible_wheel, + find_uv, get_build_verbosity_extra_flags, get_pip_version, install_certifi_script, @@ -168,6 +169,14 @@ def setup_python( environment: ParsedEnvironment, build_frontend: BuildFrontendName, ) -> dict[str, str]: + if build_frontend == "build[uv]" and Version(python_configuration.version) < Version("3.8"): + build_frontend = "build" + + uv_path = find_uv() + use_uv = build_frontend == "build[uv]" and Version(python_configuration.version) >= Version( + "3.8" + ) + tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") @@ -183,7 +192,11 @@ def setup_python( log.step("Setting up build environment...") venv_path = tmp / "venv" env = virtualenv( - python_configuration.version, base_python, venv_path, dependency_constraint_flags + python_configuration.version, + base_python, + venv_path, + dependency_constraint_flags, + use_uv=use_uv, ) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() @@ -200,32 +213,38 @@ def setup_python( # upgrade pip to the version matching our constraints # if necessary, reinstall it to ensure that it's available on PATH as 'pip' - call( - "python", - "-m", - "pip", - "install", - "--upgrade", - "pip", - *dependency_constraint_flags, - env=env, - cwd=venv_path, - ) + if build_frontend == "build[uv]": + assert uv_path is not None + pip = [str(uv_path), "pip"] + else: + pip = ["python", "-m", "pip"] + + if not use_uv: + call( + *pip, + "install", + "--upgrade", + "pip", + *dependency_constraint_flags, + env=env, + cwd=venv_path, + ) # Apply our environment after pip is ready env = environment.as_dictionary(prev_environment=env) # check what pip version we're on - assert (venv_bin_path / "pip").exists() - call("which", "pip", env=env) - call("pip", "--version", env=env) - which_pip = call("which", "pip", env=env, capture_stdout=True).strip() - if which_pip != str(venv_bin_path / "pip"): - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) + if not use_uv: + assert (venv_bin_path / "pip").exists() + call("which", "pip", env=env) + call("pip", "--version", env=env) + which_pip = call("which", "pip", env=env, capture_stdout=True).strip() + if which_pip != str(venv_bin_path / "pip"): + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) # check what Python version we're on call("which", "python", env=env) @@ -304,6 +323,18 @@ def setup_python( *dependency_constraint_flags, env=env, ) + elif build_frontend == "build[uv]": + assert uv_path is not None + call( + uv_path, + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv, uv]", + *dependency_constraint_flags, + env=env, + ) else: assert_never(build_frontend) @@ -336,6 +367,14 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") + use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version( + "3.8" + ) + uv_path = find_uv() + if use_uv and uv_path is None: + msg = "uv not found" + raise AssertionError(msg) + pip = ["pip"] if not use_uv else [str(uv_path), "pip"] log.build_start(config.identifier) identifier_tmp_dir = tmp_path / config.identifier @@ -360,7 +399,8 @@ def build(options: Options, tmp_path: Path) -> None: build_options.environment, build_frontend.name, ) - pip_version = get_pip_version(env) + if not use_uv: + pip_version = get_pip_version(env) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -386,14 +426,16 @@ def build(options: Options, tmp_path: Path) -> None: extra_flags += build_frontend.args build_env = env.copy() - build_env["VIRTUALENV_PIP"] = pip_version + if not use_uv: + build_env["VIRTUALENV_PIP"] = pip_version if build_options.dependency_constraints: constraint_path = build_options.dependency_constraints.get_for_python_version( config.version ) - user_constraints = build_env.get("PIP_CONSTRAINT") - our_constraints = constraint_path.as_uri() - build_env["PIP_CONSTRAINT"] = " ".join( + pip_constraint = "UV_CONSTRAINT" if use_uv else "PIP_CONSTRAINT" + user_constraints = build_env.get(pip_constraint) + our_constraints = str(constraint_path) if use_uv else constraint_path.as_uri() + build_env[pip_constraint] = " ".join( c for c in [user_constraints, our_constraints] if c ) @@ -412,10 +454,12 @@ def build(options: Options, tmp_path: Path) -> None: *extra_flags, env=build_env, ) - elif build_frontend.name == "build": + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) + if use_uv: + extra_flags.append("--installer=uv") call( "python", "-m", @@ -545,7 +589,8 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + if not use_uv: + call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" @@ -562,12 +607,15 @@ def build(options: Options, tmp_path: Path) -> None: call_with_arch = functools.partial(call, *arch_prefix) shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") - # Use pip version from the initial env to ensure determinism - venv_args = ["--no-periodic-update", f"--pip={pip_version}"] - # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones - if Version(config.version) < Version("3.12"): - venv_args.extend(("--setuptools=embed", "--wheel=embed")) - call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) + if use_uv: + call_with_arch("uv", "venv", venv_dir, "--python=python", env=env) + else: + # Use pip version from the initial env to ensure determinism + venv_args = ["--no-periodic-update", f"--pip={pip_version}"] + # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( @@ -608,7 +656,7 @@ def build(options: Options, tmp_path: Path) -> None: virtualenv_env_install_wheel = virtualenv_env call_with_arch( - "pip", + *pip, "install", f"{repaired_wheel}{build_options.test_extras}", env=virtualenv_env_install_wheel, @@ -617,7 +665,7 @@ def build(options: Options, tmp_path: Path) -> None: # test the wheel if build_options.test_requires: call_with_arch( - "pip", + *pip, "install", *build_options.test_requires, env=virtualenv_env_install_wheel, diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index e9e75a5fe..780e43cee 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -114,7 +114,7 @@ def setup_python( log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(python_configuration.version, base_python, venv_path, []) + env = virtualenv(python_configuration.version, base_python, venv_path, [], use_uv=False) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index 434ad8757..c842117c7 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -28,6 +28,8 @@ tomli==2.0.1 # via build typing-extensions==4.12.2 # via delocate +uv==0.2.8 + # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in zipp==3.19.2 diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index f2521f8bb..0ca4abac6 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -24,5 +24,7 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate +uv==0.2.8 + # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt index f2521f8bb..0ca4abac6 100644 --- a/cibuildwheel/resources/constraints-python312.txt +++ b/cibuildwheel/resources/constraints-python312.txt @@ -24,5 +24,7 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate +uv==0.2.8 + # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index 5fcc3ef67..704287adc 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -24,5 +24,7 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate +uv==0.2.8 + # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index 434ad8757..c842117c7 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -28,6 +28,8 @@ tomli==2.0.1 # via build typing-extensions==4.12.2 # via delocate +uv==0.2.8 + # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in zipp==3.19.2 diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 434ad8757..c842117c7 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -28,6 +28,8 @@ tomli==2.0.1 # via build typing-extensions==4.12.2 # via delocate +uv==0.2.8 + # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in zipp==3.19.2 diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 4f01a4b8c..21831fd4e 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -2,4 +2,5 @@ pip>=24.1b1 ; python_version >= '3.13' pip ; python_version < '3.13' build delocate +uv; python_version >= '3.8' virtualenv diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index f2521f8bb..0ca4abac6 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -24,5 +24,7 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate +uv==0.2.8 + # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 2d127f2bc..86d244a77 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -40,20 +40,21 @@ from .typing import PathOrStr, PlatformName __all__ = [ - "resources_dir", "MANYLINUX_ARCHS", + "cached_property", "call", - "shell", + "chdir", "find_compatible_wheel", + "find_uv", "format_safe", - "prepare_command", "get_build_verbosity_extra_flags", + "prepare_command", "read_python_configs", + "resources_dir", "selector_matches", - "strtobool", - "cached_property", - "chdir", + "shell", "split_config_settings", + "strtobool", ] resources_dir: Final[Path] = Path(__file__).parent / "resources" @@ -200,7 +201,9 @@ def get_build_verbosity_extra_flags(level: int) -> list[str]: return [] -def split_config_settings(config_settings: str, frontend: Literal["pip", "build"]) -> list[str]: +def split_config_settings( + config_settings: str, frontend: Literal["pip", "build", "build[uv]"] +) -> list[str]: config_settings_list = shlex.split(config_settings) s = "s" if frontend == "pip" else "" return [f"--config-setting{s}={setting}" for setting in config_settings_list] @@ -435,7 +438,7 @@ def options_summary(self) -> Any: return self.base_file_path.name -BuildFrontendName = Literal["pip", "build"] +BuildFrontendName = Literal["pip", "build", "build[uv]"] @dataclass(frozen=True) @@ -447,8 +450,8 @@ class BuildFrontendConfig: def from_config_string(config_string: str) -> BuildFrontendConfig: config_dict = parse_key_value_string(config_string, ["name"], ["args"]) name = " ".join(config_dict["name"]) - if name not in {"pip", "build"}: - msg = f"Unrecognised build frontend {name}, only 'pip' and 'build' are supported" + if name not in {"pip", "build", "build[uv]"}: + msg = f"Unrecognised build frontend {name}, only 'pip', 'build', and 'build[uv]' are supported" raise ValueError(msg) name = typing.cast(BuildFrontendName, name) @@ -693,45 +696,60 @@ def _parse_constraints_for_virtualenv( def virtualenv( - version: str, python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr] + version: str, + python: Path, + venv_path: Path, + dependency_constraint_flags: Sequence[PathOrStr], + *, + use_uv: bool, ) -> dict[str, str]: + """ + Create a virtual environment. If `use_uv` is True, + dependency_constraint_flags are ignored since nothing is installed in the + venv. Otherwise, pip is installed, and setuptools + wheel if Python < 3.12. + """ assert python.exists() - virtualenv_app = _ensure_virtualenv(version) - allowed_seed_packages = ["pip", "setuptools", "wheel"] - constraints = _parse_constraints_for_virtualenv( - allowed_seed_packages, dependency_constraint_flags - ) - additional_flags: list[str] = [] - for package in allowed_seed_packages: - if package in constraints: - additional_flags.append(f"--{package}={constraints[package]}") - else: - additional_flags.append(f"--no-{package}") - - # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual - # environment. The initial cost is a bit higher but reusing is much faster. - # Windows does not always allow symlinks so just disabling for now. - # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the - # version of pip that will end-up installed. - # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder - if ( - not IS_WIN - and constraints["pip"] != "embed" - and Version(constraints["pip"]) >= Version("19.3") - ): - additional_flags.append("--symlink-app-data") - - call( - sys.executable, - "-sS", # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125 - virtualenv_app, - "--activators=", - "--no-periodic-update", - *additional_flags, - "--python", - python, - venv_path, - ) + + if use_uv: + call("uv", "venv", venv_path, "--python", python) + else: + virtualenv_app = _ensure_virtualenv(version) + allowed_seed_packages = ["pip", "setuptools", "wheel"] + constraints = _parse_constraints_for_virtualenv( + allowed_seed_packages, dependency_constraint_flags + ) + additional_flags: list[str] = [] + for package in allowed_seed_packages: + if package in constraints: + additional_flags.append(f"--{package}={constraints[package]}") + else: + additional_flags.append(f"--no-{package}") + + # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual + # environment. The initial cost is a bit higher but reusing is much faster. + # Windows does not always allow symlinks so just disabling for now. + # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the + # version of pip that will end-up installed. + # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder + if ( + not IS_WIN + and constraints["pip"] != "embed" + and Version(constraints["pip"]) >= Version("19.3") + ): + additional_flags.append("--symlink-app-data") + + call( + sys.executable, + "-sS", # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125 + virtualenv_app, + "--activators=", + "--no-periodic-update", + *additional_flags, + "--python", + python, + venv_path, + ) + paths = [str(venv_path), str(venv_path / "Scripts")] if IS_WIN else [str(venv_path / "bin")] env = os.environ.copy() env["PATH"] = os.pathsep.join([*paths, env["PATH"]]) @@ -877,3 +895,14 @@ def parse_key_value_string( result[field_name] += values return dict(result) + + +def find_uv() -> Path | None: + # Prefer uv in our environment + with contextlib.suppress(ImportError, FileNotFoundError): + from uv import find_uv_bin + + return Path(find_uv_bin()) + + uv_on_path = shutil.which("uv") + return Path(uv_on_path) if uv_on_path else None diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f5daa28c6..75d724398 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -31,6 +31,7 @@ download, extract_zip, find_compatible_wheel, + find_uv, get_build_verbosity_extra_flags, get_pip_version, move_file, @@ -244,10 +245,19 @@ def setup_python( raise ValueError(msg) assert base_python.exists() + use_uv = build_frontend == "build[uv]" and Version(python_configuration.version) >= Version( + "3.8" + ) + uv_path = find_uv() + log.step("Setting up build environment...") venv_path = tmp / "venv" env = virtualenv( - python_configuration.version, base_python, venv_path, dependency_constraint_flags + python_configuration.version, + base_python, + venv_path, + dependency_constraint_flags, + use_uv=use_uv, ) # set up environment variables for run_with_env @@ -257,17 +267,18 @@ def setup_python( # upgrade pip to the version matching our constraints # if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe' - call( - "python", - "-m", - "pip", - "install", - "--upgrade", - "pip", - *dependency_constraint_flags, - env=env, - cwd=venv_path, - ) + if not use_uv: + call( + "python", + "-m", + "pip", + "install", + "--upgrade", + "pip", + *dependency_constraint_flags, + env=env, + cwd=venv_path, + ) # update env with results from CIBW_ENVIRONMENT env = environment.as_dictionary(prev_environment=env) @@ -285,16 +296,17 @@ def setup_python( sys.exit(1) # check what pip version we're on - assert (venv_path / "Scripts" / "pip.exe").exists() - where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip() - if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"): - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) + if not use_uv: + assert (venv_path / "Scripts" / "pip.exe").exists() + where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip() + if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"): + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) - call("pip", "--version", env=env) + call("pip", "--version", env=env) log.step("Installing build tools...") if build_frontend == "build": @@ -306,6 +318,17 @@ def setup_python( *dependency_constraint_flags, env=env, ) + elif build_frontend == "build[uv]": + assert uv_path is not None + call( + uv_path, + "pip", + "install", + "--upgrade", + "build[virtualenv]", + *dependency_constraint_flags, + env=env, + ) if python_libs_base: # Set up the environment for various backends to enable cross-compilation @@ -340,6 +363,9 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") + use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version( + "3.8" + ) log.build_start(config.identifier) identifier_tmp_dir = tmp_path / config.identifier @@ -362,7 +388,8 @@ def build(options: Options, tmp_path: Path) -> None: build_options.environment, build_frontend.name, ) - pip_version = get_pip_version(env) + if not use_uv: + pip_version = get_pip_version(env) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -391,7 +418,9 @@ def build(options: Options, tmp_path: Path) -> None: extra_flags += build_frontend.args build_env = env.copy() - build_env["VIRTUALENV_PIP"] = pip_version + if not use_uv: + build_env["VIRTUALENV_PIP"] = pip_version + if build_options.dependency_constraints: constraints_path = build_options.dependency_constraints.get_for_python_version( config.version @@ -411,6 +440,7 @@ def build(options: Options, tmp_path: Path) -> None: build_env["PIP_CONSTRAINT"] = " ".join( c for c in [our_constraints, user_constraints] if c ) + build_env["UV_CONSTRAINT"] = build_env["PIP_CONSTRAINT"] if build_frontend.name == "pip": extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity) @@ -427,10 +457,12 @@ def build(options: Options, tmp_path: Path) -> None: *extra_flags, env=build_env, ) - elif build_frontend.name == "build": + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) + if use_uv: + extra_flags.append("--installer=uv") call( "python", "-m", @@ -484,15 +516,20 @@ def build(options: Options, tmp_path: Path) -> None: log.step("Testing wheel...") # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + if not use_uv: + call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + venv_dir = identifier_tmp_dir / "venv-test" - # Use pip version from the initial env to ensure determinism - venv_args = ["--no-periodic-update", f"--pip={pip_version}"] - # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones - if Version(config.version) < Version("3.12"): - venv_args.extend(("--setuptools=embed", "--wheel=embed")) - call("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) + if use_uv: + call("uv", "venv", venv_dir, "--python=python", env=env) + else: + # Use pip version from the initial env to ensure determinism + venv_args = ["--no-periodic-update", f"--pip={pip_version}"] + # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + call("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( @@ -514,9 +551,11 @@ def build(options: Options, tmp_path: Path) -> None: ) shell(before_test_prepared, env=virtualenv_env) + pip = ["uv", "pip"] if use_uv else ["pip"] + # install the wheel call( - "pip", + *pip, "install", str(repaired_wheel) + build_options.test_extras, env=virtualenv_env, @@ -524,7 +563,7 @@ def build(options: Options, tmp_path: Path) -> None: # test the wheel if build_options.test_requires: - call("pip", "install", *build_options.test_requires, env=virtualenv_env) + call(*pip, "install", *build_options.test_requires, env=virtualenv_env) # run the tests from a temp dir, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel diff --git a/pyproject.toml b/pyproject.toml index 23f8c8c45..d7a5a7e40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ test = [ "tomli_w", "validate-pyproject", ] +uv = ["uv"] [project.scripts] cibuildwheel = "cibuildwheel.__main__:main" From d825136b04c46e8518ba3032d347434de14ff1b7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 6 Jun 2024 15:51:06 -0400 Subject: [PATCH 02/14] Apply suggestions from code review --- .github/workflows/test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7c501cab..edd86dee7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,9 +15,6 @@ concurrency: group: test-${{ github.ref }} cancel-in-progress: true -env: - CIBW_BUILD_FRONTEND: "build[uv]" - jobs: lint: name: Linters (mypy, flake8, etc.) From 25a45d6648c4bd8443e0cbc59ce4b54589145471 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 7 Jun 2024 01:13:48 -0400 Subject: [PATCH 03/14] ci: use uv in a few places Signed-off-by: Henry Schreiner --- .github/workflows/test.yml | 1 + test/conftest.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edd86dee7..9f50ea949 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,6 +84,7 @@ jobs: output-dir: wheelhouse env: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_BUILD_FRONTEND: 'build[uv]' - name: Run a sample build (GitHub Action, only) uses: ./ diff --git a/test/conftest.py b/test/conftest.py index 08b0ee11f..93b6de69b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,7 +6,7 @@ import pytest -from cibuildwheel.util import detect_ci_provider +from cibuildwheel.util import detect_ci_provider, find_uv from .utils import EMULATED_ARCHS, platform @@ -34,6 +34,10 @@ def pytest_addoption(parser) -> None: def build_frontend_env(request) -> dict[str, str]: if platform == "pyodide": pytest.skip("Can't use pip as build frontend for pyodide platform") + + if request.param["CIBW_BUILD_FRONTEND"] == "build" and find_uv() is not None: + return {"CIBW_BUILD_FRONTEND": "build[uv]"} + return request.param # type: ignore[no-any-return] From c57cf3d40bc24df45ab6fa8f8202311cd2fd31d6 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 7 Jun 2024 01:42:03 -0400 Subject: [PATCH 04/14] tests: drop pip from check Signed-off-by: Henry Schreiner --- test/test_dependency_versions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 17edd10f3..318ea46c0 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -101,7 +101,6 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env): project_with_expected_version_checks.generate(project_dir) tool_versions = { - "pip": "23.1.2", "delocate": "0.10.3", } @@ -109,7 +108,6 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env): constraints_file.write_text( textwrap.dedent( """ - pip=={pip} delocate=={delocate} """.format(**tool_versions) ) From 93dc643c2d8dc296171ca13713c356f3bab9aedd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 7 Jun 2024 03:12:53 -0400 Subject: [PATCH 05/14] tests: skip uv on the dep constraints tests Signed-off-by: Henry Schreiner --- test/conftest.py | 11 ++++++++--- test/test_dependency_versions.py | 10 ++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 93b6de69b..3fbf1d67c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -31,14 +31,19 @@ def pytest_addoption(parser) -> None: @pytest.fixture( params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"] ) -def build_frontend_env(request) -> dict[str, str]: +def build_frontend_env_nouv(request) -> dict[str, str]: if platform == "pyodide": pytest.skip("Can't use pip as build frontend for pyodide platform") - if request.param["CIBW_BUILD_FRONTEND"] == "build" and find_uv() is not None: + return request.param # type: ignore[no-any-return] + + +@pytest.fixture() +def build_frontend_env(build_frontend_env_nouv: dict[str, str]) -> dict[str, str]: + if build_frontend_env_nouv["CIBW_BUILD_FRONTEND"] == "build" and find_uv() is not None: return {"CIBW_BUILD_FRONTEND": "build[uv]"} - return request.param # type: ignore[no-any-return] + return build_frontend_env_nouv @pytest.fixture() diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 318ea46c0..defefc747 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -53,7 +53,7 @@ def get_versions_from_constraint_file(constraint_file): @pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.10"]) -def test_pinned_versions(tmp_path, python_version, build_frontend_env): +def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") if python_version == "3.6" and utils.platform == "macos" and platform.machine() == "arm64": @@ -79,7 +79,7 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env): add_env={ "CIBW_BUILD": build_pattern, "CIBW_ENVIRONMENT": cibw_environment_option, - **build_frontend_env, + **build_frontend_env_nouv, }, ) @@ -93,7 +93,7 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env): assert set(actual_wheels) == set(expected_wheels) -def test_dependency_constraints_file(tmp_path, build_frontend_env): +def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -101,6 +101,7 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env): project_with_expected_version_checks.generate(project_dir) tool_versions = { + "pip": "23.1.2", "delocate": "0.10.3", } @@ -108,6 +109,7 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env): constraints_file.write_text( textwrap.dedent( """ + pip=={pip} delocate=={delocate} """.format(**tool_versions) ) @@ -128,7 +130,7 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env): "CIBW_ENVIRONMENT": cibw_environment_option, "CIBW_DEPENDENCY_VERSIONS": str(constraints_file), "CIBW_SKIP": "cp36-*", - **build_frontend_env, + **build_frontend_env_nouv, }, ) From 2ea1c04723cd3d902b70be09111ca4d8596c2f47 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 8 Jun 2024 01:01:18 -0400 Subject: [PATCH 06/14] fix: pull out windows workaround Signed-off-by: Henry Schreiner --- cibuildwheel/macos.py | 8 +++----- cibuildwheel/util.py | 31 ++++++++++++++++++++++++++++++- cibuildwheel/windows.py | 18 ++---------------- pyproject.toml | 1 + 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index f742baac7..fd6d5e45a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -30,6 +30,7 @@ BuildSelector, NonPlatformWheelError, call, + combine_constraints, detect_ci_provider, download, find_compatible_wheel, @@ -432,11 +433,8 @@ def build(options: Options, tmp_path: Path) -> None: constraint_path = build_options.dependency_constraints.get_for_python_version( config.version ) - pip_constraint = "UV_CONSTRAINT" if use_uv else "PIP_CONSTRAINT" - user_constraints = build_env.get(pip_constraint) - our_constraints = str(constraint_path) if use_uv else constraint_path.as_uri() - build_env[pip_constraint] = " ".join( - c for c in [user_constraints, our_constraints] if c + combine_constraints( + build_env, constraint_path, identifier_tmp_dir if use_uv else None ) if build_frontend.name == "pip": diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 86d244a77..bff2baf4d 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -16,7 +16,7 @@ import typing import urllib.request from collections import defaultdict -from collections.abc import Generator, Iterable, Mapping, Sequence +from collections.abc import Generator, Iterable, Mapping, MutableMapping, Sequence from dataclasses import dataclass from enum import Enum from functools import cached_property, lru_cache @@ -44,6 +44,7 @@ "cached_property", "call", "chdir", + "combine_constraints", "find_compatible_wheel", "find_uv", "format_safe", @@ -900,9 +901,37 @@ def parse_key_value_string( def find_uv() -> Path | None: # Prefer uv in our environment with contextlib.suppress(ImportError, FileNotFoundError): + # pylint: disable-next=import-outside-toplevel from uv import find_uv_bin return Path(find_uv_bin()) uv_on_path = shutil.which("uv") return Path(uv_on_path) if uv_on_path else None + + +def combine_constraints( + env: MutableMapping[str, str], /, constraints_path: Path, tmp_dir: Path | None +) -> None: + """ + This will workaround a bug in pip<=21.1.1 or uv<=0.2.0 if a tmp_dir is given. + If set to None, this will use the modern URI method. + """ + + if tmp_dir: + if " " in str(constraints_path): + assert " " not in str(tmp_dir) + tmp_file = tmp_dir / "constraints.txt" + tmp_file.write_bytes(constraints_path.read_bytes()) + constraints_path = tmp_file + our_constraints = str(constraints_path) + else: + our_constraints = ( + constraints_path.as_uri() if " " in str(constraints_path) else str(constraints_path) + ) + + user_constraints = env.get("PIP_CONSTRAINT") + + env["UV_CONSTRAINT"] = env["PIP_CONSTRAINT"] = " ".join( + c for c in [our_constraints, user_constraints] if c + ) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 75d724398..263e8cf9a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -28,6 +28,7 @@ BuildSelector, NonPlatformWheelError, call, + combine_constraints, download, extract_zip, find_compatible_wheel, @@ -425,22 +426,7 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( config.version ) - # Bug in pip <= 21.1.3 - we can't have a space in the - # constraints file, and pip doesn't support drive letters - # in uhi. After probably pip 21.2, we can use uri. For - # now, use a temporary file. - if " " in str(constraints_path): - assert " " not in str(identifier_tmp_dir) - tmp_file = identifier_tmp_dir / "constraints.txt" - tmp_file.write_bytes(constraints_path.read_bytes()) - constraints_path = tmp_file - - our_constraints = str(constraints_path) - user_constraints = build_env.get("PIP_CONSTRAINT") - build_env["PIP_CONSTRAINT"] = " ".join( - c for c in [our_constraints, user_constraints] if c - ) - build_env["UV_CONSTRAINT"] = build_env["PIP_CONSTRAINT"] + combine_constraints(build_env, constraints_path, identifier_tmp_dir) if build_frontend.name == "pip": extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity) diff --git a/pyproject.toml b/pyproject.toml index d7a5a7e40..22557937f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,6 +177,7 @@ messages_control.disable = [ "wrong-import-position", "unused-argument", # Handled by Ruff "broad-exception-raised", # Could be improved eventually + "consider-using-in", # MyPy can't narrow "in" ] [tool.ruff] From 09bd9ba9c1e0bec7092e1fd435cc00070fb2b65b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 8 Jun 2024 01:20:06 -0400 Subject: [PATCH 07/14] chore: docs and remove uv pinning as it's not installed Signed-off-by: Henry Schreiner --- bin/generate_schema.py | 8 +++-- .../resources/cibuildwheel.schema.json | 10 ++++-- .../resources/constraints-python310.txt | 2 -- .../resources/constraints-python311.txt | 2 -- .../resources/constraints-python312.txt | 2 -- .../resources/constraints-python313.txt | 2 -- .../resources/constraints-python38.txt | 2 -- .../resources/constraints-python39.txt | 2 -- cibuildwheel/resources/constraints.in | 1 - cibuildwheel/resources/constraints.txt | 2 -- docs/options.md | 31 +++++++++++++++++-- 11 files changed, 41 insertions(+), 23 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index a1ac23c3e..dee1aad35 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -48,19 +48,21 @@ type: string_array build-frontend: default: default - description: Set the tool to use to build, either "pip" (default for now) or "build" + description: Set the tool to use to build, either "pip" (default for now), "build", or "build[uv]" oneOf: - - enum: [pip, build, default] + - enum: [pip, build, "build[uv]", default] - type: string pattern: '^pip; ?args:' - type: string pattern: '^build; ?args:' + - type: string + pattern: '^build\[uv\]; ?args:' - type: object additionalProperties: false required: [name] properties: name: - enum: [pip, build] + enum: [pip, build, "build[uv]"] args: type: array items: diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 088b5d542..976751a55 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -96,12 +96,13 @@ }, "build-frontend": { "default": "default", - "description": "Set the tool to use to build, either \"pip\" (default for now) or \"build\"", + "description": "Set the tool to use to build, either \"pip\" (default for now), \"build\", or \"build[uv]\"", "oneOf": [ { "enum": [ "pip", "build", + "build[uv]", "default" ] }, @@ -113,6 +114,10 @@ "type": "string", "pattern": "^build; ?args:" }, + { + "type": "string", + "pattern": "^build\\[uv\\]; ?args:" + }, { "type": "object", "additionalProperties": false, @@ -123,7 +128,8 @@ "name": { "enum": [ "pip", - "build" + "build", + "build[uv]" ] }, "args": { diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index c842117c7..434ad8757 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -28,8 +28,6 @@ tomli==2.0.1 # via build typing-extensions==4.12.2 # via delocate -uv==0.2.8 - # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in zipp==3.19.2 diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index 0ca4abac6..f2521f8bb 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -24,7 +24,5 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate -uv==0.2.8 - # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt index 0ca4abac6..f2521f8bb 100644 --- a/cibuildwheel/resources/constraints-python312.txt +++ b/cibuildwheel/resources/constraints-python312.txt @@ -24,7 +24,5 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate -uv==0.2.8 - # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index 704287adc..5fcc3ef67 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -24,7 +24,5 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate -uv==0.2.8 - # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index c842117c7..434ad8757 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -28,8 +28,6 @@ tomli==2.0.1 # via build typing-extensions==4.12.2 # via delocate -uv==0.2.8 - # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in zipp==3.19.2 diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index c842117c7..434ad8757 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -28,8 +28,6 @@ tomli==2.0.1 # via build typing-extensions==4.12.2 # via delocate -uv==0.2.8 - # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in zipp==3.19.2 diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 21831fd4e..4f01a4b8c 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -2,5 +2,4 @@ pip>=24.1b1 ; python_version >= '3.13' pip ; python_version < '3.13' build delocate -uv; python_version >= '3.8' virtualenv diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 0ca4abac6..f2521f8bb 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -24,7 +24,5 @@ pyproject-hooks==1.1.0 # via build typing-extensions==4.12.2 # via delocate -uv==0.2.8 - # via -r cibuildwheel/resources/constraints.in virtualenv==20.26.2 # via -r cibuildwheel/resources/constraints.in diff --git a/docs/options.md b/docs/options.md index 8eaa60943..b322317a6 100644 --- a/docs/options.md +++ b/docs/options.md @@ -624,7 +624,7 @@ This option can also be set using the [command-line option](#command-line) `--pr ## Build customization ### `CIBW_BUILD_FRONTEND` {: #build-frontend} -> Set the tool to use to build, either "pip" (default for now) or "build" +> Set the tool to use to build, either "pip" (default for now), "build", or "build[uv]" Options: @@ -636,17 +636,30 @@ Default: `pip` Choose which build frontend to use. Can either be "pip", which will run `python -m pip wheel`, or "build", which will run `python -m build --wheel`. +You can also use "build[uv]", which will use an external [uv][] everywhere +possible, both through `--installer=uv` passed to build, as well as when making +all build and test environments. This will generally speed up cibuildwheel. +Make sure you have an external uv on Windows and macOS, either by +pre-installing it, or installing cibuildwheel with the uv extra, +`cibuildwheel[uv]`. `uv` will not be used for Python 3.6 or Python 3.7. You +cannot use uv currently on Windows for ARM or for musllinux on s390x as +binaries are not provided by uv. Legacy dependencies like setuptools on Python +< 3.12 and pip are not installed if using uv. + +Pyodide ignores this setting, as only "build" is supported. + You can specify extra arguments to pass to `pip wheel` or `build` using the optional `args` option. !!! tip - Until v2.0.0, [pip] was the only way to build wheels, and is still the + Until v2.0.0, [pip][] was the only way to build wheels, and is still the default. However, we expect that at some point in the future, cibuildwheel - will change the default to [build], in line with the PyPA's recommendation. + will change the default to [build][], in line with the PyPA's recommendation. If you want to try `build` before this, you can use this option. [pip]: https://pip.pypa.io/en/stable/cli/pip_wheel/ [build]: https://github.com/pypa/build/ +[uv]: https://github.com/astral-sh/uv #### Examples @@ -661,6 +674,12 @@ optional `args` option. # supply an extra argument to 'pip wheel' CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation" + + # Use uv and build + CIBW_BUILD_FRONTEND: "build[uv]" + + # Use uv and build with an argument + CIBW_BUILD_FRONTEND: "build[uv]; args: --no-isolation" ``` !!! tab examples "pyproject.toml" @@ -675,6 +694,12 @@ optional `args` option. # supply an extra argument to 'pip wheel' build-frontend = { name = "pip", args = ["--no-build-isolation"] } + + # Use uv and build + build-frontend = "build[uv]" + + # Use uv and build with an argument + build-frontend = { name = "build[uv]", args = ["--no-isolation"] } ``` ### `CIBW_CONFIG_SETTINGS` {: #config-settings} From 567418f374a400d7baa21ad5fc477eea3f89c953 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Jun 2024 14:09:36 +0200 Subject: [PATCH 08/14] chore: use `build_frontend_env` fixture in test_universal2_testing_on_arm64 --- test/test_macos_archs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index d29118b60..23ba59e0a 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -172,7 +172,7 @@ def test_universal2_testing_on_x86_64(tmp_path, capfd, skip_arm64_test): assert set(actual_wheels) == set(expected_wheels) -def test_universal2_testing_on_arm64(tmp_path, capfd): +def test_universal2_testing_on_arm64(build_frontend_env, tmp_path, capfd): # cibuildwheel should test the universal2 wheel on both x86_64 and arm64, when run on arm64 if utils.platform != "macos": pytest.skip("this test is only relevant to macos") @@ -194,8 +194,8 @@ def test_universal2_testing_on_arm64(tmp_path, capfd): ) captured = capfd.readouterr() - assert "running tests on arm64" in captured.out - assert "running tests on x86_64" in captured.out + assert "running tests on arm64 with numpy" in captured.out + assert "running tests on x86_64 with numpy" in captured.out python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) expected_wheels = [w for w in ALL_MACOS_WHEELS if python_tag in w and "universal2" in w] From b2be90938552256d0e08fc74871bc678b12bc896 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Jun 2024 16:41:49 +0200 Subject: [PATCH 09/14] fix invalid test --- test/test_macos_archs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index fdeaee83a..c76aacc51 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -189,6 +189,7 @@ def test_universal2_testing_on_arm64(build_frontend_env, tmp_path, capfd): # check that a native dependency is correctly installed, once per each testing arch "CIBW_TEST_REQUIRES": "numpy", "CIBW_TEST_COMMAND": '''python -c "import numpy, platform; print(f'running tests on {platform.machine()} with numpy {numpy.__version__}')"''', + **build_frontend_env, }, single_python=True, ) From 859db872a1932df4560d77bf15b72f5cd0e39ba2 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Jun 2024 17:33:53 +0200 Subject: [PATCH 10/14] fix: testing macOS x86_64 wheels from arm64 with uv --- cibuildwheel/macos.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index b5d405a81..874f9518d 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -627,10 +627,12 @@ def build(options: Options, tmp_path: Path) -> None: venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" arch_prefix = [] + uv_arch_args = [] if testing_arch != machine_arch: if machine_arch == "arm64" and testing_arch == "x86_64": # rosetta2 will provide the emulation with just the arch prefix. arch_prefix = ["arch", "-x86_64"] + uv_arch_args = ["--python-platform", "x86_64-apple-darwin"] else: msg = f"don't know how to emulate {testing_arch} on {machine_arch}" raise RuntimeError(msg) @@ -640,8 +642,10 @@ def build(options: Options, tmp_path: Path) -> None: shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") if use_uv: - call_with_arch("uv", "venv", venv_dir, "--python=python", env=env) + pip_install = functools.partial(call, *pip, "install", *uv_arch_args) + call("uv", "venv", venv_dir, "--python=python", env=env) else: + pip_install = functools.partial(call_with_arch, *pip, "install") # Use pip version from the initial env to ensure determinism venv_args = ["--no-periodic-update", f"--pip={pip_version}"] # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones @@ -687,18 +691,14 @@ def build(options: Options, tmp_path: Path) -> None: else: virtualenv_env_install_wheel = virtualenv_env - call_with_arch( - *pip, - "install", + pip_install( f"{repaired_wheel}{build_options.test_extras}", env=virtualenv_env_install_wheel, ) # test the wheel if build_options.test_requires: - call_with_arch( - *pip, - "install", + pip_install( *build_options.test_requires, env=virtualenv_env_install_wheel, ) From 5c6ca48d568aea19aa5c05e6a27d2d787a7dc0ed Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 9 Jun 2024 12:20:18 +0200 Subject: [PATCH 11/14] use pillow rather than numpy in test_universal2_testing_on_arm64 --- test/test_macos_archs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index c76aacc51..3229b447a 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -187,16 +187,16 @@ def test_universal2_testing_on_arm64(build_frontend_env, tmp_path, capfd): add_env={ "CIBW_ARCHS": "universal2", # check that a native dependency is correctly installed, once per each testing arch - "CIBW_TEST_REQUIRES": "numpy", - "CIBW_TEST_COMMAND": '''python -c "import numpy, platform; print(f'running tests on {platform.machine()} with numpy {numpy.__version__}')"''', + "CIBW_TEST_REQUIRES": "pillow", # pillow provides wheels for macOS 10.10, not 10.9 + "CIBW_TEST_COMMAND": '''python -c "import PIL, platform; print(f'running tests on {platform.machine()} with pillow {PIL.__version__}')"''', **build_frontend_env, }, single_python=True, ) captured = capfd.readouterr() - assert "running tests on arm64 with numpy" in captured.out - assert "running tests on x86_64 with numpy" in captured.out + assert "running tests on arm64 with pillow" in captured.out + assert "running tests on x86_64 with pillow" in captured.out python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) expected_wheels = [w for w in ALL_MACOS_WHEELS if python_tag in w and "universal2" in w] From d0b66ffc55d7ae5a3571263e01eb3c5a95d2e41c Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 9 Jun 2024 14:22:09 +0200 Subject: [PATCH 12/14] make the test fail --- test/test_macos_archs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 3229b447a..53d90d197 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -187,7 +187,7 @@ def test_universal2_testing_on_arm64(build_frontend_env, tmp_path, capfd): add_env={ "CIBW_ARCHS": "universal2", # check that a native dependency is correctly installed, once per each testing arch - "CIBW_TEST_REQUIRES": "pillow", # pillow provides wheels for macOS 10.10, not 10.9 + "CIBW_TEST_REQUIRES": "--only-binary :all: pillow>=10.3", # pillow>=10.3 provides wheels for macOS 10.10, not 10.9 "CIBW_TEST_COMMAND": '''python -c "import PIL, platform; print(f'running tests on {platform.machine()} with pillow {PIL.__version__}')"''', **build_frontend_env, }, From 740fd579a243a584c9fe26c249fdb657804ff0b1 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 9 Jun 2024 14:40:10 +0200 Subject: [PATCH 13/14] fix: override MACOSX_DEPLOYMENT_TARGET in test environment --- cibuildwheel/macos.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 874f9518d..cfe197136 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -50,6 +50,7 @@ ) +@functools.lru_cache(maxsize=None) def get_macos_version() -> tuple[int, int]: """ Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) @@ -61,9 +62,29 @@ def get_macos_version() -> tuple[int, int]: """ version_str, _, _ = platform.mac_ver() version = tuple(map(int, version_str.split(".")[:2])) + if (10, 15) < version < (11, 0): + # When built against an older macOS SDK, Python will report macOS 10.16 + # instead of the real version. + version_str = call( + sys.executable, + "-sS", + "-c", + "import platform; print(platform.mac_ver()[0])", + env={"SYSTEM_VERSION_COMPAT": "0"}, + capture_stdout=True, + ) + version = tuple(map(int, version_str.split(".")[:2])) return typing.cast(Tuple[int, int], version) +@functools.lru_cache(maxsize=None) +def get_test_macosx_deployment_target() -> str: + version = get_macos_version() + if version >= (11, 0): + return f"{version[0]}.0" + return "{}.{}".format(*version) + + def get_macos_sdks() -> list[str]: output = call("xcodebuild", "-showsdks", capture_stdout=True) return [m.group(1) for m in re.finditer(r"-sdk (macosx\S+)", output)] @@ -654,6 +675,7 @@ def build(options: Options, tmp_path: Path) -> None: call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) virtualenv_env = env.copy() + virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target() virtualenv_env["PATH"] = os.pathsep.join( [ str(venv_dir / "bin"), From a1c4ac4cb3d543d6171ecd97b55701a779cd83ee Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 9 Jun 2024 15:14:58 +0200 Subject: [PATCH 14/14] fix: use f-string --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index cfe197136..890ca89e9 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -82,7 +82,7 @@ def get_test_macosx_deployment_target() -> str: version = get_macos_version() if version >= (11, 0): return f"{version[0]}.0" - return "{}.{}".format(*version) + return f"{version[0]}.{version[1]}" def get_macos_sdks() -> list[str]: