From d394386318c554707946a2ad99d8bc1e5e99757c Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Fri, 15 Jan 2021 15:40:03 +0100 Subject: [PATCH] feat: update python versions as part of update_dependencies (#496) * Update python versions as part of update_dependencies * refactor: pulling out to update_pythons * refactor: rewrite and expect install * feat: support macos too, logging output * WIP: update * refactor: drive from original file * Remove unused variable * Output a diff instead of the result file, to review changes more easily * fix: minor cleanup Co-authored-by: Henry Fredrick Schreiner Co-authored-by: Henry Schreiner Co-authored-by: Joe Rickerby --- .github/workflows/update-dependencies.yml | 6 +- .pre-commit-config.yaml | 10 +- bin/update_dependencies.py | 6 +- bin/update_pythons.py | 310 ++++++++++++++++++++++ cibuildwheel/extra.py | 24 ++ cibuildwheel/typing.py | 7 +- setup.cfg | 4 +- unit_test/build_ids_test.py | 37 ++- 8 files changed, 384 insertions(+), 20 deletions(-) create mode 100755 bin/update_pythons.py create mode 100644 cibuildwheel/extra.py diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index 1b77f6575..c483b270d 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -17,9 +17,11 @@ jobs: python-version: 3.9 architecture: x64 - name: Install dependencies - run: python -m pip install requests pip-tools - - name: Run update + run: python -m pip install ".[dev]" + - name: "Run update: dependencies" run: python ./bin/update_dependencies.py + - name: "Run update: python configs" + run: python ./bin/update_pythons.py --force - name: Create Pull Request if: github.ref == 'refs/heads/master' uses: peter-evans/create-pull-request@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 020d0068f..623e5d605 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,12 +16,20 @@ repos: hooks: - id: isort +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + files: ^bin/update_pythons.py$ + args: ["--line-length=120"] + - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.790 hooks: - id: mypy - files: ^(cibuildwheel/|test/|bin/projects.py|unit_test/) + files: ^(cibuildwheel/|test/|bin/projects.py|bin/update_pythons.py|unit_test/) pass_filenames: false + additional_dependencies: [packaging, click] - repo: https://github.com/asottile/pyupgrade rev: v2.7.4 diff --git a/bin/update_dependencies.py b/bin/update_dependencies.py index 62717c313..bd113eb56 100755 --- a/bin/update_dependencies.py +++ b/bin/update_dependencies.py @@ -28,8 +28,8 @@ '--output-file', f'cibuildwheel/resources/constraints-python{python_version}.txt' ]) else: - image = 'quay.io/pypa/manylinux2010_x86_64:latest' - subprocess.check_call(['docker', 'pull', image]) + image_runner = 'quay.io/pypa/manylinux2010_x86_64:latest' + subprocess.check_call(['docker', 'pull', image_runner]) for python_version in PYTHON_VERSIONS: abi_flags = '' if int(python_version) >= 38 else 'm' python_path = f'/opt/python/cp{python_version}-cp{python_version}{abi_flags}/bin/' @@ -37,7 +37,7 @@ 'docker', 'run', '--rm', '-e', 'CUSTOM_COMPILE_COMMAND', '-v', f'{os.getcwd()}:/volume', - '--workdir', '/volume', image, + '--workdir', '/volume', image_runner, 'bash', '-c', f'{python_path}pip install pip-tools &&' f'{python_path}pip-compile --allow-unsafe --upgrade ' diff --git a/bin/update_pythons.py b/bin/update_pythons.py new file mode 100755 index 000000000..04f075de6 --- /dev/null +++ b/bin/update_pythons.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +import copy +import difflib +import logging +from pathlib import Path +from typing import Dict, Optional, Union + +import click +import requests +import rich +import toml +from packaging.specifiers import Specifier +from packaging.version import Version +from rich.logging import RichHandler +from rich.syntax import Syntax + +from cibuildwheel.extra import InlineArrayDictEncoder +from cibuildwheel.typing import Final, Literal, TypedDict + +log = logging.getLogger("cibw") + +# Looking up the dir instead of using utils.resources_dir +# since we want to write to it. +DIR: Final[Path] = Path(__file__).parent.parent.resolve() +RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources" + + +ArchStr = Literal["32", "64"] + + +class ConfigWinCP(TypedDict): + identifier: str + version: str + arch: str + + +class ConfigWinPP(TypedDict): + identifier: str + version: str + arch: str + url: str + + +class ConfigMacOS(TypedDict): + identifier: str + version: str + url: str + + +AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS] + + +# The following set of "Versions" classes allow the initial call to the APIs to +# be cached and reused in the `update_version_*` methods. + + +class WindowsVersions: + def __init__(self, arch_str: ArchStr) -> None: + + response = requests.get("https://api.nuget.org/v3/index.json") + response.raise_for_status() + api_info = response.json() + + for resource in api_info["resources"]: + if resource["@type"] == "PackageBaseAddress/3.0.0": + endpoint = resource["@id"] + + ARCH_DICT = {"32": "win32", "64": "win_amd64"} + PACKAGE_DICT = {"32": "pythonx86", "64": "python"} + + self.arch_str = arch_str + self.arch = ARCH_DICT[arch_str] + package = PACKAGE_DICT[arch_str] + + response = requests.get(f"{endpoint}{package}/index.json") + response.raise_for_status() + cp_info = response.json() + + versions = (Version(v) for v in cp_info["versions"]) + self.versions = sorted(v for v in versions if not v.is_devrelease) + + def update_version_windows(self, spec: Specifier) -> Optional[ConfigWinCP]: + versions = sorted(v for v in self.versions if spec.contains(v)) + if not all(v.is_prerelease for v in versions): + versions = [v for v in versions if not v.is_prerelease] + log.debug(f"Windows {self.arch} {spec} has {', '.join(str(v) for v in versions)}") + + if not versions: + return None + + version = versions[-1] + identifier = f"cp{version.major}{version.minor}-{self.arch}" + result = ConfigWinCP( + identifier=identifier, + version=str(version), + arch=self.arch_str, + ) + return result + + +class PyPyVersions: + def __init__(self, arch_str: ArchStr): + + response = requests.get("https://downloads.python.org/pypy/versions.json") + response.raise_for_status() + + releases = [r for r in response.json() if r["pypy_version"] != "nightly"] + for release in releases: + release["pypy_version"] = Version(release["pypy_version"]) + release["python_version"] = Version(release["python_version"]) + + self.releases = [ + r for r in releases if not r["pypy_version"].is_prerelease and not r["pypy_version"].is_devrelease + ] + self.arch = arch_str + + def update_version_windows(self, spec: Specifier) -> ConfigWinCP: + if self.arch != "32": + raise RuntimeError("64 bit releases not supported yet on Windows") + + releases = [r for r in self.releases if spec.contains(r["python_version"])] + releases = sorted(releases, key=lambda r: r["pypy_version"]) + + if not releases: + raise RuntimeError(f"PyPy Win {self.arch} not found for {spec}! {self.releases}") + + release = releases[-1] + version = release["python_version"] + identifier = f"pp{version.major}{version.minor}-win32" + + (url,) = [rf["download_url"] for rf in release["files"] if "" in rf["platform"] == "win32"] + + return ConfigWinPP( + identifier=identifier, + version=f"{version.major}.{version.minor}", + arch="32", + url=url, + ) + + def update_version_macos(self, spec: Specifier) -> ConfigMacOS: + if self.arch != "64": + raise RuntimeError("Other archs not supported yet on macOS") + + releases = [r for r in self.releases if spec.contains(r["python_version"])] + releases = sorted(releases, key=lambda r: r["pypy_version"]) + + if not releases: + raise RuntimeError(f"PyPy macOS {self.arch} not found for {spec}!") + + release = releases[-1] + version = release["python_version"] + identifier = f"pp{version.major}{version.minor}-macosx_x86_64" + + (url,) = [ + rf["download_url"] for rf in release["files"] if "" in rf["platform"] == "darwin" and rf["arch"] == "x64" + ] + + return ConfigMacOS( + identifier=identifier, + version=f"{version.major}.{version.minor}", + url=url, + ) + + +class CPythonVersions: + def __init__(self, plat_arch: str, file_ident: str) -> None: + + response = requests.get("https://www.python.org/api/v2/downloads/release/?is_published=true") + response.raise_for_status() + + releases_info = response.json() + + self.versions_dict: Dict[Version, int] = {} + for release in releases_info: + # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") + version = Version(release["name"][7:]) + + if not version.is_prerelease and not version.is_devrelease: + uri = int(release["resource_uri"].rstrip("/").split("/")[-1]) + self.versions_dict[version] = uri + + self.file_ident = file_ident + self.plat_arch = plat_arch + + def update_version_macos(self, spec: Specifier) -> Optional[ConfigMacOS]: + + sorted_versions = sorted(v for v in self.versions_dict if spec.contains(v)) + + for version in reversed(sorted_versions): + # Find the first patch version that contains the requested file + uri = self.versions_dict[version] + response = requests.get(f"https://www.python.org/api/v2/downloads/release_file/?release={uri}") + response.raise_for_status() + file_info = response.json() + + urls = [rf["url"] for rf in file_info if self.file_ident in rf["url"]] + if urls: + return ConfigMacOS( + identifier=f"cp{version.major}{version.minor}-{self.plat_arch}", + version=f"{version.major}.{version.minor}", + url=urls[0], + ) + + return None + + +# This is a universal interface to all the above Versions classes. Given an +# identifier, it updates a config dict. + + +class AllVersions: + def __init__(self) -> None: + self.windows_32 = WindowsVersions("32") + self.windows_64 = WindowsVersions("64") + self.windows_pypy = PyPyVersions("32") + + self.macos_6 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.6.pkg") + self.macos_9 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.9.pkg") + self.macos_u2 = CPythonVersions(plat_arch="macosx_universal2", file_ident="macos11.0.pkg") + self.macos_pypy = PyPyVersions("64") + + def update_config(self, config: Dict[str, str]) -> None: + identifier = config["identifier"] + version = Version(config["version"]) + spec = Specifier(f"=={version.major}.{version.minor}.*") + log.info(f"Reading in '{identifier}' -> {spec} @ {version}") + orig_config = copy.copy(config) + config_update: Optional[AnyConfig] + + # We need to use ** in update due to MyPy (probably a bug) + if "macosx_x86_64" in identifier: + if identifier.startswith("pp"): + config_update = self.macos_pypy.update_version_macos(spec) + else: + config_update = self.macos_9.update_version_macos(spec) or self.macos_6.update_version_macos(spec) + assert config_update is not None, f"MacOS {spec} not found!" + config.update(**config_update) + elif "win32" in identifier: + if identifier.startswith("pp"): + config.update(**self.windows_pypy.update_version_windows(spec)) + else: + config_update = self.windows_32.update_version_windows(spec) + if config_update: + config.update(**config_update) + elif "win_amd64" in identifier: + config_update = self.windows_64.update_version_windows(spec) + if config_update: + config.update(**config_update) + + if config != orig_config: + log.info(f" Updated {orig_config} to {config}") + + +@click.command() +@click.option("--force", is_flag=True) +@click.option("--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False)) +def update_pythons(force: bool, level: str) -> None: + + logging.basicConfig( + level="INFO", + format="%(message)s", + datefmt="[%X]", + handlers=[RichHandler(rich_tracebacks=True, markup=True)], + ) + log.setLevel(level) + + all_versions = AllVersions() + toml_file_path = RESOURCES_DIR / "build-platforms.toml" + + original_toml = toml_file_path.read_text() + configs = toml.loads(original_toml) + + for config in configs["windows"]["python_configurations"]: + all_versions.update_config(config) + + for config in configs["macos"]["python_configurations"]: + all_versions.update_config(config) + + result_toml = toml.dumps(configs, encoder=InlineArrayDictEncoder()) # type: ignore + + rich.print() # spacer + + if original_toml == result_toml: + rich.print("[green]Check complete, Python configurations unchanged.") + return + + rich.print("Python configurations updated.") + rich.print("Changes:") + rich.print() + + toml_relpath = toml_file_path.relative_to(DIR).as_posix() + diff_lines = difflib.unified_diff( + original_toml.splitlines(keepends=True), + result_toml.splitlines(keepends=True), + fromfile=toml_relpath, + tofile=toml_relpath, + ) + rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light")) + rich.print() + + if force: + toml_file_path.write_text(result_toml) + rich.print("[green]TOML file updated.") + else: + rich.print("[yellow]File left unchanged. Use --force flag to update.") + + +if __name__ == "__main__": + update_pythons() diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py new file mode 100644 index 000000000..3c2ef4fbe --- /dev/null +++ b/cibuildwheel/extra.py @@ -0,0 +1,24 @@ +""" +These are utilities for the `/bin` scripts, not for the `cibuildwheel` program. +""" + +from typing import Any, Dict + +import toml.encoder +from packaging.version import Version + + +class InlineArrayDictEncoder(toml.encoder.TomlEncoder): # type: ignore + def __init__(self) -> None: + super().__init__() + self.dump_funcs[Version] = lambda v: f'"{v}"' + + def dump_sections(self, o: Dict[str, Any], sup: str) -> Any: + if all(isinstance(a, list) for a in o.values()): + val = "" + for k, v in o.items(): + inner = ",\n ".join(self.dump_inline_table(d_i).strip() for d_i in v) + val += f"{k} = [\n {inner},\n]\n" + return val, self._dict() + else: + return super().dump_sections(o, sup) diff --git a/cibuildwheel/typing.py b/cibuildwheel/typing.py index 03a1c7c1a..9a650d546 100644 --- a/cibuildwheel/typing.py +++ b/cibuildwheel/typing.py @@ -4,9 +4,12 @@ from typing import TYPE_CHECKING, NoReturn, Set, Union if sys.version_info < (3, 8): - from typing_extensions import Final, Literal + from typing_extensions import Final, Literal, TypedDict else: - from typing import Final, Literal + from typing import Final, Literal, TypedDict + + +__all__ = ("Final", "Literal", "TypedDict", "Set", "Union", "PopenBytes", "PathOrStr", "PlatformName", "PLATFORMS", "assert_never") if TYPE_CHECKING: diff --git a/setup.cfg b/setup.cfg index 698935072..2220ab453 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,8 @@ dev = pyyaml requests typing-extensions + packaging>=20.8 + rich>=9.6 [options.packages.find] include = @@ -81,7 +83,7 @@ junit_family=xunit2 [mypy] python_version = 3.6 -files = cibuildwheel,test,unit_test +files = cibuildwheel,test,unit_test,bin warn_unused_configs = True warn_redundant_casts = True diff --git a/unit_test/build_ids_test.py b/unit_test/build_ids_test.py index 99c5b57ed..1993efd03 100644 --- a/unit_test/build_ids_test.py +++ b/unit_test/build_ids_test.py @@ -1,19 +1,11 @@ +import pytest import toml -from toml.encoder import TomlEncoder from cibuildwheel.util import resources_dir +Version = pytest.importorskip("packaging.version").Version -class InlineArrayDictEncoder(TomlEncoder): - def dump_sections(self, o: dict, sup: str): - if all(isinstance(a, list) for a in o.values()): - val = "" - for k, v in o.items(): - inner = ",\n ".join(self.dump_inline_table(d_i).strip() for d_i in v) - val += f"{k} = [\n {inner},\n]\n" - return val, self._dict() - else: - return super().dump_sections(o, sup) +from cibuildwheel.extra import InlineArrayDictEncoder # noqa: E402 def test_compare_configs(): @@ -26,3 +18,26 @@ def test_compare_configs(): print(new_txt) assert new_txt == txt + + +def test_dump_with_Version(): + example = { + "windows": { + "python_configurations": [ + {"identifier": "cp27-win32", "version": Version("2.7.18"), "arch": "32"}, + {"identifier": "cp27-win_amd64", "version": Version("2.7.18"), "arch": "64"}, + ] + } + } + + result = """\ +[windows] +python_configurations = [ + { identifier = "cp27-win32", version = "2.7.18", arch = "32" }, + { identifier = "cp27-win_amd64", version = "2.7.18", arch = "64" }, +] +""" + + output = toml.dumps(example, encoder=InlineArrayDictEncoder()) + print(output) + assert output == result