From dc29bb85b711ca6349813c977cb35bb667a458d3 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 21 Dec 2020 22:01:44 +0100 Subject: [PATCH 1/9] Update python versions as part of update_dependencies --- .github/workflows/update-dependencies.yml | 2 +- bin/update_dependencies.py | 238 +++++++++++++++++++++- setup.cfg | 1 + 3 files changed, 239 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index 1b77f6575..6881a3aa2 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -17,7 +17,7 @@ jobs: python-version: 3.9 architecture: x64 - name: Install dependencies - run: python -m pip install requests pip-tools + run: python -m pip install requests pip-tools packaging>=20.8 - name: Run update run: python ./bin/update_dependencies.py - name: Create Pull Request diff --git a/bin/update_dependencies.py b/bin/update_dependencies.py index 62717c313..969084904 100755 --- a/bin/update_dependencies.py +++ b/bin/update_dependencies.py @@ -5,8 +5,9 @@ import shutil import subprocess import sys -from collections import namedtuple +from collections import namedtuple, defaultdict +import packaging.version import requests os.chdir(os.path.dirname(__file__)) @@ -117,3 +118,238 @@ with open('cibuildwheel/resources/pinned_docker_images.cfg', 'w') as f: config.write(f) + + +# update Python on windows +# CPython uses nugget +# c.f. https://docs.microsoft.com/en-us/nuget/api/overview +response = requests.get('https://api.nuget.org/v3/index.json') +response.raise_for_status() +api_info = response.json() +for resource in api_info['resources']: + if resource['@type'] == 'PackageBaseAddress/3.0.0': + endpoint = resource['@id'] +cp_versions = {'64': [], '32': [] } +for id, package in [('64', 'python'), ('32', 'pythonx86')]: + response = requests.get(f'{endpoint}{package}/index.json') + response.raise_for_status() + cp_info = response.json() + for version_str in cp_info['versions']: + version = packaging.version.parse(version_str) + if version.is_devrelease: + continue + cp_versions[id].append(version) + cp_versions[id].sort() +# PyPy is downloaded from https://downloads.python.org/pypy +response = requests.get('https://downloads.python.org/pypy/versions.json') +response.raise_for_status() +pp_realeases = response.json() +pp_versions = defaultdict(list) +for pp_realease in pp_realeases: + if pp_realease['pypy_version'] == 'nightly': + continue + version = packaging.version.parse(pp_realease['pypy_version']) + python_version = packaging.version.parse(pp_realease['python_version']) + python_version = f'{python_version.major}.{python_version.minor}' + url = None + for file in pp_realease['files']: + if f"{file['platform']}-{file['arch']}" == 'win32-x86': + url = file['download_url'] + break + if url: + pp_versions[python_version].append((version, url)) + +# load windows.py +with open('cibuildwheel/windows.py', 'rt') as f: + lines = f.readlines() +# hugly search pattern, package configuration shall probably done otherwise if we want to do this +for index, line in enumerate(lines): + if 'PythonConfiguration' in line and 'url=None' in line and "identifier='cp3" in line: + if "arch='32'" in line: + id='32' + else: + id='64' + start = line.index("version='") + 9 + end = line.index("'", start) + current_version = packaging.version.parse(line[start:end]) + new_version = current_version + max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') + allow_prerelease = False + if current_version.is_prerelease: + release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') + if release_version in cp_versions[id]: + new_version = release_version + else: + allow_prerelease = True + max_version = release_version + + for version in cp_versions[id]: + if version.is_prerelease and not allow_prerelease: + continue + if version > new_version and version < max_version: + new_version = version + lines[index] = line[:start] + str(new_version) + line[end:] + elif 'PythonConfiguration' in line and "identifier='pp" in line: + start = line.index("version='") + 9 + end = line.index("'", start) + id = line[start:end] + start = line.index("url='") + 5 + end = line.index("'", start) + current_url = line[start:end] + _, current_version_str, _ = current_url.split('/')[-1].split('-') + current_version = packaging.version.parse(current_version_str) + new_version = current_version + new_url = current_url + max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') + allow_prerelease = False + if current_version.is_prerelease: + release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') + found_release = False + for version, url in pp_versions[id]: + if release_version == version: + new_version = release_version + new_url = url + found_release = True + break + if not found_release: + allow_prerelease = True + max_version = release_version + + for version, url in pp_versions[id]: + if version.is_prerelease and not allow_prerelease: + continue + if version > new_version and version < max_version: + new_url = url + new_version = version + lines[index] = line[:start] + new_url + line[end:] + +with open('cibuildwheel/windows.py', 'wt') as f: + f.writelines(lines) + + +# update Python on macOS +# Cpython +# c.f. https://github.com/python/pythondotorg/issues/1352 +response = requests.get('https://www.python.org/api/v2/downloads/release/?version=3&is_published=true') +response.raise_for_status() +release_info = response.json() +cp_versions = {} +for release in release_info: + if not release['is_published']: + continue + parts = release['name'].split() + if parts[0].lower() != 'python': + continue + assert len(parts) == 2 + version = packaging.version.parse(parts[1]) + cp_versions[release['resource_uri']] = [version] + +response = requests.get('https://www.python.org/api/v2/downloads/release_file/?os=2') +response.raise_for_status() +file_info = response.json() + +for file in file_info: + key = file['release'] + if key not in cp_versions.keys(): + continue + cp_versions[key].append(file['url']) + +# PyPy is downloaded from https://downloads.python.org/pypy +response = requests.get('https://downloads.python.org/pypy/versions.json') +response.raise_for_status() +pp_realeases = response.json() +pp_versions = defaultdict(list) +for pp_realease in pp_realeases: + if pp_realease['pypy_version'] == 'nightly': + continue + version = packaging.version.parse(pp_realease['pypy_version']) + python_version = packaging.version.parse(pp_realease['python_version']) + python_version = f'{python_version.major}.{python_version.minor}' + url = None + for file in pp_realease['files']: + if f"{file['platform']}-{file['arch']}" == 'darwin-x64': + url = file['download_url'] + break + if url: + pp_versions[python_version].append((version, url)) + +# load macos.py +with open('cibuildwheel/macos.py', 'rt') as f: + lines = f.readlines() +# hugly search pattern, package configuration shall probably done otherwise if we want to do this +for index, line in enumerate(lines): + if 'PythonConfiguration' in line and "identifier='cp3" in line: + start = line.index("url='") + 5 + end = line.index("'", start) + current_url = line[start:end] + _, current_version_str, installer_kind = current_url.split('/')[-1].split('-') + current_version = packaging.version.parse(current_version_str) + new_version = current_version + new_url = current_url + max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') + allow_prerelease = False + if current_version.is_prerelease: + release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') + found_release = False + for version in cp_versions.values(): + if release_version == version[0]: + # find installer + found_url = False + for url in version[1:]: + if url.endswith(installer_kind): + new_url = url + found_url = True + break + if found_url: + new_version = release_version + found_release = True + break + if not found_release: + allow_prerelease = True + max_version = release_version + + for version in cp_versions.values(): + if version[0].is_prerelease and not allow_prerelease: + continue + if version[0] > new_version and version[0] < max_version: + # check installer kind + for url in version[1:]: + if url.endswith(installer_kind): + new_url = url + new_version = version[0] + lines[index] = line[:start] + new_url + line[end:] + elif 'PythonConfiguration' in line and "identifier='pp" in line: + start = line.index("version='") + 9 + end = line.index("'", start) + id = line[start:end] + start = line.index("url='") + 5 + end = line.index("'", start) + current_url = line[start:end] + _, current_version_str, _ = current_url.split('/')[-1].split('-') + current_version = packaging.version.parse(current_version_str) + new_version = current_version + new_url = current_url + max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') + allow_prerelease = False + if current_version.is_prerelease: + release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') + found_release = False + for version, url in pp_versions[id]: + if release_version == version: + new_version = release_version + new_url = url + found_release = True + break + if not found_release: + allow_prerelease = True + max_version = release_version + + for version, url in pp_versions[id]: + if version.is_prerelease and not allow_prerelease: + continue + if version > new_version and version < max_version: + new_url = url + new_version = version + lines[index] = line[:start] + new_url + line[end:] +with open('cibuildwheel/macos.py', 'wt') as f: + f.writelines(lines) diff --git a/setup.cfg b/setup.cfg index 698935072..fcc3cd4c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,7 @@ dev = pyyaml requests typing-extensions + packaging>=20.8 [options.packages.find] include = From ec91fe16fa7d4834d292c022055632c183d6ef20 Mon Sep 17 00:00:00 2001 From: Henry Fredrick Schreiner Date: Mon, 11 Jan 2021 11:35:49 -0500 Subject: [PATCH 2/9] refactor: pulling out to update_pythons --- .github/workflows/update-dependencies.yml | 6 +- .pre-commit-config.yaml | 9 +- bin/update_dependencies.py | 244 +--------------------- bin/update_pythons.py | 179 ++++++++++++++++ setup.cfg | 2 +- 5 files changed, 196 insertions(+), 244 deletions(-) create mode 100755 bin/update_pythons.py diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index 6881a3aa2..836eb9b7f 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -17,8 +17,10 @@ jobs: python-version: 3.9 architecture: x64 - name: Install dependencies - run: python -m pip install requests pip-tools packaging>=20.8 - - name: Run update + run: python -m pip install ".[dev]" + - name: "Run update: dependencies" + run: python ./bin/update_dependencies.py + - name: "Run update: python configs" run: python ./bin/update_dependencies.py - name: Create Pull Request if: github.ref == 'refs/heads/master' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 020d0068f..2d0a118cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,12 +16,19 @@ repos: hooks: - id: isort +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + files: ^bin/update_pythons.py$ + - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.790 hooks: - id: mypy - files: ^(cibuildwheel/|test/|bin/projects.py|unit_test/) + files: ^(cibuildwheel/|test/|bin/projects.py|bin/update_pythons.py|unit_test/) pass_filenames: false + additional_dependencies: [packaging] - repo: https://github.com/asottile/pyupgrade rev: v2.7.4 diff --git a/bin/update_dependencies.py b/bin/update_dependencies.py index 969084904..bd113eb56 100755 --- a/bin/update_dependencies.py +++ b/bin/update_dependencies.py @@ -5,9 +5,8 @@ import shutil import subprocess import sys -from collections import namedtuple, defaultdict +from collections import namedtuple -import packaging.version import requests os.chdir(os.path.dirname(__file__)) @@ -29,8 +28,8 @@ '--output-file', f'cibuildwheel/resources/constraints-python{python_version}.txt' ]) else: - image = 'quay.io/pypa/manylinux2010_x86_64:latest' - subprocess.check_call(['docker', 'pull', image]) + image_runner = 'quay.io/pypa/manylinux2010_x86_64:latest' + subprocess.check_call(['docker', 'pull', image_runner]) for python_version in PYTHON_VERSIONS: abi_flags = '' if int(python_version) >= 38 else 'm' python_path = f'/opt/python/cp{python_version}-cp{python_version}{abi_flags}/bin/' @@ -38,7 +37,7 @@ 'docker', 'run', '--rm', '-e', 'CUSTOM_COMPILE_COMMAND', '-v', f'{os.getcwd()}:/volume', - '--workdir', '/volume', image, + '--workdir', '/volume', image_runner, 'bash', '-c', f'{python_path}pip install pip-tools &&' f'{python_path}pip-compile --allow-unsafe --upgrade ' @@ -118,238 +117,3 @@ with open('cibuildwheel/resources/pinned_docker_images.cfg', 'w') as f: config.write(f) - - -# update Python on windows -# CPython uses nugget -# c.f. https://docs.microsoft.com/en-us/nuget/api/overview -response = requests.get('https://api.nuget.org/v3/index.json') -response.raise_for_status() -api_info = response.json() -for resource in api_info['resources']: - if resource['@type'] == 'PackageBaseAddress/3.0.0': - endpoint = resource['@id'] -cp_versions = {'64': [], '32': [] } -for id, package in [('64', 'python'), ('32', 'pythonx86')]: - response = requests.get(f'{endpoint}{package}/index.json') - response.raise_for_status() - cp_info = response.json() - for version_str in cp_info['versions']: - version = packaging.version.parse(version_str) - if version.is_devrelease: - continue - cp_versions[id].append(version) - cp_versions[id].sort() -# PyPy is downloaded from https://downloads.python.org/pypy -response = requests.get('https://downloads.python.org/pypy/versions.json') -response.raise_for_status() -pp_realeases = response.json() -pp_versions = defaultdict(list) -for pp_realease in pp_realeases: - if pp_realease['pypy_version'] == 'nightly': - continue - version = packaging.version.parse(pp_realease['pypy_version']) - python_version = packaging.version.parse(pp_realease['python_version']) - python_version = f'{python_version.major}.{python_version.minor}' - url = None - for file in pp_realease['files']: - if f"{file['platform']}-{file['arch']}" == 'win32-x86': - url = file['download_url'] - break - if url: - pp_versions[python_version].append((version, url)) - -# load windows.py -with open('cibuildwheel/windows.py', 'rt') as f: - lines = f.readlines() -# hugly search pattern, package configuration shall probably done otherwise if we want to do this -for index, line in enumerate(lines): - if 'PythonConfiguration' in line and 'url=None' in line and "identifier='cp3" in line: - if "arch='32'" in line: - id='32' - else: - id='64' - start = line.index("version='") + 9 - end = line.index("'", start) - current_version = packaging.version.parse(line[start:end]) - new_version = current_version - max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') - allow_prerelease = False - if current_version.is_prerelease: - release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') - if release_version in cp_versions[id]: - new_version = release_version - else: - allow_prerelease = True - max_version = release_version - - for version in cp_versions[id]: - if version.is_prerelease and not allow_prerelease: - continue - if version > new_version and version < max_version: - new_version = version - lines[index] = line[:start] + str(new_version) + line[end:] - elif 'PythonConfiguration' in line and "identifier='pp" in line: - start = line.index("version='") + 9 - end = line.index("'", start) - id = line[start:end] - start = line.index("url='") + 5 - end = line.index("'", start) - current_url = line[start:end] - _, current_version_str, _ = current_url.split('/')[-1].split('-') - current_version = packaging.version.parse(current_version_str) - new_version = current_version - new_url = current_url - max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') - allow_prerelease = False - if current_version.is_prerelease: - release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') - found_release = False - for version, url in pp_versions[id]: - if release_version == version: - new_version = release_version - new_url = url - found_release = True - break - if not found_release: - allow_prerelease = True - max_version = release_version - - for version, url in pp_versions[id]: - if version.is_prerelease and not allow_prerelease: - continue - if version > new_version and version < max_version: - new_url = url - new_version = version - lines[index] = line[:start] + new_url + line[end:] - -with open('cibuildwheel/windows.py', 'wt') as f: - f.writelines(lines) - - -# update Python on macOS -# Cpython -# c.f. https://github.com/python/pythondotorg/issues/1352 -response = requests.get('https://www.python.org/api/v2/downloads/release/?version=3&is_published=true') -response.raise_for_status() -release_info = response.json() -cp_versions = {} -for release in release_info: - if not release['is_published']: - continue - parts = release['name'].split() - if parts[0].lower() != 'python': - continue - assert len(parts) == 2 - version = packaging.version.parse(parts[1]) - cp_versions[release['resource_uri']] = [version] - -response = requests.get('https://www.python.org/api/v2/downloads/release_file/?os=2') -response.raise_for_status() -file_info = response.json() - -for file in file_info: - key = file['release'] - if key not in cp_versions.keys(): - continue - cp_versions[key].append(file['url']) - -# PyPy is downloaded from https://downloads.python.org/pypy -response = requests.get('https://downloads.python.org/pypy/versions.json') -response.raise_for_status() -pp_realeases = response.json() -pp_versions = defaultdict(list) -for pp_realease in pp_realeases: - if pp_realease['pypy_version'] == 'nightly': - continue - version = packaging.version.parse(pp_realease['pypy_version']) - python_version = packaging.version.parse(pp_realease['python_version']) - python_version = f'{python_version.major}.{python_version.minor}' - url = None - for file in pp_realease['files']: - if f"{file['platform']}-{file['arch']}" == 'darwin-x64': - url = file['download_url'] - break - if url: - pp_versions[python_version].append((version, url)) - -# load macos.py -with open('cibuildwheel/macos.py', 'rt') as f: - lines = f.readlines() -# hugly search pattern, package configuration shall probably done otherwise if we want to do this -for index, line in enumerate(lines): - if 'PythonConfiguration' in line and "identifier='cp3" in line: - start = line.index("url='") + 5 - end = line.index("'", start) - current_url = line[start:end] - _, current_version_str, installer_kind = current_url.split('/')[-1].split('-') - current_version = packaging.version.parse(current_version_str) - new_version = current_version - new_url = current_url - max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') - allow_prerelease = False - if current_version.is_prerelease: - release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') - found_release = False - for version in cp_versions.values(): - if release_version == version[0]: - # find installer - found_url = False - for url in version[1:]: - if url.endswith(installer_kind): - new_url = url - found_url = True - break - if found_url: - new_version = release_version - found_release = True - break - if not found_release: - allow_prerelease = True - max_version = release_version - - for version in cp_versions.values(): - if version[0].is_prerelease and not allow_prerelease: - continue - if version[0] > new_version and version[0] < max_version: - # check installer kind - for url in version[1:]: - if url.endswith(installer_kind): - new_url = url - new_version = version[0] - lines[index] = line[:start] + new_url + line[end:] - elif 'PythonConfiguration' in line and "identifier='pp" in line: - start = line.index("version='") + 9 - end = line.index("'", start) - id = line[start:end] - start = line.index("url='") + 5 - end = line.index("'", start) - current_url = line[start:end] - _, current_version_str, _ = current_url.split('/')[-1].split('-') - current_version = packaging.version.parse(current_version_str) - new_version = current_version - new_url = current_url - max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}') - allow_prerelease = False - if current_version.is_prerelease: - release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}') - found_release = False - for version, url in pp_versions[id]: - if release_version == version: - new_version = release_version - new_url = url - found_release = True - break - if not found_release: - allow_prerelease = True - max_version = release_version - - for version, url in pp_versions[id]: - if version.is_prerelease and not allow_prerelease: - continue - if version > new_version and version < max_version: - new_url = url - new_version = version - lines[index] = line[:start] + new_url + line[end:] -with open('cibuildwheel/macos.py', 'wt') as f: - f.writelines(lines) diff --git a/bin/update_pythons.py b/bin/update_pythons.py new file mode 100755 index 000000000..7b8a779bc --- /dev/null +++ b/bin/update_pythons.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 + +import sys +from collections import defaultdict +from itertools import groupby +from pathlib import Path +from typing import Dict, List, Tuple + +import requests +import toml +from packaging.version import Version + +if sys.version_info < (3, 8): + from typing_extensions import TypedDict +else: + from typing import TypedDict + +# Use pretty printing for debugging +# from rich import print + + +allow_prerelease = False + + +class InlineArrayDictEncoder(toml.encoder.TomlEncoder): + def dump_sections(self, o: dict, sup: str): + if all(isinstance(a, list) for a in o.values()): + val = "" + for k, v in o.items(): + inner = ",\n ".join(self.dump_inline_table(d_i).strip() for d_i in v) + val += f"{k} = [\n {inner},\n]\n" + return val, self._dict() + else: + return super().dump_sections(o, sup) + + +DIR = Path(__file__).parent.parent.resolve() +RESOURCES_DIR = DIR / "cibuildwheel/resources" + + +class ConfigWinCP(TypedDict): + identifier: str + version: str + arch: str + + +class ConfigWinPP(TypedDict): + identifier: str + version: str + arch: str + url: str + + +def get_cpython_windows() -> Dict[str, List[Version]]: + response = requests.get("https://api.nuget.org/v3/index.json") + response.raise_for_status() + api_info = response.json() + + for resource in api_info["resources"]: + if resource["@type"] == "PackageBaseAddress/3.0.0": + endpoint = resource["@id"] + + cp_versions: Dict[str, List[Version]] = {"64": [], "32": []} + for id, package in [("64", "python"), ("32", "pythonx86")]: + response = requests.get(f"{endpoint}{package}/index.json") + response.raise_for_status() + cp_info = response.json() + + for version_str in cp_info["versions"]: + version = Version(version_str) + if version.is_devrelease: + continue + if not allow_prerelease and version.is_prerelease: + continue + cp_versions[id].append(version) + cp_versions[id].sort() + + return cp_versions + + +def get_pypy_windows( + plat_arch: str = "win32-x86", +) -> Dict[str, List[Tuple[Version, str]]]: + + response = requests.get("https://downloads.python.org/pypy/versions.json") + response.raise_for_status() + pp_realeases = response.json() + pp_versions = defaultdict(list) + + for pp_realease in pp_realeases: + if pp_realease["pypy_version"] == "nightly": + continue + version = Version(pp_realease["pypy_version"]) + python_version = Version(pp_realease["python_version"]) + python_version_str = f"{python_version.major}.{python_version.minor}" + url = None + for file in pp_realease["files"]: + if f"{file['platform']}-{file['arch']}" == plat_arch: + url = file["download_url"] + break + if url: + pp_versions[python_version_str].append((version, url)) + + return pp_versions + + +# Debugging printout: +# print(get_cpython_windows()) +# print() +# print("[bold]Getting PyPy") +# print(get_pypy_windows()) + +ARCH_DICT = {"32": "win32", "64": "win_amd64"} + + +def build_ids_cp(in_dict: Dict[str, List[Version]]) -> List[ConfigWinCP]: + items: List[ConfigWinCP] = [] + for arch in in_dict: + for minor, grp in groupby(in_dict[arch], lambda v: v.minor): + # Filter pre-releases, unless it's all pre-releases + if not all(v.is_devrelease for v in grp): + grp = filter(lambda v: not v.is_devrelease, grp) + + version = sorted(grp)[-1] + identifier = f"cp3{minor}-{ARCH_DICT[arch]}" + + items.append( + ConfigWinCP( + identifier=identifier, + version=str(version), + arch=arch, + ) + ) + + return items + + +def build_ids_pp(in_dict: Dict[str, List[Tuple[Version, str]]]) -> List[ConfigWinPP]: + items: List[ConfigWinPP] = [] + for vers, matches in in_dict.items(): + vers_id = vers.replace(".", "") + if not all(v[0].is_devrelease for v in matches): + matches = list(filter(lambda v: not v[0].is_devrelease, matches)) + + version, url = sorted(matches, key=lambda v: v[0])[-1] + identifier = f"pp{vers_id}-win32" + + items.append( + ConfigWinPP( + identifier=identifier, + version=vers, + arch="32", + url=url, + ) + ) + + return items + + +windows_configs = [ + *build_ids_cp(get_cpython_windows()), + *build_ids_pp(get_pypy_windows()), +] + +configs = toml.load(RESOURCES_DIR / "build-platforms.toml") +origpy2 = list( + filter( + lambda c: c["identifier"].startswith("cp27"), + configs["windows"]["python_configurations"], + ) +) + +items = origpy2 + windows_configs +new_configs = sorted(items, key=lambda x: (x["identifier"][:3], x["identifier"][3:])) + +configs["windows"]["python_configurations"] = new_configs + +with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: + toml.dump(configs, f, encoder=InlineArrayDictEncoder()) # type: ignore diff --git a/setup.cfg b/setup.cfg index fcc3cd4c8..dc9fed9fa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,7 +82,7 @@ junit_family=xunit2 [mypy] python_version = 3.6 -files = cibuildwheel,test,unit_test +files = cibuildwheel,test,unit_test,bin warn_unused_configs = True warn_redundant_casts = True From c98c279ba7b6fd6790f8afd41fcdf0a1194b56a8 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 12 Jan 2021 12:09:24 -0500 Subject: [PATCH 3/9] refactor: rewrite and expect install --- .github/workflows/update-dependencies.yml | 2 +- .pre-commit-config.yaml | 1 + bin/update_pythons.py | 229 +++++++++++----------- cibuildwheel/extra.py | 24 +++ cibuildwheel/typing.py | 7 +- unit_test/build_ids_test.py | 37 ++-- 6 files changed, 173 insertions(+), 127 deletions(-) create mode 100644 cibuildwheel/extra.py diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index 836eb9b7f..e5bc58b13 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -21,7 +21,7 @@ jobs: - name: "Run update: dependencies" run: python ./bin/update_dependencies.py - name: "Run update: python configs" - run: python ./bin/update_dependencies.py + run: python ./bin/update_pythons.py --inplace - name: Create Pull Request if: github.ref == 'refs/heads/master' uses: peter-evans/create-pull-request@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d0a118cc..c32a6c04c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,6 +21,7 @@ repos: hooks: - id: black files: ^bin/update_pythons.py$ + args: ["--line-length=120"] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.790 diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 7b8a779bc..a4faead91 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -1,57 +1,40 @@ #!/usr/bin/env python3 -import sys -from collections import defaultdict from itertools import groupby from pathlib import Path -from typing import Dict, List, Tuple +from typing import List +import click import requests import toml from packaging.version import Version -if sys.version_info < (3, 8): - from typing_extensions import TypedDict -else: - from typing import TypedDict - -# Use pretty printing for debugging -# from rich import print - - -allow_prerelease = False - - -class InlineArrayDictEncoder(toml.encoder.TomlEncoder): - def dump_sections(self, o: dict, sup: str): - if all(isinstance(a, list) for a in o.values()): - val = "" - for k, v in o.items(): - inner = ",\n ".join(self.dump_inline_table(d_i).strip() for d_i in v) - val += f"{k} = [\n {inner},\n]\n" - return val, self._dict() - else: - return super().dump_sections(o, sup) - +from cibuildwheel.extra import InlineArrayDictEncoder +from cibuildwheel.typing import PlatformName, TypedDict +# Looking up the dir instead of using utils.resources_dir +# since we want to write to it. DIR = Path(__file__).parent.parent.resolve() RESOURCES_DIR = DIR / "cibuildwheel/resources" -class ConfigWinCP(TypedDict): +class AnyConfig(TypedDict): identifier: str - version: str + version: Version + + +class ConfigWinCP(AnyConfig): arch: str -class ConfigWinPP(TypedDict): - identifier: str - version: str +class ConfigWinPP(AnyConfig): arch: str url: str -def get_cpython_windows() -> Dict[str, List[Version]]: +def get_cpython_windows() -> List[ConfigWinCP]: + ARCH_DICT = {"32": "win32", "64": "win_amd64"} + response = requests.get("https://api.nuget.org/v3/index.json") response.raise_for_status() api_info = response.json() @@ -60,120 +43,140 @@ def get_cpython_windows() -> Dict[str, List[Version]]: if resource["@type"] == "PackageBaseAddress/3.0.0": endpoint = resource["@id"] - cp_versions: Dict[str, List[Version]] = {"64": [], "32": []} - for id, package in [("64", "python"), ("32", "pythonx86")]: + items: List[ConfigWinCP] = [] + + for arch, package in [("64", "python"), ("32", "pythonx86")]: response = requests.get(f"{endpoint}{package}/index.json") response.raise_for_status() cp_info = response.json() for version_str in cp_info["versions"]: version = Version(version_str) + if version.is_devrelease: continue - if not allow_prerelease and version.is_prerelease: - continue - cp_versions[id].append(version) - cp_versions[id].sort() - return cp_versions + identifier = f"cp{version.major}{version.minor}-{ARCH_DICT[arch]}" + + items.append( + ConfigWinCP( + identifier=identifier, + version=version, + arch=arch, + ) + ) + return items -def get_pypy_windows( - plat_arch: str = "win32-x86", -) -> Dict[str, List[Tuple[Version, str]]]: +def get_pypy(platform: PlatformName) -> List[AnyConfig]: response = requests.get("https://downloads.python.org/pypy/versions.json") response.raise_for_status() - pp_realeases = response.json() - pp_versions = defaultdict(list) + pp_releases = response.json() - for pp_realease in pp_realeases: - if pp_realease["pypy_version"] == "nightly": - continue - version = Version(pp_realease["pypy_version"]) - python_version = Version(pp_realease["python_version"]) - python_version_str = f"{python_version.major}.{python_version.minor}" - url = None - for file in pp_realease["files"]: - if f"{file['platform']}-{file['arch']}" == plat_arch: - url = file["download_url"] - break - if url: - pp_versions[python_version_str].append((version, url)) + items: List[AnyConfig] = [] - return pp_versions + for pp_release in pp_releases: + if pp_release["pypy_version"] == "nightly": + continue + pypy_version = Version(pp_release["pypy_version"]) + if pypy_version.is_prerelease or pypy_version.is_devrelease: + continue -# Debugging printout: -# print(get_cpython_windows()) -# print() -# print("[bold]Getting PyPy") -# print(get_pypy_windows()) + version = Version(pp_release["python_version"]) + + for rf in pp_release["files"]: + if platform == "windows": + if rf["platform"] == "win32" and rf["arch"] == "x86": + identifier = f"pp{version.major}{version.minor}-win32" + items.append( + ConfigWinPP( + identifier=identifier, + version=Version(f"{version.major}.{version.minor}"), + arch="32", + url=rf["download_url"], + ) + ) + break -ARCH_DICT = {"32": "win32", "64": "win_amd64"} + return items -def build_ids_cp(in_dict: Dict[str, List[Version]]) -> List[ConfigWinCP]: - items: List[ConfigWinCP] = [] - for arch in in_dict: - for minor, grp in groupby(in_dict[arch], lambda v: v.minor): - # Filter pre-releases, unless it's all pre-releases - if not all(v.is_devrelease for v in grp): - grp = filter(lambda v: not v.is_devrelease, grp) +def sort_and_filter_configs( + orig_items: List[AnyConfig], + *, + prereleases: bool = False, +) -> List[AnyConfig]: - version = sorted(grp)[-1] - identifier = f"cp3{minor}-{ARCH_DICT[arch]}" + items: List[AnyConfig] = [] - items.append( - ConfigWinCP( - identifier=identifier, - version=str(version), - arch=arch, - ) - ) + # Groupby requires pre-grouped input + orig_items = sorted(orig_items, key=lambda x: x["identifier"]) - return items + for _, grp in groupby(orig_items, lambda x: x["identifier"]): + # Never select dev releases + choices = list(filter(lambda x: not x["version"].is_devrelease, grp)) + # Filter pre-releases, unless it's all pre-releases + if not all(x["version"].is_prerelease for x in grp): + choices = list(filter(lambda x: not x["version"].is_prerelease, choices)) -def build_ids_pp(in_dict: Dict[str, List[Tuple[Version, str]]]) -> List[ConfigWinPP]: - items: List[ConfigWinPP] = [] - for vers, matches in in_dict.items(): - vers_id = vers.replace(".", "") - if not all(v[0].is_devrelease for v in matches): - matches = list(filter(lambda v: not v[0].is_devrelease, matches)) + # Select the highest choice unless there are none + _url = "url" # Needed for MyPy, see https://github.com/python/mypy/issues/9902 + choices = sorted(choices, key=lambda x: (x["version"], x.get(_url))) + if not choices: + continue + best_choice = choices[-1] - version, url = sorted(matches, key=lambda v: v[0])[-1] - identifier = f"pp{vers_id}-win32" + # Only allow a pre-release if they are all prereleases, and we've asked for them + if best_choice["version"].is_prerelease and not prereleases: + continue - items.append( - ConfigWinPP( - identifier=identifier, - version=vers, - arch="32", - url=url, - ) - ) + items.append(best_choice) - return items + return sorted( + items, + key=lambda x: ( + x["identifier"][:3], + x["version"].minor, + x["identifier"].split("-")[-1], + ), + ) -windows_configs = [ - *build_ids_cp(get_cpython_windows()), - *build_ids_pp(get_pypy_windows()), +CLASSIC_WINDOWS: List[ConfigWinCP] = [ + {"identifier": "cp27-win32", "version": Version("2.7.18"), "arch": "32"}, + {"identifier": "cp27-win_amd64", "version": Version("2.7.18"), "arch": "64"}, ] -configs = toml.load(RESOURCES_DIR / "build-platforms.toml") -origpy2 = list( - filter( - lambda c: c["identifier"].startswith("cp27"), - configs["windows"]["python_configurations"], - ) -) -items = origpy2 + windows_configs -new_configs = sorted(items, key=lambda x: (x["identifier"][:3], x["identifier"][3:])) +@click.command() +@click.option("--inplace", is_flag=True) +@click.option("--prereleases", is_flag=True) +@click.option("--all", is_flag=True) +def update_pythons(inplace: bool, prereleases: bool, all: bool) -> None: + windows_configs: List[AnyConfig] = [ + *CLASSIC_WINDOWS, + *get_cpython_windows(), + *get_pypy("windows"), + ] + + if not all: + windows_configs = sort_and_filter_configs( + windows_configs, + prereleases=prereleases, + ) + + configs = toml.load(RESOURCES_DIR / "build-platforms.toml") + configs["windows"]["python_configurations"] = windows_configs + + if inplace: + with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: + toml.dump(configs, f, encoder=InlineArrayDictEncoder()) # type: ignore + else: + print(toml.dumps(configs, encoder=InlineArrayDictEncoder())) # type: ignore -configs["windows"]["python_configurations"] = new_configs -with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: - toml.dump(configs, f, encoder=InlineArrayDictEncoder()) # type: ignore +if __name__ == "__main__": + update_pythons() diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py new file mode 100644 index 000000000..3c2ef4fbe --- /dev/null +++ b/cibuildwheel/extra.py @@ -0,0 +1,24 @@ +""" +These are utilities for the `/bin` scripts, not for the `cibuildwheel` program. +""" + +from typing import Any, Dict + +import toml.encoder +from packaging.version import Version + + +class InlineArrayDictEncoder(toml.encoder.TomlEncoder): # type: ignore + def __init__(self) -> None: + super().__init__() + self.dump_funcs[Version] = lambda v: f'"{v}"' + + def dump_sections(self, o: Dict[str, Any], sup: str) -> Any: + if all(isinstance(a, list) for a in o.values()): + val = "" + for k, v in o.items(): + inner = ",\n ".join(self.dump_inline_table(d_i).strip() for d_i in v) + val += f"{k} = [\n {inner},\n]\n" + return val, self._dict() + else: + return super().dump_sections(o, sup) diff --git a/cibuildwheel/typing.py b/cibuildwheel/typing.py index 03a1c7c1a..9a650d546 100644 --- a/cibuildwheel/typing.py +++ b/cibuildwheel/typing.py @@ -4,9 +4,12 @@ from typing import TYPE_CHECKING, NoReturn, Set, Union if sys.version_info < (3, 8): - from typing_extensions import Final, Literal + from typing_extensions import Final, Literal, TypedDict else: - from typing import Final, Literal + from typing import Final, Literal, TypedDict + + +__all__ = ("Final", "Literal", "TypedDict", "Set", "Union", "PopenBytes", "PathOrStr", "PlatformName", "PLATFORMS", "assert_never") if TYPE_CHECKING: diff --git a/unit_test/build_ids_test.py b/unit_test/build_ids_test.py index 99c5b57ed..1993efd03 100644 --- a/unit_test/build_ids_test.py +++ b/unit_test/build_ids_test.py @@ -1,19 +1,11 @@ +import pytest import toml -from toml.encoder import TomlEncoder from cibuildwheel.util import resources_dir +Version = pytest.importorskip("packaging.version").Version -class InlineArrayDictEncoder(TomlEncoder): - def dump_sections(self, o: dict, sup: str): - if all(isinstance(a, list) for a in o.values()): - val = "" - for k, v in o.items(): - inner = ",\n ".join(self.dump_inline_table(d_i).strip() for d_i in v) - val += f"{k} = [\n {inner},\n]\n" - return val, self._dict() - else: - return super().dump_sections(o, sup) +from cibuildwheel.extra import InlineArrayDictEncoder # noqa: E402 def test_compare_configs(): @@ -26,3 +18,26 @@ def test_compare_configs(): print(new_txt) assert new_txt == txt + + +def test_dump_with_Version(): + example = { + "windows": { + "python_configurations": [ + {"identifier": "cp27-win32", "version": Version("2.7.18"), "arch": "32"}, + {"identifier": "cp27-win_amd64", "version": Version("2.7.18"), "arch": "64"}, + ] + } + } + + result = """\ +[windows] +python_configurations = [ + { identifier = "cp27-win32", version = "2.7.18", arch = "32" }, + { identifier = "cp27-win_amd64", version = "2.7.18", arch = "64" }, +] +""" + + output = toml.dumps(example, encoder=InlineArrayDictEncoder()) + print(output) + assert output == result From 9abe6f75be4f6cf8d975a5ef7e653bb810a422f2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 12 Jan 2021 15:48:51 -0500 Subject: [PATCH 4/9] feat: support macos too, logging output --- .pre-commit-config.yaml | 2 +- bin/update_pythons.py | 135 ++++++++++++++++++++++++++++++++++++---- setup.cfg | 1 + 3 files changed, 126 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c32a6c04c..623e5d605 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: mypy files: ^(cibuildwheel/|test/|bin/projects.py|bin/update_pythons.py|unit_test/) pass_filenames: false - additional_dependencies: [packaging] + additional_dependencies: [packaging, click] - repo: https://github.com/asottile/pyupgrade rev: v2.7.4 diff --git a/bin/update_pythons.py b/bin/update_pythons.py index a4faead91..4a72dc10d 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -1,21 +1,30 @@ #!/usr/bin/env python3 +import logging from itertools import groupby from pathlib import Path from typing import List import click import requests +import rich import toml +from packaging.specifiers import SpecifierSet from packaging.version import Version +from rich.logging import RichHandler +from rich.syntax import Syntax from cibuildwheel.extra import InlineArrayDictEncoder -from cibuildwheel.typing import PlatformName, TypedDict +from cibuildwheel.typing import Final, PlatformName, TypedDict + +log = logging.getLogger("cibw") # Looking up the dir instead of using utils.resources_dir # since we want to write to it. -DIR = Path(__file__).parent.parent.resolve() -RESOURCES_DIR = DIR / "cibuildwheel/resources" +DIR: Final[Path] = Path(__file__).parent.parent.resolve() +RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources" + +CIBW_SUPPORTED_PYTHONS: Final[SpecifierSet] = SpecifierSet(">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*") class AnyConfig(TypedDict): @@ -32,7 +41,12 @@ class ConfigWinPP(AnyConfig): url: str +class ConfigMacOS(AnyConfig): + url: str + + def get_cpython_windows() -> List[ConfigWinCP]: + log.info("[bold]Collecting Windows CPython from nuget") ARCH_DICT = {"32": "win32", "64": "win_amd64"} response = requests.get("https://api.nuget.org/v3/index.json") @@ -65,10 +79,12 @@ def get_cpython_windows() -> List[ConfigWinCP]: arch=arch, ) ) + log.debug(items[-1]) return items def get_pypy(platform: PlatformName) -> List[AnyConfig]: + log.info("[bold]Collecting PyPy from python.org") response = requests.get("https://downloads.python.org/pypy/versions.json") response.raise_for_status() @@ -98,11 +114,71 @@ def get_pypy(platform: PlatformName) -> List[AnyConfig]: url=rf["download_url"], ) ) + log.debug(items[-1]) + break + elif platform == "macos": + if rf["platform"] == "darwin" and rf["arch"] == "x64": + identifier = f"pp{version.major}{version.minor}-macosx_x86_64" + items.append( + ConfigMacOS( + identifier=identifier, + version=Version(f"{version.major}.{version.minor}"), + url=rf["download_url"], + ) + ) + log.debug(items[-1]) break return items +def _get_id(resource_uri: str) -> int: + return int(resource_uri.rstrip("/").split("/")[-1]) + + +def get_cpython( + plat_arch: str, + file_ident: str, + versions: SpecifierSet = CIBW_SUPPORTED_PYTHONS, +) -> List[ConfigMacOS]: + log.info(f"[bold]Collecting {plat_arch} CPython from Python.org") + + response = requests.get("https://www.python.org/api/v2/downloads/release/?is_published=true") + response.raise_for_status() + + releases_info = response.json() + # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") + known_versions = {Version(release["name"][7:]): _get_id(release["resource_uri"]) for release in releases_info} + + items: List[ConfigMacOS] = [] + + sorted_versions = sorted((v for v in known_versions if versions.contains(v) and not v.is_prerelease), reverse=True) + # Group is a list of sorted patch versions + for pair, group in groupby(sorted_versions, lambda x: (x.major, x.minor)): + log.info(f"[bold]Working on {pair[0]}.{pair[1]}") + # Find the first patch version that contains the requested file + for version in group: + uri = known_versions[version] + + log.info(f" Checking {version}") + response = requests.get(f"https://www.python.org/api/v2/downloads/release_file/?release={uri}") + response.raise_for_status() + file_info = response.json() + + canidate_files = [rf["url"] for rf in file_info if file_ident in rf["url"]] + if canidate_files: + items.append( + ConfigMacOS( + identifier=f"cp{version.major}{version.minor}-{plat_arch}", + version=version, + url=canidate_files[0], + ) + ) + log.info("[green] Found!") + break + return items + + def sort_and_filter_configs( orig_items: List[AnyConfig], *, @@ -154,28 +230,65 @@ def sort_and_filter_configs( @click.command() @click.option("--inplace", is_flag=True) @click.option("--prereleases", is_flag=True) -@click.option("--all", is_flag=True) -def update_pythons(inplace: bool, prereleases: bool, all: bool) -> None: +@click.option("--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False)) +def update_pythons(inplace: bool, prereleases: bool, level: str) -> None: + + logging.basicConfig( + level="INFO", + format="%(message)s", + datefmt="[%X]", + handlers=[RichHandler(rich_tracebacks=True, markup=True)], + ) + log.setLevel(level) + windows_configs: List[AnyConfig] = [ *CLASSIC_WINDOWS, *get_cpython_windows(), *get_pypy("windows"), ] - if not all: - windows_configs = sort_and_filter_configs( - windows_configs, - prereleases=prereleases, - ) + windows_configs = sort_and_filter_configs( + windows_configs, + prereleases=prereleases, + ) + + macos_configs = [ + *get_cpython( + plat_arch="macosx_x86_64", + file_ident="macosx10.9.pkg", + ), + *get_cpython( + plat_arch="macosx_x86_64", + file_ident="macosx10.6.pkg", + versions=SpecifierSet("==3.5.*"), + ), + *get_pypy("macos"), + ] + + # For universal2: + # plat_arch="macosx_universal2", + # file_ident="macos11.0.pkg", + # versions=SpecifierSet(">=3.8"), + + macos_configs = sort_and_filter_configs( + macos_configs, + prereleases=prereleases, + ) + + for config in macos_configs: + config["version"] = Version("{0.major}.{0.minor}".format(config["version"])) configs = toml.load(RESOURCES_DIR / "build-platforms.toml") configs["windows"]["python_configurations"] = windows_configs + configs["macos"]["python_configurations"] = macos_configs if inplace: with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: toml.dump(configs, f, encoder=InlineArrayDictEncoder()) # type: ignore else: - print(toml.dumps(configs, encoder=InlineArrayDictEncoder())) # type: ignore + output = toml.dumps(configs, encoder=InlineArrayDictEncoder()) # type: ignore + rich.print(Syntax(output, "toml", theme="ansi_light")) + log.info("File not changed, use --inplace flag to update.") if __name__ == "__main__": diff --git a/setup.cfg b/setup.cfg index dc9fed9fa..2220ab453 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ dev = requests typing-extensions packaging>=20.8 + rich>=9.6 [options.packages.find] include = From 1de0718b4d24d699c35c16b33960b549853685c0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 14 Jan 2021 16:34:33 -0500 Subject: [PATCH 5/9] WIP: update --- bin/update_pythons.py | 336 ++++++++++++++++++------------------------ 1 file changed, 142 insertions(+), 194 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 4a72dc10d..da0f5485d 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -3,19 +3,19 @@ import logging from itertools import groupby from pathlib import Path -from typing import List +from typing import Dict, List, Optional import click import requests import rich -import toml +import tomlkit from packaging.specifiers import SpecifierSet from packaging.version import Version from rich.logging import RichHandler from rich.syntax import Syntax from cibuildwheel.extra import InlineArrayDictEncoder -from cibuildwheel.typing import Final, PlatformName, TypedDict +from cibuildwheel.typing import Final, Literal, PlatformName, TypedDict log = logging.getLogger("cibw") @@ -27,209 +27,169 @@ CIBW_SUPPORTED_PYTHONS: Final[SpecifierSet] = SpecifierSet(">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*") -class AnyConfig(TypedDict): - identifier: str - version: Version +ArchStr = Literal["32", "64"] -class ConfigWinCP(AnyConfig): +class ConfigWinCP(TypedDict): + identifier: str + version: Version arch: str -class ConfigWinPP(AnyConfig): +class ConfigWinPP(TypedDict): + identifier: str + version: Version arch: str url: str -class ConfigMacOS(AnyConfig): +class ConfigMacOS(TypedDict): + identifier: str + version: Version url: str -def get_cpython_windows() -> List[ConfigWinCP]: - log.info("[bold]Collecting Windows CPython from nuget") - ARCH_DICT = {"32": "win32", "64": "win_amd64"} +AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS] - response = requests.get("https://api.nuget.org/v3/index.json") - response.raise_for_status() - api_info = response.json() - for resource in api_info["resources"]: - if resource["@type"] == "PackageBaseAddress/3.0.0": - endpoint = resource["@id"] +class WindowsVersions: + def __init__(self, arch_str: ArchStr) -> None: + + response = requests.get("https://api.nuget.org/v3/index.json") + response.raise_for_status() + api_info = response.json() - items: List[ConfigWinCP] = [] + for resource in api_info["resources"]: + if resource["@type"] == "PackageBaseAddress/3.0.0": + endpoint = resource["@id"] + + ARCH_DICT = {"32": "win32", "64": "win_amd64"} + PACKAGE_DICT = {"32": "pythonx86", "64": "python"} + + arch = ARCH_DICT[arch_str] + package = PACKAGE_DICT[arch_str] - for arch, package in [("64", "python"), ("32", "pythonx86")]: response = requests.get(f"{endpoint}{package}/index.json") response.raise_for_status() cp_info = response.json() - for version_str in cp_info["versions"]: - version = Version(version_str) + versions = (Version(v) for v in cp_info["versions"]) + self.versions = sorted(v for v in versions if not v.is_devrelease) - if version.is_devrelease: - continue + def update_version(self, spec: Specifier) -> Optional[ConfigWinCP]: + versions = sorted(v for v in self.versions if spec.contains(v)) + if not all(v.is_prerelease for v in versions): + versions = [v for v in versions if not v.is_prerelease] + log.debug(versions) - identifier = f"cp{version.major}{version.minor}-{ARCH_DICT[arch]}" + if not versions: + return None - items.append( - ConfigWinCP( - identifier=identifier, - version=version, - arch=arch, - ) - ) - log.debug(items[-1]) - return items - - -def get_pypy(platform: PlatformName) -> List[AnyConfig]: - log.info("[bold]Collecting PyPy from python.org") - - response = requests.get("https://downloads.python.org/pypy/versions.json") - response.raise_for_status() - pp_releases = response.json() - - items: List[AnyConfig] = [] - - for pp_release in pp_releases: - - if pp_release["pypy_version"] == "nightly": - continue - pypy_version = Version(pp_release["pypy_version"]) - if pypy_version.is_prerelease or pypy_version.is_devrelease: - continue - - version = Version(pp_release["python_version"]) - - for rf in pp_release["files"]: - if platform == "windows": - if rf["platform"] == "win32" and rf["arch"] == "x86": - identifier = f"pp{version.major}{version.minor}-win32" - items.append( - ConfigWinPP( - identifier=identifier, - version=Version(f"{version.major}.{version.minor}"), - arch="32", - url=rf["download_url"], - ) - ) - log.debug(items[-1]) - break - elif platform == "macos": - if rf["platform"] == "darwin" and rf["arch"] == "x64": - identifier = f"pp{version.major}{version.minor}-macosx_x86_64" - items.append( - ConfigMacOS( - identifier=identifier, - version=Version(f"{version.major}.{version.minor}"), - url=rf["download_url"], - ) - ) - log.debug(items[-1]) - break - - return items + version = versions[-1] + identifier = f"cp{version.major}{version.minor}-{ARCH_DICT[arch]}" + result = ConfigWinCP( + identifier=identifier, + version=version, + arch=arch, + ) + return result + + +class PyPyVersions: + def __init__(self, arch_str: ArchStr): + + response = requests.get("https://downloads.python.org/pypy/versions.json") + response.raise_for_status() + + releases = [r for r in response.json() if r["pypy_version"] != "nightly"] + for release in releases: + release["pypy_version"] = Version(release["pypy_version"]) + release["python_version"] = Version(release["python_version"]) + + self.releases = [r for r in releases if not r["pypy_version"].is_prerelease and r["pypy_version"].is_devrelease] + self.arch == arch_str + + def update_version_windows(self, spec: Specifier) -> Optional[ConfigWinCP]: + if self.arch != "32": + raise RuntimeError("64 bit releases not supported yet on Windows") + + releases = [r for r in releases if spec.contains(r["python_verison"])] + releases = sorted(releases, key=lambda r: r["pypy_version"]) + + if not releases: + return None + + release = releases[-1] + version = release["python_version"] + identifier = f"pp{version.major}{version.minor}-win32" + return ConfigWinPP( + identifier=identifier, + version=Version(f"{version.major}.{version.minor}"), + arch="32", + url=r["download_url"], + ) + + def update_version_macos(self, spec: Specifier) -> Optional[ConfigMacOS]: + if self.arch != "64": + raise RuntimeError("Other archs not supported yet on macOS") + + releases = [r for r in releases if spec.contains(r["python_verison"])] + releases = sorted(releases, key=lambda r: r["pypy_version"]) + + if not releases: + return None + + release = releases[-1] + version = release["python_version"] + identifier = f"pp{version.major}{version.minor}-win32" + + return ConfigMacOS( + identifier=identifier, + version=Version(f"{version.major}.{version.minor}"), + url=rf["download_url"], + ) def _get_id(resource_uri: str) -> int: return int(resource_uri.rstrip("/").split("/")[-1]) -def get_cpython( - plat_arch: str, - file_ident: str, - versions: SpecifierSet = CIBW_SUPPORTED_PYTHONS, -) -> List[ConfigMacOS]: - log.info(f"[bold]Collecting {plat_arch} CPython from Python.org") +class CPythonVersions: + def __init__(self, plat_arch: str, file_ident: str) -> None: + + response = requests.get("https://www.python.org/api/v2/downloads/release/?is_published=true") + response.raise_for_status() - response = requests.get("https://www.python.org/api/v2/downloads/release/?is_published=true") - response.raise_for_status() + releases_info = response.json() - releases_info = response.json() - # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") - known_versions = {Version(release["name"][7:]): _get_id(release["resource_uri"]) for release in releases_info} + # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") + known_versions = {Version(release["name"][7:]): _get_id(release["resource_uri"]) for release in releases_info} + self.versions = sorted(v for v in known_versions if not (v.is_prerelease or v.is_devrelease)) - items: List[ConfigMacOS] = [] + def update_python_macos(self, spec: Specifier) -> Optional[ConfigMacOS]: - sorted_versions = sorted((v for v in known_versions if versions.contains(v) and not v.is_prerelease), reverse=True) - # Group is a list of sorted patch versions - for pair, group in groupby(sorted_versions, lambda x: (x.major, x.minor)): - log.info(f"[bold]Working on {pair[0]}.{pair[1]}") - # Find the first patch version that contains the requested file - for version in group: - uri = known_versions[version] + sorted_versions = [v for v in self.versions if spec.contains(v)] - log.info(f" Checking {version}") + for version in reversed(versions): + # Find the first patch version that contains the requested file + uri = self.versions[version] response = requests.get(f"https://www.python.org/api/v2/downloads/release_file/?release={uri}") response.raise_for_status() file_info = response.json() canidate_files = [rf["url"] for rf in file_info if file_ident in rf["url"]] if canidate_files: - items.append( - ConfigMacOS( - identifier=f"cp{version.major}{version.minor}-{plat_arch}", - version=version, - url=canidate_files[0], - ) + return ConfigMacOS( + identifier=f"cp{version.major}{version.minor}-{plat_arch}", + version=version, + url=canidate_files[0], ) - log.info("[green] Found!") - break - return items - - -def sort_and_filter_configs( - orig_items: List[AnyConfig], - *, - prereleases: bool = False, -) -> List[AnyConfig]: - - items: List[AnyConfig] = [] - - # Groupby requires pre-grouped input - orig_items = sorted(orig_items, key=lambda x: x["identifier"]) - - for _, grp in groupby(orig_items, lambda x: x["identifier"]): - # Never select dev releases - choices = list(filter(lambda x: not x["version"].is_devrelease, grp)) - - # Filter pre-releases, unless it's all pre-releases - if not all(x["version"].is_prerelease for x in grp): - choices = list(filter(lambda x: not x["version"].is_prerelease, choices)) - - # Select the highest choice unless there are none - _url = "url" # Needed for MyPy, see https://github.com/python/mypy/issues/9902 - choices = sorted(choices, key=lambda x: (x["version"], x.get(_url))) - if not choices: - continue - best_choice = choices[-1] - - # Only allow a pre-release if they are all prereleases, and we've asked for them - if best_choice["version"].is_prerelease and not prereleases: - continue - - items.append(best_choice) - - return sorted( - items, - key=lambda x: ( - x["identifier"][:3], - x["version"].minor, - x["identifier"].split("-")[-1], - ), - ) - -CLASSIC_WINDOWS: List[ConfigWinCP] = [ - {"identifier": "cp27-win32", "version": Version("2.7.18"), "arch": "32"}, - {"identifier": "cp27-win_amd64", "version": Version("2.7.18"), "arch": "64"}, -] + return None @click.command() @click.option("--inplace", is_flag=True) -@click.option("--prereleases", is_flag=True) @click.option("--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False)) def update_pythons(inplace: bool, prereleases: bool, level: str) -> None: @@ -241,46 +201,34 @@ def update_pythons(inplace: bool, prereleases: bool, level: str) -> None: ) log.setLevel(level) - windows_configs: List[AnyConfig] = [ - *CLASSIC_WINDOWS, - *get_cpython_windows(), - *get_pypy("windows"), - ] + windows_32 = WindowsVersions("32") + windows_64 = WindowsVersions("64") + windows_pypy = PyPyVersions("32") - windows_configs = sort_and_filter_configs( - windows_configs, - prereleases=prereleases, - ) + macos_6 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.6.pkg") - macos_configs = [ - *get_cpython( - plat_arch="macosx_x86_64", - file_ident="macosx10.9.pkg", - ), - *get_cpython( - plat_arch="macosx_x86_64", - file_ident="macosx10.6.pkg", - versions=SpecifierSet("==3.5.*"), - ), - *get_pypy("macos"), - ] - - # For universal2: - # plat_arch="macosx_universal2", - # file_ident="macos11.0.pkg", - # versions=SpecifierSet(">=3.8"), - - macos_configs = sort_and_filter_configs( - macos_configs, - prereleases=prereleases, + macos_9 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.9.pkg") + + macos_u2 = CPythonVersions( + plat_arch="macosx_universal2", + file_ident="macos11.0.pkg", ) - for config in macos_configs: - config["version"] = Version("{0.major}.{0.minor}".format(config["version"])) + macos_pypy = PyPyVersions("64") configs = toml.load(RESOURCES_DIR / "build-platforms.toml") - configs["windows"]["python_configurations"] = windows_configs - configs["macos"]["python_configurations"] = macos_configs + + for config in configs["windows"]["python_configurations"]: + version = Version(config["version"]) + spec = Specifier(f"=={version.major}.{version.minor}.*") + arch = config["arch"] + cpython = config["identifier"].startswith("cp") + + for config in configs["macos"]["python_configurations"]: + version = Version(config["version"]) + spec = Specifier(f"=={version.major}.{version.minor}.*") + arch = "64" + pypy = config["identifier"].startswith("pp") if inplace: with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: From 230538c63fdef3652343b6191e4016236dcfe129 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Jan 2021 00:54:18 -0500 Subject: [PATCH 6/9] refactor: drive from original file --- bin/update_pythons.py | 169 ++++++++++++++++++++++++++---------------- 1 file changed, 106 insertions(+), 63 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index da0f5485d..da527ff6f 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 +import copy import logging -from itertools import groupby from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, Optional, Union import click import requests import rich -import tomlkit -from packaging.specifiers import SpecifierSet +import toml +from packaging.specifiers import Specifier, SpecifierSet from packaging.version import Version from rich.logging import RichHandler from rich.syntax import Syntax from cibuildwheel.extra import InlineArrayDictEncoder -from cibuildwheel.typing import Final, Literal, PlatformName, TypedDict +from cibuildwheel.typing import Final, Literal, TypedDict log = logging.getLogger("cibw") @@ -32,20 +32,20 @@ class ConfigWinCP(TypedDict): identifier: str - version: Version + version: str arch: str class ConfigWinPP(TypedDict): identifier: str - version: Version + version: str arch: str url: str class ConfigMacOS(TypedDict): identifier: str - version: Version + version: str url: str @@ -66,7 +66,8 @@ def __init__(self, arch_str: ArchStr) -> None: ARCH_DICT = {"32": "win32", "64": "win_amd64"} PACKAGE_DICT = {"32": "pythonx86", "64": "python"} - arch = ARCH_DICT[arch_str] + self.arch_str = arch_str + self.arch = ARCH_DICT[arch_str] package = PACKAGE_DICT[arch_str] response = requests.get(f"{endpoint}{package}/index.json") @@ -76,21 +77,21 @@ def __init__(self, arch_str: ArchStr) -> None: versions = (Version(v) for v in cp_info["versions"]) self.versions = sorted(v for v in versions if not v.is_devrelease) - def update_version(self, spec: Specifier) -> Optional[ConfigWinCP]: + def update_version_windows(self, spec: Specifier) -> Optional[ConfigWinCP]: versions = sorted(v for v in self.versions if spec.contains(v)) if not all(v.is_prerelease for v in versions): versions = [v for v in versions if not v.is_prerelease] - log.debug(versions) + log.debug(f"Windows {self.arch} {spec} has {', '.join(str(v) for v in versions)}") if not versions: return None version = versions[-1] - identifier = f"cp{version.major}{version.minor}-{ARCH_DICT[arch]}" + identifier = f"cp{version.major}{version.minor}-{self.arch}" result = ConfigWinCP( identifier=identifier, - version=version, - arch=arch, + version=str(version), + arch=self.arch_str, ) return result @@ -106,47 +107,56 @@ def __init__(self, arch_str: ArchStr): release["pypy_version"] = Version(release["pypy_version"]) release["python_version"] = Version(release["python_version"]) - self.releases = [r for r in releases if not r["pypy_version"].is_prerelease and r["pypy_version"].is_devrelease] - self.arch == arch_str + self.releases = [ + r for r in releases if not r["pypy_version"].is_prerelease and not r["pypy_version"].is_devrelease + ] + self.arch = arch_str - def update_version_windows(self, spec: Specifier) -> Optional[ConfigWinCP]: + def update_version_windows(self, spec: Specifier) -> ConfigWinCP: if self.arch != "32": raise RuntimeError("64 bit releases not supported yet on Windows") - releases = [r for r in releases if spec.contains(r["python_verison"])] + releases = [r for r in self.releases if spec.contains(r["python_version"])] releases = sorted(releases, key=lambda r: r["pypy_version"]) if not releases: - return None + raise RuntimeError(f"PyPy Win {self.arch} not found for {spec}! {self.releases}") release = releases[-1] version = release["python_version"] identifier = f"pp{version.major}{version.minor}-win32" + + (url,) = [rf["download_url"] for rf in release["files"] if "" in rf["platform"] == "win32"] + return ConfigWinPP( identifier=identifier, - version=Version(f"{version.major}.{version.minor}"), + version=f"{version.major}.{version.minor}", arch="32", - url=r["download_url"], + url=url, ) - def update_version_macos(self, spec: Specifier) -> Optional[ConfigMacOS]: + def update_version_macos(self, spec: Specifier) -> ConfigMacOS: if self.arch != "64": raise RuntimeError("Other archs not supported yet on macOS") - releases = [r for r in releases if spec.contains(r["python_verison"])] + releases = [r for r in self.releases if spec.contains(r["python_version"])] releases = sorted(releases, key=lambda r: r["pypy_version"]) if not releases: - return None + raise RuntimeError(f"PyPy macOS {self.arch} not found for {spec}!") release = releases[-1] version = release["python_version"] - identifier = f"pp{version.major}{version.minor}-win32" + identifier = f"pp{version.major}{version.minor}-macosx_x86_64" + + (url,) = [ + rf["download_url"] for rf in release["files"] if "" in rf["platform"] == "darwin" and rf["arch"] == "x64" + ] return ConfigMacOS( identifier=identifier, - version=Version(f"{version.major}.{version.minor}"), - url=rf["download_url"], + version=f"{version.major}.{version.minor}", + url=url, ) @@ -162,36 +172,89 @@ def __init__(self, plat_arch: str, file_ident: str) -> None: releases_info = response.json() - # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") - known_versions = {Version(release["name"][7:]): _get_id(release["resource_uri"]) for release in releases_info} - self.versions = sorted(v for v in known_versions if not (v.is_prerelease or v.is_devrelease)) + self.versions_dict: Dict[Version, int] = {} + for release in releases_info: + # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") + version = Version(release["name"][7:]) + + if not version.is_prerelease and not version.is_devrelease: + uri = int(release["resource_uri"].rstrip("/").split("/")[-1]) + self.versions_dict[version] = uri - def update_python_macos(self, spec: Specifier) -> Optional[ConfigMacOS]: + self.file_ident = file_ident + self.plat_arch = plat_arch + + def update_version_macos(self, spec: Specifier) -> Optional[ConfigMacOS]: - sorted_versions = [v for v in self.versions if spec.contains(v)] + sorted_versions = sorted(v for v in self.versions_dict if spec.contains(v)) - for version in reversed(versions): + for version in reversed(sorted_versions): # Find the first patch version that contains the requested file - uri = self.versions[version] + uri = self.versions_dict[version] response = requests.get(f"https://www.python.org/api/v2/downloads/release_file/?release={uri}") response.raise_for_status() file_info = response.json() - canidate_files = [rf["url"] for rf in file_info if file_ident in rf["url"]] - if canidate_files: + urls = [rf["url"] for rf in file_info if self.file_ident in rf["url"]] + if urls: return ConfigMacOS( - identifier=f"cp{version.major}{version.minor}-{plat_arch}", - version=version, - url=canidate_files[0], + identifier=f"cp{version.major}{version.minor}-{self.plat_arch}", + version=f"{version.major}.{version.minor}", + url=urls[0], ) return None +class AllVersions: + def __init__(self) -> None: + self.windows_32 = WindowsVersions("32") + self.windows_64 = WindowsVersions("64") + self.windows_pypy = PyPyVersions("32") + + self.macos_6 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.6.pkg") + self.macos_9 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.9.pkg") + self.macos_u2 = CPythonVersions( + plat_arch="macosx_universal2", + file_ident="macos11.0.pkg", + ) + self.macos_pypy = PyPyVersions("64") + + def update_config(self, config: Dict[str, str]) -> None: + identifier = config["identifier"] + version = Version(config["version"]) + spec = Specifier(f"=={version.major}.{version.minor}.*") + log.info(f"Reading in '{identifier}' -> {spec} @ {version}") + orig_config = copy.copy(config) + config_update: Optional[AnyConfig] + + if "macosx_x86_64" in identifier: + if identifier.startswith("pp"): + config_update = self.macos_pypy.update_version_macos(spec) + else: + config_update = self.macos_9.update_version_macos(spec) or self.macos_6.update_version_macos(spec) + assert config_update is not None, f"MacOS {spec} not found!" + config.update(**config_update) + elif "win32" in identifier: + if identifier.startswith("pp"): + config.update(**self.windows_pypy.update_version_windows(spec)) + else: + config_update = self.windows_32.update_version_windows(spec) + if config_update: + config.update(**config_update) + elif "win_amd64" in identifier: + config_update = self.windows_64.update_version_windows(spec) + if config_update: + config.update(**config_update) + + if config != orig_config: + log.info(f" Updated {orig_config} to {config}") + + @click.command() @click.option("--inplace", is_flag=True) @click.option("--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False)) -def update_pythons(inplace: bool, prereleases: bool, level: str) -> None: +def update_pythons(inplace: bool, level: str) -> None: logging.basicConfig( level="INFO", @@ -201,34 +264,14 @@ def update_pythons(inplace: bool, prereleases: bool, level: str) -> None: ) log.setLevel(level) - windows_32 = WindowsVersions("32") - windows_64 = WindowsVersions("64") - windows_pypy = PyPyVersions("32") - - macos_6 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.6.pkg") - - macos_9 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.9.pkg") - - macos_u2 = CPythonVersions( - plat_arch="macosx_universal2", - file_ident="macos11.0.pkg", - ) - - macos_pypy = PyPyVersions("64") - + all_versions = AllVersions() configs = toml.load(RESOURCES_DIR / "build-platforms.toml") for config in configs["windows"]["python_configurations"]: - version = Version(config["version"]) - spec = Specifier(f"=={version.major}.{version.minor}.*") - arch = config["arch"] - cpython = config["identifier"].startswith("cp") + all_versions.update_config(config) for config in configs["macos"]["python_configurations"]: - version = Version(config["version"]) - spec = Specifier(f"=={version.major}.{version.minor}.*") - arch = "64" - pypy = config["identifier"].startswith("pp") + all_versions.update_config(config) if inplace: with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: From 06321bea7fe9524919a3ecab94b72ee81b254e94 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 15 Jan 2021 13:12:57 +0000 Subject: [PATCH 7/9] Remove unused variable --- bin/update_pythons.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index da527ff6f..483725bca 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -9,7 +9,7 @@ import requests import rich import toml -from packaging.specifiers import Specifier, SpecifierSet +from packaging.specifiers import Specifier from packaging.version import Version from rich.logging import RichHandler from rich.syntax import Syntax @@ -24,8 +24,6 @@ DIR: Final[Path] = Path(__file__).parent.parent.resolve() RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources" -CIBW_SUPPORTED_PYTHONS: Final[SpecifierSet] = SpecifierSet(">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*") - ArchStr = Literal["32", "64"] From a2a9b571c49ce0b6d166a9bb439edfd6d0295770 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 15 Jan 2021 13:34:10 +0000 Subject: [PATCH 8/9] Output a diff instead of the result file, to review changes more easily --- .github/workflows/update-dependencies.yml | 2 +- bin/update_pythons.py | 42 ++++++++++++++++++----- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index e5bc58b13..c483b270d 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -21,7 +21,7 @@ jobs: - name: "Run update: dependencies" run: python ./bin/update_dependencies.py - name: "Run update: python configs" - run: python ./bin/update_pythons.py --inplace + run: python ./bin/update_pythons.py --force - name: Create Pull Request if: github.ref == 'refs/heads/master' uses: peter-evans/create-pull-request@v3 diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 483725bca..9be9f1f74 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import copy +import difflib import logging from pathlib import Path from typing import Dict, Optional, Union @@ -250,9 +251,9 @@ def update_config(self, config: Dict[str, str]) -> None: @click.command() -@click.option("--inplace", is_flag=True) +@click.option("--force", is_flag=True) @click.option("--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False)) -def update_pythons(inplace: bool, level: str) -> None: +def update_pythons(force: bool, level: str) -> None: logging.basicConfig( level="INFO", @@ -263,7 +264,10 @@ def update_pythons(inplace: bool, level: str) -> None: log.setLevel(level) all_versions = AllVersions() - configs = toml.load(RESOURCES_DIR / "build-platforms.toml") + toml_file_path = RESOURCES_DIR / "build-platforms.toml" + + original_toml = toml_file_path.read_text() + configs = toml.loads(original_toml) for config in configs["windows"]["python_configurations"]: all_versions.update_config(config) @@ -271,13 +275,33 @@ def update_pythons(inplace: bool, level: str) -> None: for config in configs["macos"]["python_configurations"]: all_versions.update_config(config) - if inplace: - with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: - toml.dump(configs, f, encoder=InlineArrayDictEncoder()) # type: ignore + result_toml = toml.dumps(configs, encoder=InlineArrayDictEncoder()) # type: ignore + + rich.print() # spacer + + if original_toml == result_toml: + rich.print("[green]Check complete, Python configurations unchanged.") + exit() + + rich.print("Python configurations updated.") + rich.print("Changes:") + rich.print() + + toml_relpath = toml_file_path.relative_to(DIR).as_posix() + diff_lines = difflib.unified_diff( + original_toml.splitlines(keepends=True), + result_toml.splitlines(keepends=True), + fromfile=toml_relpath, + tofile=toml_relpath, + ) + rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light")) + rich.print() + + if force: + toml_file_path.write_text(result_toml) + rich.print("[green]TOML file updated.") else: - output = toml.dumps(configs, encoder=InlineArrayDictEncoder()) # type: ignore - rich.print(Syntax(output, "toml", theme="ansi_light")) - log.info("File not changed, use --inplace flag to update.") + rich.print("[yellow]File left unchanged. Use --force flag to update.") if __name__ == "__main__": From 8d56879ad0ed54c44bc3814d71b0cb99c6316766 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Jan 2021 08:43:58 -0500 Subject: [PATCH 9/9] fix: minor cleanup --- bin/update_pythons.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 9be9f1f74..04f075de6 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -51,6 +51,10 @@ class ConfigMacOS(TypedDict): AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS] +# The following set of "Versions" classes allow the initial call to the APIs to +# be cached and reused in the `update_version_*` methods. + + class WindowsVersions: def __init__(self, arch_str: ArchStr) -> None: @@ -159,10 +163,6 @@ def update_version_macos(self, spec: Specifier) -> ConfigMacOS: ) -def _get_id(resource_uri: str) -> int: - return int(resource_uri.rstrip("/").split("/")[-1]) - - class CPythonVersions: def __init__(self, plat_arch: str, file_ident: str) -> None: @@ -205,6 +205,10 @@ def update_version_macos(self, spec: Specifier) -> Optional[ConfigMacOS]: return None +# This is a universal interface to all the above Versions classes. Given an +# identifier, it updates a config dict. + + class AllVersions: def __init__(self) -> None: self.windows_32 = WindowsVersions("32") @@ -213,10 +217,7 @@ def __init__(self) -> None: self.macos_6 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.6.pkg") self.macos_9 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.9.pkg") - self.macos_u2 = CPythonVersions( - plat_arch="macosx_universal2", - file_ident="macos11.0.pkg", - ) + self.macos_u2 = CPythonVersions(plat_arch="macosx_universal2", file_ident="macos11.0.pkg") self.macos_pypy = PyPyVersions("64") def update_config(self, config: Dict[str, str]) -> None: @@ -227,6 +228,7 @@ def update_config(self, config: Dict[str, str]) -> None: orig_config = copy.copy(config) config_update: Optional[AnyConfig] + # We need to use ** in update due to MyPy (probably a bug) if "macosx_x86_64" in identifier: if identifier.startswith("pp"): config_update = self.macos_pypy.update_version_macos(spec) @@ -281,7 +283,7 @@ def update_pythons(force: bool, level: str) -> None: if original_toml == result_toml: rich.print("[green]Check complete, Python configurations unchanged.") - exit() + return rich.print("Python configurations updated.") rich.print("Changes:")