diff --git a/conda-lock.yml b/conda-lock.yml index bcdb14a9a5..06f476b55b 100644 --- a/conda-lock.yml +++ b/conda-lock.yml @@ -3,9 +3,9 @@ metadata: - url: conda-forge used_env_vars: [] content_hash: - linux-64: 47e27eadf78b024984ba2246127d65e7524c60bcb407b978498c1135cec23162 - osx-64: 5e63598ecaacbfd5f9f0e91975968b3d3bc12fae95d5cee457dd9b23b569ac98 - osx-arm64: 0f204f64b92bd922585ecf919b03561a9dccf5481b70ff29f89ea97b40c8ffb8 + linux-64: 4e740f8af6de809a90d97ae11e9665ea4057e2b62f165ae5425e72178f3c7659 + osx-64: 52e22739c47ecf742d0e09cb3959ecaecbdffd18c089951fb20d642bfb8d154f + osx-arm64: d685097d2634b4be5015607c52d7a2b23b3eb609a18ba28962c6b7250db1f760 platforms: - osx-arm64 - linux-64 @@ -8583,6 +8583,99 @@ package: platform: osx-arm64 url: https://conda.anaconda.org/conda-forge/noarch/msrest-0.6.21-pyh44b312d_0.tar.bz2 version: 0.6.21 + - category: main + dependencies: + __glibc: '>=2.17,<3.0.a0' + libgcc: '>=13' + mypy_extensions: '>=1.0.0' + psutil: '>=4.0' + python: '>=3.11,<3.12.0a0' + python_abi: 3.11.* + typing_extensions: '>=4.1.0' + hash: + md5: 0111eaad55bea1e607d90d4f84089f74 + sha256: 460f3eb43160dc9e9a1f2aeabdf8cd809aefcf776c33ffd155f9bd2420a005c5 + manager: conda + name: mypy + optional: false + platform: linux-64 + url: + https://conda.anaconda.org/conda-forge/linux-64/mypy-1.13.0-py311h9ecbd09_0.conda + version: 1.13.0 + - category: main + dependencies: + __osx: '>=10.13' + mypy_extensions: '>=1.0.0' + psutil: '>=4.0' + python: '>=3.11,<3.12.0a0' + python_abi: 3.11.* + typing_extensions: '>=4.1.0' + hash: + md5: 83a6e2bb0ea4940434b4d1402c92d3c4 + sha256: ddca5a4235df88b876769dc191a5a41a453d2983bcb56104c0821d252e1abbb4 + manager: conda + name: mypy + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/mypy-1.13.0-py311h1314207_0.conda + version: 1.13.0 + - category: main + dependencies: + __osx: '>=11.0' + mypy_extensions: '>=1.0.0' + psutil: '>=4.0' + python: '>=3.11,<3.12.0a0' + python_abi: 3.11.* + typing_extensions: '>=4.1.0' + hash: + md5: 8b3f3c83db062e4970ba7a04890c83d5 + sha256: 23c4fec92c5926baf3fe6ca8c0ad8b3d8ada66786d5b697d6e641d8885ac8ac6 + manager: conda + name: mypy + optional: false + platform: osx-arm64 + url: + https://conda.anaconda.org/conda-forge/osx-arm64/mypy-1.13.0-py311hae2e1ce_0.conda + version: 1.13.0 + - category: main + dependencies: + python: '>=3.5' + hash: + md5: 4eccaeba205f0aed9ac3a9ea58568ca3 + sha256: f240217476e148e825420c6bc3a0c0efb08c0718b7042fae960400c02af858a3 + manager: conda + name: mypy_extensions + optional: false + platform: linux-64 + url: + https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda + version: 1.0.0 + - category: main + dependencies: + python: '>=3.5' + hash: + md5: 4eccaeba205f0aed9ac3a9ea58568ca3 + sha256: f240217476e148e825420c6bc3a0c0efb08c0718b7042fae960400c02af858a3 + manager: conda + name: mypy_extensions + optional: false + platform: osx-64 + url: + https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda + version: 1.0.0 + - category: main + dependencies: + python: '>=3.5' + hash: + md5: 4eccaeba205f0aed9ac3a9ea58568ca3 + sha256: f240217476e148e825420c6bc3a0c0efb08c0718b7042fae960400c02af858a3 + manager: conda + name: mypy_extensions + optional: false + platform: osx-arm64 + url: + https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda + version: 1.0.0 - category: main dependencies: __glibc: '>=2.17,<3.0.a0' diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index c4d94b9caf..7f259ff23b 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -9,7 +9,7 @@ import traceback import typing from dataclasses import dataclass -from typing import Literal, cast +from typing import Any, Literal, cast from urllib.error import URLError from uuid import uuid4 @@ -70,12 +70,15 @@ ) from .migrators_types import MigrationUidTypedDict -from .models.pr_json import PullRequestData, PullRequestInfoSpecial, PullRequestState +from .models.pr_json import ( + PullRequestData, + PullRequestInfoSpecial, + PullRequestState, +) logger = logging.getLogger(__name__) BOT_HOME_DIR: str = os.getcwd() -START_TIME = None TIMEOUT = int(os.environ.get("TIMEOUT", 600)) # migrator runs on loop so avoid any seeds at current time should that happen @@ -424,9 +427,9 @@ def _check_and_process_solvability( def get_spoofed_closed_pr_info() -> PullRequestInfoSpecial: return PullRequestInfoSpecial( - id=str(uuid4()), + id=uuid4(), merged_at="never issued", - state="closed", + state=PullRequestState.CLOSED, ) @@ -437,7 +440,10 @@ def run_with_tmpdir( rerender: bool = True, base_branch: str = "main", **kwargs: typing.Any, -) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: +) -> ( + tuple[MigrationUidTypedDict, LazyJson | Literal[False]] + | tuple[Literal[False], Literal[False]] +): """ For a given feedstock and migration run the migration in a temporary directory that will be deleted after the migration is complete. @@ -465,7 +471,10 @@ def run( rerender: bool = True, base_branch: str = "main", **kwargs: typing.Any, -) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: +) -> ( + tuple[MigrationUidTypedDict, LazyJson | Literal[False]] + | tuple[Literal[False], Literal[False]] +): """For a given feedstock and migration run the migration Parameters @@ -601,6 +610,10 @@ def run( and pr_data.state != PullRequestState.CLOSED and rerender_info.rerender_comment ): + if pr_data.number is None: + raise ValueError( + f"Unexpected GitHub API response: PR number is missing for PR ID {pr_data.id}." + ) git_backend.comment_on_pull_request( repo_owner=context.git_repo_owner, repo_name=context.git_repo_name, @@ -608,12 +621,14 @@ def run( comment=rerender_info.rerender_comment, ) + pr_lazy_json: LazyJson | Literal[False] if pr_data: - pr_lazy_json = LazyJson( + pr_lazy_json_present = LazyJson( os.path.join("pr_json", f"{pr_data.id}.json"), ) - with pr_lazy_json as __edit_pr_lazy_json: + with pr_lazy_json_present as __edit_pr_lazy_json: __edit_pr_lazy_json.update(**pr_data.model_dump(mode="json")) + pr_lazy_json = pr_lazy_json_present else: pr_lazy_json = False @@ -624,7 +639,10 @@ def run( context.attrs, migrator_name, is_version=is_version_migration ) - return migration_run_data["migrate_return_value"], pr_lazy_json + migrate_return_value: MigrationUidTypedDict = migration_run_data[ + "migrate_return_value" + ] + return migrate_return_value, pr_lazy_json def _compute_time_per_migrator(migrators): @@ -633,8 +651,8 @@ def _compute_time_per_migrator(migrators): for migrator in tqdm.tqdm(migrators, ncols=80, desc="computing time per migrator"): if isinstance(migrator, Version): _num_nodes = 0 - for node_name in migrator.effective_graph.nodes: - with migrator.effective_graph.nodes[node_name]["payload"] as attrs: + for node_name in migrator.effective_graph.nodes: # type: ignore[union-attr] # TODO: effective_graph can be None + with migrator.effective_graph.nodes[node_name]["payload"] as attrs: # type: ignore[union-attr] # TODO: effective_graph can be None with attrs["version_pr_info"] as vpri: _attempts = vpri.get("new_version_attempts", {}).get( vpri.get("new_version", ""), @@ -644,7 +662,7 @@ def _compute_time_per_migrator(migrators): _num_nodes += 1 _num_nodes = max( _num_nodes, - min(PR_LIMIT * 4, len(migrator.effective_graph.nodes)), + min(PR_LIMIT * 4, len(migrator.effective_graph.nodes)), # type: ignore[union-attr] # TODO: effective_graph can be None ) num_nodes.append(_num_nodes) else: @@ -684,23 +702,6 @@ def _compute_time_per_migrator(migrators): return num_nodes, time_per_migrator, tot_time_per_migrator -def _over_time_limit(): - _now = time.time() - print( - """\ - -=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~> -=~> elpased time %ds (timeout %ds) -=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~>=~> - -""" - % (_now - START_TIME, TIMEOUT), - flush=True, - end="", - ) - return _now - START_TIME > TIMEOUT - - def _run_migrator_on_feedstock_branch( attrs, base_branch, @@ -730,7 +731,7 @@ def _run_migrator_on_feedstock_branch( # if migration successful if migrator_uid: with attrs["pr_info"] as pri: - d = frozen_to_json_friendly(migrator_uid) + d: Any = frozen_to_json_friendly(migrator_uid) # if we have the PR already do nothing if d["data"] in [ existing_pr["data"] for existing_pr in pri.get("PRed", []) @@ -738,7 +739,7 @@ def _run_migrator_on_feedstock_branch( pass else: if not pr_json: - pr_json = { + pr_json = { # type: ignore[assignment] # TODO: incompatible with LazyJson "state": "closed", "head": { "ref": "", @@ -847,15 +848,15 @@ def _run_migrator_on_feedstock_branch( return good_prs, break_loop -def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit): +def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit, start_time: float): curr_time = time.time() backend = github_backend() api_req = backend.get_api_requests_left() - if curr_time - START_TIME > TIMEOUT: + if curr_time - start_time > TIMEOUT: logger.info( "BOT TIMEOUT: breaking after %d seconds (limit %d)", - curr_time - START_TIME, + curr_time - start_time, TIMEOUT, ) return True @@ -885,7 +886,9 @@ def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit): return False -def _run_migrator(migrator, mctx, temp, time_per, git_backend: GitPlatformBackend): +def _run_migrator( + migrator, mctx, temp, time_per, git_backend: GitPlatformBackend, start_time: float +): _mg_start = time.time() migrator_name = get_migrator_name(migrator) @@ -939,7 +942,9 @@ def _run_migrator(migrator, mctx, temp, time_per, git_backend: GitPlatformBacken flush=True, ) - if _is_migrator_done(_mg_start, good_prs, time_per, migrator.pr_limit): + if _is_migrator_done( + _mg_start, good_prs, time_per, migrator.pr_limit, start_time + ): return 0 for node_name in possible_nodes: @@ -956,7 +961,9 @@ def _run_migrator(migrator, mctx, temp, time_per, git_backend: GitPlatformBacken ): # Don't let CI timeout, break ahead of the timeout so we make certain # to write to the repo - if _is_migrator_done(_mg_start, good_prs, time_per, migrator.pr_limit): + if _is_migrator_done( + _mg_start, good_prs, time_per, migrator.pr_limit, start_time + ): break base_branches = migrator.get_possible_feedstock_branches(attrs) @@ -1201,8 +1208,7 @@ def _update_graph_with_pr_info(): def main(ctx: CliContext) -> None: - global START_TIME - START_TIME = time.time() + start_time = time.time() _setup_limits() @@ -1260,7 +1266,12 @@ def main(ctx: CliContext) -> None: for mg_ind, migrator in enumerate(migrators): good_prs = _run_migrator( - migrator, mctx, temp, time_per_migrator[mg_ind], git_backend + migrator, + mctx, + temp, + time_per_migrator[mg_ind], + git_backend, + start_time=start_time, ) if good_prs > 0: pass diff --git a/conda_forge_tick/container_cli.py b/conda_forge_tick/container_cli.py index 6158a46d8d..fba12a92f5 100644 --- a/conda_forge_tick/container_cli.py +++ b/conda_forge_tick/container_cli.py @@ -199,9 +199,11 @@ def _migrate_feedstock(*, feedstock_name, default_branch, attrs, input_kwargs): logger = logging.getLogger("conda_forge_tick.container") with tempfile.TemporaryDirectory() as tmpdir: - input_fs_dir = glob.glob("/cf_feedstock_ops_dir/*-feedstock") - assert len(input_fs_dir) == 1, f"expected one feedstock, got {input_fs_dir}" - input_fs_dir = input_fs_dir[0] + input_fs_dir_list = glob.glob("/cf_feedstock_ops_dir/*-feedstock") + assert ( + len(input_fs_dir_list) == 1 + ), f"expected one feedstock, got {input_fs_dir_list}" + input_fs_dir = input_fs_dir_list[0] logger.debug( f"input container feedstock dir {input_fs_dir}: {os.listdir(input_fs_dir)}" ) @@ -253,9 +255,11 @@ def _update_version(*, version, hash_type): logger = logging.getLogger("conda_forge_tick.container") with tempfile.TemporaryDirectory() as tmpdir: - input_fs_dir = glob.glob("/cf_feedstock_ops_dir/*-feedstock") - assert len(input_fs_dir) == 1, f"expected one feedstock, got {input_fs_dir}" - input_fs_dir = input_fs_dir[0] + input_fs_dir_list = glob.glob("/cf_feedstock_ops_dir/*-feedstock") + assert ( + len(input_fs_dir_list) == 1 + ), f"expected one feedstock, got {input_fs_dir_list}" + input_fs_dir = input_fs_dir_list[0] logger.debug( f"input container feedstock dir {input_fs_dir}: {os.listdir(input_fs_dir)}" ) @@ -472,12 +476,14 @@ def parse_meta_yaml( "--cbc-path", type=str, default=None, help="The path to global pinning file." ) def parse_recipe_yaml( + log_level, for_pinning, platform_arch, cbc_path, ): return _run_bot_task( _parse_recipe_yaml, + log_level=log_level, existing_feedstock_node_attrs=None, for_pinning=for_pinning, platform_arch=platform_arch, diff --git a/conda_forge_tick/make_migrators.py b/conda_forge_tick/make_migrators.py index 5ec38c0194..df141b0799 100644 --- a/conda_forge_tick/make_migrators.py +++ b/conda_forge_tick/make_migrators.py @@ -16,12 +16,11 @@ MutableSet, Sequence, Set, - Union, cast, ) if typing.TYPE_CHECKING: - from .migrators_types import PackageName + from .migrators_types import BuildTypedDict, PackageName import networkx as nx import tqdm @@ -104,7 +103,6 @@ def add_replacement_migrator( old_pkg: "PackageName", new_pkg: "PackageName", rationale: str, - alt_migrator: Union[Migrator, None] = None, ) -> None: """Adds a migrator to replace one package with another. @@ -144,26 +142,15 @@ def add_replacement_migrator( # post plucking we can have several strange cases, lets remove all selfloops total_graph.remove_edges_from(nx.selfloop_edges(total_graph)) - if alt_migrator is not None: - migrators.append( - alt_migrator( - old_pkg=old_pkg, - new_pkg=new_pkg, - rationale=rationale, - pr_limit=PR_LIMIT, - graph=total_graph, - ), - ) - else: - migrators.append( - Replacement( - old_pkg=old_pkg, - new_pkg=new_pkg, - rationale=rationale, - pr_limit=PR_LIMIT, - graph=total_graph, - ), - ) + migrators.append( + Replacement( + old_pkg=old_pkg, + new_pkg=new_pkg, + rationale=rationale, + pr_limit=PR_LIMIT, + graph=total_graph, + ), + ) def add_arch_migrate(migrators: MutableSequence[Migrator], gx: nx.DiGraph) -> None: @@ -211,7 +198,7 @@ def add_rebuild_migration_yaml( migrators: MutableSequence[Migrator], gx: nx.DiGraph, package_names: Sequence[str], - output_to_feedstock: Mapping[str, str], + output_to_feedstock: Mapping[str, set[str]], excluded_feedstocks: MutableSet[str], exclude_pinned_pkgs: bool, migration_yaml: str, @@ -407,7 +394,7 @@ def migration_factory( for yaml_file, _ in migration_yamls ] - output_to_feedstock = gx.graph["outputs_lut"] + output_to_feedstock: Mapping[str, set[str]] = gx.graph["outputs_lut"] all_package_names = set( sum( ( @@ -592,13 +579,13 @@ def create_migration_yaml_creator( # find the most stringent max pin for this feedstock if any pin_spec = "" for block in [meta_yaml] + meta_yaml.get("outputs", []) or []: - build = block.get("build", {}) or {} + build: BuildTypedDict = block.get("build", {}) or {} # parse back to dict possible_p_dicts = [] if isinstance(build.get("run_exports", None), MutableMapping): - for _, v in build.get("run_exports", {}).items(): - for p in v: + for _, v in build.get("run_exports", {}).items(): # type: ignore[union-attr] # TODO: the isinstance check above does not lead to correct type inference + for p in v: # type: ignore[union-attr] # TODO: the isinstance check above does not lead to correct type inference possible_p_dicts.append(parse_munged_run_export(p)) else: for p in build.get("run_exports", []) or []: diff --git a/conda_forge_tick/migration_runner.py b/conda_forge_tick/migration_runner.py index e07127a81f..6731cc5532 100644 --- a/conda_forge_tick/migration_runner.py +++ b/conda_forge_tick/migration_runner.py @@ -257,7 +257,7 @@ def run_migration_local( migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs) data["commit_message"] = migrator.commit_message(feedstock_ctx) - data["pr_body"] = migrator.pr_body(feedstock_ctx) + data["pr_body"] = migrator.pr_body() data["pr_title"] = migrator.pr_title(feedstock_ctx) return data diff --git a/conda_forge_tick/migrators/arch.py b/conda_forge_tick/migrators/arch.py index 9b744d6b62..fbd8d93422 100644 --- a/conda_forge_tick/migrators/arch.py +++ b/conda_forge_tick/migrators/arch.py @@ -1,7 +1,7 @@ import os import typing from textwrap import dedent -from typing import Any, Optional, Sequence +from typing import Any, Collection, Literal, Optional, Sequence import networkx as nx @@ -106,10 +106,10 @@ class ArchRebuild(GraphMigrator): def __init__( self, graph: nx.DiGraph = None, - name: Optional[str] = None, + name: str = "aarch64 and ppc64le addition", pr_limit: int = 0, piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None, - target_packages: Optional[Sequence[str]] = None, + target_packages: Optional[Collection[str]] = None, effective_graph: nx.DiGraph = None, _do_init: bool = True, ): @@ -201,7 +201,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: with pushd(recipe_dir + "/.."): self.set_build_number("recipe/meta.yaml") with open("conda-forge.yml") as f: @@ -248,8 +248,8 @@ class OSXArm(GraphMigrator): # We purposefully don't want to bump build number for this migrator bump_number = 0 - ignored_packages = set() - excluded_dependencies = set() + ignored_packages: set[str] = set() + excluded_dependencies: set[str] = set() arches = ["osx_arm64"] @@ -261,10 +261,10 @@ class OSXArm(GraphMigrator): def __init__( self, graph: nx.DiGraph = None, - name: Optional[str] = None, + name: str = "arm osx addition", pr_limit: int = 0, piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None, - target_packages: Optional[Sequence[str]] = None, + target_packages: Optional[Collection[str]] = None, effective_graph: nx.DiGraph = None, _do_init: bool = True, ): @@ -316,12 +316,14 @@ def __init__( # are not added to the graph _filter_excluded_deps(graph, self.excluded_dependencies) - target_packages = set(target_packages) + target_packages_modified = set(target_packages) # filter the graph down to the target packages - if target_packages: - target_packages.add("python") # hack that is ~harmless? - _cut_to_target_packages(graph, target_packages) + if target_packages_modified: + target_packages_modified.add("python") # hack that is ~harmless? + _cut_to_target_packages(graph, target_packages_modified) + + target_packages = target_packages_modified # filter out stub packages and ignored packages _filter_stubby_and_ignored_nodes(graph, self.ignored_packages) @@ -374,7 +376,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: with pushd(recipe_dir + "/.."): self.set_build_number("recipe/meta.yaml") with open("conda-forge.yml") as f: diff --git a/conda_forge_tick/migrators/broken_rebuild.py b/conda_forge_tick/migrators/broken_rebuild.py index fb3d137188..cf4ab0e7ce 100644 --- a/conda_forge_tick/migrators/broken_rebuild.py +++ b/conda_forge_tick/migrators/broken_rebuild.py @@ -4,6 +4,7 @@ from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.migrators.core import Migrator +from conda_forge_tick.migrators_types import AttrsTypedDict BROKEN_PACKAGES = """\ linux-ppc64le/adios2-2.7.1-mpi_mpich_py36ha1d8cba_0.tar.bz2 @@ -371,7 +372,7 @@ def order( """ return graph - def filter(self, attrs) -> bool: + def filter(self, attrs: AttrsTypedDict, not_bad_str_start: str = "") -> bool: return ( super().filter(attrs) or attrs["feedstock_name"] not in self.feedstocks_to_migrate diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 9dbf91e1e6..f01ce14551 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -6,7 +6,7 @@ import re import typing from pathlib import Path -from typing import Any, List, Sequence, Set +from typing import Any, List, Literal, Sequence, Set import dateutil.parser import networkx as nx @@ -469,7 +469,7 @@ def run_post_piggyback_migrations( def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: """Perform the migration, updating the ``meta.yaml`` Parameters @@ -486,16 +486,11 @@ def migrate( """ return self.migrator_uid(attrs) - def pr_body( - self, feedstock_ctx: ClonedFeedstockContext, add_label_text=True - ) -> str: - """Create a PR message body - - Returns - ------- - body: str - The body of the PR message - :param feedstock_ctx: + @staticmethod + def custom_pr_body(add_label_text: bool = False): + """ + Create a PR message body, where the label text is optional. + :param add_label_text: Whether to add an explanatory text for the bot-rerun label """ body = "{}\n\n" @@ -522,6 +517,16 @@ def pr_body( ) return body + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: + """ + Children override this method to provide a custom PR body. + + By default, identical to the custom_pr_body method, but with the add_label_text parameter set to True. + + :param feedstock_ctx: The feedstock context (might be needed by subclasses) + """ + return self.custom_pr_body(add_label_text=True) + def commit_message(self, feedstock_ctx: FeedstockContext) -> str: """Create a commit message :param feedstock_ctx: diff --git a/conda_forge_tick/migrators/cross_compile.py b/conda_forge_tick/migrators/cross_compile.py index 79275234bc..ca7c0a0700 100644 --- a/conda_forge_tick/migrators/cross_compile.py +++ b/conda_forge_tick/migrators/cross_compile.py @@ -44,9 +44,9 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No return directories = set() with pushd(cb_work_dir): - for dp, dn, fn in os.walk("."): - for f in fn: - if f != "config.sub": + for dp, dn, filename in os.walk("."): + for name in filename: + if name != "config.sub": continue if os.path.exists(os.path.join(dp, "config.guess")): directories.add(dp) diff --git a/conda_forge_tick/migrators/cstdlib.py b/conda_forge_tick/migrators/cstdlib.py index e8e9b7e1fe..aa76b5beda 100644 --- a/conda_forge_tick/migrators/cstdlib.py +++ b/conda_forge_tick/migrators/cstdlib.py @@ -95,18 +95,18 @@ def _process_section(output_index, attrs, lines): # regexes here. So leave a marker that we can skip on last_line_was_build = True line_build = i - elif pat_compiler_c.search(line): + elif match := pat_compiler_c.match(line): line_compiler_c = i - indent_c = pat_compiler_c.match(line).group("indent") - selector_c = pat_compiler_c.match(line).group("selector") or "" - elif pat_compiler_m2c.search(line): + indent_c = match.group("indent") + selector_c = match.group("selector") or "" + elif match := pat_compiler_m2c.search(line): line_compiler_m2c = i - indent_m2c = pat_compiler_m2c.match(line).group("indent") - selector_m2c = pat_compiler_m2c.match(line).group("selector") or "" - elif pat_compiler_other.search(line): + indent_m2c = match.group("indent") + selector_m2c = match.group("selector") or "" + elif match := pat_compiler_other.search(line): line_compiler_other = i - indent_other = pat_compiler_other.match(line).group("indent") - selector_other = pat_compiler_other.match(line).group("selector") or "" + indent_other = match.group("indent") + selector_other = match.group("selector") or "" elif re.match(r"^\s*host:.*", line): line_host = i elif re.match(r"^\s*run:.*", line): diff --git a/conda_forge_tick/migrators/flang.py b/conda_forge_tick/migrators/flang.py index 4c93245832..2084115f79 100644 --- a/conda_forge_tick/migrators/flang.py +++ b/conda_forge_tick/migrators/flang.py @@ -53,8 +53,8 @@ def _process_section(lines): langs = {pat_comp.sub(r"\g", x) for x in lines[begin : end + 1]} # if we caught a comment, lang will be "" - langs = sorted([x for x in langs if x]) - comp_block_new = [f'{indent}- {{{{ compiler("{lang}") }}}}' for lang in langs] + langs_list = sorted([x for x in langs if x]) + comp_block_new = [f'{indent}- {{{{ compiler("{lang}") }}}}' for lang in langs_list] comp_block_new = [f'{indent}- {{{{ stdlib("c") }}}}'] + comp_block_new new_lines = lines[:begin] + comp_block_new + lines[end + 1 :] diff --git a/conda_forge_tick/migrators/libboost.py b/conda_forge_tick/migrators/libboost.py index 356c7ef269..d18972ea01 100644 --- a/conda_forge_tick/migrators/libboost.py +++ b/conda_forge_tick/migrators/libboost.py @@ -4,7 +4,7 @@ from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema -def _slice_into_output_sections(meta_yaml_lines, attrs): +def _slice_into_output_sections(meta_yaml_lines: list[str], attrs): """ Turn a recipe into slices corresponding to the outputs. @@ -77,7 +77,7 @@ def _slice_into_output_sections(meta_yaml_lines, attrs): final_sections = {} final_sections[-1] = sections[-1] # we always keep the first global section final_output_index = 0 - carried_lines = [] + carried_lines: list[str] = [] for output_index in range(len(sections) - 1): section = sections[output_index] if any(re_name.match(line) for line in section): diff --git a/conda_forge_tick/migrators/license.py b/conda_forge_tick/migrators/license.py index 707d9cf114..dee23e4da2 100644 --- a/conda_forge_tick/migrators/license.py +++ b/conda_forge_tick/migrators/license.py @@ -179,7 +179,7 @@ def _munge_licenses(lparts): def _scrape_license_string(pkg): - d = {} + d: dict[str, str | list[str] | None] = {} if pkg.startswith("r-"): pkg = pkg[2:] @@ -218,7 +218,7 @@ def _scrape_license_string(pkg): d["license_file"] = [ lf for lf in cmeta.meta.get("about", {}).get("license_file", []) ] - if len(d["license_file"]) == 0: + if len(d["license_file"]) == 0: # type: ignore[arg-type] # this is not a typed dict d["license_file"] = None if "cran_license" in d: @@ -303,8 +303,8 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No or attrs.get("name", "").startswith("r-") ) and "r-base" in attrs["raw_meta_yaml"]: if attrs.get("feedstock_name", None) is not None: - if attrs.get("feedstock_name", None).endswith("-feedstock"): - name = attrs.get("feedstock_name")[: -len("-feedstock")] + if attrs.get("feedstock_name", None).endswith("-feedstock"): # type: ignore[union-attr] # this is not a typed dict + name = attrs.get("feedstock_name")[: -len("-feedstock")] # type: ignore[arg-type,index] # this is not a typed dict else: name = attrs.get("feedstock_name") else: diff --git a/conda_forge_tick/migrators/matplotlib_base.py b/conda_forge_tick/migrators/matplotlib_base.py index e71631de85..7acf51cb9d 100644 --- a/conda_forge_tick/migrators/matplotlib_base.py +++ b/conda_forge_tick/migrators/matplotlib_base.py @@ -1,14 +1,18 @@ import copy import os import typing -from typing import Any +from typing import Any, Literal from conda_forge_tick.migrators.core import _parse_bad_attr, _skip_due_to_schema from conda_forge_tick.migrators.replacement import Replacement from conda_forge_tick.utils import frozen_to_json_friendly if typing.TYPE_CHECKING: - from ..migrators_types import AttrsTypedDict, MigrationUidTypedDict + from ..migrators_types import ( + AttrsTypedDict, + MigrationUidTypedDict, + RequirementsTypedDict, + ) class MatplotlibBase(Replacement): @@ -40,7 +44,7 @@ def parse_already_pred() -> bool: _is_pred = parse_already_pred() _is_bad = _parse_bad_attr(attrs, not_bad_str_start) - requirements = attrs.get("requirements", {}) + requirements: RequirementsTypedDict = attrs.get("requirements", {}) rq = ( requirements.get("build", set()) | requirements.get("host", set()) @@ -59,7 +63,7 @@ def parse_already_pred() -> bool: def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: yum_pth = os.path.join(recipe_dir, "yum_requirements.txt") if not os.path.exists(yum_pth): yum_lines = [] diff --git a/conda_forge_tick/migrators/migration_yaml.py b/conda_forge_tick/migrators/migration_yaml.py index 4c5b0fe058..aef8f8add4 100644 --- a/conda_forge_tick/migrators/migration_yaml.py +++ b/conda_forge_tick/migrators/migration_yaml.py @@ -6,7 +6,7 @@ import time import typing from collections import defaultdict -from typing import Any, List, MutableSet, Optional, Sequence, Set +from typing import Any, Collection, Literal, MutableSet, Optional, Sequence, Set import networkx as nx @@ -129,8 +129,8 @@ def __init__( name: str, graph: nx.DiGraph = None, pr_limit: int = 50, - top_level: Set["PackageName"] = None, - cycles: Optional[Sequence["PackageName"]] = None, + top_level: Set["PackageName"] | None = None, + cycles: Optional[Collection["PackageName"]] = None, migration_number: Optional[int] = None, bump_number: int = 1, piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None, @@ -246,7 +246,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: # if conda-forge-pinning update the pins and close the migration if attrs.get("name", "") == "conda-forge-pinning": # read up the conda build config @@ -459,7 +459,7 @@ def __init__( pr_limit: int = 1, bump_number: int = 1, effective_graph: nx.DiGraph = None, - pinnings: Optional[List[int]] = None, + pinnings: list[str] | None = None, **kwargs: Any, ): if pinnings is None: @@ -520,7 +520,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: migration_yaml_dict = { "__migrator": { "build_number": 1, @@ -664,7 +664,7 @@ def all_noarch(attrs, only_python=False): def create_rebuild_graph( gx: nx.DiGraph, package_names: Sequence[str], - excluded_feedstocks: MutableSet[str] = None, + excluded_feedstocks: MutableSet[str] | None = None, exclude_pinned_pkgs: bool = True, include_noarch: bool = False, include_build: bool = False, @@ -676,7 +676,7 @@ def create_rebuild_graph( # where numpy needs to be rebuilt despite being pinned. if exclude_pinned_pkgs: for node in package_names: - excluded_feedstocks.update(gx.graph["outputs_lut"].get(node, {node})) + excluded_feedstocks |= gx.graph["outputs_lut"].get(node, {node}) included_nodes = set() diff --git a/conda_forge_tick/migrators/mpi_pin_run_as_build.py b/conda_forge_tick/migrators/mpi_pin_run_as_build.py index f843d7edd1..14f256fbbf 100644 --- a/conda_forge_tick/migrators/mpi_pin_run_as_build.py +++ b/conda_forge_tick/migrators/mpi_pin_run_as_build.py @@ -9,8 +9,8 @@ def _parse_cbc_mpi(lines): in_prab = False - prab_indent = None - mpi_indent = None + prab_indent: int | None = None + mpi_indent: int | None = None new_lines = [] for _line in lines: if _line.endswith("\n"): @@ -42,7 +42,7 @@ def _parse_cbc_mpi(lines): if mpi_indent is not None: continue - if curr_indent <= prab_indent: + if prab_indent is not None and curr_indent <= prab_indent: in_prab = False prab_indent = None diff --git a/conda_forge_tick/migrators/pip_check.py b/conda_forge_tick/migrators/pip_check.py index 8a6a3ab16b..33a906db01 100644 --- a/conda_forge_tick/migrators/pip_check.py +++ b/conda_forge_tick/migrators/pip_check.py @@ -17,7 +17,7 @@ SELECTOR_RE = re.compile(r"^(\s*)(\S*):\s*(\S*)\s*#\s*\[(.*)\]") -def _munge_key(key, selector): +def _munge_key(key, selector) -> str: return ( key + "__conda-selector_" @@ -47,7 +47,11 @@ def _round_trip_value(val): return s.read().split(":")[1].strip() -def _munge_line(line, mapping, groups): +def _munge_line( + line: str, + mapping: dict[tuple[str, str, str, str], str], + groups: dict[str, tuple[str, str, str, str]], +) -> str: m = SELECTOR_RE.match(line) if m: spc, key, val, selector = m.group(1, 2, 3, 4) @@ -148,9 +152,10 @@ def _adjust_test_dict(meta, key, mapping, groups, parent_group=None): class PipCheckMigrator(MiniMigrator): def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: """run pip check if we see python in any host sections""" - build_host = ( - attrs["requirements"].get("host", set()) - or attrs["requirements"].get("build", set()) + # the nested types in AttrsTypedDict are incorrect + build_host: set[str] = ( + attrs["requirements"].get("host", set()) # type: ignore[assignment] + or attrs["requirements"].get("build", set()) # type: ignore[assignment] or set() ) return "python" not in build_host or _skip_due_to_schema( @@ -159,8 +164,8 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None: with pushd(recipe_dir): - mapping = {} - groups = {} + mapping: dict[tuple[str, str, str, str], str] = {} + groups: dict[str, tuple[str, str, str, str]] = {} with open("meta.yaml") as fp: lines = [] for line in fp.readlines(): diff --git a/conda_forge_tick/migrators/replacement.py b/conda_forge_tick/migrators/replacement.py index 71ccf2dce8..4ce9bdcdd9 100644 --- a/conda_forge_tick/migrators/replacement.py +++ b/conda_forge_tick/migrators/replacement.py @@ -2,7 +2,7 @@ import os import re import typing -from typing import Any, Sequence +from typing import Any, Literal, Sequence import networkx as nx @@ -109,7 +109,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: with open(os.path.join(recipe_dir, "meta.yaml")) as f: raw = f.read() lines = raw.splitlines() diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 758422e360..b853e951bd 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -5,7 +5,7 @@ import random import typing import warnings -from typing import Any, List, Sequence +from typing import Any, List, Literal, Sequence import conda.exceptions import networkx as nx @@ -68,9 +68,8 @@ def __init__(self, python_nodes, *args, **kwargs): self._init_kwargs = copy.deepcopy(kwargs) self.python_nodes = python_nodes - if "check_solvable" in kwargs: - kwargs.pop("check_solvable") - super().__init__(*args, **kwargs, check_solvable=False) + kwargs["check_solvable"] = False + super().__init__(*args, **kwargs) self._new_version = None self._reset_effective_graph() @@ -192,7 +191,7 @@ def migrate( attrs: "AttrsTypedDict", hash_type: str = "sha256", **kwargs: Any, - ) -> "MigrationUidTypedDict": + ) -> MigrationUidTypedDict | Literal[False]: version = attrs["new_version"] with open(os.path.join(recipe_dir, "meta.yaml")) as fp: @@ -219,19 +218,17 @@ def migrate( ) ) - def pr_body( - self, feedstock_ctx: ClonedFeedstockContext, add_label_text: bool = False - ) -> str: - if feedstock_ctx.feedstock_name in self.effective_graph.nodes: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: + if feedstock_ctx.feedstock_name in self.effective_graph.nodes: # type: ignore[union-attr] # TODO: effective_graph shouldn't be allowed to be None pred = [ ( name, - self.effective_graph.nodes[name]["payload"]["version_pr_info"][ + self.effective_graph.nodes[name]["payload"]["version_pr_info"][ # type: ignore[union-attr] # TODO: effective_graph shouldn't be allowed to be None "new_version" ], ) for name in list( - self.effective_graph.predecessors(feedstock_ctx.feedstock_name), + self.effective_graph.predecessors(feedstock_ctx.feedstock_name), # type: ignore[union-attr] # TODO: effective_graph shouldn't be allowed to be None ) ] else: @@ -319,7 +316,7 @@ def pr_body( body += self._hint_and_maybe_update_deps(feedstock_ctx) - return super().pr_body(feedstock_ctx, add_label_text=False).format(body) + return super().custom_pr_body(add_label_text=False).format(body) def _hint_and_maybe_update_deps(self, feedstock_ctx: ClonedFeedstockContext): update_deps = get_keys_default( @@ -438,8 +435,7 @@ def _get_attempts_r(node, seen): @functools.lru_cache(maxsize=1024) def _get_attempts(node): if _has_solver_checks(node): - seen = set() - return _get_attempts_r(node, seen) + return _get_attempts_r(node, seen=set()) else: return _get_attempts_nr(node) diff --git a/conda_forge_tick/migrators_types.py b/conda_forge_tick/migrators_types.py index a2574c0ebc..1970ecfeb6 100644 --- a/conda_forge_tick/migrators_types.py +++ b/conda_forge_tick/migrators_types.py @@ -64,7 +64,7 @@ class MetaYamlOutputs(TypedDict, total=False): requirements: "RequirementsTypedDict" test: "TestTypedDict" # TODO: Not entirely sure this is right - build: BuildRunExportsDict + build: BuildTypedDict class RecipeTypedDict(TypedDict, total=False): @@ -84,6 +84,7 @@ class MigrationUidTypedDict(TypedDict, total=False): migrator_version: int name: str migrator_object_version: int + pin_version: str # Used by version migrators version: str @@ -94,9 +95,10 @@ class PackageTypedDict(TypedDict): class RequirementsTypedDict(TypedDict, total=False): - build: List[str] - host: List[str] - run: List[str] + build: set[str] + host: set[str] + run: set[str] + test: set[str] class SourceTypedDict(TypedDict, total=False): @@ -127,6 +129,7 @@ class AttrsTypedDict_(TypedDict, total=False): package: PackageTypedDict raw_meta_yaml: str req: Set[str] + name: str platforms: List[str] pr_info: typing.Any requirements: RequirementsTypedDict @@ -135,6 +138,7 @@ class AttrsTypedDict_(TypedDict, total=False): version: str new_version: Union[str, bool] archived: bool + outputs_names: set[str] PRed: List[PRedElementTypedDict] version_pr_info: typing.Any # Legacy types in here diff --git a/conda_forge_tick/models/pr_json.py b/conda_forge_tick/models/pr_json.py index be0f884e5a..266060babd 100644 --- a/conda_forge_tick/models/pr_json.py +++ b/conda_forge_tick/models/pr_json.py @@ -2,7 +2,7 @@ from enum import StrEnum from typing import ClassVar, Literal -from pydantic import UUID4, AnyHttpUrl, Field, TypeAdapter +from pydantic import UUID4, AnyHttpUrl, Field from pydantic_extra_types.color import Color from conda_forge_tick.models.common import ( @@ -167,4 +167,4 @@ class PullRequestInfoSpecial(StrictBaseModel): state: Literal[PullRequestState.CLOSED] -PullRequestData = TypeAdapter(PullRequestDataValid | PullRequestInfoSpecial) +PullRequestData = PullRequestDataValid | PullRequestInfoSpecial diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index f0d026f39f..f87982e6e1 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -118,7 +118,8 @@ def write_version_migrator_status(migrator, mctx): ) with open("./status/version_status.json", "w") as f: - old_out = out.copy() + old_out: dict[str, dict[str, str] | set[str]] = {} + old_out.update(out) old_out["queued"] = set(out["queued"].keys()) old_out["errored"] = set(out["errors"].keys()) json.dump(old_out, f, sort_keys=True, indent=2, default=_sorted_set_json) diff --git a/environment.yml b/environment.yml index 06205e212d..4ccccb7877 100644 --- a/environment.yml +++ b/environment.yml @@ -71,3 +71,4 @@ dependencies: - pytest-split - setuptools_scm >=8 - python-build + - mypy diff --git a/mypy.ini b/mypy.ini index db1c7f4c73..36d1351e14 100644 --- a/mypy.ini +++ b/mypy.ini @@ -55,3 +55,18 @@ ignore_missing_imports = True [mypy-distributed.*] ignore_missing_imports = True + +[mypy-conda_feedstock_ops.*] +ignore_missing_imports = True + +[mypy-tqdm.*] +ignore_missing_imports = True + +[mypy-requests.*] +ignore_missing_imports = True + +[mypy-dateutil.*] +ignore_missing_imports = True + +[mypy-conda_forge_feedstock_ops.*] +ignore_missing_imports = True diff --git a/tests/model/test_validate.py b/tests/model/test_validate.py index cd5e424092..ac278d0a58 100644 --- a/tests/model/test_validate.py +++ b/tests/model/test_validate.py @@ -215,4 +215,4 @@ def test_model_invalid(model: PerPackageModel, invalid_feedstock: str): def test_validate_pr_json(pr_json: str): - PullRequestData.validate_json(pr_json) + TypeAdapter(PullRequestData).validate_json(pr_json) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 0540a47ba8..47d623df55 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -544,7 +544,7 @@ def run_test_migration( if isinstance(m, Version): pass else: - assert prb in m.pr_body(None) + assert prb in m.pr_body() try: if "new_version" in kwargs: pmy["version_pr_info"] = {"new_version": kwargs["new_version"]} @@ -680,7 +680,8 @@ def test_all_noarch(meta, is_all_noarch): "meta,is_all_noarch", [ ( - json.loads("""\ + json.loads( + """\ { "about": { "description": "NetworkX is a Python language software package for the creation,\\nmanipulation, and study of the structure, dynamics, and functions of complex\\nnetworks.", @@ -744,11 +745,13 @@ def test_all_noarch(meta, is_all_noarch): "sha256": "307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", "url": "https://pypi.org/packages/source/n/networkx/networkx-3.4.2.tar.gz" } - }"""), + }""" + ), True, ), ( - json.loads("""\ + json.loads( + """\ { "about": { "description": "This is a python extension ", @@ -817,7 +820,8 @@ def test_all_noarch(meta, is_all_noarch): "pytest" ] } - }"""), + }""" + ), False, ), ], diff --git a/tests/test_migrators_v1.py b/tests/test_migrators_v1.py index 0cf98b78cf..4308e6150f 100644 --- a/tests/test_migrators_v1.py +++ b/tests/test_migrators_v1.py @@ -271,7 +271,7 @@ def run_test_migration( if isinstance(m, Version): pass else: - assert prb in m.pr_body(None) + assert prb in m.pr_body() try: if "new_version" in kwargs: pmy["version_pr_info"] = {"new_version": kwargs["new_version"]}