Merge pull request #667 from bioconda/update-pinnings-fixes

update-pinnings fixes

bioconda_utils/bioconda_utils-conda_build_config.yaml

@@ -25,10 +25,10 @@ bamtools:
   - 2.5.1
 
 # NOTE: Workaround https://github.com/conda/conda-build/issues/3974 we slightly alter the values
-#       from conda-forge-pinnings here (appending a space that should be ignored later on).
+#       from conda-forge-pinnings here (inserting '.*' or ' ' which should be ignored later on).
 r_base:
-  - '4.0 '
+  - 4.0.*
 python:
-  - '2.7.* *_cpython '
-  - '3.6.* *_cpython '
-  - '3.7.* *_cpython '
+  - 2.7.*  *_cpython
+  - 3.6.*  *_cpython
+  - 3.7.*  *_cpython

bioconda_utils/cli.py

@@ -541,6 +541,9 @@ def dag(recipe_folder, config, packages="*", format='gml', hide_singletons=False
      help="""Bump package build numbers even if the only applicable pinning
      change is the python version. This is generally required unless you plan
      on building everything.""")
+@arg('--skip-variants',
+     nargs='*',
+     help='Skip packages that use one of the given variant keys.')
 @arg('--cache', help='''To speed up debugging, use repodata cached locally in
      the provided filename. If the file does not exist, it will be created the
      first time.''')
@@ -550,13 +553,15 @@ def dag(recipe_folder, config, packages="*", format='gml', hide_singletons=False
 def update_pinning(recipe_folder, config, packages="*",
                    skip_additional_channels=None,
                    bump_only_python=False,
+                   skip_variants=None,
                    cache=None):
     """Bump a package build number and all dependencies as required due
     to a change in pinnings
     """
     config = utils.load_config(config)
     if skip_additional_channels:
         config['channels'] += skip_additional_channels
+    skip_variants = frozenset(skip_variants or ())
 
     if cache:
         utils.RepoData().set_cache(cache)
@@ -578,11 +583,13 @@ def update_pinning(recipe_folder, config, packages="*",
     hadErrors = set()
     bumpErrors = set()
 
-    needs_bump = partial(update_pinnings.check, build_config=build_config)
+    needs_bump = partial(
+        update_pinnings.check, build_config=build_config, skip_variant_keys=skip_variants,
+    )
 
     State = update_pinnings.State
 
-    for status, recip in zip(utils.parallel_iter(needs_bump, dag, "Processing..."), dag):
+    for status, recip in utils.parallel_iter(needs_bump, dag, "Processing..."):
         logger.debug("Recipe %s status: %s", recip, status)
         stats[status] += 1
         if status.needs_bump(bump_only_python):

bioconda_utils/update_pinnings.py

@@ -2,25 +2,42 @@
 Determine which packages need updates after pinning change
 """
 
-import re
-import sys
-import os.path
-import logging
-import collections
 import enum
+import logging
+import string
 
-import networkx as nx
-
-from .utils import RepoData, load_conda_build_config, parallel_iter
+from .utils import RepoData
+# FIXME: trim_build_only_deps is not exported via conda_build.api!
+#        Re-implement it here or ask upstream to export that functionality.
+from conda_build.metadata import trim_build_only_deps
 
 # for type checking
+from typing import AbstractSet
 from .recipe import Recipe, RecipeError
 from conda_build.metadata import MetaData
 
 
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
+def skip_for_variants(meta: MetaData, variant_keys: AbstractSet[str]) -> bool:
+    """Check if the recipe uses any given variant keys
+
+    Args:
+      meta: Variant MetaData object
+
+    Returns:
+      True if any variant key from variant_keys is used
+    """
+    # This is the same behavior as in
+    # conda_build.metadata.Metadata.get_hash_contents but without leaving out
+    # "build_string_excludes" (python, r_base, etc.).
+    dependencies = set(meta.get_used_vars())
+    trim_build_only_deps(meta, dependencies)
+
+    return not dependencies.isdisjoint(variant_keys)
+
+
 def will_build_variant(meta: MetaData) -> bool:
     """Check if the recipe variant will be built as currently rendered
 
@@ -127,36 +144,60 @@ def failed(self) -> bool:
         return self & self.FAIL
 
 
-def check(recipe: Recipe, build_config, keep_metas=False) -> State:
+allowed_build_string_characters = frozenset(
+    string.digits + string.ascii_uppercase + string.ascii_lowercase + '_.'
+)
+
+
+def has_invalid_build_string(meta: MetaData) -> bool:
+    build_string = meta.build_id()
+    return not (build_string and set(build_string).issubset(allowed_build_string_characters))
+
+
+def check(
+    recipe: Recipe,
+    build_config,
+    keep_metas=False,
+    skip_variant_keys: AbstractSet[str] = frozenset(),
+) -> State:
     """Determine if a given recipe should have its build number increments
     (bumped) due to a recent change in pinnings.
 
     Args:
       recipe: The recipe to check
       build_config: conda build config object
       keep_metas: If true, `Recipe.conda_release` is not called
+      skip_variant_keys: Variant keys to skip a recipe for if they are used
 
     Returns:
-      Tuple of state and a list of rendered MetaYaml variant objects
+      Tuple of state and a the input recipe
     """
     try:
         logger.debug("Calling Conda to render %s", recipe)
         metas = recipe.conda_render(config=build_config)
         logger.debug("Finished rendering %s", recipe)
     except RecipeError as exc:
         logger.error(exc)
-        return State.FAIL
+        return State.FAIL, recipe
     except Exception as exc:
         logger.exception("update_pinnings.check failed with exception in api.render(%s):", recipe)
-        return State.FAIL
+        return State.FAIL, recipe
 
     if metas is None:
         logger.error("Failed to render %s. Got 'None' from recipe.conda_render()", recipe)
-        return State.FAIL
+        return State.FAIL, recipe
+
+    if any(has_invalid_build_string(meta) for meta, _, _ in metas):
+        logger.error(
+            "Failed to get build strings for %s with bypass_env_check. "
+            "Probably needs build/skip instead of dep constraint.",
+            recipe,
+        )
+        return State.FAIL, recipe
 
     flags = State(0)
     for meta, _, _ in metas:
-        if meta.skip():
+        if meta.skip() or skip_for_variants(meta, skip_variant_keys):
             flags |= State.SKIP
         elif have_variant(meta):
             flags |= State.HAVE
@@ -170,4 +211,4 @@ def check(recipe: Recipe, build_config, keep_metas=False) -> State:
             flags |= State.BUMP
     if not keep_metas:
         recipe.conda_release()
-    return flags
+    return flags, recipe