[WIP] update to conda-build 3.10.9 (#312)

* build: rename --prelint to --lint, add --lint-only/--lint-exclude args

* requirements: update to conda-build 3.10.9

* build: remove default value for --lint-exclude

* requirements: require python >=3.6

* meta.get_section might return None (conda-build 3.10.9 compat)

* silence most flake8 nagging

* bioconductor: fix missing newline at EOF for post-link.sh

* fix typo

* use meta.get_value preferably

* make flake8 shut up already

* meta.get_value only works for path with depth 2

* lint: fix should_not_be_noarch for conda_build >3.10.3

* lint._has_compilers: also check for old compiler packages

* lint._get_deps: fix sections being unset on default

* docs: use only one job for debugging

.circleci/build-docs.sh

@@ -58,7 +58,9 @@ rm -r *
 
 # build docs and copy over to tmpdir
 cd ${DOCSOURCE}
-make clean html SPHINXOPTS="-j2" 2>&1 | grep -v "WARNING: nonlocal image URL found:"
+# TODO: reenable "-j2" when docs build fine
+make clean html 2>&1 | grep -v "WARNING: nonlocal image URL found:"
+# make clean html SPHINXOPTS="-j2" 2>&1 | grep -v "WARNING: nonlocal image URL found:"
 cp -r ${DOCHTML}/* $STAGING
 
 # commit and push

bioconda_utils/bioconda_utils-requirements.txt

@@ -2,7 +2,7 @@ anaconda-client=1.6.*
 argh=0.26.*
 beautifulsoup4=4.6.*
 conda=4.5.4
-conda-build=3.10.3
+conda-build=3.10.9
 galaxy-lib>=18.5.5
 jinja2=2.10.*
 jsonschema=2.6.*

bioconda_utils/bioconductor_skeleton.py

@@ -5,15 +5,16 @@
 import configparser
 from textwrap import dedent
 import tarfile
-import pyaml
 import hashlib
 import os
 import re
-import bs4
 from collections import OrderedDict
 import logging
+
+import bs4
+import pyaml
 import requests
-from colorlog import ColoredFormatter
+
 from . import utils
 from . import cran_skeleton
 
@@ -200,7 +201,10 @@ def find_best_bioc_version(package, version):
     for bioc_version in bioconductor_versions():
         for kind, func in zip(
             ('package', 'data'),
-            (bioconductor_tarball_url, bioconductor_annotation_data_url, bioconductor_experiment_data_url)
+            (
+                bioconductor_tarball_url, bioconductor_annotation_data_url,
+                bioconductor_experiment_data_url,
+            ),
         ):
             url = func(package, version, bioc_version)
             if requests.head(url).status_code == 200:
@@ -261,8 +265,8 @@ def __init__(self, package, bioc_version=None, pkg_version=None):
 
             htmls = {
                 'regular_package': os.path.join(
-                    base_url, self.bioc_version, 'bioc', 'html', package
-                    + '.html'),
+                    base_url, self.bioc_version, 'bioc', 'html',
+                    package + '.html'),
                 'annotation_package': os.path.join(
                     base_url, self.bioc_version, 'data', 'annotation', 'html',
                     package + '.html'),
@@ -310,7 +314,6 @@ def __init__(self, package, bioc_version=None, pkg_version=None):
 
         self.depends_on_gcc = False
 
-
     @property
     def bioarchive_url(self):
         """
@@ -337,7 +340,8 @@ def cargoport_url(self):
         elif response.status_code == 200:
             return url
         else:
-            raise PageNotFoundError("Unexpected error: {0.status_code} ({0.reason})".format(response))
+            raise PageNotFoundError(
+                "Unexpected error: {0.status_code} ({0.reason})".format(response))
 
     @property
     def bioconductor_tarball_url(self):
@@ -396,7 +400,8 @@ def tarball_url(self):
                 find_best_bioc_version(self.package, self.version)
 
             if self._tarball_url is None:
-                raise ValueError("No working URLs found for this version in any bioconductor version")
+                raise ValueError(
+                    "No working URLs found for this version in any bioconductor version")
         return self._tarball_url
 
     @property
@@ -428,7 +433,8 @@ def cached_tarball(self):
             if response.status_code == 200:
                 fout.write(response.content)
             else:
-                raise PageNotFoundError('Unexpected error {0.status_code} ({0.reason})'.format(response))
+                raise PageNotFoundError(
+                    'Unexpected error {0.status_code} ({0.reason})'.format(response))
         shutil.move(tmp, fn)
         self._cached_tarball = fn
         return fn
@@ -479,7 +485,6 @@ def depends(self):
         except KeyError:
             return []
 
-
     @property
     def linkingto(self):
         """
@@ -490,7 +495,6 @@ def linkingto(self):
         except KeyError:
             return []
 
-
     def _parse_dependencies(self, items):
         """
         The goal is to go from
@@ -595,15 +599,19 @@ def dependencies(self):
             # Modified from conda_build.skeletons.cran
             #
             with tarfile.open(self.cached_tarball) as tf:
-                need_f = any([f.name.lower().endswith(('.f', '.f90', '.f77')) for f in tf])
-                need_c = True if need_f else \
-                    any([f.name.lower().endswith('.c') for f in tf])
-                need_cxx = any([f.name.lower().endswith(('.cxx', '.cpp', '.cc', '.c++'))
-                                         for f in tf])
-                need_autotools = any([f.name.lower().endswith('/configure') for f in tf])
-                need_make = True if any((need_autotools, need_f, need_cxx, need_c)) else \
-                    any([f.name.lower().endswith(('/makefile', '/makevars'))
-                        for f in tf])
+                need_f = any(f.name.lower().endswith(('.f', '.f90', '.f77')) for f in tf)
+                if need_f:
+                    need_c = True
+                else:
+                    need_c = any(f.name.lower().endswith('.c') for f in tf)
+                need_cxx = any(
+                    f.name.lower().endswith(('.cxx', '.cpp', '.cc', '.c++')) for f in tf)
+                need_autotools = any(f.name.lower().endswith('/configure') for f in tf)
+                if any((need_autotools, need_f, need_cxx, need_c)):
+                    need_make = True
+                else:
+                    need_make = any(
+                        f.name.lower().endswith(('/makefile', '/makevars')) for f in tf)
         else:
             need_c = need_cxx = need_f = need_autotools = need_make = False
 
@@ -818,7 +826,9 @@ def write_recipe_recursive(proj, seen_dependencies, recipe_dir, config, force,
             continue
 
         if conda_name_without_version in seen_dependencies:
-            logger.debug("{} already created or in existing channels, skipping".format(conda_name_without_version))
+            logger.debug(
+                "{} already created or in existing channels, skipping"
+                .format(conda_name_without_version))
             continue
 
         seen_dependencies.update([conda_name_without_version])
@@ -1018,7 +1028,8 @@ def write_recipe(package, recipe_dir, config, force=False, bioc_version=None,
             # Install and clean up
             R CMD INSTALL --library=$PREFIX/lib/R/library $TARBALL
             rm $TARBALL
-            rmdir $STAGING""")
+            rmdir $STAGING
+            """)  # noqa: E501: line too long
         with open(os.path.join(recipe_dir, 'post-link.sh'), 'w') as fout:
             fout.write(dedent(post_link_template))
         pre_unlink_template = "R CMD REMOVE --library=$PREFIX/lib/R/library/ {0}\n".format(package)

bioconda_utils/build.py

@@ -43,8 +43,7 @@ def build(
     channels=None,
     docker_builder=None,
     _raise_error=False,
-    prelint=False,
-    df=None,
+    lint_args=None,
 ):
     """
     Build a single recipe for a single env
@@ -81,17 +80,13 @@ def build(
         Instead of returning a failed build result, raise the error instead.
         Used for testing.
 
-    prelint : bool
-        If True, then apply linting just before building. `df` should probably
-        be provided as well.
-
-    df : pandas.DataFrame
-        Dataframe of channel info, likely from linting.channel_dataframe()
+    lint_args : linting.LintArgs | None
+        If not None, then apply linting just before building.
     """
 
-    if prelint:
+    if lint_args is not None:
         logger.info('Linting recipe')
-        report = linting.lint([recipe], df)
+        report = linting.lint([recipe], lint_args)
         if report is not None:
             summarized = pandas.DataFrame(
                 dict(failed_tests=report.groupby('recipe')['check'].agg('unique')))
@@ -111,10 +106,11 @@ def build(
 
     logger.info("BUILD START %s", recipe)
 
-    # --no-build-id is needed for some very long package names that triggers the 89 character limits
-    # this option can be removed as soon as all packages are rebuild with the 255 character limit
-    # Moreover, --no-build-id will block us from using parallel builds in conda-build 2.x
-    # build_args = ["--no-build-id"]
+    # --no-build-id is needed for some very long package names that triggers
+    # the 89 character limits this option can be removed as soon as all
+    # packages are rebuild with the 255 character limit
+    # Moreover, --no-build-id will block us from using parallel builds in
+    # conda-build 2.x build_args = ["--no-build-id"]
 
     # use global variant config file (contains pinnings)
     build_args = ["--skip-existing"]
@@ -186,14 +182,13 @@ def build(
 
     logger.info('TEST START via mulled-build %s', recipe)
 
-    use_base_image = meta.get_section('extra').get('container', {})\
-                                              .get('extended-base')
+    use_base_image = meta.get_value('extra/container', {}).get('extended-base', False)
     base_image = 'bioconda/extended-base-image' if use_base_image else None
 
     mulled_images = []
     for pkg_path in pkg_paths:
         try:
-            res = pkg_test.test_package(pkg_path, base_image=base_image)
+            pkg_test.test_package(pkg_path, base_image=base_image)
         except sp.CalledProcessError as e:
             logger.error('TEST FAILED: %s', recipe)
             return BuildResult(False, None)
@@ -215,7 +210,7 @@ def build_recipes(
     anaconda_upload=False,
     mulled_upload_target=None,
     check_channels=None,
-    prelint=False,
+    lint_args=None,
 ):
     """
     Build one or many bioconda packages.
@@ -264,6 +259,8 @@ def build_recipes(
         `config['channels'][0]`). If this list is empty, then do not check any
         channels.
 
+    lint_args : linting.LintArgs | None
+        If not None, then apply linting just before building.
     """
     orig_config = config
     config = utils.load_config(config)
@@ -293,11 +290,16 @@ def build_recipes(
 
     logger.debug('recipes: %s', recipes)
 
-    if prelint:
-        logger.info("Downloading channel information to use for linting")
-        df = linting.channel_dataframe(channels=['conda-forge', 'defaults'])
-    else:
-        df = None
+    if lint_args is not None:
+        df = lint_args.df
+        if df is None:
+            logger.info("Downloading channel information to use for linting")
+            df = linting.channel_dataframe(channels=['conda-forge', 'defaults'])
+        lint_exclude = (lint_args.exclude or ())
+        if 'already_in_bioconda' not in lint_exclude:
+            lint_exclude = tuple(lint_exclude) + ('already_in_bioconda',)
+            lint_args = linting.LintArgs(lint_args.df, lint_exclude, lint_args.registry)
+        lint_args = linting.LintArgs(df, lint_exclude, lint_args.registry)
 
     dag, name2recipes = utils.get_dag(recipes, config=orig_config, blacklist=blacklist)
     recipe2name = {}
@@ -425,8 +427,7 @@ def build_recipes(
             force=force,
             channels=config['channels'],
             docker_builder=docker_builder,
-            df=df,
-            prelint=prelint,
+            lint_args=lint_args,
         )
 
         all_success &= res.success