diff --git a/conda_build/__init__.py b/conda_build/__init__.py index 1a9b54ef8e..bb67128c6d 100644 --- a/conda_build/__init__.py +++ b/conda_build/__init__.py @@ -8,6 +8,8 @@ __version__ = get_versions()['version'] del get_versions +import logging + # Sub commands added by conda-build to the conda command sub_commands = [ 'build', @@ -21,3 +23,13 @@ 'sign', 'skeleton', ] + +# Set default logging handler to avoid "No handler found" warnings. +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/conda_build/build.py b/conda_build/build.py index 71f6470f65..8dbf329d5b 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -55,10 +55,10 @@ from conda_build.post import (post_process, post_build, fix_permissions, get_build_metadata) from conda_build.utils import (rm_rf, _check_call, copy_into, on_win, get_build_folders, - silence_loggers, path_prepended, create_entry_points, + path_prepended, create_entry_points, prepend_bin_path, codec, root_script_dir, print_skip_message, ensure_list, get_lock, ExitStack, get_recipe_abspath, tmp_chdir, - expand_globs) + expand_globs, LoggingContext) from conda_build.metadata import build_string_from_metadata from conda_build.index import update_index from conda_build.create_test import (create_files, create_shell_files, @@ -75,8 +75,6 @@ else: shell_path = '/bin/bash' -log = logging.getLogger(__file__) - def prefix_files(prefix): ''' @@ -593,82 +591,78 @@ def create_env(prefix, specs, config, clear_cache=True): Create a conda envrionment for the given prefix and specs. ''' if config.debug: - logging.getLogger("conda").setLevel(logging.DEBUG) - logging.getLogger("binstar").setLevel(logging.DEBUG) - logging.getLogger("install").setLevel(logging.DEBUG) - logging.getLogger("conda.install").setLevel(logging.DEBUG) - logging.getLogger("fetch").setLevel(logging.DEBUG) - logging.getLogger("print").setLevel(logging.DEBUG) - logging.getLogger("progress").setLevel(logging.DEBUG) - logging.getLogger("dotupdate").setLevel(logging.DEBUG) - logging.getLogger("stdoutlog").setLevel(logging.DEBUG) - logging.getLogger("requests").setLevel(logging.DEBUG) + logging.getLogger("conda_build").setLevel(logging.DEBUG) + external_logger_context = LoggingContext(logging.DEBUG) else: - silence_loggers(show_warnings_and_errors=True) - - if os.path.isdir(prefix): - rm_rf(prefix) + logging.getLogger("conda_build").setLevel(logging.INFO) + external_logger_context = LoggingContext(logging.ERROR) - specs = list(specs) - for feature, value in feature_list: - if value: - specs.append('%s@' % feature) + with external_logger_context: + log = logging.getLogger(__name__) - if specs: # Don't waste time if there is nothing to do - log.debug("Creating environment in %s", prefix) - log.debug(str(specs)) + if os.path.isdir(prefix): + rm_rf(prefix) - with path_prepended(prefix): - locks = [] - try: - cc.pkgs_dirs = cc.pkgs_dirs[:1] - locked_folders = cc.pkgs_dirs + list(config.bldpkgs_dirs) - for folder in locked_folders: - if not os.path.isdir(folder): - os.makedirs(folder) - lock = get_lock(folder, timeout=config.timeout) - if not folder.endswith('pkgs'): - update_index(folder, config=config, lock=lock, could_be_mirror=False) - locks.append(lock) - - with ExitStack() as stack: - for lock in locks: - stack.enter_context(lock) - index = get_build_index(config=config, clear_cache=True) - - actions = plan.install_actions(prefix, index, specs) - if config.disable_pip: - actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('pip-')] # noqa - actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('setuptools-')] # noqa - plan.display_actions(actions, index) - if on_win: - for k, v in os.environ.items(): - os.environ[k] = str(v) - plan.execute_actions(actions, index, verbose=config.debug) - except (SystemExit, PaddingError, LinkError) as exc: - if (("too short in" in str(exc) or - 'post-link failed for: openssl' in str(exc) or - isinstance(exc, PaddingError)) and - config.prefix_length > 80): - if config.prefix_length_fallback: - log.warn("Build prefix failed with prefix length %d", config.prefix_length) - log.warn("Error was: ") - log.warn(str(exc)) - log.warn("One or more of your package dependencies needs to be rebuilt " - "with a longer prefix length.") - log.warn("Falling back to legacy prefix length of 80 characters.") - log.warn("Your package will not install into prefixes > 80 characters.") - config.prefix_length = 80 - - # Set this here and use to create environ - # Setting this here is important because we use it below (symlink) - prefix = config.build_prefix - - create_env(prefix, specs, config=config, - clear_cache=clear_cache) - else: - raise - warn_on_old_conda_build(index=index) + specs = list(specs) + for feature, value in feature_list: + if value: + specs.append('%s@' % feature) + + if specs: # Don't waste time if there is nothing to do + log.debug("Creating environment in %s", prefix) + log.debug(str(specs)) + + with path_prepended(prefix): + locks = [] + try: + cc.pkgs_dirs = cc.pkgs_dirs[:1] + locked_folders = cc.pkgs_dirs + list(config.bldpkgs_dirs) + for folder in locked_folders: + if not os.path.isdir(folder): + os.makedirs(folder) + lock = get_lock(folder, timeout=config.timeout) + if not folder.endswith('pkgs'): + update_index(folder, config=config, lock=lock, could_be_mirror=False) + locks.append(lock) + + with ExitStack() as stack: + for lock in locks: + stack.enter_context(lock) + index = get_build_index(config=config, clear_cache=True) + + actions = plan.install_actions(prefix, index, specs) + if config.disable_pip: + actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('pip-')] # noqa + actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('setuptools-')] # noqa + plan.display_actions(actions, index) + if on_win: + for k, v in os.environ.items(): + os.environ[k] = str(v) + plan.execute_actions(actions, index, verbose=config.debug) + except (SystemExit, PaddingError, LinkError) as exc: + if (("too short in" in str(exc) or + 'post-link failed for: openssl' in str(exc) or + isinstance(exc, PaddingError)) and + config.prefix_length > 80): + if config.prefix_length_fallback: + log.warn("Build prefix failed with prefix length %d", config.prefix_length) + log.warn("Error was: ") + log.warn(str(exc)) + log.warn("One or more of your package dependencies needs to be rebuilt " + "with a longer prefix length.") + log.warn("Falling back to legacy prefix length of 80 characters.") + log.warn("Your package will not install into prefixes > 80 characters.") + config.prefix_length = 80 + + # Set this here and use to create environ + # Setting this here is important because we use it below (symlink) + prefix = config.build_prefix + + create_env(prefix, specs, config=config, + clear_cache=clear_cache) + else: + raise + warn_on_old_conda_build(index=index) # ensure prefix exists, even if empty, i.e. when specs are empty if not isdir(prefix): @@ -685,7 +679,7 @@ def get_installed_conda_build_version(): vers_inst = [dist.split('::', 1)[-1].rsplit('-', 2)[1] for dist in root_linked if dist.split('::', 1)[-1].rsplit('-', 2)[0] == 'conda-build'] if not len(vers_inst) == 1: - log.warn("Could not detect installed version of conda-build") + logging.getLogger(__name__).warn("Could not detect installed version of conda-build") return None return vers_inst[0] @@ -696,7 +690,8 @@ def get_conda_build_index_versions(index): try: pkgs = r.get_pkgs(MatchSpec('conda-build')) except (NoPackagesFound, NoPackagesFoundError): - log.warn("Could not find any versions of conda-build in the channels") + logging.getLogger(__name__).warn("Could not find any versions of conda-build " + "in the channels") return [pkg.version for pkg in pkgs] @@ -835,6 +830,8 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F print_skip_message(m) return [] + log = logging.getLogger(__name__) + with path_prepended(config.build_prefix): env = environ.get_dict(config=config, m=m) env["CONDA_BUILD_STATE"] = "BUILD" @@ -1097,6 +1094,7 @@ def clean_pkg_cache(dist, timeout): for pkg_id in [dist, 'local::' + dist]: assert pkg_id not in package_cache() except AssertionError: + log = logging.getLogger(__name__) log.debug("Conda caching error: %s package remains in cache after removal", dist) log.debug("Clearing package cache to compensate") cache = package_cache() @@ -1499,7 +1497,8 @@ def handle_pypi_upload(f, config): try: subprocess.check_call() except: - log.warn("wheel upload failed - is twine installed? Is this package registered?") + logging.getLogger(__name__).warn("wheel upload failed - is twine installed?" + " Is this package registered?") def print_build_intermediate_warning(config): diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 30ef1c70e0..c639575b3e 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -19,7 +19,7 @@ get_render_parser, bldpkg_path) from conda_build.conda_interface import cc, add_parser_channels, url_path import conda_build.source as source -from conda_build.utils import silence_loggers, print_skip_message +from conda_build.utils import print_skip_message, LoggingContext from conda_build.config import Config on_win = (sys.platform == 'win32') @@ -221,12 +221,12 @@ def parse_args(args): def output_action(recipe, config): - silence_loggers(show_warnings_and_errors=False) - metadata, _, _ = api.render(recipe, config=config) - if metadata.skip(): - print_skip_message(metadata) - else: - print(bldpkg_path(metadata)) + with LoggingContext(logging.CRITICAL + 1): + metadata, _, _ = api.render(recipe, config=config) + if metadata.skip(): + print_skip_message(metadata) + else: + print(bldpkg_path(metadata)) def source_action(metadata, config): diff --git a/conda_build/cli/main_render.py b/conda_build/cli/main_render.py index b35fcb0276..3548ba7f6f 100644 --- a/conda_build/cli/main_render.py +++ b/conda_build/cli/main_render.py @@ -16,7 +16,7 @@ from conda_build.completers import (RecipeCompleter, PythonVersionCompleter, RVersionsCompleter, LuaVersionsCompleter, NumPyVersionCompleter) from conda_build.config import Config -from conda_build.utils import silence_loggers +from conda_build.utils import LoggingContext on_win = (sys.platform == 'win32') @@ -127,14 +127,12 @@ def execute(args): config = Config() set_language_env_vars(args, p, config) - metadata, _, _ = render_recipe(args.recipe, no_download_source=args.no_source, config=config) - if args.output: - logging.basicConfig(level=logging.ERROR) - silence_loggers(show_warnings_and_errors=False) - print(bldpkg_path(metadata)) - else: - logging.basicConfig(level=logging.INFO) - print(output_yaml(metadata, args.file)) + with LoggingContext(logging.CRITICAL + 1): + metadata, _, _ = render_recipe(args.recipe, no_download_source=args.no_source, config=config) + if args.output: + print(bldpkg_path(metadata)) + else: + print(output_yaml(metadata, args.file)) def main(): diff --git a/conda_build/config.py b/conda_build/config.py index 8150da9ac6..f743e86339 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -16,7 +16,6 @@ from .utils import get_build_folders, rm_rf -log = logging.getLogger(__file__) on_win = (sys.platform == 'win32') DEFAULT_PREFIX_LENGTH = 255 @@ -404,7 +403,8 @@ def __enter__(self): def __exit__(self, e_type, e_value, traceback): if not getattr(self, 'dirty') and not getattr(self, 'keep_old_work') and e_type is None: - log.info("--keep-old-work flag not specified. Removing source and build files.\n") + logging.getLogger(__name__).info("--keep-old-work flag not specified. " + "Removing source and build files.\n") self.clean() diff --git a/conda_build/environ.py b/conda_build/environ.py index b68af26eb7..16dcd015d6 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -19,8 +19,6 @@ from conda_build.features import feature_list from conda_build.utils import prepend_bin_path, ensure_list -log = logging.getLogger(__file__) - def get_perl_ver(config): return str(config.CONDA_PERL) @@ -50,6 +48,7 @@ def get_lua_include_dir(config): def verify_git_repo(git_dir, git_url, config, expected_rev='HEAD'): env = os.environ.copy() + log = logging.getLogger(__name__) if config.verbose: stderr = None else: @@ -138,6 +137,7 @@ def get_git_info(repo, config): :return: """ d = {} + log = logging.getLogger(__name__) if config.verbose: stderr = None diff --git a/conda_build/inspect.py b/conda_build/inspect.py index 78fd6d6970..8ec481fe80 100644 --- a/conda_build/inspect.py +++ b/conda_build/inspect.py @@ -23,9 +23,6 @@ from conda_build.utils import groupby, getter, comma_join, rm_rf -log = logging.getLogger(__file__) - - def which_prefix(path): """ given the path (to a (presumably) conda installed file) return the @@ -144,6 +141,7 @@ def replace_path(binary, path, prefix): def test_installable(channel='defaults'): success = True + log = logging.getLogger(__name__) has_py = re.compile(r'py(\d)(\d)') for platform in ['osx-64', 'linux-32', 'linux-64', 'win-32', 'win-64']: log.info("######## Testing platform %s ########", platform) @@ -238,8 +236,8 @@ def inspect_linkages(packages, prefix=sys.prefix, untracked=False, if path.startswith(prefix): deps = list(which_package(path)) if len(deps) > 1: - log.warn("Warning: %s comes from multiple packages: %s", path, - comma_join(deps)) + logging.getLogger(__name__).warn("Warning: %s comes from multiple " + "packages: %s", path, comma_join(deps)) if not deps: if exists(path): depmap['untracked'].append((lib, path.split(prefix + diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 3341287796..37c5908f33 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -4,6 +4,7 @@ import json import logging import os +import re import sys import jinja2 @@ -12,8 +13,6 @@ from .environ import get_dict as get_environ from .metadata import select_lines, ns_cfg -log = logging.getLogger(__file__) - class UndefinedNeverFail(jinja2.Undefined): """ @@ -77,6 +76,7 @@ def get_source(self, environment, template): def load_setup_py_data(config, setup_file='setup.py', from_recipe_dir=False, recipe_dir=None, permit_undefined_jinja=True): _setuptools_data = {} + log = logging.getLogger(__name__) def setup(**kw): _setuptools_data.update(kw) @@ -152,6 +152,7 @@ def setup(**kw): def load_setuptools(config, setup_file='setup.py', from_recipe_dir=False, recipe_dir=None, permit_undefined_jinja=True): + log = logging.getLogger(__name__) log.warn("Deprecation notice: the load_setuptools function has been renamed to " "load_setup_py_data. load_setuptools will be removed in a future release.") return load_setup_py_data(config=config, setup_file=setup_file, from_recipe_dir=from_recipe_dir, @@ -167,8 +168,8 @@ def load_npm(): def load_file_regex(config, load_file, regex_pattern, from_recipe_dir=False, recipe_dir=None, permit_undefined_jinja=True): - import re match = False + log = logging.getLogger(__name__) cd_to_work = False diff --git a/conda_build/metadata.py b/conda_build/metadata.py index b9abdd15a6..1a64fc536b 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1,10 +1,11 @@ from __future__ import absolute_import, division, print_function +import glob import logging import os +from os.path import isfile, join import re import sys -from os.path import isfile, join from .conda_interface import iteritems, PY3, text_type from .conda_interface import memoized, md5_file @@ -31,7 +32,6 @@ 'files of conda recipes)') on_win = (sys.platform == 'win32') -log = logging.getLogger(__file__) def ns_cfg(config): diff --git a/conda_build/sign.py b/conda_build/sign.py index 764c3c3649..8e5731d185 100644 --- a/conda_build/sign.py +++ b/conda_build/sign.py @@ -21,8 +21,6 @@ $ conda install -n root pycrypto """) -log = logging.getLogger(__file__) - def keygen(name, size=2048): print("Generating public/private key pair (%d bits)..." % size) @@ -103,6 +101,7 @@ def verify(path): It raises SignatureError when the signature file, or the public key does not exist. """ + log = logging.getLogger(__name__) sig_path = path + '.sig' if not isfile(sig_path): log.error("signature does not exist: %s", sig_path) diff --git a/conda_build/source.py b/conda_build/source.py index 268b8c4470..1969286514 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -33,8 +33,6 @@ git_submod_re = re.compile(r'(?:.+)\.(.+)\.(?:.+)\s(.+)') -log = logging.getLogger(__file__) - def download_to_cache(meta, config): ''' Download a source to the local cache. ''' @@ -374,7 +372,7 @@ def get_repository_info(recipe_path): time.ctime(os.path.getmtime( join(recipe_path, "meta.yaml")))) except CalledProcessError: - log.debug("Failed to checkout source in " + recipe_path) + logging.getLogger(__name__).debug("Failed to checkout source in " + recipe_path) return "{}, last modified {}".format(recipe_path, time.ctime(os.path.getmtime( join(recipe_path, "meta.yaml")))) diff --git a/conda_build/utils.py b/conda_build/utils.py index 129f43c60b..390fefee20 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -41,8 +41,6 @@ from contextlib2 import ExitStack # NOQA -log = logging.getLogger(__file__) - # elsewhere, kept here for reduced duplication. NOQA because it is not used in this file. from .conda_interface import rm_rf # NOQA @@ -50,7 +48,6 @@ codec = getpreferredencoding() or 'utf-8' on_win = sys.platform == "win32" -log = logging.getLogger(__file__) root_script_dir = os.path.join(root_dir, 'Scripts' if on_win else 'bin') @@ -92,7 +89,8 @@ def get_recipe_abspath(recipe): def copy_into(src, dst, timeout=90, symlinks=False, lock=None): - "Copy all the files and directories in src to the directory dst" + """Copy all the files and directories in src to the directory dst""" + log = logging.getLogger(__name__) if isdir(src): merge_tree(src, dst, symlinks, timeout=timeout, lock=lock) @@ -448,25 +446,6 @@ def get_build_folders(croot): return glob(os.path.join(croot, "*" + "[0-9]" * 10 + "*")) -def silence_loggers(show_warnings_and_errors=True): - if show_warnings_and_errors: - log_level = logging.WARN - else: - log_level = logging.CRITICAL + 1 - logging.getLogger(os.path.dirname(__file__)).setLevel(log_level) - # This squelches a ton of conda output that is not hugely relevant - logging.getLogger("conda").setLevel(log_level) - logging.getLogger("binstar").setLevel(log_level) - logging.getLogger("install").setLevel(log_level + 10) - logging.getLogger("conda.install").setLevel(log_level + 10) - logging.getLogger("fetch").setLevel(log_level) - logging.getLogger("print").setLevel(log_level) - logging.getLogger("progress").setLevel(log_level) - logging.getLogger("dotupdate").setLevel(log_level) - logging.getLogger("stdoutlog").setLevel(log_level) - logging.getLogger("requests").setLevel(log_level) - - def prepend_bin_path(env, prefix, prepend_prefix=False): # bin_dirname takes care of bin on *nix, Scripts on win env['PATH'] = join(prefix, bin_dirname) + os.pathsep + env['PATH'] @@ -675,3 +654,32 @@ def find_recipe(path): elif not results: raise IOError("No meta.yaml or conda.yaml files found in %s" % path) return results[0] + + +class LoggingContext(object): + loggers = ['conda', 'binstar', 'install', 'conda.install', 'fetch', 'print', 'progress', + 'dotupdate', 'stdoutlog', 'requests'] + + def __init__(self, level=logging.WARN, handler=None, close=True): + self.level = level + self.old_levels = {} + self.handler = handler + self.close = close + + def __enter__(self): + for logger in LoggingContext.loggers: + log = logging.getLogger(logger) + self.old_levels[logger] = log.level + log.setLevel(self.level if ('install' not in logger or + self.level < logging.INFO) else self.level + 10) + if self.handler: + self.logger.addHandler(self.handler) + + def __exit__(self, et, ev, tb): + for logger, level in self.old_levels.items(): + logging.getLogger(logger).setLevel(level) + if self.handler: + self.logger.removeHandler(self.handler) + if self.handler and self.close: + self.handler.close() + # implicit return of None => don't swallow exceptions