From 6235b25c5d02fe73b1d42ed371cec6d874aa5ae8 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Sep 2018 11:16:43 -0500 Subject: [PATCH 1/3] fix CONDA_BUILD_STATE not being set in load_setup_py_data --- conda_build/build.py | 5 +- conda_build/jinja_context.py | 4 +- conda_build/metadata.py | 70 +++++++++---------- conda_build/render.py | 1 + .../metadata/state_variables/meta.yaml | 5 ++ .../metadata/state_variables/setup.py | 30 ++++++++ 6 files changed, 73 insertions(+), 42 deletions(-) create mode 100644 tests/test-recipes/metadata/state_variables/setup.py diff --git a/conda_build/build.py b/conda_build/build.py index 43352f685d..b18a1d9981 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1733,10 +1733,7 @@ def _construct_metadata_for_test_from_package(package, config): local_channel = os.path.dirname(local_pkg_location) # update indices in the channel - for pattern in ('win-*', 'linux-*', 'osx-*', 'noarch'): - for folder in glob(os.path.join(local_channel, pattern)): - log.info("Updating index at %s to make package installable with dependencies" % folder) - update_index(folder, verbose=config.verbose) + update_index(local_channel, verbose=config.verbose) try: metadata = render_recipe(os.path.join(info_dir, 'recipe'), config=config, diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 52f006476e..84d9cb9af4 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -107,6 +107,8 @@ def load_setup_py_data(m, setup_file='setup.py', from_recipe_dir=False, recipe_d origin_setup_script = os.path.join(os.path.dirname(__file__), '_load_setup_py_data.py') dest_setup_script = os.path.join(m.config.work_dir, '_load_setup_py_data.py') copy_into(origin_setup_script, dest_setup_script) + env = get_environ(m) + env["CONDA_BUILD_STATE"] = "RENDER" if os.path.isfile(m.config.build_python): args = [m.config.build_python, dest_setup_script, m.config.work_dir, setup_file] if from_recipe_dir: @@ -115,7 +117,7 @@ def load_setup_py_data(m, setup_file='setup.py', from_recipe_dir=False, recipe_d args.extend(['--recipe-dir', recipe_dir]) if permit_undefined_jinja: args.append('--permit-undefined-jinja') - check_call_env(args, env=get_environ(m)) + check_call_env(args, env=env) # this is a file that the subprocess will have written with open(os.path.join(m.config.work_dir, 'conda_build_loaded_setup_py.json')) as f: _setuptools_data = json.load(f) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 61019b249d..08d41c58dc 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -915,45 +915,38 @@ def parse_again(self, permit_undefined_jinja=False, allow_no_other_outputs=False log.warn("using unsupported internal conda-build function `parse_again`. Please use " "conda_build.api.render instead.") - os.environ["CONDA_BUILD_STATE"] = "RENDER" append_sections_file = None clobber_sections_file = None - try: - # we sometimes create metadata from dictionaries, in which case we'll have no path - if self.meta_path: - self.meta = parse(self._get_contents(permit_undefined_jinja, - allow_no_other_outputs=allow_no_other_outputs, - bypass_env_check=bypass_env_check), - config=self.config, - path=self.meta_path) - - append_sections_file = os.path.join(self.path, 'recipe_append.yaml') - clobber_sections_file = os.path.join(self.path, 'recipe_clobber.yaml') - - append_sections_file = self.config.append_sections_file or append_sections_file - if append_sections_file and not os.path.isfile(append_sections_file): - log.debug('input append sections file did not exist: %s', append_sections_file) - append_sections_file = None - clobber_sections_file = self.config.clobber_sections_file or clobber_sections_file - if clobber_sections_file and not os.path.isfile(clobber_sections_file): - log.debug('input clobber sections file did not exist: %s', clobber_sections_file) - clobber_sections_file = None - - if append_sections_file: - self.append_metadata_sections(append_sections_file, merge=True) - if clobber_sections_file: - self.append_metadata_sections(clobber_sections_file, merge=False) - if self.config.bootstrap: - dependencies = _get_dependencies_from_environment(self.config.bootstrap) - self.append_metadata_sections(dependencies, merge=True) - - if self.meta.get('build', {}).get('error_overlinking', False): - self.config.error_overlinking = self.meta['build']['error_overlinking'] - except: - raise - finally: - del os.environ["CONDA_BUILD_STATE"] - pass + # we sometimes create metadata from dictionaries, in which case we'll have no path + if self.meta_path: + self.meta = parse(self._get_contents(permit_undefined_jinja, + allow_no_other_outputs=allow_no_other_outputs, + bypass_env_check=bypass_env_check), + config=self.config, + path=self.meta_path) + + append_sections_file = os.path.join(self.path, 'recipe_append.yaml') + clobber_sections_file = os.path.join(self.path, 'recipe_clobber.yaml') + + append_sections_file = self.config.append_sections_file or append_sections_file + if append_sections_file and not os.path.isfile(append_sections_file): + log.debug('input append sections file did not exist: %s', append_sections_file) + append_sections_file = None + clobber_sections_file = self.config.clobber_sections_file or clobber_sections_file + if clobber_sections_file and not os.path.isfile(clobber_sections_file): + log.debug('input clobber sections file did not exist: %s', clobber_sections_file) + clobber_sections_file = None + + if append_sections_file: + self.append_metadata_sections(append_sections_file, merge=True) + if clobber_sections_file: + self.append_metadata_sections(clobber_sections_file, merge=False) + if self.config.bootstrap: + dependencies = _get_dependencies_from_environment(self.config.bootstrap) + self.append_metadata_sections(dependencies, merge=True) + + if self.meta.get('build', {}).get('error_overlinking', False): + self.config.error_overlinking = self.meta['build']['error_overlinking'] self.validate_features() self.ensure_no_pip_requirements() @@ -1497,6 +1490,7 @@ def _get_contents(self, permit_undefined_jinja, allow_no_other_outputs=False, else: template = env.from_string("") + os.environ["CONDA_BUILD_STATE"] = "RENDER" rendered = template.render(environment=env) if permit_undefined_jinja: @@ -1509,6 +1503,8 @@ def _get_contents(self, permit_undefined_jinja, allow_no_other_outputs=False, ex = "Failed to run jinja context function" sys.exit("Error: Failed to render jinja template in {}:\n{}" .format(self.meta_path, str(ex))) + finally: + del os.environ["CONDA_BUILD_STATE"] return rendered def __unicode__(self): diff --git a/conda_build/render.py b/conda_build/render.py index de16a56e2e..dda8f06f6c 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -674,6 +674,7 @@ def distribute_variants(metadata, variants, permit_unsatisfiable_variants=False, def expand_outputs(metadata_tuples): """Obtain all metadata objects for all outputs from recipe. Useful for outputting paths.""" expanded_outputs = OrderedDict() + for (_m, download, reparse) in metadata_tuples: for (output_dict, m) in _m.get_output_metadata_set(permit_unsatisfiable_variants=False): expanded_outputs[m.dist()] = (output_dict, m) diff --git a/tests/test-recipes/metadata/state_variables/meta.yaml b/tests/test-recipes/metadata/state_variables/meta.yaml index 31625e0537..d00632edec 100644 --- a/tests/test-recipes/metadata/state_variables/meta.yaml +++ b/tests/test-recipes/metadata/state_variables/meta.yaml @@ -1,8 +1,13 @@ +{% set data = load_setup_py_data() %} + package: # should be defined as "RENDER" during rendering phase name: {{ CONDA_BUILD_STATE|lower }} version: 1.0 +source: + path: . + build: script: - test "$PKG_NAME" = "render" # [unix] diff --git a/tests/test-recipes/metadata/state_variables/setup.py b/tests/test-recipes/metadata/state_variables/setup.py new file mode 100644 index 0000000000..2f8660659e --- /dev/null +++ b/tests/test-recipes/metadata/state_variables/setup.py @@ -0,0 +1,30 @@ +import os +from setuptools import setup +# from distutils.core import setup + +if not os.getenv("CONDA_BUILD_STATE") == "RENDER": + raise ValueError("Conda build state not set correctly") + +setup( + name="conda-build-test-project", + version='1.0', + author="Continuum Analytics, Inc.", + url="https://github.com/conda/conda-build", + license="BSD", + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + ], + description="test package for testing conda-build", + packages=['conda_build_test'], + scripts=[ + 'bin/test-script-setup.py', + ], +) From 6768530d3d35d6d46c2d5e71595550e97f180388 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Sep 2018 11:24:52 -0500 Subject: [PATCH 2/3] detect and fallback to old subdir update_index behavior --- conda_build/index.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/conda_build/index.py b/conda_build/index.py index a12014885d..738d5a1e9a 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -237,6 +237,17 @@ def _ensure_valid_channel(local_folder, subdir): os.makedirs(path) +def update_subdir_index(dir_path, subdir, check_md5=False, channel_name=None, threads=None, verbose=True): + """ + Update all index files in dir_path with changed packages. + + :param check_md5: Whether to check MD5s instead of mtimes for determining + if a package changed. + :type check_md5: bool + """ + return ChannelIndex(dir_path, channel_name, deep_integrity_check=check_md5, threads=threads).index_subdir(subdir, verbose=verbose) + + def update_index(dir_path, check_md5=False, channel_name=None, patch_generator=None, threads=None, verbose=True): """ If dir_path contains a directory named 'noarch', the path tree therein is treated @@ -248,20 +259,14 @@ def update_index(dir_path, check_md5=False, channel_name=None, patch_generator=N information will be updated. """ + base_path, dirname = os.path.split(dir_path) + if dirname in DEFAULT_SUBDIRS: + log.warn("The update_index function has changed to index all subdirs at once. You're pointing it at a single subdir. " + "Please update your code to point it at the channel root, rather than a subdir.") + return update_subdir_index(base_path, dirname, check_md5=check_md5, channel_name=channel_name, threads=threads, verbose=verbose) return ChannelIndex(dir_path, channel_name, deep_integrity_check=check_md5, threads=threads).index(patch_generator=patch_generator, verbose=verbose) -def update_subdir_index(dir_path, subdir, check_md5=False, channel_name=None, threads=None, verbose=True): - """ - Update all index files in dir_path with changed packages. - - :param check_md5: Whether to check MD5s instead of mtimes for determining - if a package changed. - :type check_md5: bool - """ - return ChannelIndex(dir_path, channel_name, deep_integrity_check=check_md5, threads=threads).index_subdir(subdir, verbose=verbose) - - def _determine_namespace(info): if info.get('namespace'): namespace = info['namespace'] From 903cb63e323c569183faf2bf0a59da8ec92287ec Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Sep 2018 11:52:54 -0500 Subject: [PATCH 3/3] fix test_test_extra_dep (replace click dep with imagesize) --- tests/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6f793756c4..9f009f7c33 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -621,12 +621,12 @@ def test_render_with_python_arg_reduces_subspace(capfd): def test_test_extra_dep(testing_metadata): - testing_metadata.meta['test']['imports'] = ['click'] + testing_metadata.meta['test']['imports'] = ['imagesize'] api.output_yaml(testing_metadata, 'meta.yaml') output = api.build(testing_metadata, notest=True, anaconda_upload=False)[0] # tests version constraints. CLI would quote this - "click <6.7" - args = [output, '-t', '--extra-deps', 'click <6.7'] + args = [output, '-t', '--extra-deps', 'imagesize <1.0'] # extra_deps will add it in main_build.execute(args)