From ccdb5ab2ae1b59366eb561793f308ac54d87bc50 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Fri, 16 Sep 2016 11:43:23 -0500 Subject: [PATCH 001/156] Remove rather than assert on rpaths outside prefix --- conda_build/post.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index ae0774d473..789a55a196 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -319,11 +319,12 @@ def mk_relative_linux(f, prefix, rpaths=('lib',)): elif old.startswith('/'): # Test if this absolute path is outside of prefix. That is fatal. relpath = os.path.relpath(old, prefix) - assert not relpath.startswith('..' + os.sep), \ - 'rpath {0} is outside prefix {1}'.format(old, prefix) - relpath = '$ORIGIN/' + os.path.relpath(old, origin) - if relpath not in new: - new.append(relpath) + if relpath.startswith('..' + os.sep): + print('Warning: rpath {0} is outside prefix {1} (removing it)'.format(old, prefix)) + else: + relpath = '$ORIGIN/' + os.path.relpath(old, origin) + if relpath not in new: + new.append(relpath) # Ensure that the asked-for paths are also in new. for rpath in rpaths: if not rpath.startswith('/'): From c70d9ed70caf0977601ce508a223d989baa6c3c2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 27 Sep 2016 11:30:13 -0500 Subject: [PATCH 002/156] add conda_versions info file to record build-time vers of conda, c-b --- conda_build/build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda_build/build.py b/conda_build/build.py index 74dbcc3417..24d33a8da3 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -26,6 +26,8 @@ import filelock +# used to get version +import conda from .conda_interface import cc from .conda_interface import envs_dirs, root_dir from .conda_interface import plan @@ -353,6 +355,10 @@ def create_info_files(m, files, config, prefix): for f in files: fo.write(f + '\n') + with open(join(config.info_dir, 'conda_versions'), **mode_dict) as fo: + fo.write("Conda version at build time: {0}\n".format(conda.__version__)) + fo.write("Conda-build version at build time: {0}\n".format(__version__)) + detect_and_record_prefix_files(m, files, prefix, config) write_no_link(m, config, files) From b8056581db372bf1784a451a25e2422996f6b38a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 27 Sep 2016 20:19:21 -0500 Subject: [PATCH 003/156] add initial test files to guide development --- .../alternate_type_wheel/meta.yaml | 16 +++++++++ .../meta.yaml | 21 ++++++++++++ .../run_test.py | 2 ++ .../split-packages/copying_files/meta.yaml | 33 +++++++++++++++++++ .../split-packages/copying_files/run_test.py | 1 + .../copying_files/test_subpackage1.py | 18 ++++++++++ .../jinja2_subpackage_name/meta.yaml | 10 ++++++ .../noarch_subpackage/meta.yaml | 14 ++++++++ .../overlapping_files/meta.yaml | 20 +++++++++++ .../script_autodetect_interpreter/meta.yaml | 19 +++++++++++ .../subpackage1.py | 4 +++ .../subpackage2.sh | 2 ++ .../subpackage3.unrecognized | 0 .../script_install_files/meta.yaml | 15 +++++++++ .../script_install_files/subpackage1.py | 4 +++ .../script_install_files/test_subpackage1.py | 6 ++++ 16 files changed, 185 insertions(+) create mode 100644 tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml create mode 100644 tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml create mode 100644 tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py create mode 100644 tests/test-recipes/split-packages/copying_files/meta.yaml create mode 120000 tests/test-recipes/split-packages/copying_files/run_test.py create mode 100644 tests/test-recipes/split-packages/copying_files/test_subpackage1.py create mode 100644 tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml create mode 100644 tests/test-recipes/split-packages/noarch_subpackage/meta.yaml create mode 100644 tests/test-recipes/split-packages/overlapping_files/meta.yaml create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized create mode 100644 tests/test-recipes/split-packages/script_install_files/meta.yaml create mode 100644 tests/test-recipes/split-packages/script_install_files/subpackage1.py create mode 100644 tests/test-recipes/split-packages/script_install_files/test_subpackage1.py diff --git a/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml b/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml new file mode 100644 index 0000000000..d149b3dcc5 --- /dev/null +++ b/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml @@ -0,0 +1,16 @@ +package: + name: split_packages_alternate_type_wheel + version: 1.0 + +requirements: + run: + - my_script_subpackage + +outputs: + - name: my_script_subpackage + script: subpackage1.py + script_interpreter: python + type: wheel + test: + script: test_subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml new file mode 100644 index 0000000000..e0bf52eb60 --- /dev/null +++ b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml @@ -0,0 +1,21 @@ +# Test that our composite run requirements for the parent package are the union of subpackage +# requirements + +package: + name: split_packages_compose_run_requirements_from_subpackages + version: 1.0 + +requirements: + run: + - my_script_subpackage + - my_script_subpackage_2 + +outputs: + - name: my_script_subpackage + requirements: + - cython + - name: my_script_subpackage_2 + requirements: + - click + +# tests are in run_test.py, and they check that the packages above are both installed diff --git a/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py new file mode 100644 index 0000000000..5088a1f4b2 --- /dev/null +++ b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py @@ -0,0 +1,2 @@ +import cython +import click diff --git a/tests/test-recipes/split-packages/copying_files/meta.yaml b/tests/test-recipes/split-packages/copying_files/meta.yaml new file mode 100644 index 0000000000..c400a4b9cd --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/meta.yaml @@ -0,0 +1,33 @@ +package: + name: split_packages_file_list + version: 1.0 + +build: + script: + # test copying filename + - echo weee > $PREFIX/subpackage_file1 # [unix] + - echo weee > %PREFIX%/subpackage_file1 # [win] + # test copying by folder name + - mkdir $PREFIX/somedir # [unix] + - mkdir %PREFIX%/somedir # [win] + - echo weee > $PREFIX/somedir/subpackage_file1 # [unix] + - echo weee > %PREFIX%/somedir/subpackage_file1 # [win] + # test glob patterns + - echo weee > $PREFIX/subpackage_file1.ext # [unix] + - echo weee > $PREFIX/subpackage_file2.ext # [unix] + - echo weee > %PREFIX%/subpackage_file1.ext # [win] + - echo weee > %PREFIX%/subpackage_file2.ext # [win] + +requirements: + run: + - my_script_subpackage + +outputs: + - name: my_script_subpackage + files: + - subpackage_file1 + - somedir + - *.ext + test: + script: test_subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/copying_files/run_test.py b/tests/test-recipes/split-packages/copying_files/run_test.py new file mode 120000 index 0000000000..829b62d71c --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/run_test.py @@ -0,0 +1 @@ +test_subpackage1.py \ No newline at end of file diff --git a/tests/test-recipes/split-packages/copying_files/test_subpackage1.py b/tests/test-recipes/split-packages/copying_files/test_subpackage1.py new file mode 100644 index 0000000000..e096bc0c90 --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/test_subpackage1.py @@ -0,0 +1,18 @@ +import os + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" + +filename = os.path.join(os.environ['PREFIX'], 'subdir', 'subpackage_file1') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1.ext') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" + + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file2.ext') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" diff --git a/tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml b/tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml new file mode 100644 index 0000000000..da5ae9c950 --- /dev/null +++ b/tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml @@ -0,0 +1,10 @@ +package: + name: split_packages_jinja2_subpackage_name + version: 1.0 + +requirements: + run: + - {{ PKG_NAME }}_subpackage + +outputs: + - name: {{ PKG_NAME }}_subpackage diff --git a/tests/test-recipes/split-packages/noarch_subpackage/meta.yaml b/tests/test-recipes/split-packages/noarch_subpackage/meta.yaml new file mode 100644 index 0000000000..9b307273ee --- /dev/null +++ b/tests/test-recipes/split-packages/noarch_subpackage/meta.yaml @@ -0,0 +1,14 @@ +package: + name: split_packages_jinja2_subpackage_name + version: 1.0 + +requirements: + run: + - pkg_subpackage + - pkg_subpackage_python_noarch + +outputs: + - name: pkg_subpackage + noarch: True + - name: pkg_subpackage_python_noarch + noarch: python diff --git a/tests/test-recipes/split-packages/overlapping_files/meta.yaml b/tests/test-recipes/split-packages/overlapping_files/meta.yaml new file mode 100644 index 0000000000..0cdcc36de0 --- /dev/null +++ b/tests/test-recipes/split-packages/overlapping_files/meta.yaml @@ -0,0 +1,20 @@ +# this test is to make sure that we raise an error when more than one subpackage +# contains the same file. This is important to avoid, as conda does nothing smart +# about keeping files that are installed by two packages when one is removed. + +package: + name: split_packages_script_overlapping_files + version: 1.0 + +requirements: + run: + - my_script_subpackage + - my_script_subpackage_2 + +outputs: + - name: my_script_subpackage + script: subpackage1.py + script_interpreter: python + - name: my_script_subpackage_2 + script: subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml b/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml new file mode 100644 index 0000000000..08d7a25776 --- /dev/null +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml @@ -0,0 +1,19 @@ +package: + name: split_packages_autodetect_interpreter + version: 1.0 + +requirements: + run: + - my_script_subpackage + - my_script_subpackage_shell + - my_script_subpackage_unrecognized + +outputs: + - name: my_script_subpackage + script: subpackage1.py + # Assume that on Windows, we have bash available here + - name: my_script_subpackage_shell + script: subpackage2.sh + # what happens when we have an unrecognized script type? + - name: my_script_subpackage_unrecognized + script: subpackage3.unrecognized diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py new file mode 100644 index 0000000000..09ec8a7540 --- /dev/null +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py @@ -0,0 +1,4 @@ +import os + +with open(os.path.join(os.environ['PREFIX'], 'subpackage_file_1'), 'w') as f: + f.write("weeee") diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh new file mode 100644 index 0000000000..ce59ebf7a5 --- /dev/null +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo weeee > $PREFIX/subpackage_file_2 diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test-recipes/split-packages/script_install_files/meta.yaml b/tests/test-recipes/split-packages/script_install_files/meta.yaml new file mode 100644 index 0000000000..af39bc4b15 --- /dev/null +++ b/tests/test-recipes/split-packages/script_install_files/meta.yaml @@ -0,0 +1,15 @@ +package: + name: split_packages_script_install_files + version: 1.0 + +requirements: + run: + - my_script_subpackage + +outputs: + - name: my_script_subpackage + script: subpackage1.py + script_interpreter: python + test: + script: test_subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/script_install_files/subpackage1.py b/tests/test-recipes/split-packages/script_install_files/subpackage1.py new file mode 100644 index 0000000000..09ec8a7540 --- /dev/null +++ b/tests/test-recipes/split-packages/script_install_files/subpackage1.py @@ -0,0 +1,4 @@ +import os + +with open(os.path.join(os.environ['PREFIX'], 'subpackage_file_1'), 'w') as f: + f.write("weeee") diff --git a/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py b/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py new file mode 100644 index 0000000000..96ed1deba1 --- /dev/null +++ b/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py @@ -0,0 +1,6 @@ +import os + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') + +assert os.path.isfile(filename) +assert open(filename).read() == "weeee" From f7cea462676a3fdb7b1ede33a6dca964cc194061 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 28 Sep 2016 09:34:06 -0500 Subject: [PATCH 004/156] convert popen args to bytestrings if unicode --- conda_build/source.py | 2 +- conda_build/utils.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/conda_build/source.py b/conda_build/source.py index 4cc22dc9f0..5fbac03c2e 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -13,7 +13,7 @@ from .conda_interface import hashsum_file from conda_build.os_utils import external -from conda_build.utils import (tar_xf, unzip, safe_print_unicode, copy_into, on_win, +from conda_build.utils import (tar_xf, unzip, safe_print_unicode, copy_into, on_win, codec, check_output_env, check_call_env, convert_path_for_cygwin_or_msys2) # legacy exports for conda diff --git a/conda_build/utils.py b/conda_build/utils.py index a134205b2f..c2c94e576e 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -455,7 +455,13 @@ def _func_defaulting_env_to_os_environ(func, *popenargs, **kwargs): kwargs = kwargs.copy() env_copy = os.environ.copy() kwargs.update({'env': env_copy}) - return func(*popenargs, **kwargs) + args = [] + for arg in popenargs: + # arguments to subprocess need to be bytestrings + if hasattr(arg, 'encode'): + arg = arg.encode(codec) + args.append(arg) + return func(*args, **kwargs) def check_call_env(*popenargs, **kwargs): From c2e1116b4d1ae10f3e6b3ce942cdf9124c10976d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 28 Sep 2016 09:52:19 -0500 Subject: [PATCH 005/156] fix perl file access error on win skeleton cpan --- conda_build/skeletons/cpan.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conda_build/skeletons/cpan.py b/conda_build/skeletons/cpan.py index 8ae0569399..0ab2418327 100644 --- a/conda_build/skeletons/cpan.py +++ b/conda_build/skeletons/cpan.py @@ -157,12 +157,11 @@ def get_cpan_api_url(url, colons): url = url.replace("::", "-") with PerlTmpDownload(url) as json_path: try: - dist_json_file = gzip.open(json_path) - output = dist_json_file.read() + with gzip.open(json_path) as dist_json_file: + output = dist_json_file.read() if hasattr(output, "decode"): output = output.decode('utf-8-sig') rel_dict = json.loads(output) - dist_json_file.close() except IOError: rel_dict = json.loads(open(json_path).read()) return rel_dict From e9378ba13e40bbe6b08a7e1ba16d1988eddef86e Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 13 Sep 2016 13:49:19 -0500 Subject: [PATCH 006/156] Refactor code that copies site-packages So as to support `noarch: python` which also needs to be able to do this --- conda_build/build.py | 6 ++++ conda_build/noarch_python.py | 66 ++++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 74dbcc3417..0e0e9dcd75 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -714,9 +714,15 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F croot=config.croot) create_info_files(m, sorted(files2 - files1), config=config, prefix=config.build_prefix) + if m.get_value('build/noarch_python'): import conda_build.noarch_python as noarch_python noarch_python.transform(m, sorted(files2 - files1), config.build_prefix) + elif str(m.get_value('build/noarch')).lower() == "python": + + + import conda_build.noarch_python as noarch_python + noarch_python.populate_files(m, sorted(files2 - files1), config.build_prefix) files3 = prefix_files(prefix=config.build_prefix) fix_permissions(files3 - files1, config.build_prefix) diff --git a/conda_build/noarch_python.py b/conda_build/noarch_python.py index 4771381774..ac9de13f5a 100644 --- a/conda_build/noarch_python.py +++ b/conda_build/noarch_python.py @@ -97,31 +97,41 @@ def handle_file(f, d, prefix): _error_exit("Error: Don't know how to handle file: %s" % f) -def transform(m, files, prefix): - assert 'py_' in m.dist() +def setup_bin_dir(prefix): + bin_dir = join(prefix, 'bin') + _force_dir(bin_dir) + return bin_dir + + +def setup_scripts_dir(prefix): + scripts_dir = join(prefix, 'Scripts') + _force_dir(scripts_dir) + return scripts_dir + + +def populate_files(m, files, prefix): + setup_bin_dir(prefix) + setup_scripts_dir(prefix) name = m.name() - bin_dir = join(prefix, 'bin') - _force_dir(bin_dir) + bin_dir = setup_bin_dir(prefix) + scripts_dir = setup_scripts_dir(prefix) # Create *nix prelink script # Note: it's important to use LF newlines or it wont work if we build on Win with open(join(bin_dir, '.%s-pre-link.sh' % name), 'wb') as fo: fo.write('''\ -#!/bin/bash -$PREFIX/bin/python $SOURCE_DIR/link.py -'''.encode('utf-8')) - - scripts_dir = join(prefix, 'Scripts') - _force_dir(scripts_dir) + #!/bin/bash + $PREFIX/bin/python $SOURCE_DIR/link.py + '''.encode('utf-8')) # Create windows prelink script (be nice and use Windows newlines) with open(join(scripts_dir, '.%s-pre-link.bat' % name), 'wb') as fo: fo.write('''\ -@echo off -"%PREFIX%\\python.exe" "%SOURCE_DIR%\\link.py" -'''.replace('\n', '\r\n').encode('utf-8')) + @echo off + "%PREFIX%\\python.exe" "%SOURCE_DIR%\\link.py" + '''.replace('\n', '\r\n').encode('utf-8')) d = {'dist': m.dist(), 'site-packages': [], @@ -138,13 +148,35 @@ def transform(m, files, prefix): for i, fn in enumerate(fns): fns[i] = fn.replace('\\', '/') - # Find our way to this directory - this_dir = dirname(__file__) - # copy in windows exe shims if there are any python-scripts if d['python-scripts']: for fn in 'cli-32.exe', 'cli-64.exe': - shutil.copyfile(join(this_dir, fn), join(prefix, fn)) + shutil.copyfile(join(dirname(__file__), fn), join(prefix, fn)) + + # Find our way to this directory + this_dir = dirname(__file__) + + # Read the local _link.py + with open(join(this_dir, '_link.py')) as fi: + link_code = fi.read() + + # Write the package metadata, and bumper with code for linking + with open(join(prefix, 'link.py'), 'w') as fo: + fo.write('DATA = ') + json.dump(d, fo, indent=2, sort_keys=True) + fo.write('\n## END DATA\n\n') + fo.write(link_code) + + return d + + +def transform(m, files, prefix): + assert 'py_' in m.dist() + + d = populate_files(m, files, prefix) + + # Find our way to this directory + this_dir = dirname(__file__) # Read the local _link.py with open(join(this_dir, '_link.py')) as fi: From 0e9e4fd290c60b4b49394eb3a9d786f483b42cca Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 13 Sep 2016 15:45:34 -0500 Subject: [PATCH 007/156] Remove pyc file if building a noarch package --- conda_build/post.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index b94ee0b69c..2b217a6e42 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -135,7 +135,7 @@ def remove_easy_install_pth(files, prefix, config, preserve_egg_dir=False): def rm_py_along_so(prefix): - "remove .py (.pyc) files alongside .so or .pyd files" + """remove .py (.pyc) files alongside .so or .pyd files""" for root, _, files in os.walk(prefix): for fn in files: if fn.endswith(('.so', '.pyd')): @@ -160,6 +160,13 @@ def rm_pyo(files, prefix): os.unlink(os.path.join(prefix, fn)) +def rm_pyc(files, prefix): + re_pyc = re.compile(r'.*(?:\.pyc$)') + for fn in files: + if re_pyc.match(fn): + os.unlink(os.path.join(prefix, fn)) + + def compile_missing_pyc(files, cwd, python_exe): compile_files = [] for fn in files: @@ -182,9 +189,14 @@ def compile_missing_pyc(files, cwd, python_exe): call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd) -def post_process(files, prefix, config, preserve_egg_dir=False): +def post_process(files, prefix, config, preserve_egg_dir=False, noarch=False): + # TODO: verify that files isn't changing + # eg - does `files` reflect the true state of the filesystem? rm_pyo(files, prefix) - compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python) + if not noarch: + compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python) + else: + rm_pyc(files, prefix) remove_easy_install_pth(files, prefix, config, preserve_egg_dir=preserve_egg_dir) rm_py_along_so(prefix) From 6ca9cb8097acfa83d7bbda6721aa1574f48c0e6d Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 14 Sep 2016 10:06:34 -0500 Subject: [PATCH 008/156] Treat noarch_python and noarch: python similar --- conda_build/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 0e0e9dcd75..f0d7ba5123 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -242,7 +242,10 @@ def detect_and_record_prefix_files(m, files, prefix, config): else: files_with_prefix = [] - if files_with_prefix and not m.get_value('build/noarch_python'): + is_noarch = m.get_value('build/noarch_python') or \ + str(m.get_value('build/noarch')).lower() == "python" + + if files_with_prefix and not is_noarch: if on_win: # Paths on Windows can contain spaces, so we need to quote the # paths. Fortunately they can't contain quotes, so we don't have @@ -719,8 +722,6 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F import conda_build.noarch_python as noarch_python noarch_python.transform(m, sorted(files2 - files1), config.build_prefix) elif str(m.get_value('build/noarch')).lower() == "python": - - import conda_build.noarch_python as noarch_python noarch_python.populate_files(m, sorted(files2 - files1), config.build_prefix) From 213c616a7433aa4fb244b4e65fdfb4700699d72e Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 14 Sep 2016 11:12:37 -0500 Subject: [PATCH 009/156] Check is noarch: python --- conda_build/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index f0d7ba5123..62f89a14c7 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -350,7 +350,9 @@ def create_info_files(m, files, config, prefix): files = [_f.replace('\\', '/') for _f in files] with open(join(config.info_dir, 'files'), **mode_dict) as fo: - if m.get_value('build/noarch_python'): + is_noarch = m.get_value('build/noarch_python') or \ + (str(m.get_value('build/noarch')).lower() == "python") + if is_noarch: fo.write('\n') else: for f in files: From 7798f6ecbbbd26c4d3465b68a6f7f6666abe373c Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 14 Sep 2016 11:49:13 -0500 Subject: [PATCH 010/156] Don't create in link script, Scripts or bin folder --- conda_build/noarch_python.py | 72 ++++++++++++------------------------ 1 file changed, 23 insertions(+), 49 deletions(-) diff --git a/conda_build/noarch_python.py b/conda_build/noarch_python.py index ac9de13f5a..408bf0382e 100644 --- a/conda_build/noarch_python.py +++ b/conda_build/noarch_python.py @@ -97,42 +97,7 @@ def handle_file(f, d, prefix): _error_exit("Error: Don't know how to handle file: %s" % f) -def setup_bin_dir(prefix): - bin_dir = join(prefix, 'bin') - _force_dir(bin_dir) - return bin_dir - - -def setup_scripts_dir(prefix): - scripts_dir = join(prefix, 'Scripts') - _force_dir(scripts_dir) - return scripts_dir - - def populate_files(m, files, prefix): - setup_bin_dir(prefix) - setup_scripts_dir(prefix) - - name = m.name() - - bin_dir = setup_bin_dir(prefix) - scripts_dir = setup_scripts_dir(prefix) - - # Create *nix prelink script - # Note: it's important to use LF newlines or it wont work if we build on Win - with open(join(bin_dir, '.%s-pre-link.sh' % name), 'wb') as fo: - fo.write('''\ - #!/bin/bash - $PREFIX/bin/python $SOURCE_DIR/link.py - '''.encode('utf-8')) - - # Create windows prelink script (be nice and use Windows newlines) - with open(join(scripts_dir, '.%s-pre-link.bat' % name), 'wb') as fo: - fo.write('''\ - @echo off - "%PREFIX%\\python.exe" "%SOURCE_DIR%\\link.py" - '''.replace('\n', '\r\n').encode('utf-8')) - d = {'dist': m.dist(), 'site-packages': [], 'python-scripts': [], @@ -153,26 +118,35 @@ def populate_files(m, files, prefix): for fn in 'cli-32.exe', 'cli-64.exe': shutil.copyfile(join(dirname(__file__), fn), join(prefix, fn)) - # Find our way to this directory - this_dir = dirname(__file__) - - # Read the local _link.py - with open(join(this_dir, '_link.py')) as fi: - link_code = fi.read() - - # Write the package metadata, and bumper with code for linking - with open(join(prefix, 'link.py'), 'w') as fo: - fo.write('DATA = ') - json.dump(d, fo, indent=2, sort_keys=True) - fo.write('\n## END DATA\n\n') - fo.write(link_code) - return d def transform(m, files, prefix): assert 'py_' in m.dist() + bin_dir = join(prefix, 'bin') + _force_dir(bin_dir) + + scripts_dir = join(prefix, 'Scripts') + _force_dir(scripts_dir) + + name = m.name() + + # Create *nix prelink script + # Note: it's important to use LF newlines or it wont work if we build on Win + with open(join(bin_dir, '.%s-pre-link.sh' % name), 'wb') as fo: + fo.write('''\ + #!/bin/bash + $PREFIX/bin/python $SOURCE_DIR/link.py + '''.encode('utf-8')) + + # Create windows prelink script (be nice and use Windows newlines) + with open(join(scripts_dir, '.%s-pre-link.bat' % name), 'wb') as fo: + fo.write('''\ + @echo off + "%PREFIX%\\python.exe" "%SOURCE_DIR%\\link.py" + '''.replace('\n', '\r\n').encode('utf-8')) + d = populate_files(m, files, prefix) # Find our way to this directory From 8aec41dc0e872a5007f04b05fd79f95dc05bdd60 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 14 Sep 2016 14:27:53 -0500 Subject: [PATCH 011/156] Write info file when building noarch: python --- conda_build/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 62f89a14c7..bce0b7f400 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -350,10 +350,11 @@ def create_info_files(m, files, config, prefix): files = [_f.replace('\\', '/') for _f in files] with open(join(config.info_dir, 'files'), **mode_dict) as fo: - is_noarch = m.get_value('build/noarch_python') or \ - (str(m.get_value('build/noarch')).lower() == "python") - if is_noarch: + if m.get_value('build/noarch_python'): fo.write('\n') + elif str(m.get_value('build/noarch')).lower() == "python": + for f in files: + fo.write(f[f.find("site-packages"):] + '\n') else: for f in files: fo.write(f + '\n') From 1bc53607a116a416a23d0ea65a9c363a00811647 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 14 Sep 2016 15:38:02 -0500 Subject: [PATCH 012/156] Post process noarch packages --- conda_build/build.py | 7 ++++--- conda_build/post.py | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index bce0b7f400..0f0863cf7d 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -701,9 +701,10 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F files2 = prefix_files(prefix=config.build_prefix) post_process(sorted(files2 - files1), - prefix=config.build_prefix, - config=config, - preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir'))) + prefix=config.build_prefix, + config=config, + preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')), + noarch= m.get_value('build/noarch')) # The post processing may have deleted some files (like easy-install.pth) files2 = prefix_files(prefix=config.build_prefix) diff --git a/conda_build/post.py b/conda_build/post.py index 2b217a6e42..fe99489d91 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -190,8 +190,6 @@ def compile_missing_pyc(files, cwd, python_exe): def post_process(files, prefix, config, preserve_egg_dir=False, noarch=False): - # TODO: verify that files isn't changing - # eg - does `files` reflect the true state of the filesystem? rm_pyo(files, prefix) if not noarch: compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python) From 207bf11707b5d25b55744ab68c4b6b73c580eaac Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 16 Sep 2016 10:15:42 -0500 Subject: [PATCH 013/156] Test noarch: python package building When building this kind of package the following should happen: - copy site-packages to package - remove pyc files - inserts location of files in info/files This differs from building noarch_python: true by - does not create a Scripts or bin folder - does not create a link.py script --- .../metadata/noarch_python/meta.yaml | 28 +++++++++++++++++++ .../noarch_python_test_package/README | 1 + .../noarch_python_test_package.py | 11 ++++++++ .../noarch_python_test_package/setup.py | 17 +++++++++++ .../metadata/noarch_python/run_test.py | 24 ++++++++++++++++ tests/test_api_build.py | 7 +++++ 6 files changed, 88 insertions(+) create mode 100644 tests/test-recipes/metadata/noarch_python/meta.yaml create mode 100644 tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README create mode 100644 tests/test-recipes/metadata/noarch_python/noarch_python_test_package/noarch_python_test_package.py create mode 100644 tests/test-recipes/metadata/noarch_python/noarch_python_test_package/setup.py create mode 100644 tests/test-recipes/metadata/noarch_python/run_test.py diff --git a/tests/test-recipes/metadata/noarch_python/meta.yaml b/tests/test-recipes/metadata/noarch_python/meta.yaml new file mode 100644 index 0000000000..ff26b02158 --- /dev/null +++ b/tests/test-recipes/metadata/noarch_python/meta.yaml @@ -0,0 +1,28 @@ +package: + name: noarch_python_test_package + version: "1.0" + +source: + path: ./noarch_python_test_package + +build: + script: python setup.py install --single-version-externally-managed --record=record.txt + noarch: python + entry_points: + - noarch_python_test_package_script = noarch_python_test_package:main + +requirements: + build: + - python + - setuptools + run: + - python + +# Tests are commented out because they should fail since conda does not yet know how to install +# noarch: python packages. Once it is implemented then this should be uncommented. + +#test: +# imports: +# - noarch_python_test_package +# commands: +# - noarch_python_test_package_script diff --git a/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README b/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README new file mode 100644 index 0000000000..52d2f1286c --- /dev/null +++ b/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README @@ -0,0 +1 @@ +Simple package to test noarch package building. diff --git a/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/noarch_python_test_package.py b/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/noarch_python_test_package.py new file mode 100644 index 0000000000..1373380e34 --- /dev/null +++ b/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/noarch_python_test_package.py @@ -0,0 +1,11 @@ +""" This functions as a module but also as entry point. +""" + +answer = 142 + + +def main(): + print(answer + 100) + +if __name__ == '__main__': + main() diff --git a/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/setup.py b/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/setup.py new file mode 100644 index 0000000000..3556a445d3 --- /dev/null +++ b/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +name = 'noarch_python_test_package' + +setup( + name=name, + version='1.0', + author='Almar', + author_email='almar@notmyemail.com', + url='http://continuum.io', + license='(new) BSD', + description='testing noarch python package building', + platforms='any', + provides=[name], + py_modules=[name], + entry_points={'console_scripts': ['%s_script = %s:main' % (name, name)], }, +) diff --git a/tests/test-recipes/metadata/noarch_python/run_test.py b/tests/test-recipes/metadata/noarch_python/run_test.py new file mode 100644 index 0000000000..df3bbb5ac0 --- /dev/null +++ b/tests/test-recipes/metadata/noarch_python/run_test.py @@ -0,0 +1,24 @@ +# Tests are commented out because they should fail since conda does not yet know how to install +# noarch: python packages. Once it is implemented then this should be uncommented. + +# import os +# import subprocess +# +# import noarch_python_test_package +# +# pkgs_dir = os.path.abspath(os.path.join(os.environ["ROOT"], 'pkgs')) +# pkg_dir = os.path.join(pkgs_dir, 'noarch_python_test_package-1.0-py_0') +# +# assert os.path.isdir(pkg_dir) +# +# site_packages = os.path.join(pkg_dir, 'site-packages') +# assert os.path.isdir(site_packages) +# +# # Check module +# +# assert noarch_python_test_package.answer == 142 +# +# # Check entry point +# +# res = subprocess.check_output(['noarch_python_test_package_script']).decode('utf-8').strip() +# assert res == '242' diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 06972dcc4f..da9dc48b64 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -608,3 +608,10 @@ def test_noarch_foo_value(): metadata = json.loads(package_has_file(fn, 'info/index.json').decode()) assert 'noarch' in metadata assert metadata['noarch'] == "foo" + + +def test_noarch_python(): + recipe = os.path.join(metadata_dir, "noarch_python") + fn = api.get_output_file_path(recipe) + api.build(recipe) + assert package_has_file(fn, 'info/files') is not '' From bc881ceec4ea84ecfca6397f167d53148d3f9945 Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 16 Sep 2016 14:27:03 -0500 Subject: [PATCH 014/156] Add all files to file --- conda_build/build.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 0f0863cf7d..da29df1fe7 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -352,9 +352,6 @@ def create_info_files(m, files, config, prefix): with open(join(config.info_dir, 'files'), **mode_dict) as fo: if m.get_value('build/noarch_python'): fo.write('\n') - elif str(m.get_value('build/noarch')).lower() == "python": - for f in files: - fo.write(f[f.find("site-packages"):] + '\n') else: for f in files: fo.write(f + '\n') From a0bf05171221241745e6e5fc81819adc42ba0363 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 19 Sep 2016 12:57:05 -0500 Subject: [PATCH 015/156] Don't create entry points for noarch package Instead list the entry points in info/noarch.json This is a new file introduced to keep track of entry points in no arch packages so that they can be built by conda when the package is being installed. The form of info/noarch.json is as follows: { type: , enty_points: [], mapped_directories: } --- conda_build/build.py | 10 +++++++++- conda_build/noarch_python.py | 9 +++++++++ tests/test_api_build.py | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index da29df1fe7..120eb00f74 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -345,6 +345,12 @@ def create_info_files(m, files, config, prefix): write_info_json(m, config, mode_dict) write_about_json(m, config) + if str(m.get_value('build/noarch')).lower() == "python": + import conda_build.noarch_python as noarch_python + noarch_python.create_entry_point_information( + "python", m.get_value('build/entry_points'), config + ) + if on_win: # make sure we use '/' path separators in metadata files = [_f.replace('\\', '/') for _f in files] @@ -694,7 +700,9 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F get_build_metadata(m, config=config) create_post_scripts(m, config=config) - create_entry_points(m.get_value('build/entry_points'), config=config) + + if str(m.get_value('build/noarch')).lower() != "python": + create_entry_points(m.get_value('build/entry_points'), config=config) files2 = prefix_files(prefix=config.build_prefix) post_process(sorted(files2 - files1), diff --git a/conda_build/noarch_python.py b/conda_build/noarch_python.py index 408bf0382e..21862b0eb3 100644 --- a/conda_build/noarch_python.py +++ b/conda_build/noarch_python.py @@ -56,6 +56,15 @@ def rewrite_script(fn, prefix): return fn +def create_entry_point_information(noarch_type, entry_points, config): + entry_point_information = {} + entry_point_information['type'] = noarch_type + entry_point_information['entry_points'] = entry_points + file = os.path.join(config.info_dir, "noarch.json") + with io.open(file, 'w', encoding='utf-8') as entry_point_file: + json.dump(entry_point_information, entry_point_file, indent=4) + + def handle_file(f, d, prefix): """Process a file for inclusion in a noarch python package. """ diff --git a/tests/test_api_build.py b/tests/test_api_build.py index da9dc48b64..e5936ca499 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -615,3 +615,6 @@ def test_noarch_python(): fn = api.get_output_file_path(recipe) api.build(recipe) assert package_has_file(fn, 'info/files') is not '' + noarch = json.loads(package_has_file(fn, 'info/noarch.json').decode()) + assert 'entry_points' in noarch + assert 'type' in noarch From 4e3d3d6bff731eb4b064002dc7ddddbe092af7d8 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 20 Sep 2016 14:39:28 -0500 Subject: [PATCH 016/156] Refactor entry point information --- conda_build/noarch_python.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/conda_build/noarch_python.py b/conda_build/noarch_python.py index 21862b0eb3..44169c36af 100644 --- a/conda_build/noarch_python.py +++ b/conda_build/noarch_python.py @@ -57,12 +57,10 @@ def rewrite_script(fn, prefix): def create_entry_point_information(noarch_type, entry_points, config): - entry_point_information = {} - entry_point_information['type'] = noarch_type - entry_point_information['entry_points'] = entry_points + entry_point_information = {"type": noarch_type, "entry_points": entry_points} file = os.path.join(config.info_dir, "noarch.json") - with io.open(file, 'w', encoding='utf-8') as entry_point_file: - json.dump(entry_point_information, entry_point_file, indent=4) + with open(file, 'w') as entry_point_file: + entry_point_file.write(json.dumps(entry_point_information)) def handle_file(f, d, prefix): From 55e3c96a02e3d9774a5bffe4951887598a12d726 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 26 Sep 2016 11:24:00 -0500 Subject: [PATCH 017/156] Extract check for noarch python into a function --- conda_build/build.py | 20 +++++++++++--------- conda_build/post.py | 6 +++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 120eb00f74..6d246f33e7 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -56,6 +56,8 @@ from conda_build.exceptions import indent from conda_build.features import feature_list +import conda_build.noarch_python as noarch_python + if 'bsd' in sys.platform: shell_path = '/bin/sh' else: @@ -242,8 +244,7 @@ def detect_and_record_prefix_files(m, files, prefix, config): else: files_with_prefix = [] - is_noarch = m.get_value('build/noarch_python') or \ - str(m.get_value('build/noarch')).lower() == "python" + is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m.get_value('build/noarch')) if files_with_prefix and not is_noarch: if on_win: @@ -345,8 +346,7 @@ def create_info_files(m, files, config, prefix): write_info_json(m, config, mode_dict) write_about_json(m, config) - if str(m.get_value('build/noarch')).lower() == "python": - import conda_build.noarch_python as noarch_python + if is_noarch_python(m.get_value('build/noarch')): noarch_python.create_entry_point_information( "python", m.get_value('build/entry_points'), config ) @@ -701,7 +701,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F get_build_metadata(m, config=config) create_post_scripts(m, config=config) - if str(m.get_value('build/noarch')).lower() != "python": + if not is_noarch_python(m.get_value('build/noarch')): create_entry_points(m.get_value('build/entry_points'), config=config) files2 = prefix_files(prefix=config.build_prefix) @@ -709,7 +709,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F prefix=config.build_prefix, config=config, preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')), - noarch= m.get_value('build/noarch')) + noarch=m.get_value('build/noarch')) # The post processing may have deleted some files (like easy-install.pth) files2 = prefix_files(prefix=config.build_prefix) @@ -728,10 +728,8 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F prefix=config.build_prefix) if m.get_value('build/noarch_python'): - import conda_build.noarch_python as noarch_python noarch_python.transform(m, sorted(files2 - files1), config.build_prefix) - elif str(m.get_value('build/noarch')).lower() == "python": - import conda_build.noarch_python as noarch_python + elif is_noarch_python(m.get_value('build/noarch')): noarch_python.populate_files(m, sorted(files2 - files1), config.build_prefix) files3 = prefix_files(prefix=config.build_prefix) @@ -1146,3 +1144,7 @@ def is_package_built(metadata, config): # will be empty if none found, and evalute to False package_exists = [url for url in urls if url + '::' + metadata.pkg_fn() in index] return package_exists or metadata.pkg_fn() in index + + +def is_noarch_python(build_noarch): + return str(build_noarch).lower() == "python" diff --git a/conda_build/post.py b/conda_build/post.py index fe99489d91..e15fe38035 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -191,10 +191,10 @@ def compile_missing_pyc(files, cwd, python_exe): def post_process(files, prefix, config, preserve_egg_dir=False, noarch=False): rm_pyo(files, prefix) - if not noarch: - compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python) - else: + if noarch: rm_pyc(files, prefix) + else: + compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python) remove_easy_install_pth(files, prefix, config, preserve_egg_dir=preserve_egg_dir) rm_py_along_so(prefix) From d6fb6bf2f58c40d91939c2650fb402c271e2072f Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 26 Sep 2016 11:52:15 -0500 Subject: [PATCH 018/156] Test noarch python package building --- .../metadata/_noarch_python/meta.yaml | 19 +++++++++++++++ .../noarch_python_test_package.py | 0 .../noarch_python_test_package/setup.py | 0 .../meta.yaml | 13 ++++------ .../noarch_python_test_package.py | 11 +++++++++ .../noarch_python_test_package/setup.py | 17 +++++++++++++ .../_noarch_python_with_tests/run_test.py | 21 ++++++++++++++++ .../noarch_python_test_package/README | 1 - .../metadata/noarch_python/run_test.py | 24 ------------------- tests/test_api_build.py | 10 +++++++- 10 files changed, 82 insertions(+), 34 deletions(-) create mode 100644 tests/test-recipes/metadata/_noarch_python/meta.yaml rename tests/test-recipes/metadata/{noarch_python => _noarch_python}/noarch_python_test_package/noarch_python_test_package.py (100%) rename tests/test-recipes/metadata/{noarch_python => _noarch_python}/noarch_python_test_package/setup.py (100%) rename tests/test-recipes/metadata/{noarch_python => _noarch_python_with_tests}/meta.yaml (57%) create mode 100644 tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/noarch_python_test_package.py create mode 100644 tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/setup.py create mode 100644 tests/test-recipes/metadata/_noarch_python_with_tests/run_test.py delete mode 100644 tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README delete mode 100644 tests/test-recipes/metadata/noarch_python/run_test.py diff --git a/tests/test-recipes/metadata/_noarch_python/meta.yaml b/tests/test-recipes/metadata/_noarch_python/meta.yaml new file mode 100644 index 0000000000..9fcf2e655b --- /dev/null +++ b/tests/test-recipes/metadata/_noarch_python/meta.yaml @@ -0,0 +1,19 @@ +package: + name: noarch_python_test_package + version: "1.0" + +source: + path: ./noarch_python_test_package + +build: + script: python setup.py install --single-version-externally-managed --record=record.txt + noarch: python + entry_points: + - noarch_python_test_package_script = noarch_python_test_package:main + +requirements: + build: + - python + - setuptools + run: + - python diff --git a/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/noarch_python_test_package.py b/tests/test-recipes/metadata/_noarch_python/noarch_python_test_package/noarch_python_test_package.py similarity index 100% rename from tests/test-recipes/metadata/noarch_python/noarch_python_test_package/noarch_python_test_package.py rename to tests/test-recipes/metadata/_noarch_python/noarch_python_test_package/noarch_python_test_package.py diff --git a/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/setup.py b/tests/test-recipes/metadata/_noarch_python/noarch_python_test_package/setup.py similarity index 100% rename from tests/test-recipes/metadata/noarch_python/noarch_python_test_package/setup.py rename to tests/test-recipes/metadata/_noarch_python/noarch_python_test_package/setup.py diff --git a/tests/test-recipes/metadata/noarch_python/meta.yaml b/tests/test-recipes/metadata/_noarch_python_with_tests/meta.yaml similarity index 57% rename from tests/test-recipes/metadata/noarch_python/meta.yaml rename to tests/test-recipes/metadata/_noarch_python_with_tests/meta.yaml index ff26b02158..eec2078d05 100644 --- a/tests/test-recipes/metadata/noarch_python/meta.yaml +++ b/tests/test-recipes/metadata/_noarch_python_with_tests/meta.yaml @@ -18,11 +18,8 @@ requirements: run: - python -# Tests are commented out because they should fail since conda does not yet know how to install -# noarch: python packages. Once it is implemented then this should be uncommented. - -#test: -# imports: -# - noarch_python_test_package -# commands: -# - noarch_python_test_package_script +test: + imports: + - noarch_python_test_package + commands: + - noarch_python_test_package_script diff --git a/tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/noarch_python_test_package.py b/tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/noarch_python_test_package.py new file mode 100644 index 0000000000..1373380e34 --- /dev/null +++ b/tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/noarch_python_test_package.py @@ -0,0 +1,11 @@ +""" This functions as a module but also as entry point. +""" + +answer = 142 + + +def main(): + print(answer + 100) + +if __name__ == '__main__': + main() diff --git a/tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/setup.py b/tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/setup.py new file mode 100644 index 0000000000..3556a445d3 --- /dev/null +++ b/tests/test-recipes/metadata/_noarch_python_with_tests/noarch_python_test_package/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +name = 'noarch_python_test_package' + +setup( + name=name, + version='1.0', + author='Almar', + author_email='almar@notmyemail.com', + url='http://continuum.io', + license='(new) BSD', + description='testing noarch python package building', + platforms='any', + provides=[name], + py_modules=[name], + entry_points={'console_scripts': ['%s_script = %s:main' % (name, name)], }, +) diff --git a/tests/test-recipes/metadata/_noarch_python_with_tests/run_test.py b/tests/test-recipes/metadata/_noarch_python_with_tests/run_test.py new file mode 100644 index 0000000000..309d195d40 --- /dev/null +++ b/tests/test-recipes/metadata/_noarch_python_with_tests/run_test.py @@ -0,0 +1,21 @@ +import os +import subprocess + +import noarch_python_test_package + +pkgs_dir = os.path.abspath(os.path.join(os.environ["ROOT"], 'pkgs')) +pkg_dir = os.path.join(pkgs_dir, 'noarch_python_test_package-1.0-py_0') + +assert os.path.isdir(pkg_dir) + +site_packages = os.path.join(pkg_dir, 'site-packages') +assert os.path.isdir(site_packages) + +# Check module + +assert noarch_python_test_package.answer == 142 + +# Check entry point + +res = subprocess.check_output(['noarch_python_test_package_script']).decode('utf-8').strip() +assert res == '242' diff --git a/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README b/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README deleted file mode 100644 index 52d2f1286c..0000000000 --- a/tests/test-recipes/metadata/noarch_python/noarch_python_test_package/README +++ /dev/null @@ -1 +0,0 @@ -Simple package to test noarch package building. diff --git a/tests/test-recipes/metadata/noarch_python/run_test.py b/tests/test-recipes/metadata/noarch_python/run_test.py deleted file mode 100644 index df3bbb5ac0..0000000000 --- a/tests/test-recipes/metadata/noarch_python/run_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# Tests are commented out because they should fail since conda does not yet know how to install -# noarch: python packages. Once it is implemented then this should be uncommented. - -# import os -# import subprocess -# -# import noarch_python_test_package -# -# pkgs_dir = os.path.abspath(os.path.join(os.environ["ROOT"], 'pkgs')) -# pkg_dir = os.path.join(pkgs_dir, 'noarch_python_test_package-1.0-py_0') -# -# assert os.path.isdir(pkg_dir) -# -# site_packages = os.path.join(pkg_dir, 'site-packages') -# assert os.path.isdir(site_packages) -# -# # Check module -# -# assert noarch_python_test_package.answer == 142 -# -# # Check entry point -# -# res = subprocess.check_output(['noarch_python_test_package_script']).decode('utf-8').strip() -# assert res == '242' diff --git a/tests/test_api_build.py b/tests/test_api_build.py index e5936ca499..728579a812 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -590,6 +590,7 @@ def test_disable_pip(test_config): with pytest.raises(SystemExit): api.build(metadata) + @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="rpath fixup only done on Linux so far.") def test_rpath_linux(test_config): api.build(os.path.join(metadata_dir, "_rpath"), config=test_config) @@ -610,8 +611,15 @@ def test_noarch_foo_value(): assert metadata['noarch'] == "foo" +@pytest.mark.xfail +def test_noarch_python_with_tests(): + recipe = os.path.join(metadata_dir, "_noarch_python_with_tests") + fn = api.get_output_file_path(recipe) + api.build(recipe) + + def test_noarch_python(): - recipe = os.path.join(metadata_dir, "noarch_python") + recipe = os.path.join(metadata_dir, "_noarch_python") fn = api.get_output_file_path(recipe) api.build(recipe) assert package_has_file(fn, 'info/files') is not '' From 951e6f820364a6c4612b2a391431695eca3463b9 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 26 Sep 2016 12:38:52 -0500 Subject: [PATCH 019/156] Refactor check for noarch python --- conda_build/build.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 6d246f33e7..ec4712c194 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -244,7 +244,7 @@ def detect_and_record_prefix_files(m, files, prefix, config): else: files_with_prefix = [] - is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m.get_value('build/noarch')) + is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m) if files_with_prefix and not is_noarch: if on_win: @@ -346,7 +346,7 @@ def create_info_files(m, files, config, prefix): write_info_json(m, config, mode_dict) write_about_json(m, config) - if is_noarch_python(m.get_value('build/noarch')): + if is_noarch_python(m): noarch_python.create_entry_point_information( "python", m.get_value('build/entry_points'), config ) @@ -701,7 +701,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F get_build_metadata(m, config=config) create_post_scripts(m, config=config) - if not is_noarch_python(m.get_value('build/noarch')): + if not is_noarch_python(m): create_entry_points(m.get_value('build/entry_points'), config=config) files2 = prefix_files(prefix=config.build_prefix) @@ -729,7 +729,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F if m.get_value('build/noarch_python'): noarch_python.transform(m, sorted(files2 - files1), config.build_prefix) - elif is_noarch_python(m.get_value('build/noarch')): + elif is_noarch_python(m): noarch_python.populate_files(m, sorted(files2 - files1), config.build_prefix) files3 = prefix_files(prefix=config.build_prefix) @@ -1146,5 +1146,5 @@ def is_package_built(metadata, config): return package_exists or metadata.pkg_fn() in index -def is_noarch_python(build_noarch): - return str(build_noarch).lower() == "python" +def is_noarch_python(meta): + return str(meta.get_value('build/noarch')).lower() == "python" From 8892208bb455272c3e9af5a23c54fd1afb4b5c3a Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 26 Sep 2016 13:20:14 -0500 Subject: [PATCH 020/156] Don't include cli scripts in noarch packages --- conda_build/noarch_python.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conda_build/noarch_python.py b/conda_build/noarch_python.py index 44169c36af..b4724b0f33 100644 --- a/conda_build/noarch_python.py +++ b/conda_build/noarch_python.py @@ -120,11 +120,6 @@ def populate_files(m, files, prefix): for i, fn in enumerate(fns): fns[i] = fn.replace('\\', '/') - # copy in windows exe shims if there are any python-scripts - if d['python-scripts']: - for fn in 'cli-32.exe', 'cli-64.exe': - shutil.copyfile(join(dirname(__file__), fn), join(prefix, fn)) - return d @@ -159,6 +154,11 @@ def transform(m, files, prefix): # Find our way to this directory this_dir = dirname(__file__) + # copy in windows exe shims if there are any python-scripts + if d['python-scripts']: + for fn in 'cli-32.exe', 'cli-64.exe': + shutil.copyfile(join(this_dir, fn), join(prefix, fn)) + # Read the local _link.py with open(join(this_dir, '_link.py')) as fi: link_code = fi.read() From 767e29802f5dee82055960c16301f4cacf9de0a2 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 29 Sep 2016 09:54:10 -0500 Subject: [PATCH 021/156] Write true structure of noarch package to files --- conda_build/build.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/conda_build/build.py b/conda_build/build.py index ec4712c194..58c9f81e87 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -358,6 +358,14 @@ def create_info_files(m, files, config, prefix): with open(join(config.info_dir, 'files'), **mode_dict) as fo: if m.get_value('build/noarch_python'): fo.write('\n') + elif is_noarch_python(m): + for f in files: + if f.find("site-packages") > 0: + fo.write(f[f.find("site-packages"):] + '\n') + elif f.startswith("bin"): + fo.write(f.replace("bin", "python-scripts") + '\n') + elif f.startswith("Scripts"): + fo.write(f.replace("Scripts", "python-scripts") + '\n') else: for f in files: fo.write(f + '\n') From 5025946faa4422820408a0b3efd2ed2769fb92d8 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Fri, 30 Sep 2016 09:49:22 +0200 Subject: [PATCH 022/156] post: catch patchelf failures --- conda_build/post.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conda_build/post.py b/conda_build/post.py index 789a55a196..5d8046d47d 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -310,7 +310,11 @@ def mk_relative_linux(f, prefix, rpaths=('lib',)): origin = dirname(elf) patchelf = external.find_executable('patchelf', prefix) - existing = check_output([patchelf, '--print-rpath', elf]).decode('utf-8').splitlines()[0] + try: + existing = check_output([patchelf, '--print-rpath', elf]).decode('utf-8').splitlines()[0] + except: + print('patchelf: --print-rpath failed for %s\n' % (elf)) + return existing = existing.split(os.pathsep) new = [] for old in existing: From 111a4d818ad162c80e93714c4ab9c44fdb938a56 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 29 Sep 2016 16:49:07 -0500 Subject: [PATCH 023/156] Get entry points form setup.py --- conda_build/build.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 58c9f81e87..d1cd74fe96 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -57,6 +57,8 @@ from conda_build.features import feature_list import conda_build.noarch_python as noarch_python +from conda_build import jinja_context + if 'bsd' in sys.platform: shell_path = '/bin/sh' @@ -325,6 +327,34 @@ def write_no_link(m, config, files): fo.write(f + '\n') +def get_entry_points_from_setup_py(setup_py_data): + entry_points = setup_py_data.get("entry_points") + if isinstance(entry_points, dict): + return entry_points.get("console_scripts") + else: + import configparser + config = configparser.ConfigParser() + config.read_string(entry_points) + keys = [] + if config.sections().index("console_scripts") >= 0: + for key in config['console_scripts']: + keys.append("{0} = {1}".format(key, config["console_scripts"].get(key))) + return keys + + +def get_entry_points(config, m): + entry_point_scripts = [] + entry_point_scripts.extend(m.get_value('build/entry_points')) + setup_py_data = jinja_context.load_setup_py_data(config) + setup_py_entry_points = get_entry_points_from_setup_py(setup_py_data) + + for ep in setup_py_entry_points: + if ep not in entry_point_scripts: + entry_point_scripts.append(ep) + + return entry_point_scripts + + def create_info_files(m, files, config, prefix): ''' Creates the metadata files that will be stored in the built package. @@ -333,8 +363,6 @@ def create_info_files(m, files, config, prefix): :type m: Metadata :param files: Paths to files to include in package :type files: list of str - :param include_recipe: Whether or not to include the recipe (True by default) - :type include_recipe: bool ''' copy_recipe(m, config) @@ -346,9 +374,11 @@ def create_info_files(m, files, config, prefix): write_info_json(m, config, mode_dict) write_about_json(m, config) + entry_point_scripts = get_entry_points(config, m) + if is_noarch_python(m): noarch_python.create_entry_point_information( - "python", m.get_value('build/entry_points'), config + "python", entry_point_scripts, config ) if on_win: From 3c43979e3174af9dd9e4079d084b6964304a173c Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 30 Sep 2016 09:48:21 -0500 Subject: [PATCH 024/156] Don't write entry points to info/files --- conda_build/build.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index d1cd74fe96..a24984b193 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -355,6 +355,18 @@ def get_entry_points(config, m): return entry_point_scripts +def get_entry_point_script_names(entry_point_scripts): + scripts = [] + for entry_point in entry_point_scripts: + cmd = entry_point[:entry_point.find("= ")].strip() + if on_win: + scripts.append("Scripts\\%s-scripts.py" % cmd) + scripts.append("Scripts\\%s.exe" % cmd) + else: + scripts.append("bin/%s" % cmd) + return scripts + + def create_info_files(m, files, config, prefix): ''' Creates the metadata files that will be stored in the built package. @@ -381,6 +393,8 @@ def create_info_files(m, files, config, prefix): "python", entry_point_scripts, config ) + entry_point_script_names = get_entry_point_script_names(entry_point_scripts) + if on_win: # make sure we use '/' path separators in metadata files = [_f.replace('\\', '/') for _f in files] @@ -392,9 +406,9 @@ def create_info_files(m, files, config, prefix): for f in files: if f.find("site-packages") > 0: fo.write(f[f.find("site-packages"):] + '\n') - elif f.startswith("bin"): + elif f.startswith("bin") and (f not in entry_point_script_names): fo.write(f.replace("bin", "python-scripts") + '\n') - elif f.startswith("Scripts"): + elif f.startswith("Scripts") and (f not in entry_point_script_names): fo.write(f.replace("Scripts", "python-scripts") + '\n') else: for f in files: From d6909a02955854d6693c1a8aad73990e0a9da036 Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 30 Sep 2016 11:32:37 -0500 Subject: [PATCH 025/156] Don't link entry points --- conda_build/build.py | 31 +++++++++++++++++++------------ conda_build/noarch_python.py | 7 ++++++- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index a24984b193..c963d6a131 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -336,9 +336,9 @@ def get_entry_points_from_setup_py(setup_py_data): config = configparser.ConfigParser() config.read_string(entry_points) keys = [] - if config.sections().index("console_scripts") >= 0: - for key in config['console_scripts']: - keys.append("{0} = {1}".format(key, config["console_scripts"].get(key))) + if "console_scripts" in config.sections(): + for key in config.get('console_scripts'): + keys.append("{0} = {1}".format(key, config.get("console_scripts").get(key))) return keys @@ -346,11 +346,12 @@ def get_entry_points(config, m): entry_point_scripts = [] entry_point_scripts.extend(m.get_value('build/entry_points')) setup_py_data = jinja_context.load_setup_py_data(config) - setup_py_entry_points = get_entry_points_from_setup_py(setup_py_data) + if setup_py_data: + setup_py_entry_points = get_entry_points_from_setup_py(setup_py_data) - for ep in setup_py_entry_points: - if ep not in entry_point_scripts: - entry_point_scripts.append(ep) + for ep in setup_py_entry_points: + if ep not in entry_point_scripts: + entry_point_scripts.append(ep) return entry_point_scripts @@ -765,8 +766,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F # The post processing may have deleted some files (like easy-install.pth) files2 = prefix_files(prefix=config.build_prefix) - if any(config.meta_dir in join(config.build_prefix, f) for f in - files2 - files1): + if any(config.meta_dir in join(config.build_prefix, f) for f in files2 - files1): meta_files = (tuple(f for f in files2 - files1 if config.meta_dir in join(config.build_prefix, f)),) sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory. @@ -776,13 +776,20 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F prefix=config.build_prefix, build_python=config.build_python, croot=config.croot) - create_info_files(m, sorted(files2 - files1), config=config, - prefix=config.build_prefix) + + entry_point_script_names = get_entry_point_script_names(get_entry_points(config, m)) + if is_noarch_python(m): + pkg_files = [f for f in sorted(files2 - files1) if f not in entry_point_script_names] + else: + pkg_files = sorted(files2 - files1) + + create_info_files(m, pkg_files, config=config, prefix=config.build_prefix) if m.get_value('build/noarch_python'): noarch_python.transform(m, sorted(files2 - files1), config.build_prefix) elif is_noarch_python(m): - noarch_python.populate_files(m, sorted(files2 - files1), config.build_prefix) + noarch_python.populate_files( + m, pkg_files, config.build_prefix, entry_point_script_names) files3 = prefix_files(prefix=config.build_prefix) fix_permissions(files3 - files1, config.build_prefix) diff --git a/conda_build/noarch_python.py b/conda_build/noarch_python.py index b4724b0f33..b21697f96a 100644 --- a/conda_build/noarch_python.py +++ b/conda_build/noarch_python.py @@ -104,7 +104,7 @@ def handle_file(f, d, prefix): _error_exit("Error: Don't know how to handle file: %s" % f) -def populate_files(m, files, prefix): +def populate_files(m, files, prefix, entry_point_scripts=None): d = {'dist': m.dist(), 'site-packages': [], 'python-scripts': [], @@ -120,6 +120,11 @@ def populate_files(m, files, prefix): for i, fn in enumerate(fns): fns[i] = fn.replace('\\', '/') + if entry_point_scripts: + for entry_point in entry_point_scripts: + src = join(prefix, entry_point) + os.unlink(src) + return d From c33dd6e70331036423cc92ede3320d8f05724349 Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 30 Sep 2016 13:46:38 -0500 Subject: [PATCH 026/156] Make more compatible with Windows --- conda_build/build.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index c963d6a131..a5acbdefd6 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -332,13 +332,17 @@ def get_entry_points_from_setup_py(setup_py_data): if isinstance(entry_points, dict): return entry_points.get("console_scripts") else: - import configparser + from conda_build.conda_interface import configparser config = configparser.ConfigParser() - config.read_string(entry_points) + # ConfigParser (for python2) does not support read_string method + try: + config.read_string(entry_points) + except AttributeError: + config.read(unicode(entry_points)) keys = [] if "console_scripts" in config.sections(): - for key in config.get('console_scripts'): - keys.append("{0} = {1}".format(key, config.get("console_scripts").get(key))) + for key in config['console_scripts']: + keys.append("{0} = {1}".format(key, config["console_scripts"].get(key))) return keys @@ -361,7 +365,7 @@ def get_entry_point_script_names(entry_point_scripts): for entry_point in entry_point_scripts: cmd = entry_point[:entry_point.find("= ")].strip() if on_win: - scripts.append("Scripts\\%s-scripts.py" % cmd) + scripts.append("Scripts\\%s-script.py" % cmd) scripts.append("Scripts\\%s.exe" % cmd) else: scripts.append("bin/%s" % cmd) From 2397524bf080b160f2ba8ae2c5cee8eba5eaad41 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 1 Oct 2016 14:04:33 -0500 Subject: [PATCH 027/156] fix path walking in get_ext_files --- conda_build/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index a134205b2f..df6a110a91 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -444,10 +444,10 @@ def guess_license_family(license_name, allowed_license_families): # Return all files in dir, and all its subdirectories, ending in pattern def get_ext_files(start_path, pattern): - for _, _, files in os.walk(start_path): + for root, _, files in os.walk(start_path): for f in files: if f.endswith(pattern): - yield os.path.join(dirname, f) + yield os.path.join(root, f) def _func_defaulting_env_to_os_environ(func, *popenargs, **kwargs): From a238e09596705983b729a57672b2c499b440ca59 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 3 Oct 2016 09:04:59 -0500 Subject: [PATCH 028/156] Explain xfail --- tests/test_api_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 728579a812..febecd1ec9 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -611,7 +611,7 @@ def test_noarch_foo_value(): assert metadata['noarch'] == "foo" -@pytest.mark.xfail +@pytest.mark.xfail(reason="Conda can not yet install `noarch: python` packages") def test_noarch_python_with_tests(): recipe = os.path.join(metadata_dir, "_noarch_python_with_tests") fn = api.get_output_file_path(recipe) From e4132598a30feb9afe145d7a4151f26c736c2f35 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 3 Oct 2016 11:16:05 -0500 Subject: [PATCH 029/156] update changelog for 2.0.3 --- CHANGELOG.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a748f9e39b..15c16e0469 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,26 @@ +2016-09-27 2.0.3: +----------------- + +Enhancements: +------------- + +* add support for noarch: python #1366 + +Bug fixes: +---------- + +* convert popen args to bytestrings if unicode #1413 +* Fix perl file access error on win skeleton cpan #1414 +* Catch patchelf failures in post #1418 +* fix path walking in get_ext_files #1422 + +Contributors: +------------- + +* @mingwandroid +* @msarahan +* @soapy1 + 2016-09-27 2.0.2: ----------------- From 4ceccd5aa8edde54f183b7b57c6798d27e07f340 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 3 Oct 2016 20:03:37 -0500 Subject: [PATCH 030/156] make subdir a derived property. test. --- conda_build/config.py | 17 +++++++++++++++-- tests/test_config.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/conda_build/config.py b/conda_build/config.py index 0bd72cbe32..d851c0abde 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -12,7 +12,7 @@ import time from .conda_interface import string_types, binstar_upload -from .conda_interface import subdir, root_dir, root_writable, cc, bits, platform +from .conda_interface import root_dir, root_writable, cc, bits, platform from .utils import get_build_folders, rm_rf @@ -105,7 +105,6 @@ def env(lang, default): Setting('verbose', False), Setting('debug', False), Setting('timeout', 90), - Setting('subdir', subdir), Setting('bits', bits), Setting('platform', platform), Setting('set_build_id', True), @@ -120,6 +119,20 @@ def env(lang, default): for name, value in kwargs.items(): setattr(self, name, value) + @property + def subdir(self): + if self.platform == 'noarch': + return self.platform + else: + return "-".join([self.platform, str(self.bits)]) + + @subdir.setter + def subdir(self, value): + values = value.split('-') + self.platform = values[0] + if len(values) > 1: + self.bits = values[1] + @property def croot(self): """This is where source caches and work folders live""" diff --git a/tests/test_config.py b/tests/test_config.py index cae315bb98..608c8abae6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -39,3 +39,37 @@ def test_build_id_at_end_of_long_build_prefix(config): build_id = 'test123' config.build_id = build_id assert build_id in config.build_prefix + + +def test_create_config_with_subdir(): + config = Config(subdir='steve-128') + assert config.platform == 'steve' + assert config.subdir == 'steve-128' + + config = Config(subdir='noarch') + assert config.platform == 'noarch' + assert config.subdir == 'noarch' + + +def test_set_platform(config): + config.platform = 'steve' + bits = config.bits + assert config.subdir == 'steve-' + str(bits) + + +def test_set_subdir(config): + config.subdir = 'steve' + bits = config.bits + assert config.subdir == 'steve-' + str(bits) + assert config.platform == 'steve' + + config.subdir = 'steve-128' + assert config.subdir == 'steve-128' + assert config.platform == 'steve' + assert config.bits == '128' + + +def test_set_bits(config): + config.bits = 128 + assert config.subdir == config.platform + '-' + str(128) + assert config.bits == 128 From 92026e8af78aebc7511ce7710a3c0034669cc1e4 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 4 Oct 2016 18:56:30 -0500 Subject: [PATCH 031/156] fix regression in pypi skel for recipes with entry points. test. --- conda_build/skeletons/pypi.py | 4 ++-- tests/test_api_skeleton.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/conda_build/skeletons/pypi.py b/conda_build/skeletons/pypi.py index 4c9508145c..c542e22030 100644 --- a/conda_build/skeletons/pypi.py +++ b/conda_build/skeletons/pypi.py @@ -685,10 +685,10 @@ def get_package_metadata(package, d, data, output_dir, python_version, all_extra entry_points = pkginfo['entry_points'] else: setuptools_run = True - for section in config.sections(): + for section in _config.sections(): if section in ['console_scripts', 'gui_scripts']: value = ['%s=%s' % (option, _config.get(section, option)) - for option in config.options(section)] + for option in _config.options(section)] entry_points[section] = value if not isinstance(entry_points, dict): print("WARNING: Could not add entry points. They were:") diff --git a/tests/test_api_skeleton.py b/tests/test_api_skeleton.py index a4d140e786..9320651fc3 100644 --- a/tests/test_api_skeleton.py +++ b/tests/test_api_skeleton.py @@ -88,3 +88,8 @@ def test_pypi_version_sorting(testing_workdir, test_config): def test_list_skeletons(): skeletons = api.list_skeletons() assert set(skeletons) == set(['pypi', 'cran', 'cpan', 'luarocks']) + + +def test_pypi_with_entry_points(testing_workdir): + api.skeletonize('planemo', repo='pypi', python_version="2.7") + assert os.path.isdir('planemo') From 1f8f999f0b1add6add32d207a6206c3e46819d33 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 4 Oct 2016 21:27:35 -0500 Subject: [PATCH 032/156] don't load setup.py data when meta.yaml entry points present --- conda_build/build.py | 29 ++++++++++++++----- conda_build/utils.py | 17 +++++++++++ .../metadata/source_setup_py_data/bld.bat | 4 +-- .../metadata/source_setup_py_data/build.sh | 4 +-- .../metadata/source_setup_py_data/meta.yaml | 9 +++++- 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index a5acbdefd6..912f68f4cd 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -12,6 +12,7 @@ import mmap import os from os.path import isdir, isfile, islink, join +import re import shutil import stat import subprocess @@ -49,7 +50,8 @@ fix_permissions, get_build_metadata) from conda_build.utils import (rm_rf, _check_call, copy_into, on_win, get_build_folders, silence_loggers, path_prepended, create_entry_points, - prepend_bin_path, codec, root_script_dir, print_skip_message) + prepend_bin_path, codec, root_script_dir, print_skip_message, + sys_path_prepended) from conda_build.index import update_index from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) @@ -346,16 +348,27 @@ def get_entry_points_from_setup_py(setup_py_data): return keys +def setup_py_has_entry_points(config): + has_entry_points = False + if os.path.isfile(os.path.join(config.work_dir, 'setup.py')): + with open(os.path.join(config.work_dir, 'setup.py')) as f: + match = re.search('entry_points', f.read()) + has_entry_points = (match is not None) + return has_entry_points + + def get_entry_points(config, m): entry_point_scripts = [] entry_point_scripts.extend(m.get_value('build/entry_points')) - setup_py_data = jinja_context.load_setup_py_data(config) - if setup_py_data: - setup_py_entry_points = get_entry_points_from_setup_py(setup_py_data) - - for ep in setup_py_entry_points: - if ep not in entry_point_scripts: - entry_point_scripts.append(ep) + if not entry_point_scripts and setup_py_has_entry_points(config): + with sys_path_prepended(config): + setup_py_data = jinja_context.load_setup_py_data(config) + if setup_py_data: + setup_py_entry_points = get_entry_points_from_setup_py(setup_py_data) + + for ep in setup_py_entry_points: + if ep not in entry_point_scripts: + entry_point_scripts.append(ep) return entry_point_scripts diff --git a/conda_build/utils.py b/conda_build/utils.py index 707ccd64dc..a403756005 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -382,6 +382,23 @@ def prepend_bin_path(env, prefix, prepend_prefix=False): return env +@contextlib.contextmanager +def sys_path_prepended(config): + path_backup = sys.path[:] + if on_win: + sys.path.insert(1, os.path.join(config.build_prefix, 'lib', 'site-packages')) + else: + lib_dir = os.path.join(config.build_prefix, 'lib') + python_dir = glob(os.path.join(lib_dir, 'python[0-9\.]*')) + if python_dir: + python_dir = python_dir[0] + sys.path.insert(1, os.path.join(python_dir, 'site-packages')) + try: + yield + finally: + sys.path = path_backup + + @contextlib.contextmanager def path_prepended(prefix): old_path = os.environ['PATH'] diff --git a/tests/test-recipes/metadata/source_setup_py_data/bld.bat b/tests/test-recipes/metadata/source_setup_py_data/bld.bat index 4168d5d6f0..015fce3517 100644 --- a/tests/test-recipes/metadata/source_setup_py_data/bld.bat +++ b/tests/test-recipes/metadata/source_setup_py_data/bld.bat @@ -6,6 +6,6 @@ if errorlevel 1 exit 1 for /f "delims=" %%i in ('git describe') do set gitdesc=%%i if errorlevel 1 exit 1 echo "%gitdesc%" -if not "%gitdesc%"=="1.21.0" exit 1 +if not "%gitdesc%"=="1.21.5" exit 1 echo "%PKG_VERSION%" -if not "%PKG_VERSION%"=="1.21.0" exit 1 +if not "%PKG_VERSION%"=="1.21.5" exit 1 diff --git a/tests/test-recipes/metadata/source_setup_py_data/build.sh b/tests/test-recipes/metadata/source_setup_py_data/build.sh index ecde5ca3c3..4273b9d6d6 100644 --- a/tests/test-recipes/metadata/source_setup_py_data/build.sh +++ b/tests/test-recipes/metadata/source_setup_py_data/build.sh @@ -3,6 +3,6 @@ # Ensure we are in a git repo [ -d .git ] git describe -[ "$(git describe)" = 1.21.0 ] +[ "$(git describe)" = 1.21.5 ] echo "\$PKG_VERSION = $PKG_VERSION" -[ "${PKG_VERSION}" = 1.21.0 ] +[ "${PKG_VERSION}" = 1.21.5 ] diff --git a/tests/test-recipes/metadata/source_setup_py_data/meta.yaml b/tests/test-recipes/metadata/source_setup_py_data/meta.yaml index b5e9305abe..3fc302b510 100644 --- a/tests/test-recipes/metadata/source_setup_py_data/meta.yaml +++ b/tests/test-recipes/metadata/source_setup_py_data/meta.yaml @@ -12,8 +12,15 @@ package: source: git_url: ../../../../../conda_build_test_recipe - git_tag: 1.21.0 + git_tag: 1.21.5 + +build: + entry_points: + - entry = conda_version_test.manual_entry:main requirements: build: - python {{ PY_VER }}* + # cython inclusion here is to test https://github.com/conda/conda-build/issues/149 + # cython chosen because it is implicated somehow in setup.py complications. Numpy would also work. + - cython From 6676fca2652c12df1709d0525d183644ec0ab2db Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Wed, 5 Oct 2016 12:20:48 +0100 Subject: [PATCH 033/156] Add build/skip_compile_pyc option Some packages ship .py files that should not be compiled because either they cannot be compiled (for example they contain templates) or should not be compiled yet because the Python interpreter that will be used cannot be known at build-time. An example of such a package is Qt Creator which uses GDB's embedded Python interpreter to give a richer debugging experience. This package falls foul of both scenarios given above. --- conda_build/build.py | 3 ++- conda_build/metadata.py | 5 ++-- conda_build/post.py | 26 ++++++++++++------- .../metadata/skip_compile_pyc/README | 1 + .../metadata/skip_compile_pyc/compile_pyc.py | 2 ++ .../metadata/skip_compile_pyc/meta.yaml | 20 ++++++++++++++ .../skip_compile_pyc/skip_compile_pyc.py | 2 ++ .../skip_compile_pyc/sub/compile_pyc.py | 2 ++ .../skip_compile_pyc/sub/skip_compile_pyc.py | 2 ++ tests/test_api_build.py | 19 ++++++++++++++ 10 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 tests/test-recipes/metadata/skip_compile_pyc/README create mode 100644 tests/test-recipes/metadata/skip_compile_pyc/compile_pyc.py create mode 100644 tests/test-recipes/metadata/skip_compile_pyc/meta.yaml create mode 100644 tests/test-recipes/metadata/skip_compile_pyc/skip_compile_pyc.py create mode 100644 tests/test-recipes/metadata/skip_compile_pyc/sub/compile_pyc.py create mode 100644 tests/test-recipes/metadata/skip_compile_pyc/sub/skip_compile_pyc.py diff --git a/conda_build/build.py b/conda_build/build.py index a5acbdefd6..d2b0986733 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -766,7 +766,8 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F prefix=config.build_prefix, config=config, preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')), - noarch=m.get_value('build/noarch')) + noarch=m.get_value('build/noarch'), + skip_compile_pyc=m.get_value('build/skip_compile_pyc')) # The post processing may have deleted some files (like easy-install.pth) files2 = prefix_files(prefix=config.build_prefix) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index bc19e7401d..2797a3bff3 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -237,6 +237,7 @@ def parse(data, config, path=None): 'build/noarch_python': bool, 'build/detect_binary_files_with_prefix': bool, 'build/skip': bool, + 'build/skip_compile_pyc': list, 'app/own_environment': bool } @@ -305,8 +306,8 @@ def _git_clean(source_meta): 'features', 'track_features', 'preserve_egg_dir', 'no_link', 'binary_relocation', 'script', 'noarch', 'noarch_python', 'has_prefix_files', 'binary_has_prefix_files', 'ignore_prefix_files', - 'detect_binary_files_with_prefix', 'rpaths', 'script_env', - 'always_include_files', 'skip', 'msvc_compiler', + 'detect_binary_files_with_prefix', 'skip_compile_pyc', 'rpaths', + 'script_env', 'always_include_files', 'skip', 'msvc_compiler', 'pin_depends', 'include_recipe' # pin_depends is experimental still ], 'requirements': ['build', 'run', 'conflicts'], diff --git a/conda_build/post.py b/conda_build/post.py index 02b99bf4cd..60ff4b127b 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -8,6 +8,7 @@ import mmap import re import os +import fnmatch from os.path import (basename, dirname, join, splitext, isdir, isfile, exists, islink, realpath, relpath, normpath) import stat @@ -167,9 +168,14 @@ def rm_pyc(files, prefix): os.unlink(os.path.join(prefix, fn)) -def compile_missing_pyc(files, cwd, python_exe): +def compile_missing_pyc(files, cwd, python_exe, skip_compile_pyc=()): compile_files = [] - for fn in files: + skip_compile_pyc_n = [os.path.normpath(skip) for skip in skip_compile_pyc] + skipped_files = set() + for skip in skip_compile_pyc_n: + skipped_files.update(set(fnmatch.filter(files, skip))) + unskipped_files = set(files) - skipped_files + for fn in unskipped_files: # omit files in Library/bin, Scripts, and the root prefix - they are not generally imported if sys.platform == 'win32': if any([fn.lower().startswith(start) for start in ['library/bin', 'library\\bin', @@ -183,18 +189,20 @@ def compile_missing_pyc(files, cwd, python_exe): os.path.dirname(fn) + cache_prefix + os.path.basename(fn) + 'c' not in files): compile_files.append(fn) - if compile_files and os.path.isfile(python_exe): - print('compiling .pyc files...') - for f in compile_files: - call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd) - + if compile_files: + if not os.path.isfile(python_exe): + print('compiling .pyc files... failed as no python interpreter was found') + else: + print('compiling .pyc files...') + for f in compile_files: + call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd) -def post_process(files, prefix, config, preserve_egg_dir=False, noarch=False): +def post_process(files, prefix, config, preserve_egg_dir=False, noarch=False, skip_compile_pyc=()): rm_pyo(files, prefix) if noarch: rm_pyc(files, prefix) else: - compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python) + compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python, skip_compile_pyc=skip_compile_pyc) remove_easy_install_pth(files, prefix, config, preserve_egg_dir=preserve_egg_dir) rm_py_along_so(prefix) diff --git a/tests/test-recipes/metadata/skip_compile_pyc/README b/tests/test-recipes/metadata/skip_compile_pyc/README new file mode 100644 index 0000000000..e9a92ad4f5 --- /dev/null +++ b/tests/test-recipes/metadata/skip_compile_pyc/README @@ -0,0 +1 @@ +Simple package to test skip_compile_pyc package building. diff --git a/tests/test-recipes/metadata/skip_compile_pyc/compile_pyc.py b/tests/test-recipes/metadata/skip_compile_pyc/compile_pyc.py new file mode 100644 index 0000000000..b48e5a074e --- /dev/null +++ b/tests/test-recipes/metadata/skip_compile_pyc/compile_pyc.py @@ -0,0 +1,2 @@ +import os + diff --git a/tests/test-recipes/metadata/skip_compile_pyc/meta.yaml b/tests/test-recipes/metadata/skip_compile_pyc/meta.yaml new file mode 100644 index 0000000000..a9caff1aeb --- /dev/null +++ b/tests/test-recipes/metadata/skip_compile_pyc/meta.yaml @@ -0,0 +1,20 @@ +package: + name: skip_compile_pyc + version: "1.0" + +source: + path: . + +requirements: + build: + - python + +build: + script: + - cp -rf "${SRC_DIR}"/* "${PREFIX}"/ # [unix] + - xcopy /s %SRC_DIR% %PREFIX% # [win] + skip_compile_pyc: + # rec_glob is used to find files: + - sub/skip* + # test that path normalization happens: + - ./sub/../skip_compile_pyc.py diff --git a/tests/test-recipes/metadata/skip_compile_pyc/skip_compile_pyc.py b/tests/test-recipes/metadata/skip_compile_pyc/skip_compile_pyc.py new file mode 100644 index 0000000000..b48e5a074e --- /dev/null +++ b/tests/test-recipes/metadata/skip_compile_pyc/skip_compile_pyc.py @@ -0,0 +1,2 @@ +import os + diff --git a/tests/test-recipes/metadata/skip_compile_pyc/sub/compile_pyc.py b/tests/test-recipes/metadata/skip_compile_pyc/sub/compile_pyc.py new file mode 100644 index 0000000000..b48e5a074e --- /dev/null +++ b/tests/test-recipes/metadata/skip_compile_pyc/sub/compile_pyc.py @@ -0,0 +1,2 @@ +import os + diff --git a/tests/test-recipes/metadata/skip_compile_pyc/sub/skip_compile_pyc.py b/tests/test-recipes/metadata/skip_compile_pyc/sub/skip_compile_pyc.py new file mode 100644 index 0000000000..b48e5a074e --- /dev/null +++ b/tests/test-recipes/metadata/skip_compile_pyc/sub/skip_compile_pyc.py @@ -0,0 +1,2 @@ +import os + diff --git a/tests/test_api_build.py b/tests/test_api_build.py index febecd1ec9..46cadd3889 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -16,6 +16,7 @@ from binstar_client.errors import NotFound import pytest import yaml +import tarfile from conda_build import api, exceptions from conda_build.utils import (copy_into, on_win, check_call_env, convert_path_for_cygwin_or_msys2, @@ -626,3 +627,21 @@ def test_noarch_python(): noarch = json.loads(package_has_file(fn, 'info/noarch.json').decode()) assert 'entry_points' in noarch assert 'type' in noarch + + +def test_skip_compile_pyc(): + recipe = os.path.join(metadata_dir, "skip_compile_pyc") + fn = api.get_output_file_path(recipe) + api.build(recipe) + tf = tarfile.open(fn) + pyc_count = 0 + for f in tf.getmembers(): + filename = os.path.basename(f.name) + _, ext = os.path.splitext(filename) + basename = filename.split('.',1)[0] + if basename == 'skip_compile_pyc': + assert not ext == '.pyc', "a skip_compile_pyc .pyc was compiled: {}".format(filename) + if ext == '.pyc': + assert basename == 'compile_pyc', "an unexpected .pyc was compiled: {}".format(filename) + pyc_count = pyc_count+1 + assert pyc_count == 2, "there should be 2 .pyc files, instead there were {}".format(pyc_count) \ No newline at end of file From 1128ea0f7134792dc282cb6802782c5eb8654096 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 5 Oct 2016 09:18:49 -0500 Subject: [PATCH 034/156] revert cython setup.py test --- tests/test-recipes/metadata/source_setup_py_data/bld.bat | 4 ++-- tests/test-recipes/metadata/source_setup_py_data/build.sh | 4 ++-- tests/test-recipes/metadata/source_setup_py_data/meta.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test-recipes/metadata/source_setup_py_data/bld.bat b/tests/test-recipes/metadata/source_setup_py_data/bld.bat index 015fce3517..4168d5d6f0 100644 --- a/tests/test-recipes/metadata/source_setup_py_data/bld.bat +++ b/tests/test-recipes/metadata/source_setup_py_data/bld.bat @@ -6,6 +6,6 @@ if errorlevel 1 exit 1 for /f "delims=" %%i in ('git describe') do set gitdesc=%%i if errorlevel 1 exit 1 echo "%gitdesc%" -if not "%gitdesc%"=="1.21.5" exit 1 +if not "%gitdesc%"=="1.21.0" exit 1 echo "%PKG_VERSION%" -if not "%PKG_VERSION%"=="1.21.5" exit 1 +if not "%PKG_VERSION%"=="1.21.0" exit 1 diff --git a/tests/test-recipes/metadata/source_setup_py_data/build.sh b/tests/test-recipes/metadata/source_setup_py_data/build.sh index 4273b9d6d6..ecde5ca3c3 100644 --- a/tests/test-recipes/metadata/source_setup_py_data/build.sh +++ b/tests/test-recipes/metadata/source_setup_py_data/build.sh @@ -3,6 +3,6 @@ # Ensure we are in a git repo [ -d .git ] git describe -[ "$(git describe)" = 1.21.5 ] +[ "$(git describe)" = 1.21.0 ] echo "\$PKG_VERSION = $PKG_VERSION" -[ "${PKG_VERSION}" = 1.21.5 ] +[ "${PKG_VERSION}" = 1.21.0 ] diff --git a/tests/test-recipes/metadata/source_setup_py_data/meta.yaml b/tests/test-recipes/metadata/source_setup_py_data/meta.yaml index 3fc302b510..1f2e908d4f 100644 --- a/tests/test-recipes/metadata/source_setup_py_data/meta.yaml +++ b/tests/test-recipes/metadata/source_setup_py_data/meta.yaml @@ -12,7 +12,7 @@ package: source: git_url: ../../../../../conda_build_test_recipe - git_tag: 1.21.5 + git_tag: 1.21.0 build: entry_points: From 929c2a335a4d12608dac9d90bd3d1ddcd0058b01 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 5 Oct 2016 09:48:09 -0500 Subject: [PATCH 035/156] revert to supporting only meta.yaml entry points --- conda_build/build.py | 49 ++------------------------------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 912f68f4cd..96488f7eb7 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -12,7 +12,6 @@ import mmap import os from os.path import isdir, isfile, islink, join -import re import shutil import stat import subprocess @@ -329,50 +328,6 @@ def write_no_link(m, config, files): fo.write(f + '\n') -def get_entry_points_from_setup_py(setup_py_data): - entry_points = setup_py_data.get("entry_points") - if isinstance(entry_points, dict): - return entry_points.get("console_scripts") - else: - from conda_build.conda_interface import configparser - config = configparser.ConfigParser() - # ConfigParser (for python2) does not support read_string method - try: - config.read_string(entry_points) - except AttributeError: - config.read(unicode(entry_points)) - keys = [] - if "console_scripts" in config.sections(): - for key in config['console_scripts']: - keys.append("{0} = {1}".format(key, config["console_scripts"].get(key))) - return keys - - -def setup_py_has_entry_points(config): - has_entry_points = False - if os.path.isfile(os.path.join(config.work_dir, 'setup.py')): - with open(os.path.join(config.work_dir, 'setup.py')) as f: - match = re.search('entry_points', f.read()) - has_entry_points = (match is not None) - return has_entry_points - - -def get_entry_points(config, m): - entry_point_scripts = [] - entry_point_scripts.extend(m.get_value('build/entry_points')) - if not entry_point_scripts and setup_py_has_entry_points(config): - with sys_path_prepended(config): - setup_py_data = jinja_context.load_setup_py_data(config) - if setup_py_data: - setup_py_entry_points = get_entry_points_from_setup_py(setup_py_data) - - for ep in setup_py_entry_points: - if ep not in entry_point_scripts: - entry_point_scripts.append(ep) - - return entry_point_scripts - - def get_entry_point_script_names(entry_point_scripts): scripts = [] for entry_point in entry_point_scripts: @@ -404,7 +359,7 @@ def create_info_files(m, files, config, prefix): write_info_json(m, config, mode_dict) write_about_json(m, config) - entry_point_scripts = get_entry_points(config, m) + entry_point_scripts = m.get_value('build/entry_points') if is_noarch_python(m): noarch_python.create_entry_point_information( @@ -794,7 +749,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F build_python=config.build_python, croot=config.croot) - entry_point_script_names = get_entry_point_script_names(get_entry_points(config, m)) + entry_point_script_names = get_entry_point_script_names(m.get_value('build/entry_points')) if is_noarch_python(m): pkg_files = [f for f in sorted(files2 - files1) if f not in entry_point_script_names] else: From 073a4a0a50741b8044e60bb41eb6b19966dc9202 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Oct 2016 12:33:49 -0500 Subject: [PATCH 036/156] fall back to copying fewer attributes if copy fails with OSError --- conda_build/utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index 707ccd64dc..e1a4faa599 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -104,7 +104,16 @@ def copy_into(src, dst, timeout=90, symlinks=False): os.makedirs(os.path.dirname(dst_fn)) if lock: lock.acquire(timeout=timeout) - shutil.copy2(src, dst_fn) + # with each of these, we are copying less metadata. This seems to be necessary + # to cope with some shared filesystems with some virtual machine setups. + # See https://github.com/conda/conda-build/issues/1426 + try: + shutil.copy2(src, dst_fn) + except OSError: + try: + shutil.copy(src, dst_fn) + except OSError: + shutil.copyfile(src, dst_fn) except shutil.Error: log.debug("skipping %s - already exists in %s", os.path.basename(src), dst) finally: From 021497285556ad60c89f72a5eb8e56d1eb3c76fa Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Oct 2016 18:21:50 -0500 Subject: [PATCH 037/156] include more metadata, output to about.json --- conda_build/build.py | 14 ++++++++++---- tests/test_api_build.py | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 24d33a8da3..9aca50d2cf 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -284,6 +284,16 @@ def write_about_json(m, config): value = m.get_value('about/%s' % key) if value: d[key] = value + # for sake of reproducibility, record some conda info + conda_info = json.loads(subprocess.check_output(['conda', 'info', '--json'])) + d['conda_version'] = conda_info['conda_version'] + d['conda_build_version'] = conda_info['conda_build_version'] + d['conda_env_version'] = conda_info['conda_env_version'] + d['conda_private'] = conda_info['conda_private'] + d['offline'] = conda_info['offline'] + d['channels'] = conda_info['channels'] + d['env_vars'] = conda_info['env_vars'] + d['root_pkgs'] = json.loads(subprocess.check_output(['conda', 'list', '-n', 'root', '--json'])) json.dump(d, fo, indent=2, sort_keys=True) @@ -355,10 +365,6 @@ def create_info_files(m, files, config, prefix): for f in files: fo.write(f + '\n') - with open(join(config.info_dir, 'conda_versions'), **mode_dict) as fo: - fo.write("Conda version at build time: {0}\n".format(conda.__version__)) - fo.write("Conda-build version at build time: {0}\n".format(__version__)) - detect_and_record_prefix_files(m, files, prefix, config) write_no_link(m, config, files) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 06972dcc4f..8d422a84b8 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -10,6 +10,8 @@ import json import uuid +# for version +import conda from conda_build.conda_interface import PY3, url_path from binstar_client.commands import remove, show @@ -17,7 +19,7 @@ import pytest import yaml -from conda_build import api, exceptions +from conda_build import api, exceptions, __version__ from conda_build.utils import (copy_into, on_win, check_call_env, convert_path_for_cygwin_or_msys2, package_has_file) from conda_build.os_utils.external import find_executable @@ -608,3 +610,14 @@ def test_noarch_foo_value(): metadata = json.loads(package_has_file(fn, 'info/index.json').decode()) assert 'noarch' in metadata assert metadata['noarch'] == "foo" + + +def test_about_json_content(test_metadata): + api.build(test_metadata) + fn = api.get_output_file_path(test_metadata) + about = json.loads(package_has_file(fn, 'info/about.json').decode()) + assert 'conda_version' in about and about['conda_version'] == conda.__version__ + assert 'conda_build_version' in about and about['conda_build_version'] == __version__ + assert 'channels' in about and about['channels'] + assert 'env_vars' in about and about['env_vars'] + assert 'root_pkgs' in about and about['root_pkgs'] From 57c8993ed52c3a93f86e2bf15b6c34686a0802f1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Oct 2016 18:34:05 -0500 Subject: [PATCH 038/156] make code more backwards compatible with older conda --- conda_build/build.py | 13 +++++++++---- tests/test_api_build.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index cb6ca99bd0..5624d4fe72 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -291,15 +291,20 @@ def write_about_json(m, config): if value: d[key] = value # for sake of reproducibility, record some conda info - conda_info = json.loads(subprocess.check_output(['conda', 'info', '--json'])) + conda_info = json.loads(subprocess.check_output(['conda', 'info', '--json', '-s'])) d['conda_version'] = conda_info['conda_version'] d['conda_build_version'] = conda_info['conda_build_version'] d['conda_env_version'] = conda_info['conda_env_version'] - d['conda_private'] = conda_info['conda_private'] d['offline'] = conda_info['offline'] d['channels'] = conda_info['channels'] - d['env_vars'] = conda_info['env_vars'] - d['root_pkgs'] = json.loads(subprocess.check_output(['conda', 'list', '-n', 'root', '--json'])) + # this information will only be present in conda 4.2.10+ + try: + d['conda_private'] = conda_info['conda_private'] + d['env_vars'] = conda_info['env_vars'] + except KeyError: + pass + d['root_pkgs'] = json.loads(subprocess.check_output(['conda', 'list', '-n', + 'root', '--json'])) json.dump(d, fo, indent=2, sort_keys=True) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index b38a980345..1958aeb656 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -21,6 +21,7 @@ import tarfile from conda_build import api, exceptions, __version__ +from conda_build.build import VersionOrder from conda_build.utils import (copy_into, on_win, check_call_env, convert_path_for_cygwin_or_msys2, package_has_file) from conda_build.os_utils.external import find_executable @@ -621,7 +622,15 @@ def test_about_json_content(test_metadata): assert 'conda_version' in about and about['conda_version'] == conda.__version__ assert 'conda_build_version' in about and about['conda_build_version'] == __version__ assert 'channels' in about and about['channels'] - assert 'env_vars' in about and about['env_vars'] + try: + assert 'env_vars' in about and about['env_vars'] + except AssertionError: + # new versions of conda support this, so we should raise errors. + if VersionOrder(conda.__version__) >= VersionOrder('4.2.10'): + raise + else: + pass + assert 'root_pkgs' in about and about['root_pkgs'] From ee5e3f2846f435ce6b6306fe25c685fc43781a1d Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 6 Oct 2016 16:11:41 +0100 Subject: [PATCH 039/156] post: Fixes and a test for fix_permissions We had a bug report https://github.com/ContinuumIO/anaconda-recipes/issues/52 where for some unknown reason qt.conf has no read permissions for other, and our testing procedure did not spot this. Our fix_permissions() function wasn't forcing read access. I took the opportunity to change it in other ways too. Now, in total, it: 1. Directories are all set to 0o775. 2. Ensures the user can write to files. 3. Broadcasts user execute to group and other for files. 4. Ensures that user, group and other can read files. Also, use octal literals for these mode values. --- conda_build/_link.py | 2 +- conda_build/build.py | 4 ++-- conda_build/post.py | 15 ++++++++++++--- conda_build/utils.py | 2 +- .../test-recipes/metadata/fix_permissions/README | 1 + .../metadata/fix_permissions/meta.yaml | 11 +++++++++++ .../sub/lacks_grp_other_read_perms | 1 + tests/test_api_build.py | 11 ++++++++++- 8 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 tests/test-recipes/metadata/fix_permissions/README create mode 100644 tests/test-recipes/metadata/fix_permissions/meta.yaml create mode 100644 tests/test-recipes/metadata/fix_permissions/sub/lacks_grp_other_read_perms diff --git a/conda_build/_link.py b/conda_build/_link.py index 1a17ba612b..3d4c85ede0 100644 --- a/conda_build/_link.py +++ b/conda_build/_link.py @@ -82,7 +82,7 @@ def create_script(fn): with open(dst, 'w') as fo: fo.write('#!%s\n' % normpath(sys.executable)) fo.write(data) - os.chmod(dst, 0o755) + os.chmod(dst, 0o775) FILES.append('bin/%s' % fn) diff --git a/conda_build/build.py b/conda_build/build.py index d2b0986733..7f0e2f35a3 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -96,10 +96,10 @@ def create_post_scripts(m, config): dst_dir = join(config.build_prefix, 'Scripts' if on_win else 'bin') if not isdir(dst_dir): - os.makedirs(dst_dir, int('755', 8)) + os.makedirs(dst_dir, 0o775) dst = join(dst_dir, '.%s-%s%s' % (m.name(), tp, ext)) copy_into(src, dst, config.timeout) - os.chmod(dst, int('755', 8)) + os.chmod(dst, 0o775) def have_prefix_files(files, prefix): diff --git a/conda_build/post.py b/conda_build/post.py index 60ff4b127b..41bd2bf9e0 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -82,7 +82,7 @@ def fix_shebang(f, prefix, build_python, osx_is_app=False): print("updating shebang:", f) with io.open(path, 'w', encoding=locale.getpreferredencoding()) as fo: fo.write(new_data.decode(encoding)) - os.chmod(path, int('755', 8)) + os.chmod(path, 0o775) def write_pth(egg_path, config): @@ -197,6 +197,7 @@ def compile_missing_pyc(files, cwd, python_exe, skip_compile_pyc=()): for f in compile_files: call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd) + def post_process(files, prefix, config, preserve_egg_dir=False, noarch=False, skip_compile_pyc=()): rm_pyo(files, prefix) if noarch: @@ -395,12 +396,20 @@ def fix_permissions(files, prefix): print("Fixing permissions") for root, dirs, _ in os.walk(prefix): for dn in dirs: - lchmod(join(root, dn), int('755', 8)) + lchmod(join(root, dn), 0o775) for f in files: path = join(prefix, f) st = os.lstat(path) - lchmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w + old_mode = stat.S_IMODE(st.st_mode) + new_mode = old_mode + # broadcast execute + if old_mode & stat.S_IXUSR: + new_mode = new_mode | stat.S_IXGRP | stat.S_IXOTH + # ensure user and group can write and all can read + new_mode = new_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + if old_mode != new_mode: + lchmod(path, new_mode) def post_build(m, files, prefix, build_python, croot): diff --git a/conda_build/utils.py b/conda_build/utils.py index 707ccd64dc..d3189a0960 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -418,7 +418,7 @@ def create_entry_point(path, module, func, config): with open(path, 'w') as fo: fo.write('#!%s\n' % config.build_python) fo.write(pyscript) - os.chmod(path, int('755', 8)) + os.chmod(path, 0o775) def create_entry_points(items, config): diff --git a/tests/test-recipes/metadata/fix_permissions/README b/tests/test-recipes/metadata/fix_permissions/README new file mode 100644 index 0000000000..7486ee294f --- /dev/null +++ b/tests/test-recipes/metadata/fix_permissions/README @@ -0,0 +1 @@ +Simple package to test fix_permissions. diff --git a/tests/test-recipes/metadata/fix_permissions/meta.yaml b/tests/test-recipes/metadata/fix_permissions/meta.yaml new file mode 100644 index 0000000000..8a20707d17 --- /dev/null +++ b/tests/test-recipes/metadata/fix_permissions/meta.yaml @@ -0,0 +1,11 @@ +package: + name: fix_permissions + version: "1.0" + +source: + path: . + +build: + script: + - cp -rf "${SRC_DIR}"/* "${PREFIX}"/ # [unix] + - xcopy /s %SRC_DIR% %PREFIX% # [win] diff --git a/tests/test-recipes/metadata/fix_permissions/sub/lacks_grp_other_read_perms b/tests/test-recipes/metadata/fix_permissions/sub/lacks_grp_other_read_perms new file mode 100644 index 0000000000..70a669cc7b --- /dev/null +++ b/tests/test-recipes/metadata/fix_permissions/sub/lacks_grp_other_read_perms @@ -0,0 +1 @@ +no_one_can_read diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 46cadd3889..893fbf19d1 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -644,4 +644,13 @@ def test_skip_compile_pyc(): if ext == '.pyc': assert basename == 'compile_pyc', "an unexpected .pyc was compiled: {}".format(filename) pyc_count = pyc_count+1 - assert pyc_count == 2, "there should be 2 .pyc files, instead there were {}".format(pyc_count) \ No newline at end of file + assert pyc_count == 2, "there should be 2 .pyc files, instead there were {}".format(pyc_count) + + +def test_fix_permissions(): + recipe = os.path.join(metadata_dir, "fix_permissions") + fn = api.get_output_file_path(recipe) + api.build(recipe) + tf = tarfile.open(fn) + for f in tf.getmembers(): + assert f.mode & 0o444 == 0o444, "tar member '{}' has invalid (read) mode".format(f.name) From d92e7f517ed0e1eaff4e39fca3021e9e2a206190 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Oct 2016 18:59:01 -0500 Subject: [PATCH 040/156] test sys_path_prepended, document it a bit; flake8 --- conda_build/build.py | 6 ++---- conda_build/environ.py | 1 - conda_build/post.py | 1 - conda_build/source.py | 2 +- conda_build/utils.py | 9 ++++++--- conda_build/windows.py | 1 - tests/test_utils.py | 11 +++++++++-- 7 files changed, 18 insertions(+), 13 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 96488f7eb7..a28596c4af 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -49,8 +49,7 @@ fix_permissions, get_build_metadata) from conda_build.utils import (rm_rf, _check_call, copy_into, on_win, get_build_folders, silence_loggers, path_prepended, create_entry_points, - prepend_bin_path, codec, root_script_dir, print_skip_message, - sys_path_prepended) + prepend_bin_path, codec, root_script_dir, print_skip_message) from conda_build.index import update_index from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) @@ -58,7 +57,6 @@ from conda_build.features import feature_list import conda_build.noarch_python as noarch_python -from conda_build import jinja_context if 'bsd' in sys.platform: @@ -751,7 +749,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F entry_point_script_names = get_entry_point_script_names(m.get_value('build/entry_points')) if is_noarch_python(m): - pkg_files = [f for f in sorted(files2 - files1) if f not in entry_point_script_names] + pkg_files = [fi for fi in sorted(files2 - files1) if fi not in entry_point_script_names] else: pkg_files = sorted(files2 - files1) diff --git a/conda_build/environ.py b/conda_build/environ.py index 147f75274f..c3f94cf45e 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -14,7 +14,6 @@ from .conda_interface import root_dir, cc from conda_build.os_utils import external -from conda_build import source from conda_build import utils from conda_build.features import feature_list from conda_build.utils import prepend_bin_path diff --git a/conda_build/post.py b/conda_build/post.py index 02b99bf4cd..e6eb905596 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -26,7 +26,6 @@ from conda_build import environ from conda_build import utils -from conda_build import source if sys.platform.startswith('linux'): from conda_build.os_utils import elf diff --git a/conda_build/source.py b/conda_build/source.py index 5fbac03c2e..4cc22dc9f0 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -13,7 +13,7 @@ from .conda_interface import hashsum_file from conda_build.os_utils import external -from conda_build.utils import (tar_xf, unzip, safe_print_unicode, copy_into, on_win, codec, +from conda_build.utils import (tar_xf, unzip, safe_print_unicode, copy_into, on_win, check_output_env, check_call_env, convert_path_for_cygwin_or_msys2) # legacy exports for conda diff --git a/conda_build/utils.py b/conda_build/utils.py index a403756005..289aff27a5 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -382,13 +382,16 @@ def prepend_bin_path(env, prefix, prepend_prefix=False): return env +# not currently used. Leaving in because it may be useful for when we do things +# like load setup.py data, and we need the modules from some prefix other than +# the root prefix, which is what conda-build runs from. @contextlib.contextmanager -def sys_path_prepended(config): +def sys_path_prepended(prefix): path_backup = sys.path[:] if on_win: - sys.path.insert(1, os.path.join(config.build_prefix, 'lib', 'site-packages')) + sys.path.insert(1, os.path.join(prefix, 'lib', 'site-packages')) else: - lib_dir = os.path.join(config.build_prefix, 'lib') + lib_dir = os.path.join(prefix, 'lib') python_dir = glob(os.path.join(lib_dir, 'python[0-9\.]*')) if python_dir: python_dir = python_dir[0] diff --git a/conda_build/windows.py b/conda_build/windows.py index 056de9a4a1..9edab217de 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -14,7 +14,6 @@ from .conda_interface import bits from conda_build import environ -from conda_build import source from conda_build.utils import _check_call, root_script_dir, path_prepended diff --git a/tests/test_utils.py b/tests/test_utils.py index c0d90a1ce9..67aae01bd4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,6 @@ import unittest -import tempfile -import shutil import os +import sys import pytest @@ -20,6 +19,7 @@ def makefile(name, contents=""): f.write(contents) + @pytest.fixture(scope='function') def namespace_setup(testing_workdir, request): namespace = os.path.join(testing_workdir, 'namespace') @@ -28,6 +28,13 @@ def namespace_setup(testing_workdir, request): return testing_workdir +def test_prepend_sys_path(): + path = sys.path[:] + with utils.sys_path_prepended(sys.prefix): + assert sys.path != path + assert sys.path[1].startswith(sys.prefix) + + def test_copy_source_tree(namespace_setup): dst = os.path.join(namespace_setup, 'dest') utils.copy_into(os.path.join(namespace_setup, 'namespace'), dst) From 5c18406d935c95d0dc8b341782f9c59ecc4b36da Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Oct 2016 07:10:04 -0500 Subject: [PATCH 041/156] consolidate get_site_packages defines throughout (3->1) --- conda_build/develop.py | 21 ++------------------- conda_build/environ.py | 13 ++----------- conda_build/post.py | 10 +++++----- conda_build/utils.py | 19 ++++++++++++++++--- tests/test_develop.py | 2 ++ tests/test_utils.py | 10 ++++++++++ 6 files changed, 37 insertions(+), 38 deletions(-) diff --git a/conda_build/develop.py b/conda_build/develop.py index 7c05c54645..5668552b4e 100644 --- a/conda_build/develop.py +++ b/conda_build/develop.py @@ -13,7 +13,7 @@ from .conda_interface import string_types from conda_build.post import mk_relative_osx -from conda_build.utils import _check_call, rec_glob +from conda_build.utils import _check_call, rec_glob, get_site_packages from conda_build.os_utils.external import find_executable @@ -62,22 +62,6 @@ def write_to_conda_pth(sp_dir, pkg_path): print("added " + pkg_path) -def get_site_pkg(prefix, py_ver): - ''' - Given the path to conda environment, find the site-packages directory - - :param prefix: path to conda environment. Look here for current - environment's site-packages - :returns: absolute path to site-packages directory - ''' - # get site-packages directory - stdlib_dir = join(prefix, 'Lib' if sys.platform == 'win32' else - 'lib/python%s' % py_ver) - sp_dir = join(stdlib_dir, 'site-packages') - - return sp_dir - - def get_setup_py(path_): ''' Return full path to setup.py or exit if not found ''' # build path points to source dir, builds are placed in the @@ -162,8 +146,7 @@ def execute(recipe_dirs, prefix=sys.prefix, no_pth_file=False, assert find_executable('python', prefix=prefix) # current environment's site-packages directory - py_ver = '%d.%d' % (sys.version_info.major, sys.version_info.minor) - sp_dir = get_site_pkg(prefix, py_ver) + sp_dir = get_site_packages(prefix) if type(recipe_dirs) == string_types: recipe_dirs = [recipe_dirs] diff --git a/conda_build/environ.py b/conda_build/environ.py index c3f94cf45e..0f792a7062 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -43,19 +43,10 @@ def get_npy_ver(config): return '' -def get_stdlib_dir(config): - return join(config.build_prefix, 'Lib' if sys.platform == 'win32' else - 'lib/python%s' % get_py_ver(config)) - - def get_lua_include_dir(config): return join(config.build_prefix, "include") -def get_sp_dir(config): - return join(get_stdlib_dir(config), 'site-packages') - - def verify_git_repo(git_dir, git_url, config, expected_rev='HEAD'): env = os.environ.copy() if config.verbose: @@ -261,8 +252,8 @@ def python_vars(config): d = { 'PYTHON': config.build_python, 'PY3K': str(config.PY3K), - 'STDLIB_DIR': get_stdlib_dir(config), - 'SP_DIR': get_sp_dir(config), + 'STDLIB_DIR': utils.get_stdlib_dir(config.build_prefix), + 'SP_DIR': utils.get_site_packages(config.build_prefix), 'PY_VER': get_py_ver(config), 'CONDA_PY': str(config.CONDA_PY), } diff --git a/conda_build/post.py b/conda_build/post.py index 068b2e1667..c39db1580d 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -25,7 +25,6 @@ from .conda_interface import md5_file from .conda_interface import PY3 -from conda_build import environ from conda_build import utils if sys.platform.startswith('linux'): @@ -86,7 +85,7 @@ def fix_shebang(f, prefix, build_python, osx_is_app=False): def write_pth(egg_path, config): fn = basename(egg_path) - with open(join(environ.get_sp_dir(config), + with open(join(utils.get_site_packages(config.build_prefix), '%s.pth' % (fn.split('-')[0])), 'w') as fo: fo.write('./%s\n' % fn) @@ -97,7 +96,7 @@ def remove_easy_install_pth(files, prefix, config, preserve_egg_dir=False): itself """ absfiles = [join(prefix, f) for f in files] - sp_dir = environ.get_sp_dir(config) + sp_dir = utils.get_site_packages(prefix) for egg_path in glob(join(sp_dir, '*-py*.egg')): if isdir(egg_path): if preserve_egg_dir or not any(join(egg_path, i) in absfiles for i @@ -202,7 +201,8 @@ def post_process(files, prefix, config, preserve_egg_dir=False, noarch=False, sk if noarch: rm_pyc(files, prefix) else: - compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python, skip_compile_pyc=skip_compile_pyc) + compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python, + skip_compile_pyc=skip_compile_pyc) remove_easy_install_pth(files, prefix, config, preserve_egg_dir=preserve_egg_dir) rm_py_along_so(prefix) @@ -406,7 +406,7 @@ def fix_permissions(files, prefix): if old_mode & stat.S_IXUSR: new_mode = new_mode | stat.S_IXGRP | stat.S_IXOTH # ensure user and group can write and all can read - new_mode = new_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + new_mode = new_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # noqa if old_mode != new_mode: lchmod(path, new_mode) diff --git a/conda_build/utils.py b/conda_build/utils.py index d926017805..407ff504c2 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -344,11 +344,24 @@ def path2url(path): return urlparse.urljoin('file:', urllib.pathname2url(path)) -def get_site_packages(prefix): +def get_stdlib_dir(prefix): if sys.platform == 'win32': - sp = os.path.join(prefix, 'Lib', 'site-packages') + stdlib_dir = os.path.join(prefix, 'Lib', 'site-packages') else: - sp = os.path.join(prefix, 'lib', 'python%s' % sys.version[:3], 'site-packages') + lib_dir = os.path.join(prefix, 'lib') + stdlib_dir = glob(os.path.join(lib_dir, 'python[0-9\.]*')) + if not stdlib_dir: + stdlib_dir = '' + else: + stdlib_dir = stdlib_dir[0] + return stdlib_dir + + +def get_site_packages(prefix): + stdlib_dir = get_stdlib_dir(prefix) + sp = '' + if stdlib_dir: + sp = os.path.join(stdlib_dir, 'site-packages') return sp diff --git a/tests/test_develop.py b/tests/test_develop.py index cd7d2b6440..76e2dea00f 100644 --- a/tests/test_develop.py +++ b/tests/test_develop.py @@ -7,6 +7,8 @@ from conda_build.develop import _uninstall, write_to_conda_pth from conda_build.utils import rm_rf +from .utils import testing_workdir + import pytest diff --git a/tests/test_utils.py b/tests/test_utils.py index 67aae01bd4..1d05afbfce 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,6 +19,16 @@ def makefile(name, contents=""): f.write(contents) +@pytest.mark.skipif(utils.on_win, reason="only unix has python version in site-packages path") +def test_get_site_packages(testing_workdir): + # https://github.com/conda/conda-build/issues/1055#issuecomment-250961576 + + # crazy unreal python version that should show up in a second + crazy_path = os.path.join(testing_workdir, 'lib', 'python8.2', 'site-packages') + os.makedirs(crazy_path) + site_packages = utils.get_site_packages(testing_workdir) + assert site_packages == crazy_path + @pytest.fixture(scope='function') def namespace_setup(testing_workdir, request): From d3223b5e483432f0bfc0d4a1a1ae490230298b4d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Oct 2016 19:10:19 -0500 Subject: [PATCH 042/156] try to speed up appveyor enough to pass --- appveyor.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 3bc06e4621..cbdf6e3915 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,6 +65,7 @@ install: - copy "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat" "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\amd64\vcvarsamd64.bat" # This is an extra repo that we clone for relative path entries - cmd: pushd .. && git clone https://github.com/conda/conda_build_test_recipe && popd + - conda clean -pt # Not a .NET project, we build package in the install step instead @@ -74,19 +75,7 @@ test_script: - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - set PATH - mkdir C:\cbtmp - - py.test -v --cov conda_build --cov-report xml tests --basetemp C:\cbtmp -n 2 - # create a package (this build is done with conda-build from source, and does not test entry points) - - conda build conda.recipe --no-anaconda-upload -c conda-forge - # Create a new environment (with underscore, so that conda can be installed), to test entry points - - conda create -n _cbtest python=%PYTHON_VERSION% - - activate _cbtest - - conda render --output conda.recipe > tmpFile - - SET /p fn= < tmpFile - - DEL tmpFile - - conda install %fn% - - pip install filelock - # this build should be done using actual entry points from the package we built above. - - conda build conda.recipe --no-anaconda-upload -c conda-forge + - py.test -v --cov conda_build --cov-report xml tests --basetemp C:\cbtmp on_failure: - 7z.exe a cbtmp.7z C:\cbtmp From 07e6fe0b37e2b00307bebb7e192511c9717f6a97 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 6 Oct 2016 22:22:41 -0500 Subject: [PATCH 043/156] turn off activation for many tests; use win del to delete trash --- appveyor.yml | 4 +--- conda_build/build.py | 8 ++++++++ conda_build/cli/main_build.py | 3 --- conda_build/utils.py | 8 ++++---- .../bld.bat | 0 .../build.sh | 0 .../meta.yaml | 0 tests/test_api_build.py | 19 ++++++++++++------- tests/test_cli.py | 10 +++++----- tests/utils.py | 2 +- 10 files changed, 31 insertions(+), 23 deletions(-) rename tests/test-recipes/metadata/{set_env_var_activate_build => _set_env_var_activate_build}/bld.bat (100%) rename tests/test-recipes/metadata/{set_env_var_activate_build => _set_env_var_activate_build}/build.sh (100%) rename tests/test-recipes/metadata/{set_env_var_activate_build => _set_env_var_activate_build}/meta.yaml (100%) diff --git a/appveyor.yml b/appveyor.yml index cbdf6e3915..47dcae5248 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,8 +65,6 @@ install: - copy "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat" "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\amd64\vcvarsamd64.bat" # This is an extra repo that we clone for relative path entries - cmd: pushd .. && git clone https://github.com/conda/conda_build_test_recipe && popd - - conda clean -pt - # Not a .NET project, we build package in the install step instead build: false @@ -75,7 +73,7 @@ test_script: - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - set PATH - mkdir C:\cbtmp - - py.test -v --cov conda_build --cov-report xml tests --basetemp C:\cbtmp + - py.test -v --cov conda_build --cov-report xml tests --basetemp C:\cbtmp -n 2 on_failure: - 7z.exe a cbtmp.7z C:\cbtmp diff --git a/conda_build/build.py b/conda_build/build.py index 15fefb4218..492ffb1baf 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1005,6 +1005,14 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, to_build_recursive = [] recipe_list = deque(recipe_list) + if on_win: + trash_dir = os.path.join(os.path.dirname(sys.executable), 'pkgs', '.trash') + if os.path.isdir(trash_dir): + # We don't really care if this does a complete job. + # Cleaning up some files is better than none. + subprocess.call('del /s /q "{0}\\*.*" >nul 2>&1'.format(trash_dir), shell=True) + # delete_trash(None) + already_built = set() while recipe_list: # This loop recursively builds dependencies if recipes exist diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 982db1b700..adead0d1ff 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -201,9 +201,6 @@ def execute(args): config.clean_pkgs() return - if on_win: - delete_trash(None) - set_language_env_vars(args, parser, config=config, execute=execute) action = None diff --git a/conda_build/utils.py b/conda_build/utils.py index d926017805..de27e49417 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -405,10 +405,10 @@ def sys_path_prepended(prefix): if python_dir: python_dir = python_dir[0] sys.path.insert(1, os.path.join(python_dir, 'site-packages')) - try: - yield - finally: - sys.path = path_backup + try: + yield + finally: + sys.path = path_backup @contextlib.contextmanager diff --git a/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat b/tests/test-recipes/metadata/_set_env_var_activate_build/bld.bat similarity index 100% rename from tests/test-recipes/metadata/set_env_var_activate_build/bld.bat rename to tests/test-recipes/metadata/_set_env_var_activate_build/bld.bat diff --git a/tests/test-recipes/metadata/set_env_var_activate_build/build.sh b/tests/test-recipes/metadata/_set_env_var_activate_build/build.sh similarity index 100% rename from tests/test-recipes/metadata/set_env_var_activate_build/build.sh rename to tests/test-recipes/metadata/_set_env_var_activate_build/build.sh diff --git a/tests/test-recipes/metadata/set_env_var_activate_build/meta.yaml b/tests/test-recipes/metadata/_set_env_var_activate_build/meta.yaml similarity index 100% rename from tests/test-recipes/metadata/set_env_var_activate_build/meta.yaml rename to tests/test-recipes/metadata/_set_env_var_activate_build/meta.yaml diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 893fbf19d1..5c379a52fb 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -93,7 +93,7 @@ def test_token_upload(testing_workdir): with pytest.raises(NotFound): show.main(args) - metadata, _, _ = api.render(empty_sections) + metadata, _, _ = api.render(empty_sections, activate=False) metadata.meta['package']['name'] = '_'.join([metadata.name(), folder_uuid]) metadata.config.token = args.token @@ -112,8 +112,8 @@ def test_token_upload(testing_workdir): @pytest.mark.parametrize("service_name", ["binstar", "anaconda"]) -def test_no_anaconda_upload_condarc(service_name, testing_workdir, capfd): - api.build(empty_sections, anaconda_upload=False) +def test_no_anaconda_upload_condarc(service_name, testing_workdir, test_config, capfd): + api.build(empty_sections, config=test_config) output, error = capfd.readouterr() assert "Automatic uploading is disabled" in output, error @@ -145,19 +145,19 @@ def test_no_include_recipe_meta_yaml(test_config): # as a sanity check here. output_file = os.path.join(sys.prefix, "conda-bld", test_config.subdir, "empty_sections-0.0-0.tar.bz2") - api.build(empty_sections, anaconda_upload=False) + api.build(empty_sections, config=test_config) assert package_has_file(output_file, "info/recipe/meta.yaml") output_file = os.path.join(sys.prefix, "conda-bld", test_config.subdir, "no_include_recipe-0.0-0.tar.bz2") - api.build(os.path.join(metadata_dir, '_no_include_recipe'), anaconda_upload=False) + api.build(os.path.join(metadata_dir, '_no_include_recipe'), config=test_config) assert not package_has_file(output_file, "info/recipe/meta.yaml") -def test_early_abort(capfd): +def test_early_abort(test_config, capfd): """There have been some problems with conda-build dropping out early. Make sure we aren't causing them""" - api.build(os.path.join(metadata_dir, '_test_early_abort'), anaconda_upload=False) + api.build(os.path.join(metadata_dir, '_test_early_abort'), config=test_config) output, error = capfd.readouterr() assert "Hello World" in output @@ -174,6 +174,11 @@ def test_output_build_path_git_source(testing_workdir, test_config): def test_build_with_no_activate_does_not_activate(): api.build(os.path.join(metadata_dir, '_set_env_var_no_activate_build'), activate=False) + +def test_build_with_activate_does_activate(): + api.build(os.path.join(metadata_dir, '_set_env_var_activate_build'), activate=True) + + @pytest.mark.skipif(sys.platform == "win32", reason="no binary prefix manipulation done on windows.") def test_binary_has_prefix_files(testing_workdir, test_config): diff --git a/tests/test_cli.py b/tests/test_cli.py index a9c7541a9c..16ce623e46 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -28,7 +28,7 @@ def test_build(): - args = ['--no-anaconda-upload', os.path.join(metadata_dir, "python_run")] + args = ['--no-anaconda-upload', os.path.join(metadata_dir, "empty_sections"), '--no-activate'] main_build.execute(args) @@ -36,7 +36,7 @@ def test_build_add_channel(): """This recipe requires the blinker package, which is only on conda-forge. This verifies that the -c argument works.""" - args = ['--no-anaconda-upload', '-c', 'conda_build_test', + args = ['--no-anaconda-upload', '-c', 'conda_build_test', '--no-activate', os.path.join(metadata_dir, "_recipe_requiring_external_channel")] main_build.execute(args) @@ -45,7 +45,7 @@ def test_build_add_channel(): def test_build_without_channel_fails(testing_workdir): # remove the conda forge channel from the arguments and make sure that we fail. If we don't, # we probably have channels in condarc, and this is not a good test. - args = ['--no-anaconda-upload', + args = ['--no-anaconda-upload', '--no-activate', os.path.join(metadata_dir, "_recipe_requiring_external_channel")] main_build.execute(args) @@ -104,7 +104,7 @@ def test_slash_in_recipe_arg_keeps_build_id(testing_workdir, test_config): def test_build_no_build_id(testing_workdir, test_config, capfd): args = [os.path.join(metadata_dir, "has_prefix_files"), '--no-build-id', - '--croot', test_config.croot] + '--croot', test_config.croot, '--no-activate',] main_build.execute(args) fn = api.get_output_file_path(os.path.join(metadata_dir, "has_prefix_files"), config=test_config) @@ -136,7 +136,7 @@ def test_skeleton_pypi(testing_workdir, test_config): assert os.path.isdir('click') # ensure that recipe generated is buildable - args = ['click', '--no-anaconda-upload', '--croot', test_config.croot] + args = ['click', '--no-anaconda-upload', '--croot', test_config.croot, '--no-activate',] main_build.execute(args) diff --git a/tests/utils.py b/tests/utils.py index 98ce8ee520..674e364bda 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -53,7 +53,7 @@ def return_to_saved_path(): @pytest.fixture(scope='function') def test_config(testing_workdir, request): - return Config(croot=testing_workdir, anaconda_upload=False, verbose=True) + return Config(croot=testing_workdir, anaconda_upload=False, verbose=True, activate=False) @pytest.fixture(scope='function') From 56c1c18ab84a2305b40d1a881648fc1c832a0f4f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Oct 2016 08:14:43 -0500 Subject: [PATCH 044/156] fix potential race condition with test_no_include_recipe_meta_yaml --- tests/test_api_build.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 5c379a52fb..8f061a8c0f 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -140,16 +140,15 @@ def test_no_include_recipe_config_arg(test_metadata): assert not package_has_file(output_file, "info/recipe/meta.yaml") -def test_no_include_recipe_meta_yaml(test_config): +def test_no_include_recipe_meta_yaml(test_metadata, test_config): # first, make sure that the recipe is there by default. This test copied from above, but copied # as a sanity check here. - output_file = os.path.join(sys.prefix, "conda-bld", test_config.subdir, - "empty_sections-0.0-0.tar.bz2") - api.build(empty_sections, config=test_config) + output_file = api.get_output_file_path(test_metadata) + api.build(test_metadata) assert package_has_file(output_file, "info/recipe/meta.yaml") - output_file = os.path.join(sys.prefix, "conda-bld", test_config.subdir, - "no_include_recipe-0.0-0.tar.bz2") + output_file = api.get_output_file_path(os.path.join(metadata_dir, '_no_include_recipe'), + config=test_config) api.build(os.path.join(metadata_dir, '_no_include_recipe'), config=test_config) assert not package_has_file(output_file, "info/recipe/meta.yaml") From c67c7fe06a322895c56ecc5cbcb3aac3532dc880 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Oct 2016 09:08:34 -0500 Subject: [PATCH 045/156] fix unicode json handling for extra conda info --- conda_build/build.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 92f7b78260..cdd8091794 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -290,7 +290,10 @@ def write_about_json(m, config): if value: d[key] = value # for sake of reproducibility, record some conda info - conda_info = json.loads(subprocess.check_output(['conda', 'info', '--json', '-s'])) + conda_info = subprocess.check_output(['conda', 'info', '--json', '-s']) + if hasattr(conda_info, 'decode'): + conda_info = conda_info.decode(codec) + conda_info = json.loads(conda_info) d['conda_version'] = conda_info['conda_version'] d['conda_build_version'] = conda_info['conda_build_version'] d['conda_env_version'] = conda_info['conda_env_version'] @@ -302,8 +305,10 @@ def write_about_json(m, config): d['env_vars'] = conda_info['env_vars'] except KeyError: pass - d['root_pkgs'] = json.loads(subprocess.check_output(['conda', 'list', '-n', - 'root', '--json'])) + pkgs = subprocess.check_output(['conda', 'list', '-n', 'root', '--json']) + if hasattr(pkgs, 'decode'): + pkgs = pkgs.decode(codec) + d['root_pkgs'] = json.loads(pkgs) json.dump(d, fo, indent=2, sort_keys=True) From f7929df6b4359fef1baeedb9963891fe5f2dc176 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Oct 2016 09:46:53 -0500 Subject: [PATCH 046/156] strip token in channel urls. test. --- conda_build/build.py | 11 ++++++++++- tests/test_build.py | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index cdd8091794..d3a01d0f41 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -12,6 +12,7 @@ import mmap import os from os.path import isdir, isfile, islink, join +import re import shutil import stat import subprocess @@ -281,6 +282,10 @@ def detect_and_record_prefix_files(m, files, prefix, config): raise RuntimeError(errstr) +def sanitize_channel(channel): + return re.sub('\/t\/[a-zA-Z0-9\-]*\/', '/t//', channel) + + def write_about_json(m, config): with open(join(config.info_dir, 'about.json'), 'w') as fo: d = {} @@ -298,7 +303,11 @@ def write_about_json(m, config): d['conda_build_version'] = conda_info['conda_build_version'] d['conda_env_version'] = conda_info['conda_env_version'] d['offline'] = conda_info['offline'] - d['channels'] = conda_info['channels'] + channels = conda_info['channels'] + stripped_channels = [] + for channel in channels: + stripped_channels.append(sanitize_channel(channel)) + d['channels'] = stripped_channels # this information will only be present in conda 4.2.10+ try: d['conda_private'] = conda_info['conda_private'] diff --git a/tests/test_build.py b/tests/test_build.py index c947b84d45..f2fec12752 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -137,3 +137,8 @@ def test_warn_on_old_conda_build(test_config, capfd): available_packages=['1.0.0beta']) output, error = capfd.readouterr() assert "conda-build appears to be out of date. You have version " not in error + + +def test_sanitize_channel(): + test_url = 'https://conda.anaconda.org/t/ms-534991f2-4123-473a-b512-42025291b927/somechannel' + assert build.sanitize_channel(test_url) == 'https://conda.anaconda.org/t//somechannel' From eef13cbc24c76872fb94dc17d1348dab81e17d0e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Oct 2016 09:58:04 -0500 Subject: [PATCH 047/156] fix stdlib error in get_stdlib_dir on win --- conda_build/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index f684bc6967..a75b329785 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -346,7 +346,7 @@ def path2url(path): def get_stdlib_dir(prefix): if sys.platform == 'win32': - stdlib_dir = os.path.join(prefix, 'Lib', 'site-packages') + stdlib_dir = os.path.join(prefix, 'Lib') else: lib_dir = os.path.join(prefix, 'lib') stdlib_dir = glob(os.path.join(lib_dir, 'python[0-9\.]*')) From d09cabf30b4bab093bd35e2bbb824a0463a5a464 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Oct 2016 10:27:15 -0500 Subject: [PATCH 048/156] update changelog for 2.0.4 --- CHANGELOG.txt | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 15c16e0469..06be35e1aa 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,30 @@ +2016-10-07 2.0.4: +----------------- + +Enhancements: +------------- + +* Add build/skip_compile_pyc meta.yaml option. Use to skip compilation on pyc files listed therein. #1169 +* Add build environment metadata to about.json (conda, conda-build versions, channels, root pkgs) #1407 +* Make subdir member of config a derived property, so that setting platform or bits is more direct #1427 +* Use subprocess call to windows del function to clear .trash folder, rather than conda. Big speedup. #1438 + +Bug fixes: +---------- + +* fix regression regarding 'config' in pypi skeleton for recipes with entry points #1430 +* don't load setup.py data when considering entry points (use only info from meta.yaml) #1431 +* fall back to trying to copy files without attributes or metadata if those fail #1436 +* Fix permissions on packaged files to be user and group writable, and other readable. #1437 +* fix conda develop not respecting python version of target environment #1440 + +Contributors: +------------- + +* @mingwandroid +* @msarahan + + 2016-09-27 2.0.3: ----------------- From 8ad19ff951aad3751385a264e3883238f38bfd84 Mon Sep 17 00:00:00 2001 From: "Bryan W. Weber" Date: Sun, 9 Oct 2016 00:32:07 -0400 Subject: [PATCH 049/156] Fix link to required testing repository in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a142d25445..f4b6e80637 100644 --- a/README.rst +++ b/README.rst @@ -95,7 +95,7 @@ Testing ------- Running our test suite requires cloning one other repo at the same level as conda-build: -https://github.com/conda/conda_build_test_repo - this is necessary for relative path tests +https://github.com/conda/conda_build_test_recipe - this is necessary for relative path tests outside of conda build's build tree. The test suite runs with py.test. Some useful commands to run select tests, From fe75dde373b87506f76616d871a1026445759c23 Mon Sep 17 00:00:00 2001 From: "Bryan W. Weber" Date: Sun, 9 Oct 2016 00:33:22 -0400 Subject: [PATCH 050/156] Add function to match a regex in a given file. Return the match object from a regex search in the given file. Allows the possibility to find aribtrary text, such as version numbers that are defined outside the setup.py file --- conda_build/jinja_context.py | 39 ++++++++++++++++++++++++++++++++++++ conda_build/metadata.py | 6 ++++++ conda_build/render.py | 3 ++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 5b6ebbdba5..2b4038a377 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -167,6 +167,43 @@ def load_npm(): return json.load(pkg) +def load_file_regex(config, load_file=None, regex_pattern=None, from_recipe_dir=False, + recipe_dir=None, permit_undefined_jinja=True): + import re + match = False + + cd_to_work = False + + if load_file is None: + raise RuntimeError('File to be searched is not specified.') + + if from_recipe_dir and recipe_dir: + load_file = os.path.abspath(os.path.join(recipe_dir, load_file)) + elif os.path.exists(config.work_dir): + cd_to_work = True + cwd = os.getcwd() + os.chdir(config.work_dir) + if not os.path.isabs(load_file): + load_file = os.path.join(config.work_dir, load_file) + else: + message = ("Did not find {} file in manually specified location, and source " + "not downloaded yet.".format(load_file)) + if permit_undefined_jinja: + log.debug(message) + return {} + else: + raise RuntimeError(message) + + if os.path.isfile(load_file): + match = re.search(regex_pattern, open(load_file, 'r').read()) + + # Reset the working directory + if cd_to_work: + os.chdir(cwd) + + return match if match else None + + def context_processor(initial_metadata, recipe_dir, config, permit_undefined_jinja): """ Return a dictionary to use as context for jinja templates. @@ -185,5 +222,7 @@ def context_processor(initial_metadata, recipe_dir, config, permit_undefined_jin load_setuptools=partial(load_setuptools, config=config, recipe_dir=recipe_dir, permit_undefined_jinja=permit_undefined_jinja), load_npm=load_npm, + load_file_regex=partial(load_file_regex, config=config, recipe_dir=recipe_dir, + permit_undefined_jinja=permit_undefined_jinja), environ=environ) return ctx diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 2797a3bff3..30c380abca 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -847,6 +847,12 @@ def uses_setup_py_in_meta(self): meta_text = f.read() return "load_setup_py_data" in meta_text or "load_setuptools" in meta_text + @property + def uses_regex_in_meta(self): + with open(self.meta_path) as f: + meta_text = f.read() + return "load_file_regex" in meta_text + @property def uses_jinja(self): if not self.meta_path: diff --git a/conda_build/render.py b/conda_build/render.py index 5b895fb6a5..483057007c 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -78,7 +78,8 @@ def parse_or_try_download(metadata, no_download_source, config, need_reparse_in_env = False if (force_download or (not no_download_source and (metadata.uses_vcs_in_meta or - metadata.uses_setup_py_in_meta))): + metadata.uses_setup_py_in_meta or + metadata.uses_regex_in_meta))): # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. From 48e41a961821ecac1af14d8e3f5208b13b14a0c9 Mon Sep 17 00:00:00 2001 From: "Bryan W. Weber" Date: Sun, 9 Oct 2016 00:33:39 -0400 Subject: [PATCH 051/156] Add tests for regex matching --- .../metadata/source_regex/bld.bat | 11 ++++++++ .../metadata/source_regex/build.sh | 8 ++++++ .../metadata/source_regex/meta.yaml | 26 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 tests/test-recipes/metadata/source_regex/bld.bat create mode 100644 tests/test-recipes/metadata/source_regex/build.sh create mode 100644 tests/test-recipes/metadata/source_regex/meta.yaml diff --git a/tests/test-recipes/metadata/source_regex/bld.bat b/tests/test-recipes/metadata/source_regex/bld.bat new file mode 100644 index 0000000000..4168d5d6f0 --- /dev/null +++ b/tests/test-recipes/metadata/source_regex/bld.bat @@ -0,0 +1,11 @@ +if not exist .git exit 1 +git config core.fileMode false +if errorlevel 1 exit 1 +git describe --tags --dirty +if errorlevel 1 exit 1 +for /f "delims=" %%i in ('git describe') do set gitdesc=%%i +if errorlevel 1 exit 1 +echo "%gitdesc%" +if not "%gitdesc%"=="1.21.0" exit 1 +echo "%PKG_VERSION%" +if not "%PKG_VERSION%"=="1.21.0" exit 1 diff --git a/tests/test-recipes/metadata/source_regex/build.sh b/tests/test-recipes/metadata/source_regex/build.sh new file mode 100644 index 0000000000..ecde5ca3c3 --- /dev/null +++ b/tests/test-recipes/metadata/source_regex/build.sh @@ -0,0 +1,8 @@ +# We test the environment variables in a different recipe + +# Ensure we are in a git repo +[ -d .git ] +git describe +[ "$(git describe)" = 1.21.0 ] +echo "\$PKG_VERSION = $PKG_VERSION" +[ "${PKG_VERSION}" = 1.21.0 ] diff --git a/tests/test-recipes/metadata/source_regex/meta.yaml b/tests/test-recipes/metadata/source_regex/meta.yaml new file mode 100644 index 0000000000..a6a2ef1b2a --- /dev/null +++ b/tests/test-recipes/metadata/source_regex/meta.yaml @@ -0,0 +1,26 @@ +# This recipe exercises the use of GIT_ variables in jinja template strings, +# including use cases involving expressions such as FOO[:7] or FOO.replace(...) + +# it uses load_setup_py_data from conda_build.jinja_context to populate some fields +# with values fed from setuptools. + +{% set data = load_file_regex(load_file='meta.yaml', regex_pattern='git_tag: ([\\d.]+)', from_recipe_dir=True) %} + +package: + name: conda-build-test-get-regex-data + version: {{ data.group(1) }} + +source: + git_url: ../../../../../conda_build_test_recipe + git_tag: 1.21.0 + +build: + entry_points: + - entry = conda_version_test.manual_entry:main + +requirements: + build: + - python {{ PY_VER }}* + # cython inclusion here is to test https://github.com/conda/conda-build/issues/149 + # cython chosen because it is implicated somehow in setup.py complications. Numpy would also work. + - cython From bb04949b5d5894b45e6a1735c40a396ef08fdb7a Mon Sep 17 00:00:00 2001 From: "Bryan W. Weber" Date: Sun, 9 Oct 2016 21:08:32 -0400 Subject: [PATCH 052/156] Make load_file and regex_pattern arguments mandatory --- conda_build/jinja_context.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 2b4038a377..8e784d3745 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -167,16 +167,13 @@ def load_npm(): return json.load(pkg) -def load_file_regex(config, load_file=None, regex_pattern=None, from_recipe_dir=False, +def load_file_regex(config, load_file, regex_pattern, from_recipe_dir=False, recipe_dir=None, permit_undefined_jinja=True): import re match = False cd_to_work = False - if load_file is None: - raise RuntimeError('File to be searched is not specified.') - if from_recipe_dir and recipe_dir: load_file = os.path.abspath(os.path.join(recipe_dir, load_file)) elif os.path.exists(config.work_dir): From ed7ec4f25964e875c5577ed2e7847a4973e2abc4 Mon Sep 17 00:00:00 2001 From: "Bryan W. Weber" Date: Sun, 9 Oct 2016 21:11:26 -0400 Subject: [PATCH 053/156] Raise TypeError if load_file is not a file --- conda_build/jinja_context.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 8e784d3745..808138b1b1 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -193,6 +193,8 @@ def load_file_regex(config, load_file, regex_pattern, from_recipe_dir=False, if os.path.isfile(load_file): match = re.search(regex_pattern, open(load_file, 'r').read()) + else: + raise TypeError('{} is not a file that can be read'.format(load_file)) # Reset the working directory if cd_to_work: From 730148c1fc7ba1911cd326ab50862e046a0b0ec0 Mon Sep 17 00:00:00 2001 From: "Bryan W. Weber" Date: Sun, 9 Oct 2016 21:12:00 -0400 Subject: [PATCH 054/156] Fix comment in source_regex test --- tests/test-recipes/metadata/source_regex/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-recipes/metadata/source_regex/meta.yaml b/tests/test-recipes/metadata/source_regex/meta.yaml index a6a2ef1b2a..fa0b89443a 100644 --- a/tests/test-recipes/metadata/source_regex/meta.yaml +++ b/tests/test-recipes/metadata/source_regex/meta.yaml @@ -1,8 +1,8 @@ # This recipe exercises the use of GIT_ variables in jinja template strings, # including use cases involving expressions such as FOO[:7] or FOO.replace(...) -# it uses load_setup_py_data from conda_build.jinja_context to populate some fields -# with values fed from setuptools. +# it uses load_file_regex from conda_build.jinja_context to populate some fields +# with values fed from meta.yaml files. {% set data = load_file_regex(load_file='meta.yaml', regex_pattern='git_tag: ([\\d.]+)', from_recipe_dir=True) %} From 0e0ce68539143c3e1f6a66eb97ea40307750abcb Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 11 Oct 2016 14:56:59 -0500 Subject: [PATCH 055/156] quote paths in activation of envs --- conda_build/build.py | 6 +++--- conda_build/windows.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 841cf182ef..5ce2231aeb 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -733,8 +733,8 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F else: data = open(work_file).read() with open(work_file, 'w') as bf: - bf.write("source {conda_root}activate {build_prefix} &> " - "/dev/null\n".format(conda_root=root_script_dir + os.path.sep, + bf.write('source "{conda_root}activate" "{build_prefix}" &> ' + '/dev/null\n'.format(conda_root=root_script_dir + os.path.sep, build_prefix=config.build_prefix)) bf.write(data) else: @@ -953,7 +953,7 @@ def test(m, config, move_broken=True): with open(test_script, 'w') as tf: if config.activate: ext = ".bat" if on_win else "" - tf.write("{source} {conda_root}activate{ext} {test_env} {squelch}\n".format( + tf.write('{source} "{conda_root}activate{ext}" "{test_env}" {squelch}\n'.format( conda_root=root_script_dir + os.path.sep, source="call" if on_win else "source", ext=ext, diff --git a/conda_build/windows.py b/conda_build/windows.py index 9edab217de..ae7f6a7004 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -190,7 +190,7 @@ def build(m, bld_bat, config): fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"])) fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"])) if config.activate: - fo.write("call {conda_root}\\activate.bat {prefix}\n".format( + fo.write('call "{conda_root}\\activate.bat" "{prefix}"\n'.format( conda_root=root_script_dir, prefix=config.build_prefix)) fo.write("REM ===== end generated header =====\n") From 82267cf9d65bb120a88550a4c4373dd30e0d2f20 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 12 Oct 2016 08:45:11 -0500 Subject: [PATCH 056/156] fix source re-copy (leading to IOError) with test as separate ste --- conda_build/cli/main_build.py | 2 +- conda_build/render.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index adead0d1ff..7ec5c5afb0 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -245,7 +245,7 @@ def main(): print(str(e)) sys.exit(1) except filelock.Timeout as e: - print("File lock could on {0} not be obtained. You might need to try fewer builds at once." + print("File lock on {0} could not be obtained. You might need to try fewer builds at once." " Otherwise, run conda clean --lock".format(e.lock_file)) sys.exit(1) return diff --git a/conda_build/render.py b/conda_build/render.py index 5b895fb6a5..f1a75c1f83 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -84,7 +84,8 @@ def parse_or_try_download(metadata, no_download_source, config, # meta.yaml, and not previously installed in builder env. try: if not config.dirty: - source.provide(metadata.path, metadata.get_section('source'), config=config) + if not os.path.exists(config.work_dir) or len(os.listdir(config.work_dir)) == 0: + source.provide(metadata.path, metadata.get_section('source'), config=config) need_source_download = False try: metadata.parse_again(config=config, permit_undefined_jinja=False) From 4a6d725f26a4fa3c6ce0b40401cee8d2a47fe339 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 12 Oct 2016 10:27:29 -0500 Subject: [PATCH 057/156] replace distutils copy_tree with in-house. dirs were not being created. --- conda_build/render.py | 2 +- conda_build/source.py | 8 +++---- conda_build/utils.py | 48 ++++++++++++++++++++++++++++++++++++++---- tests/test_api_test.py | 10 +++++++++ 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/conda_build/render.py b/conda_build/render.py index f1a75c1f83..2432393d7b 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -84,7 +84,7 @@ def parse_or_try_download(metadata, no_download_source, config, # meta.yaml, and not previously installed in builder env. try: if not config.dirty: - if not os.path.exists(config.work_dir) or len(os.listdir(config.work_dir)) == 0: + if len(os.listdir(config.work_dir)) == 0: source.provide(metadata.path, metadata.get_section('source'), config=config) need_source_download = False try: diff --git a/conda_build/source.py b/conda_build/source.py index 4cc22dc9f0..e3239a5b75 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -492,13 +492,12 @@ def provide(recipe_dir, meta, config, patch=True): elif 'svn_url' in meta: svn_source(meta, config=config) elif 'path' in meta: + path = normpath(abspath(join(recipe_dir, meta.get('path')))) if config.verbose: - print("Copying %s to %s" % (abspath(join(recipe_dir, - meta.get('path'))), - config.work_dir)) + print("Copying %s to %s" % (path, config.work_dir)) # careful here: we set test path to be outside of conda-build root in setup.cfg. # If you don't do that, this is a recursive function - copy_into(abspath(join(recipe_dir, meta.get('path'))), config.work_dir, config.timeout) + copy_into(path, config.work_dir, config.timeout) else: # no source if not isdir(config.work_dir): os.makedirs(config.work_dir) @@ -507,6 +506,7 @@ def provide(recipe_dir, meta, config, patch=True): src_dir = config.work_dir for patch in meta.get('patches', []): apply_patch(src_dir, join(recipe_dir, patch), config, git) + return config.work_dir diff --git a/conda_build/utils.py b/conda_build/utils.py index a75b329785..d0741b40d0 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -3,7 +3,6 @@ from collections import defaultdict import contextlib from difflib import get_close_matches -from distutils.dir_util import copy_tree import fnmatch from glob import glob from locale import getpreferredencoding @@ -12,7 +11,9 @@ import os from os.path import dirname, getmtime, getsize, isdir, join, isfile, abspath import re +import stat import subprocess + import sys import shutil import tarfile @@ -121,18 +122,57 @@ def copy_into(src, dst, timeout=90, symlinks=False): lock.release() +# http://stackoverflow.com/a/22331852/1170370 +def copytree(src, dst, symlinks=False, ignore=None, dry_run=False): + if not os.path.exists(dst): + os.makedirs(dst) + shutil.copystat(src, dst) + lst = os.listdir(src) + if ignore: + excl = ignore(src, lst) + lst = [x for x in lst if x not in excl] + + dst_lst = [os.path.join(dst, item) for item in lst] + + if not dry_run: + for idx, item in enumerate(lst): + s = os.path.join(src, item) + d = dst_lst[idx] + if symlinks and os.path.islink(s): + if os.path.lexists(d): + os.remove(d) + os.symlink(os.readlink(s), d) + try: + st = os.lstat(s) + mode = stat.S_IMODE(st.st_mode) + os.lchmod(d, mode) + except: + pass # lchmod not available + elif os.path.isdir(s): + copytree(s, d, symlinks, ignore) + else: + try: + shutil.copy2(s, d) + except IOError: + try: + shutil.copy(s, d) + except IOError: + shutil.copyfile(s, d) + return dst_lst + + def merge_tree(src, dst, symlinks=False, timeout=90): """ Merge src into dst recursively by copying all files from src into dst. Return a list of all files copied. - Like copy_tree(src, dst), but raises an error if merging the two trees + Like copytree(src, dst), but raises an error if merging the two trees would overwrite any files. """ assert src not in dst, ("Can't merge/copy source into subdirectory of itself. Please create " "separate spaces for these things.") - new_files = copy_tree(src, dst, preserve_symlinks=symlinks, dry_run=True) + new_files = copytree(src, dst, symlinks=symlinks, dry_run=True) # do not copy lock files new_files = [f for f in new_files if not f.endswith('.conda_lock')] existing = [f for f in new_files if isfile(f)] @@ -144,7 +184,7 @@ def merge_tree(src, dst, symlinks=False, timeout=90): lock = filelock.SoftFileLock(join(src, ".conda_lock")) lock.acquire(timeout=timeout) try: - copy_tree(src, dst, preserve_symlinks=symlinks) + copytree(src, dst, symlinks=symlinks) except: raise finally: diff --git a/tests/test_api_test.py b/tests/test_api_test.py index 8015103183..c993a0856c 100644 --- a/tests/test_api_test.py +++ b/tests/test_api_test.py @@ -21,6 +21,16 @@ def test_package_test(testing_workdir, test_config): api.test(output_file, config=test_config) +def test_package_with_jinja2_does_not_redownload_source(testing_workdir, test_config): + recipe = os.path.join(metadata_dir, 'jinja2_build_str') + api.build(recipe, config=test_config, notest=True) + output_file = api.get_output_file_path(recipe, config=test_config) + # this recipe uses jinja2, which should trigger source download, except that source download + # will have already happened in the build stage. + # https://github.com/conda/conda-build/issues/1451 + api.test(output_file, config=test_config) + + def test_recipe_test(testing_workdir, test_config): # temporarily necessary because we have custom rebuilt svn for longer prefix here test_config.channel_urls = ('conda_build_test', ) From 431a5ab72be6991cc6f897bc39c067f5a25d2fc9 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 12 Oct 2016 17:48:06 -0500 Subject: [PATCH 058/156] prepend conda's PATH, in case it isn't on PATH already --- conda_build/build.py | 55 ++++++++++++++++++++--------------- conda_build/cli/main_build.py | 1 - tests/test_build.py | 25 ++++++++++++++-- tests/test_cli.py | 11 ++++++- tests/utils.py | 24 ++++++++++++++- 5 files changed, 86 insertions(+), 30 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 841cf182ef..3a45613ddc 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -28,7 +28,6 @@ import filelock # used to get version -import conda from .conda_interface import cc from .conda_interface import envs_dirs, root_dir from .conda_interface import plan @@ -295,29 +294,37 @@ def write_about_json(m, config): if value: d[key] = value # for sake of reproducibility, record some conda info - conda_info = subprocess.check_output(['conda', 'info', '--json', '-s']) - if hasattr(conda_info, 'decode'): - conda_info = conda_info.decode(codec) - conda_info = json.loads(conda_info) - d['conda_version'] = conda_info['conda_version'] - d['conda_build_version'] = conda_info['conda_build_version'] - d['conda_env_version'] = conda_info['conda_env_version'] - d['offline'] = conda_info['offline'] - channels = conda_info['channels'] - stripped_channels = [] - for channel in channels: - stripped_channels.append(sanitize_channel(channel)) - d['channels'] = stripped_channels - # this information will only be present in conda 4.2.10+ - try: - d['conda_private'] = conda_info['conda_private'] - d['env_vars'] = conda_info['env_vars'] - except KeyError: - pass - pkgs = subprocess.check_output(['conda', 'list', '-n', 'root', '--json']) - if hasattr(pkgs, 'decode'): - pkgs = pkgs.decode(codec) - d['root_pkgs'] = json.loads(pkgs) + with path_prepended(sys.prefix): + # on *nix, changed env is not actually respected without shell=True: + # http://stackoverflow.com/a/5659249/1170370 + conda_info = subprocess.check_output('conda info --json -s', + env=os.environ, shell=True) + if hasattr(conda_info, 'decode'): + conda_info = conda_info.decode(codec) + conda_info = json.loads(conda_info) + d['conda_version'] = conda_info['conda_version'] + d['conda_build_version'] = conda_info['conda_build_version'] + # conda env will be in most, but not necessarily all installations. + # Don't die if we don't see it. + if 'conda_env_version' in conda_info: + d['conda_env_version'] = conda_info['conda_env_version'] + d['offline'] = conda_info['offline'] + channels = conda_info['channels'] + stripped_channels = [] + for channel in channels: + stripped_channels.append(sanitize_channel(channel)) + d['channels'] = stripped_channels + # this information will only be present in conda 4.2.10+ + try: + d['conda_private'] = conda_info['conda_private'] + d['env_vars'] = conda_info['env_vars'] + except KeyError: + pass + pkgs = subprocess.check_output('conda list -n root --json', + env=os.environ, shell=True) + if hasattr(pkgs, 'decode'): + pkgs = pkgs.decode(codec) + d['root_pkgs'] = json.loads(pkgs) json.dump(d, fo, indent=2, sort_keys=True) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index adead0d1ff..42764eb8a5 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -18,7 +18,6 @@ from conda_build.cli.main_render import (set_language_env_vars, RecipeCompleter, render_recipe, get_render_parser, bldpkg_path) from conda_build.conda_interface import cc -from conda_build.conda_interface import delete_trash from conda_build.conda_interface import add_parser_channels import conda_build.source as source from conda_build.utils import get_recipe_abspath, silence_loggers, rm_rf, print_skip_message diff --git a/tests/test_build.py b/tests/test_build.py index f2fec12752..ddc28b0b8e 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -3,17 +3,19 @@ and is more unit-test oriented. """ -import copy +import json import os +import subprocess import sys import pytest -from conda_build import build, api +import conda +from conda_build import build, api, __version__ from conda_build.metadata import MetaData from conda_build.utils import rm_rf, on_win -from .utils import testing_workdir, test_config, test_metadata, metadata_dir +from .utils import testing_workdir, test_config, test_metadata, metadata_dir, put_bad_conda_on_path prefix_tests = {"normal": os.path.sep} if sys.platform == "win32": @@ -142,3 +144,20 @@ def test_warn_on_old_conda_build(test_config, capfd): def test_sanitize_channel(): test_url = 'https://conda.anaconda.org/t/ms-534991f2-4123-473a-b512-42025291b927/somechannel' assert build.sanitize_channel(test_url) == 'https://conda.anaconda.org/t//somechannel' + + +def test_write_about_json_without_conda_on_path(testing_workdir, test_metadata): + with put_bad_conda_on_path(testing_workdir): + # verify that the correct (bad) conda is the one we call + with pytest.raises(subprocess.CalledProcessError): + subprocess.check_output('conda -h', env=os.environ, shell=True) + build.write_about_json(test_metadata, test_metadata.config) + + output_file = os.path.join(test_metadata.config.info_dir, 'about.json') + assert os.path.isfile(output_file) + with open(output_file) as f: + about = json.load(f) + assert 'conda_version' in about + assert about['conda_version'] == conda.__version__ + assert 'conda_build_version' in about + assert about['conda_build_version'] == __version__ diff --git a/tests/test_cli.py b/tests/test_cli.py index 16ce623e46..f3d7ec8b09 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ import json import os +import subprocess import sys import yaml @@ -14,7 +15,7 @@ from conda_build import api from conda_build.utils import get_site_packages, on_win, get_build_folders, package_has_file -from .utils import testing_workdir, metadata_dir, testing_env, test_config, test_metadata +from .utils import testing_workdir, metadata_dir, testing_env, test_config, test_metadata, put_bad_conda_on_path import conda_build.cli.main_build as main_build import conda_build.cli.main_sign as main_sign @@ -32,6 +33,14 @@ def test_build(): main_build.execute(args) +# regression test for https://github.com/conda/conda-build/issues/1450 +def test_build_with_conda_not_on_path(testing_workdir): + with put_bad_conda_on_path(testing_workdir): + # using subprocess is not ideal, but it is the easiest way to ensure that PATH + # is altered the way we want here. + subprocess.check_call('conda-build {0}'.format(os.path.join(metadata_dir, "python_run")), + env=os.environ, shell=True) + def test_build_add_channel(): """This recipe requires the blinker package, which is only on conda-forge. This verifies that the -c argument works.""" diff --git a/tests/utils.py b/tests/utils.py index 674e364bda..156702b288 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,10 @@ from collections import defaultdict +import contextlib import os +import stat import subprocess import sys -import tarfile + import pytest @@ -99,3 +101,23 @@ def add_mangling(filename): filename = os.path.join(os.path.dirname(filename), '__pycache__', os.path.basename(filename)) return filename + 'c' + + +@contextlib.contextmanager +def put_bad_conda_on_path(testing_workdir): + path_backup = os.environ['PATH'] + # it is easier to add an intentionally bad path than it is to try to scrub any existing path + os.environ['PATH'] = os.pathsep.join([testing_workdir, os.environ["PATH"]]) + + exe_name = 'conda.bat' if on_win else 'conda' + out_exe = os.path.join(testing_workdir, exe_name) + with open(out_exe, 'w') as f: + f.write("exit 1") + st = os.stat(out_exe) + os.chmod(out_exe, st.st_mode | 0o111) + try: + yield + except: + raise + finally: + os.environ['PATH'] = path_backup From ab84fe512893ae2713d1962275582e3e3e694f46 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 12 Oct 2016 19:30:47 -0500 Subject: [PATCH 059/156] add test for regex without from_recipe. Fixes from comments --- conda_build/jinja_context.py | 11 +++++----- conda_build/metadata.py | 4 ++++ conda_build/render.py | 5 +---- .../metadata/source_regex/bld.bat | 3 ++- .../metadata/source_regex/build.sh | 3 ++- .../metadata/source_regex/meta.yaml | 8 ++----- .../source_regex_from_recipe_dir/bld.bat | 11 ++++++++++ .../source_regex_from_recipe_dir/build.sh | 8 +++++++ .../source_regex_from_recipe_dir/meta.yaml | 22 +++++++++++++++++++ 9 files changed, 57 insertions(+), 18 deletions(-) create mode 100644 tests/test-recipes/metadata/source_regex_from_recipe_dir/bld.bat create mode 100644 tests/test-recipes/metadata/source_regex_from_recipe_dir/build.sh create mode 100644 tests/test-recipes/metadata/source_regex_from_recipe_dir/meta.yaml diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 808138b1b1..3341287796 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -1,8 +1,3 @@ -''' -Created on Jan 16, 2014 - -@author: sean -''' from __future__ import absolute_import, division, print_function from functools import partial @@ -138,6 +133,9 @@ def setup(**kw): if os.path.isfile(setup_file): code = compile(open(setup_file).read(), setup_file, 'exec', dont_inherit=1) exec(code, ns, ns) + else: + if not permit_undefined_jinja: + raise TypeError('{} is not a file that can be read'.format(setup_file)) sys.modules['versioneer'] = versioneer @@ -194,7 +192,8 @@ def load_file_regex(config, load_file, regex_pattern, from_recipe_dir=False, if os.path.isfile(load_file): match = re.search(regex_pattern, open(load_file, 'r').read()) else: - raise TypeError('{} is not a file that can be read'.format(load_file)) + if not permit_undefined_jinja: + raise TypeError('{} is not a file that can be read'.format(load_file)) # Reset the working directory if cd_to_work: diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 30c380abca..d9bad92759 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -853,6 +853,10 @@ def uses_regex_in_meta(self): meta_text = f.read() return "load_file_regex" in meta_text + @property + def needs_source_for_render(self): + return self.uses_vcs_in_meta or self.uses_setup_py_in_meta or self.uses_regex_in_meta + @property def uses_jinja(self): if not self.meta_path: diff --git a/conda_build/render.py b/conda_build/render.py index 483057007c..b44c21da5a 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -77,10 +77,7 @@ def parse_or_try_download(metadata, no_download_source, config, force_download=False): need_reparse_in_env = False - if (force_download or (not no_download_source and (metadata.uses_vcs_in_meta or - metadata.uses_setup_py_in_meta or - metadata.uses_regex_in_meta))): - + if (force_download or (not no_download_source and metadata.needs_source_for_render)): # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. try: diff --git a/tests/test-recipes/metadata/source_regex/bld.bat b/tests/test-recipes/metadata/source_regex/bld.bat index 4168d5d6f0..a9e80149d6 100644 --- a/tests/test-recipes/metadata/source_regex/bld.bat +++ b/tests/test-recipes/metadata/source_regex/bld.bat @@ -7,5 +7,6 @@ for /f "delims=" %%i in ('git describe') do set gitdesc=%%i if errorlevel 1 exit 1 echo "%gitdesc%" if not "%gitdesc%"=="1.21.0" exit 1 +:: This looks weird, but it reflects accurately the meta.yaml in conda_build_test_recipe at 1.21.0 tag echo "%PKG_VERSION%" -if not "%PKG_VERSION%"=="1.21.0" exit 1 +if not "%PKG_VERSION%"=="1.20.2" exit 1 diff --git a/tests/test-recipes/metadata/source_regex/build.sh b/tests/test-recipes/metadata/source_regex/build.sh index ecde5ca3c3..718dd174d6 100644 --- a/tests/test-recipes/metadata/source_regex/build.sh +++ b/tests/test-recipes/metadata/source_regex/build.sh @@ -4,5 +4,6 @@ [ -d .git ] git describe [ "$(git describe)" = 1.21.0 ] +# This looks weird, but it reflects accurately the meta.yaml in conda_build_test_recipe at 1.21.0 tag echo "\$PKG_VERSION = $PKG_VERSION" -[ "${PKG_VERSION}" = 1.21.0 ] +[ "${PKG_VERSION}" = 1.20.2 ] diff --git a/tests/test-recipes/metadata/source_regex/meta.yaml b/tests/test-recipes/metadata/source_regex/meta.yaml index fa0b89443a..1e1a34873d 100644 --- a/tests/test-recipes/metadata/source_regex/meta.yaml +++ b/tests/test-recipes/metadata/source_regex/meta.yaml @@ -1,10 +1,9 @@ -# This recipe exercises the use of GIT_ variables in jinja template strings, -# including use cases involving expressions such as FOO[:7] or FOO.replace(...) +# This recipe exercises the use of regex-supplied variables in jinja template strings, # it uses load_file_regex from conda_build.jinja_context to populate some fields # with values fed from meta.yaml files. -{% set data = load_file_regex(load_file='meta.yaml', regex_pattern='git_tag: ([\\d.]+)', from_recipe_dir=True) %} +{% set data = load_file_regex(load_file='meta.yaml', regex_pattern='git_tag: ([\\d.]+)') %} package: name: conda-build-test-get-regex-data @@ -21,6 +20,3 @@ build: requirements: build: - python {{ PY_VER }}* - # cython inclusion here is to test https://github.com/conda/conda-build/issues/149 - # cython chosen because it is implicated somehow in setup.py complications. Numpy would also work. - - cython diff --git a/tests/test-recipes/metadata/source_regex_from_recipe_dir/bld.bat b/tests/test-recipes/metadata/source_regex_from_recipe_dir/bld.bat new file mode 100644 index 0000000000..4168d5d6f0 --- /dev/null +++ b/tests/test-recipes/metadata/source_regex_from_recipe_dir/bld.bat @@ -0,0 +1,11 @@ +if not exist .git exit 1 +git config core.fileMode false +if errorlevel 1 exit 1 +git describe --tags --dirty +if errorlevel 1 exit 1 +for /f "delims=" %%i in ('git describe') do set gitdesc=%%i +if errorlevel 1 exit 1 +echo "%gitdesc%" +if not "%gitdesc%"=="1.21.0" exit 1 +echo "%PKG_VERSION%" +if not "%PKG_VERSION%"=="1.21.0" exit 1 diff --git a/tests/test-recipes/metadata/source_regex_from_recipe_dir/build.sh b/tests/test-recipes/metadata/source_regex_from_recipe_dir/build.sh new file mode 100644 index 0000000000..ecde5ca3c3 --- /dev/null +++ b/tests/test-recipes/metadata/source_regex_from_recipe_dir/build.sh @@ -0,0 +1,8 @@ +# We test the environment variables in a different recipe + +# Ensure we are in a git repo +[ -d .git ] +git describe +[ "$(git describe)" = 1.21.0 ] +echo "\$PKG_VERSION = $PKG_VERSION" +[ "${PKG_VERSION}" = 1.21.0 ] diff --git a/tests/test-recipes/metadata/source_regex_from_recipe_dir/meta.yaml b/tests/test-recipes/metadata/source_regex_from_recipe_dir/meta.yaml new file mode 100644 index 0000000000..57e002f047 --- /dev/null +++ b/tests/test-recipes/metadata/source_regex_from_recipe_dir/meta.yaml @@ -0,0 +1,22 @@ +# This recipe exercises the use of regex-supplied variables in jinja template strings, + +# it uses load_file_regex from conda_build.jinja_context to populate some fields +# with values fed from meta.yaml files. + +{% set data = load_file_regex(load_file='meta.yaml', regex_pattern='git_tag: ([\\d.]+)', from_recipe_dir=True) %} + +package: + name: conda-build-test-get-regex-data + version: {{ data.group(1) }} + +source: + git_url: ../../../../../conda_build_test_recipe + git_tag: 1.21.0 + +build: + entry_points: + - entry = conda_version_test.manual_entry:main + +requirements: + build: + - python {{ PY_VER }}* From 7d91edd2c8195385efa9de9d64eea8c26531ba34 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 12 Oct 2016 21:58:00 -0500 Subject: [PATCH 060/156] provide full path to conda instead of path prepend. Fix win. --- conda_build/build.py | 60 +++++++++++++++++++++----------------------- tests/test_build.py | 2 -- tests/test_cli.py | 3 ++- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 3a45613ddc..f64b3f7ce0 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -293,38 +293,36 @@ def write_about_json(m, config): value = m.get_value('about/%s' % key) if value: d[key] = value + + bin_path = os.path.join(sys.prefix, "Scripts" if on_win else "bin", 'conda') + # for sake of reproducibility, record some conda info - with path_prepended(sys.prefix): - # on *nix, changed env is not actually respected without shell=True: - # http://stackoverflow.com/a/5659249/1170370 - conda_info = subprocess.check_output('conda info --json -s', - env=os.environ, shell=True) - if hasattr(conda_info, 'decode'): - conda_info = conda_info.decode(codec) - conda_info = json.loads(conda_info) - d['conda_version'] = conda_info['conda_version'] - d['conda_build_version'] = conda_info['conda_build_version'] - # conda env will be in most, but not necessarily all installations. - # Don't die if we don't see it. - if 'conda_env_version' in conda_info: - d['conda_env_version'] = conda_info['conda_env_version'] - d['offline'] = conda_info['offline'] - channels = conda_info['channels'] - stripped_channels = [] - for channel in channels: - stripped_channels.append(sanitize_channel(channel)) - d['channels'] = stripped_channels - # this information will only be present in conda 4.2.10+ - try: - d['conda_private'] = conda_info['conda_private'] - d['env_vars'] = conda_info['env_vars'] - except KeyError: - pass - pkgs = subprocess.check_output('conda list -n root --json', - env=os.environ, shell=True) - if hasattr(pkgs, 'decode'): - pkgs = pkgs.decode(codec) - d['root_pkgs'] = json.loads(pkgs) + conda_info = subprocess.check_output([bin_path, 'info', '--json', '-s']) + if hasattr(conda_info, 'decode'): + conda_info = conda_info.decode(codec) + conda_info = json.loads(conda_info) + d['conda_version'] = conda_info['conda_version'] + d['conda_build_version'] = conda_info['conda_build_version'] + # conda env will be in most, but not necessarily all installations. + # Don't die if we don't see it. + if 'conda_env_version' in conda_info: + d['conda_env_version'] = conda_info['conda_env_version'] + d['offline'] = conda_info['offline'] + channels = conda_info['channels'] + stripped_channels = [] + for channel in channels: + stripped_channels.append(sanitize_channel(channel)) + d['channels'] = stripped_channels + # this information will only be present in conda 4.2.10+ + try: + d['conda_private'] = conda_info['conda_private'] + d['env_vars'] = conda_info['env_vars'] + except KeyError: + pass + pkgs = subprocess.check_output([bin_path, 'list', '-n', 'root', '--json']) + if hasattr(pkgs, 'decode'): + pkgs = pkgs.decode(codec) + d['root_pkgs'] = json.loads(pkgs) json.dump(d, fo, indent=2, sort_keys=True) diff --git a/tests/test_build.py b/tests/test_build.py index ddc28b0b8e..3301755bb8 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -158,6 +158,4 @@ def test_write_about_json_without_conda_on_path(testing_workdir, test_metadata): with open(output_file) as f: about = json.load(f) assert 'conda_version' in about - assert about['conda_version'] == conda.__version__ assert 'conda_build_version' in about - assert about['conda_build_version'] == __version__ diff --git a/tests/test_cli.py b/tests/test_cli.py index f3d7ec8b09..d34d9b742b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,8 @@ from conda_build import api from conda_build.utils import get_site_packages, on_win, get_build_folders, package_has_file -from .utils import testing_workdir, metadata_dir, testing_env, test_config, test_metadata, put_bad_conda_on_path +from .utils import (testing_workdir, metadata_dir, testing_env, test_config, test_metadata, + put_bad_conda_on_path) import conda_build.cli.main_build as main_build import conda_build.cli.main_sign as main_sign From 83045c66cfab37f8d49bb99d14617bd7c21a9456 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 13 Oct 2016 09:21:30 -0500 Subject: [PATCH 061/156] fix version argument to skel pypi not working --- conda_build/api.py | 7 +++++++ tests/test_api_skeleton.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/conda_build/api.py b/conda_build/api.py index 18c03499a8..3a412e22b8 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -165,6 +165,13 @@ def skeletonize(packages, repo, output_dir=".", version=None, recursive=False, """Generate a conda recipe from an external repo. Translates metadata from external sources into expected conda recipe format.""" + if version: + kwargs.update({'version': version}) + if recursive: + kwargs.update({'version': version}) + if output_dir != ".": + kwargs.update({'output_dir': output_dir}) + # here we're dumping all extra kwargs as attributes on the config object. We'll extract # only relevant ones below config = get_or_merge_config(config, **kwargs) diff --git a/tests/test_api_skeleton.py b/tests/test_api_skeleton.py index 9320651fc3..805f8b0fe9 100644 --- a/tests/test_api_skeleton.py +++ b/tests/test_api_skeleton.py @@ -93,3 +93,11 @@ def test_list_skeletons(): def test_pypi_with_entry_points(testing_workdir): api.skeletonize('planemo', repo='pypi', python_version="2.7") assert os.path.isdir('planemo') + + +def test_pypi_with_version_arg(testing_workdir): + # regression test for https://github.com/conda/conda-build/issues/1442 + api.skeletonize('PrettyTable', 'pypi', version='0.7.2') + with open('prettytable/meta.yaml') as f: + actual = yaml.load(f) + assert parse_version(actual['package']['version']) == parse_version("0.7.2") From 701d6f4e54b9a909258dd4e6e3023358f631e3c4 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 13 Oct 2016 10:46:00 -0500 Subject: [PATCH 062/156] change metadata test output filenames to avoid race conditions --- tests/test_cli.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 16ce623e46..b5f8dd9708 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -178,31 +178,35 @@ def test_metapackage(test_config, testing_workdir): """the metapackage command creates a package with runtime dependencies specified on the CLI""" args = ['metapackage_test', '1.0', '-d', 'bzip2'] main_metapackage.execute(args) - test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, 'metapackage_test-1.0-0.tar.bz2') + test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, + 'metapackage_test-1.0-0.tar.bz2') assert os.path.isfile(test_path) def test_metapackage_build_number(test_config, testing_workdir): """the metapackage command creates a package with runtime dependencies specified on the CLI""" - args = ['metapackage_test', '1.0', '-d', 'bzip2', '--build-number', '1'] + args = ['metapackage_test_build_number', '1.0', '-d', 'bzip2', '--build-number', '1'] main_metapackage.execute(args) - test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, 'metapackage_test-1.0-1.tar.bz2') + test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, + 'metapackage_test_build_number-1.0-1.tar.bz2') assert os.path.isfile(test_path) def test_metapackage_build_string(test_config, testing_workdir): """the metapackage command creates a package with runtime dependencies specified on the CLI""" - args = ['metapackage_test', '1.0', '-d', 'bzip2', '--build-string', 'frank'] + args = ['metapackage_test_build_string', '1.0', '-d', 'bzip2', '--build-string', 'frank'] main_metapackage.execute(args) - test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, 'metapackage_test-1.0-frank.tar.bz2') + test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, + 'metapackage_test_build_string-1.0-frank.tar.bz2') assert os.path.isfile(test_path) def test_metapackage_metadata(test_config, testing_workdir): - args = ['metapackage_test', '1.0', '-d', 'bzip2', "--home", "http://abc.com", "--summary", "wee", - "--license", "BSD"] + args = ['metapackage_test_metadata', '1.0', '-d', 'bzip2', "--home", "http://abc.com", + "--summary", "wee", "--license", "BSD"] main_metapackage.execute(args) - test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, 'metapackage_test-1.0-0.tar.bz2') + test_path = os.path.join(sys.prefix, "conda-bld", test_config.subdir, + 'metapackage_test_metadata-1.0-0.tar.bz2') assert os.path.isfile(test_path) info = json.loads(package_has_file(test_path, 'info/index.json').decode('utf-8')) assert info['license'] == 'BSD' From 01e4d3d0493a2b172afddbe5b92164c8332095d7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 13 Oct 2016 11:51:20 -0500 Subject: [PATCH 063/156] fix infinite recursion when missing dependency is R or python --- conda_build/build.py | 31 +++++++++++++++++-------------- conda_build/conda_interface.py | 6 +++++- conda_build/utils.py | 2 +- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 0e0355e7d2..2fd808bc60 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -36,7 +36,8 @@ from .conda_interface import package_cache from .conda_interface import prefix_placeholder, linked, symlink_conda from .conda_interface import url_path -from .conda_interface import Resolve, MatchSpec, NoPackagesFound, Unsatisfiable +from .conda_interface import (Resolve, MatchSpec, NoPackagesFound, Unsatisfiable, + CondaValueError, NoPackagesFoundError) from .conda_interface import TemporaryDirectory from .conda_interface import get_rc_urls, get_local_urls from .conda_interface import VersionOrder @@ -559,7 +560,7 @@ def get_conda_build_index_versions(index): pkgs = [] try: pkgs = r.get_pkgs(MatchSpec('conda-build')) - except NoPackagesFound: + except (NoPackagesFound, NoPackagesFoundError): log.warn("Could not find any versions of conda-build in the channels") return [pkg.version for pkg in pkgs] @@ -739,8 +740,9 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F data = open(work_file).read() with open(work_file, 'w') as bf: bf.write('source "{conda_root}activate" "{build_prefix}" &> ' - '/dev/null\n'.format(conda_root=root_script_dir + os.path.sep, - build_prefix=config.build_prefix)) + '/dev/null\n'.format(conda_root=root_script_dir + + os.path.sep, + build_prefix=config.build_prefix)) bf.write(data) else: if not isfile(work_file): @@ -1050,6 +1052,7 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, # delete_trash(None) already_built = set() + extra_help = "" while recipe_list: # This loop recursively builds dependencies if recipes exist if build_only: @@ -1092,13 +1095,8 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, config=recipe_config) if not notest and ok_to_test: test(metadata, config=recipe_config) - except (NoPackagesFound, Unsatisfiable) as e: + except (NoPackagesFound, NoPackagesFoundError, Unsatisfiable, CondaValueError) as e: error_str = str(e) - # Typically if a conflict is with one of these - # packages, the other package needs to be rebuilt - # (e.g., a conflict with 'python 3.5*' and 'x' means - # 'x' isn't build for Python 3.5 and needs to be - # rebuilt). skip_names = ['python', 'r'] add_recipes = [] # add the failed one back in at the beginning - but its deps may come before it @@ -1108,12 +1106,17 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, continue pkg = line.lstrip(' - ').split(' -> ')[-1] pkg = pkg.strip().split(' ')[0] - if pkg in skip_names: - continue if pkg in to_build_recursive: raise RuntimeError("Can't build {0} due to unsatisfiable dependencies:\n" - .format(recipe) + error_str) + .format(recipe) + error_str + "\n" + extra_help) + + if pkg in skip_names: + to_build_recursive.append(pkg) + extra_help = """Typically if a conflict is with the Python or R +packages, the other package needs to be rebuilt +(e.g., a conflict with 'python 3.5*' and 'x' means +'x' isn't build for Python 3.5 and needs to be rebuilt.""" recipe_glob = glob(os.path.join(recipe_parent_dir, pkg)) if recipe_glob: @@ -1125,7 +1128,7 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, add_recipes.append(recipe_dir) else: raise RuntimeError("Can't build {0} due to unsatisfiable dependencies:\n" - .format(recipe) + error_str) + .format(recipe) + error_str + "\n\n" + extra_help) recipe_list.extendleft(add_recipes) # outputs message, or does upload, depending on value of args.anaconda_upload diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index b5af8365ca..61536f7f6e 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -18,7 +18,7 @@ rm_rf, symlink_conda, rm_fetched, package_cache) # NOQA from conda.lock import Locked # NOQA from conda.misc import untracked, walk_prefix # NOQA -from conda.resolve import MatchSpec, NoPackagesFound, Resolve, Unsatisfiable, normalized_version # NOQA +from conda.resolve import MatchSpec, NoPackagesFound, Resolve, Unsatisfiable, normalized_version, CondaValueError # NOQA from conda.signature import KEYS, KEYS_DIR, hash_file, verify # NOQA from conda.utils import human_bytes, hashsum_file, md5_file, memoized, unix_path_to_win, win_path_to_unix, url_path # NOQA import conda.config as cc # NOQA @@ -52,6 +52,7 @@ load_condarc = lambda fn: conda.base.context.reset_context([fn]) PaddingError = conda.exceptions.PaddingError LinkError = conda.exceptions.LinkError + NoPackagesFoundError = conda.exceptions.NoPackagesFoundError else: from conda.config import get_default_urls, non_x86_linux_machines, load_condarc # NOQA @@ -78,6 +79,9 @@ class PaddingError(Exception): class LinkError(Exception): pass + class NoPackagesFoundError(Exception): + pass + class SignatureError(Exception): pass diff --git a/conda_build/utils.py b/conda_build/utils.py index d0741b40d0..d279234874 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -147,7 +147,7 @@ def copytree(src, dst, symlinks=False, ignore=None, dry_run=False): mode = stat.S_IMODE(st.st_mode) os.lchmod(d, mode) except: - pass # lchmod not available + pass # lchmod not available elif os.path.isdir(s): copytree(s, d, symlinks, ignore) else: From ab78f2151887c8146a675a380b4b70372bd66ef8 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 13 Oct 2016 14:08:39 -0500 Subject: [PATCH 064/156] release notes for 2.0.5 --- CHANGELOG.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 06be35e1aa..588bb84af0 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,27 @@ +2016-10-13 2.0.5: +----------------- + +Enhancements: +------------- + +* Add new jinja function for extracting information from files with regular expressions #1443 + +Bug fixes: +---------- + +* Quote paths in activation of build and test envs #1448 +* Fix source re-copy (leading to IOError) with test as a separate step #1452 +* Call conda with an absolute path when gathering metadata for package about.json #1455 +* Don't strictly require conda-env to be present for about.json data #1455 +* Fix version argument to skeletons not being respected #1456 +* Fix infinite recursion when missing dependency is either r or python #1458 + +Contributors: +------------- + +* @bryanwweber +* @msarahan + 2016-10-07 2.0.4: ----------------- From 7fc115c77a06bdfd3c8526565fd9d13bf7a31a8b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 13 Oct 2016 17:31:43 -0500 Subject: [PATCH 065/156] fix compatibility with conda 4.1.x --- .travis.yml | 13 +++++++++++-- conda_build/build.py | 6 +++--- conda_build/conda_interface.py | 6 +++++- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index c68807b0f9..93da447f55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,11 @@ matrix: # env: CANARY=true # os: linux - python: '3.5' - env: FLAKE8=true + env: CONDA=4.1 + os: linux + - python: '3.5' + env: + - FLAKE8=true os: linux install: @@ -22,12 +26,17 @@ install: - export PATH="$HOME/miniconda/bin:$PATH" - hash -r - conda config --set always_yes yes + - conda config --set auto_update_conda False - conda update -q --all - if [[ "$CANARY" == "true" ]]; then conda install -y -q -c conda-canary conda; conda config --set always_yes yes; + else + if [ -n "$CONDA" ]; then + conda install -y --no-deps conda=${CONDA} requests; + fi fi - - conda install -q --force --no-deps conda requests + - conda install -q anaconda-client requests filelock jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 - pip install pkginfo - if [[ "$FLAKE8" == "true" ]]; then diff --git a/conda_build/build.py b/conda_build/build.py index 2fd808bc60..9e4a0a21f5 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -36,12 +36,12 @@ from .conda_interface import package_cache from .conda_interface import prefix_placeholder, linked, symlink_conda from .conda_interface import url_path -from .conda_interface import (Resolve, MatchSpec, NoPackagesFound, Unsatisfiable, - CondaValueError, NoPackagesFoundError) +from .conda_interface import Resolve, MatchSpec, Unsatisfiable from .conda_interface import TemporaryDirectory from .conda_interface import get_rc_urls, get_local_urls from .conda_interface import VersionOrder -from .conda_interface import PaddingError, LinkError +from .conda_interface import (PaddingError, LinkError, CondaValueError, + NoPackagesFoundError, NoPackagesFound) from conda_build import __version__ from conda_build import environ, source, tarcheck diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 61536f7f6e..d565ef8173 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -18,7 +18,7 @@ rm_rf, symlink_conda, rm_fetched, package_cache) # NOQA from conda.lock import Locked # NOQA from conda.misc import untracked, walk_prefix # NOQA -from conda.resolve import MatchSpec, NoPackagesFound, Resolve, Unsatisfiable, normalized_version, CondaValueError # NOQA +from conda.resolve import MatchSpec, NoPackagesFound, Resolve, Unsatisfiable, normalized_version # NOQA from conda.signature import KEYS, KEYS_DIR, hash_file, verify # NOQA from conda.utils import human_bytes, hashsum_file, md5_file, memoized, unix_path_to_win, win_path_to_unix, url_path # NOQA import conda.config as cc # NOQA @@ -53,6 +53,7 @@ PaddingError = conda.exceptions.PaddingError LinkError = conda.exceptions.LinkError NoPackagesFoundError = conda.exceptions.NoPackagesFoundError + CondaValueError = conda.exceptions.CondaValueError else: from conda.config import get_default_urls, non_x86_linux_machines, load_condarc # NOQA @@ -82,6 +83,9 @@ class LinkError(Exception): class NoPackagesFoundError(Exception): pass + class CondaValueError(Exception): + pass + class SignatureError(Exception): pass From 472836a092fa16ad07716697ead3a8676746f066 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 13 Oct 2016 18:29:27 -0500 Subject: [PATCH 066/156] 2.0.6 changelog --- CHANGELOG.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 588bb84af0..64c18350d8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,17 @@ +2016-10-13 2.0.6: +----------------- + +Bug fixes: +---------- + +* fix erroneous import that was only compatible with conda 4.2.x #1460 + +Contributors: +------------- + +* @msarahan + + 2016-10-13 2.0.5: ----------------- @@ -22,6 +36,7 @@ Contributors: * @bryanwweber * @msarahan + 2016-10-07 2.0.4: ----------------- From 64a9a4a6930f69ff0ae5ebf675958bfa852dc4e5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 19 Oct 2016 09:01:35 -0500 Subject: [PATCH 067/156] fix metapackage not going through api build --- conda_build/metapackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/metapackage.py b/conda_build/metapackage.py index 56c9284be9..0f84bcc30c 100644 --- a/conda_build/metapackage.py +++ b/conda_build/metapackage.py @@ -6,7 +6,7 @@ def create_metapackage(name, version, entry_points=(), build_string=None, build_number=0, dependencies=(), home=None, license_name=None, summary=None, config=None): # local import to avoid circular import, we provid create_metapackage in api - from conda_build.build import build + from conda_build.api import build if not config: config = Config() From d2c3dfeb3f889c1588cebc0b92eadf968fd61c3e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 19 Oct 2016 10:05:42 -0500 Subject: [PATCH 068/156] restore script exe creation on windows --- conda_build/windows.py | 36 +++++++++++++++++- .../_script_win_creates_exe/meta.yaml | 13 +++++++ .../metadata/_script_win_creates_exe/setup.py | 5 +++ .../_script_win_creates_exe/test-script | 3 ++ tests/test_api_build.py | 37 ++++++++++++------- 5 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 tests/test-recipes/metadata/_script_win_creates_exe/meta.yaml create mode 100644 tests/test-recipes/metadata/_script_win_creates_exe/setup.py create mode 100644 tests/test-recipes/metadata/_script_win_creates_exe/test-script diff --git a/conda_build/windows.py b/conda_build/windows.py index ae7f6a7004..b3d046b646 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -2,7 +2,7 @@ import os import sys -from os.path import isdir, join +from os.path import isdir, join, dirname, isfile # importing setuptools patches distutils so that it knows how to find VC for python 2.7 import setuptools # noqa @@ -14,7 +14,7 @@ from .conda_interface import bits from conda_build import environ -from conda_build.utils import _check_call, root_script_dir, path_prepended +from conda_build.utils import _check_call, root_script_dir, path_prepended, copy_into assert sys.platform == 'win32' @@ -30,6 +30,36 @@ } +def fix_staged_scripts(scripts_dir): + """ + Fixes scripts which have been installed unix-style to have a .bat + helper + """ + if not isdir(scripts_dir): + return + for fn in os.listdir(scripts_dir): + # process all the extensionless files + if not isfile(join(scripts_dir, fn)) or '.' in fn: + continue + + with open(join(scripts_dir, fn)) as f: + line = f.readline().lower() + # If it's a #!python script + if not (line.startswith('#!') and 'python' in line.lower()): + continue + print('Adjusting unix-style #! script %s, ' + 'and adding a .bat file for it' % fn) + # copy it with a .py extension (skipping that first #! line) + with open(join(scripts_dir, fn + '-script.py'), 'w') as fo: + fo.write(f.read()) + # now create the .exe file + copy_into(join(dirname(__file__), 'cli-%d.exe' % bits), + join(scripts_dir, fn + '.exe')) + + # remove the original script + os.remove(join(scripts_dir, fn)) + + def build_vcvarsall_vs_path(version): """ Given the Visual Studio version, returns the default path to the @@ -198,3 +228,5 @@ def build(m, bld_bat, config): cmd = ['cmd.exe', '/c', 'bld.bat'] _check_call(cmd, cwd=src_dir) + + fix_staged_scripts(join(config.build_prefix, 'Scripts')) diff --git a/tests/test-recipes/metadata/_script_win_creates_exe/meta.yaml b/tests/test-recipes/metadata/_script_win_creates_exe/meta.yaml new file mode 100644 index 0000000000..62250598a0 --- /dev/null +++ b/tests/test-recipes/metadata/_script_win_creates_exe/meta.yaml @@ -0,0 +1,13 @@ +package: + name: script_win_creates_exe + version: 1.0 + +source: + path: . + +build: + script: python setup.py install + +requirements: + build: + - python diff --git a/tests/test-recipes/metadata/_script_win_creates_exe/setup.py b/tests/test-recipes/metadata/_script_win_creates_exe/setup.py new file mode 100644 index 0000000000..1411131d15 --- /dev/null +++ b/tests/test-recipes/metadata/_script_win_creates_exe/setup.py @@ -0,0 +1,5 @@ +from distutils.core import setup +setup(name='foobar', + version='1.0', + scripts=['test-script'] + ) diff --git a/tests/test-recipes/metadata/_script_win_creates_exe/test-script b/tests/test-recipes/metadata/_script_win_creates_exe/test-script new file mode 100644 index 0000000000..d1ddee3885 --- /dev/null +++ b/tests/test-recipes/metadata/_script_win_creates_exe/test-script @@ -0,0 +1,3 @@ +#!/usr/bin/env python +import sys +print(sys.version) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index bd93ce2fbd..d9974ce8b3 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -610,10 +610,10 @@ def test_noarch_none_value(testing_workdir, test_config): api.build(recipe, config=test_config) -def test_noarch_foo_value(): +def test_noarch_foo_value(test_config): recipe = os.path.join(metadata_dir, "noarch_foo") - fn = api.get_output_file_path(recipe) - api.build(recipe) + fn = api.get_output_file_path(recipe, test_config) + api.build(recipe, config=test_config) metadata = json.loads(package_has_file(fn, 'info/index.json').decode()) assert 'noarch' in metadata assert metadata['noarch'] == "foo" @@ -639,25 +639,25 @@ def test_about_json_content(test_metadata): @pytest.mark.xfail(reason="Conda can not yet install `noarch: python` packages") -def test_noarch_python_with_tests(): +def test_noarch_python_with_tests(test_config): recipe = os.path.join(metadata_dir, "_noarch_python_with_tests") - api.build(recipe) + api.build(recipe, config=test_config) -def test_noarch_python(): +def test_noarch_python(test_config): recipe = os.path.join(metadata_dir, "_noarch_python") - fn = api.get_output_file_path(recipe) - api.build(recipe) + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) assert package_has_file(fn, 'info/files') is not '' noarch = json.loads(package_has_file(fn, 'info/noarch.json').decode()) assert 'entry_points' in noarch assert 'type' in noarch -def test_skip_compile_pyc(): +def test_skip_compile_pyc(test_config): recipe = os.path.join(metadata_dir, "skip_compile_pyc") - fn = api.get_output_file_path(recipe) - api.build(recipe) + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) tf = tarfile.open(fn) pyc_count = 0 for f in tf.getmembers(): @@ -672,10 +672,19 @@ def test_skip_compile_pyc(): assert pyc_count == 2, "there should be 2 .pyc files, instead there were {}".format(pyc_count) -def test_fix_permissions(): +def test_fix_permissions(test_config): recipe = os.path.join(metadata_dir, "fix_permissions") - fn = api.get_output_file_path(recipe) - api.build(recipe) + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) tf = tarfile.open(fn) for f in tf.getmembers(): assert f.mode & 0o444 == 0o444, "tar member '{}' has invalid (read) mode".format(f.name) + + +@pytest.mark.skipif(not on_win, reason="windows-only functionality") +def test_script_win_creates_exe(test_config): + recipe = os.path.join(metadata_dir, "_script_win_creates_exe") + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) + assert package_has_file(fn, 'Scripts/test-script.exe') + assert package_has_file(fn, 'Scripts/test-script-script.py') From 6b6c022928805f611209ed7c9febbe837364807d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 19 Oct 2016 10:40:17 -0500 Subject: [PATCH 069/156] fix noarch value propagation from meta.yaml to config --- conda_build/build.py | 2 +- conda_build/render.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index 9e4a0a21f5..45a2b20371 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -248,7 +248,7 @@ def detect_and_record_prefix_files(m, files, prefix, config): else: files_with_prefix = [] - is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m) + is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m) or m.get_value('build/noarch') if files_with_prefix and not is_noarch: if on_win: diff --git a/conda_build/render.py b/conda_build/render.py index 34a3eb9da4..69f033e238 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -109,6 +109,8 @@ def parse_or_try_download(metadata, no_download_source, config, metadata.parse_until_resolved(config=config) except exceptions.UnableToParseMissingSetuptoolsDependencies: need_reparse_in_env = True + if metadata.get_value('build/noarch'): + config.noarch = True return metadata, need_source_download, need_reparse_in_env From 7d1ed378746f93af1d3060c1d60384c0c7d4d73e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 19 Oct 2016 13:31:40 -0500 Subject: [PATCH 070/156] fix calls to subprocess not converting unicode to str --- conda_build/utils.py | 10 +++++----- tests/test-recipes/metadata/patch/meta.yaml | 7 +++++++ tests/test-recipes/metadata/patch/somefile | 1 + tests/test-recipes/metadata/patch/test.patch | 7 +++++++ 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 tests/test-recipes/metadata/patch/meta.yaml create mode 100644 tests/test-recipes/metadata/patch/somefile create mode 100644 tests/test-recipes/metadata/patch/test.patch diff --git a/conda_build/utils.py b/conda_build/utils.py index d279234874..d35f8b29fc 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -537,20 +537,20 @@ def _func_defaulting_env_to_os_environ(func, *popenargs, **kwargs): kwargs = kwargs.copy() env_copy = os.environ.copy() kwargs.update({'env': env_copy}) - args = [] + _args = [] for arg in popenargs: # arguments to subprocess need to be bytestrings if hasattr(arg, 'encode'): arg = arg.encode(codec) - args.append(arg) - return func(*args, **kwargs) + _args.append(str(arg)) + return func(_args, **kwargs) -def check_call_env(*popenargs, **kwargs): +def check_call_env(popenargs, **kwargs): return _func_defaulting_env_to_os_environ(subprocess.check_call, *popenargs, **kwargs) -def check_output_env(*popenargs, **kwargs): +def check_output_env(popenargs, **kwargs): return _func_defaulting_env_to_os_environ(subprocess.check_output, *popenargs, **kwargs) diff --git a/tests/test-recipes/metadata/patch/meta.yaml b/tests/test-recipes/metadata/patch/meta.yaml new file mode 100644 index 0000000000..abf07b9ad7 --- /dev/null +++ b/tests/test-recipes/metadata/patch/meta.yaml @@ -0,0 +1,7 @@ +package: + name: test_patch + version: 1.0 + +source: + path: . + patches: test.patch diff --git a/tests/test-recipes/metadata/patch/somefile b/tests/test-recipes/metadata/patch/somefile new file mode 100644 index 0000000000..8baef1b4ab --- /dev/null +++ b/tests/test-recipes/metadata/patch/somefile @@ -0,0 +1 @@ +abc diff --git a/tests/test-recipes/metadata/patch/test.patch b/tests/test-recipes/metadata/patch/test.patch new file mode 100644 index 0000000000..09f117fed3 --- /dev/null +++ b/tests/test-recipes/metadata/patch/test.patch @@ -0,0 +1,7 @@ +diff --git a/tests/test-recipes/metadata/patch/somefile b/tests/test-recipes/metadata/patch/somefile +index 8baef1b..190a180 100644 +--- a/tests/test-recipes/metadata/patch/somefile ++++ b/tests/test-recipes/metadata/patch/somefile +@@ -1 +1 @@ +-abc ++123 From 472497ef1cab78c553e6345841275436e93f846d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 19 Oct 2016 13:38:13 -0500 Subject: [PATCH 071/156] make subprocess argument conversion 'native' for each python --- conda_build/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index d35f8b29fc..036e77e5cb 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -540,8 +540,10 @@ def _func_defaulting_env_to_os_environ(func, *popenargs, **kwargs): _args = [] for arg in popenargs: # arguments to subprocess need to be bytestrings - if hasattr(arg, 'encode'): + if sys.version_info.major < 3 and hasattr(arg, 'encode'): arg = arg.encode(codec) + elif sys.version_info.major >= 3 and hasattr(arg, 'decode'): + arg = arg.decode(codec) _args.append(str(arg)) return func(_args, **kwargs) From 168f0181cfece3bb2992e6f103156f1b08a5e14c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 19 Oct 2016 14:06:59 -0500 Subject: [PATCH 072/156] make list values more forgiving to be either strings or lists --- conda_build/api.py | 8 +------- conda_build/build.py | 5 +++-- conda_build/create_test.py | 10 +++++----- conda_build/environ.py | 4 ++-- conda_build/metadata.py | 11 ++++++----- conda_build/source.py | 5 +++-- conda_build/utils.py | 7 +++++++ tests/test-recipes/metadata/patch/meta.yaml | 3 ++- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index 3a412e22b8..9e4956f65f 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -20,13 +20,7 @@ # make the Config class available in the api namespace from conda_build.config import Config, get_or_merge_config, DEFAULT_PREFIX_LENGTH as _prefix_length - - -def _ensure_list(recipe_arg): - from .conda_interface import string_types - if isinstance(recipe_arg, string_types) or not hasattr(recipe_arg, '__iter__'): - recipe_arg = [recipe_arg] - return recipe_arg +from conda_build.utils import ensure_list as _ensure_list def render(recipe_path, config=None, **kwargs): diff --git a/conda_build/build.py b/conda_build/build.py index 45a2b20371..6a589b96af 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -52,7 +52,8 @@ fix_permissions, get_build_metadata) from conda_build.utils import (rm_rf, _check_call, copy_into, on_win, get_build_folders, silence_loggers, path_prepended, create_entry_points, - prepend_bin_path, codec, root_script_dir, print_skip_message) + prepend_bin_path, codec, root_script_dir, print_skip_message, + ensure_list) from conda_build.index import update_index from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) @@ -921,7 +922,7 @@ def test(m, config, move_broken=True): # add packages listed in the run environment and test/requires specs.extend(ms.spec for ms in m.ms_depends('run')) - specs += m.get_value('test/requires', []) + specs += ensure_list(m.get_value('test/requires', [])) if py_files: # as the tests are run by python, ensure that python is installed. diff --git a/conda_build/create_test.py b/conda_build/create_test.py index 932e9edf39..cdd7bc1a5d 100644 --- a/conda_build/create_test.py +++ b/conda_build/create_test.py @@ -9,7 +9,7 @@ from os.path import join, exists, isdir import sys -from conda_build.utils import copy_into, get_ext_files, on_win +from conda_build.utils import copy_into, get_ext_files, on_win, ensure_list from conda_build import source @@ -47,14 +47,14 @@ def create_files(dir_path, m, config): True if it has. """ has_files = False - for fn in m.get_value('test/files', []): + for fn in ensure_list(m.get_value('test/files', [])): has_files = True path = join(m.path, fn) copy_into(path, join(dir_path, fn), config.timeout) # need to re-download source in order to do tests if m.get_value('test/source_files') and not isdir(config.work_dir): source.provide(m.path, m.get_section('source'), config=config) - for pattern in m.get_value('test/source_files', []): + for pattern in ensure_list(m.get_value('test/source_files', [])): if on_win and '\\' in pattern: raise RuntimeError("test/source_files paths must use / " "as the path delimiter on Windows") @@ -83,7 +83,7 @@ def create_shell_files(dir_path, m, config): with open(join(dir_path, name), 'a') as f: f.write('\n\n') - for cmd in m.get_value('test/commands', []): + for cmd in ensure_list(m.get_value('test/commands', [])): f.write(cmd) f.write('\n') if sys.platform == 'win32': @@ -100,7 +100,7 @@ def create_py_files(dir_path, m): fo.write(header + '\n') fo.write("print('===== testing package: %s =====')\n" % m.dist()) - for name in m.get_value('test/imports', []): + for name in ensure_list(m.get_value('test/imports', [])): fo.write('print("import: %r")\n' % name) fo.write('import %s\n' % name) fo.write('\n') diff --git a/conda_build/environ.py b/conda_build/environ.py index 0f792a7062..af7aa09d95 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -16,7 +16,7 @@ from conda_build.os_utils import external from conda_build import utils from conda_build.features import feature_list -from conda_build.utils import prepend_bin_path +from conda_build.utils import prepend_bin_path, ensure_list log = logging.getLogger(__file__) @@ -285,7 +285,7 @@ def lua_vars(config): def meta_vars(meta, config): d = {} - for var_name in meta.get_value('build/script_env', []): + for var_name in ensure_list(meta.get_value('build/script_env', [])): value = os.getenv(var_name) if value is None: warnings.warn( diff --git a/conda_build/metadata.py b/conda_build/metadata.py index d9bad92759..3ef76ef5e1 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -15,7 +15,7 @@ from conda_build import exceptions from conda_build.features import feature_list from conda_build.config import Config -from conda_build.utils import rec_glob +from conda_build.utils import rec_glob, ensure_list try: import yaml @@ -209,6 +209,7 @@ def parse(data, config, path=None): default_structs = { 'source/patches': list, 'build/entry_points': list, + 'build/script': list, 'build/script_env': list, 'build/features': list, 'build/track_features': list, @@ -633,7 +634,7 @@ def build_id(self): res.append(s + v.strip('*')) break - features = self.get_value('build/features', []) + features = ensure_list(self.get_value('build/features', [])) if res: res.append('_') if features: @@ -698,7 +699,7 @@ def info_index(self): return d def has_prefix_files(self): - ret = self.get_value('build/has_prefix_files', []) + ret = ensure_list(self.get_value('build/has_prefix_files', [])) if not isinstance(ret, list): raise RuntimeError('build/has_prefix_files should be a list of paths') if sys.platform == 'win32': @@ -718,7 +719,7 @@ def ignore_prefix_files(self): return ret def always_include_files(self): - files = self.get_value('build/always_include_files', []) + files = ensure_list(self.get_value('build/always_include_files', [])) if any('\\' in i for i in files): raise RuntimeError("build/always_include_files paths must use / " "as the path delimiter on Windows") @@ -730,7 +731,7 @@ def include_recipe(self): return self.get_value('build/include_recipe', True) def binary_has_prefix_files(self): - ret = self.get_value('build/binary_has_prefix_files', []) + ret = ensure_list(self.get_value('build/binary_has_prefix_files', [])) if not isinstance(ret, list): raise RuntimeError('build/binary_has_prefix_files should be a list of paths') if sys.platform == 'win32': diff --git a/conda_build/source.py b/conda_build/source.py index e3239a5b75..268b8c4470 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -13,7 +13,7 @@ from .conda_interface import hashsum_file from conda_build.os_utils import external -from conda_build.utils import (tar_xf, unzip, safe_print_unicode, copy_into, on_win, +from conda_build.utils import (tar_xf, unzip, safe_print_unicode, copy_into, on_win, ensure_list, check_output_env, check_call_env, convert_path_for_cygwin_or_msys2) # legacy exports for conda @@ -504,7 +504,8 @@ def provide(recipe_dir, meta, config, patch=True): if patch: src_dir = config.work_dir - for patch in meta.get('patches', []): + patches = ensure_list(meta.get('patches', [])) + for patch in patches: apply_patch(src_dir, join(recipe_dir, patch), config, git) return config.work_dir diff --git a/conda_build/utils.py b/conda_build/utils.py index 036e77e5cb..882930380b 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -594,3 +594,10 @@ def package_has_file(package_path, file_path): except tarfile.ReadError: raise RuntimeError("Could not extract metadata from %s. " "File probably corrupt." % package_path) + + +def ensure_list(arg): + from .conda_interface import string_types + if isinstance(arg, string_types) or not hasattr(arg, '__iter__'): + arg = [arg] + return arg diff --git a/tests/test-recipes/metadata/patch/meta.yaml b/tests/test-recipes/metadata/patch/meta.yaml index abf07b9ad7..43ab8632c9 100644 --- a/tests/test-recipes/metadata/patch/meta.yaml +++ b/tests/test-recipes/metadata/patch/meta.yaml @@ -4,4 +4,5 @@ package: source: path: . - patches: test.patch + patches: + - test.patch From 87c4e11106d676dad4cf5cbb1dee86f0176ae52e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 19 Oct 2016 14:32:06 -0500 Subject: [PATCH 073/156] fix noarch test (config arg needed to be passed with key) --- conda_build/render.py | 2 +- tests/test_api_build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/render.py b/conda_build/render.py index 69f033e238..5a4f20250d 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -153,10 +153,10 @@ def render_recipe(recipe_path, config, no_download_source=False): sys.stderr.write(e.error_msg()) sys.exit(1) - config.noarch = m.get_value('build/noarch') m, need_download, need_reparse_in_env = parse_or_try_download(m, no_download_source=no_download_source, config=config) + config.noarch = bool(m.get_value('build/noarch')) if need_cleanup: rm_rf(recipe_dir) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index d9974ce8b3..588d342398 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -612,7 +612,7 @@ def test_noarch_none_value(testing_workdir, test_config): def test_noarch_foo_value(test_config): recipe = os.path.join(metadata_dir, "noarch_foo") - fn = api.get_output_file_path(recipe, test_config) + fn = api.get_output_file_path(recipe, config=test_config) api.build(recipe, config=test_config) metadata = json.loads(package_has_file(fn, 'info/index.json').decode()) assert 'noarch' in metadata From 70ce7248a5a4c5a3375cb3be5370ddbf7c81ec12 Mon Sep 17 00:00:00 2001 From: Julien Schueller Date: Mon, 24 Oct 2016 16:33:45 +0200 Subject: [PATCH 074/156] Add SHLIB_EXT environment variable to the build Often seen in feedstocks as DYN_EXT, SO_EXT, DYNAMIC_EXT, EXT, SONAME, LIBEXT, etc --- conda_build/environ.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/conda_build/environ.py b/conda_build/environ.py index af7aa09d95..710d9f13dc 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -348,6 +348,17 @@ def get_cpu_count(): return "1" +def get_shlib_ext(): + # Return the shared library extension. + if sys.platform == 'win32': + return '.dll' + elif sys.platform == 'darwin': + return '.dylib' + elif sys.platform.startswith('linux'): + return '.so' + else: + raise NotImplementedError(sys.platform) + def windows_vars(prefix): library_prefix = join(prefix, 'Library') drive, tail = prefix.split(':') @@ -405,6 +416,8 @@ def system_vars(env_dict, prefix, config): else: d['CPU_COUNT'] = get_cpu_count() + d['SHLIB_EXT'] = get_shlib_ext() + if "LANG" in os.environ: d['LANG'] = os.environ['LANG'] d['PATH'] = os.environ.copy()['PATH'] From 1c158fdf5d8e571276344c10d6f4aa41b9188217 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Mon, 24 Oct 2016 13:22:31 +0100 Subject: [PATCH 075/156] detect_binary_files_with_prefix: Add more testing The `detect_binary_files_with_prefix: true` case now also checks info/has_prefix in the tarfile. Add a `detect_binary_files_with_prefix: false` test, currently marked as xfail. --- .../_detect_binary_files_with_prefix/bld.bat | 1 + .../build.sh | 0 .../meta.yaml | 0 .../run_test.sh | 0 .../write_binary_has_prefix.py | 0 .../bld.bat | 1 + .../build.sh | 1 + .../meta.yaml | 6 ++++ .../run_test.sh | 3 ++ .../write_binary_has_prefix.py | 10 ++++++ tests/test_api_build.py | 35 +++++++++++++++++++ 11 files changed, 57 insertions(+) create mode 100644 tests/test-recipes/metadata/_detect_binary_files_with_prefix/bld.bat rename tests/test-recipes/metadata/{detect_binary_files_with_prefix => _detect_binary_files_with_prefix}/build.sh (100%) rename tests/test-recipes/metadata/{detect_binary_files_with_prefix => _detect_binary_files_with_prefix}/meta.yaml (100%) rename tests/test-recipes/metadata/{detect_binary_files_with_prefix => _detect_binary_files_with_prefix}/run_test.sh (100%) rename tests/test-recipes/metadata/{detect_binary_files_with_prefix => _detect_binary_files_with_prefix}/write_binary_has_prefix.py (100%) create mode 100644 tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/bld.bat create mode 100644 tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/build.sh create mode 100644 tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/meta.yaml create mode 100644 tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/run_test.sh create mode 100644 tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/write_binary_has_prefix.py diff --git a/tests/test-recipes/metadata/_detect_binary_files_with_prefix/bld.bat b/tests/test-recipes/metadata/_detect_binary_files_with_prefix/bld.bat new file mode 100644 index 0000000000..9f72a3c26e --- /dev/null +++ b/tests/test-recipes/metadata/_detect_binary_files_with_prefix/bld.bat @@ -0,0 +1 @@ +python %RECIPE_DIR%\write_binary_has_prefix.py diff --git a/tests/test-recipes/metadata/detect_binary_files_with_prefix/build.sh b/tests/test-recipes/metadata/_detect_binary_files_with_prefix/build.sh similarity index 100% rename from tests/test-recipes/metadata/detect_binary_files_with_prefix/build.sh rename to tests/test-recipes/metadata/_detect_binary_files_with_prefix/build.sh diff --git a/tests/test-recipes/metadata/detect_binary_files_with_prefix/meta.yaml b/tests/test-recipes/metadata/_detect_binary_files_with_prefix/meta.yaml similarity index 100% rename from tests/test-recipes/metadata/detect_binary_files_with_prefix/meta.yaml rename to tests/test-recipes/metadata/_detect_binary_files_with_prefix/meta.yaml diff --git a/tests/test-recipes/metadata/detect_binary_files_with_prefix/run_test.sh b/tests/test-recipes/metadata/_detect_binary_files_with_prefix/run_test.sh similarity index 100% rename from tests/test-recipes/metadata/detect_binary_files_with_prefix/run_test.sh rename to tests/test-recipes/metadata/_detect_binary_files_with_prefix/run_test.sh diff --git a/tests/test-recipes/metadata/detect_binary_files_with_prefix/write_binary_has_prefix.py b/tests/test-recipes/metadata/_detect_binary_files_with_prefix/write_binary_has_prefix.py similarity index 100% rename from tests/test-recipes/metadata/detect_binary_files_with_prefix/write_binary_has_prefix.py rename to tests/test-recipes/metadata/_detect_binary_files_with_prefix/write_binary_has_prefix.py diff --git a/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/bld.bat b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/bld.bat new file mode 100644 index 0000000000..9f72a3c26e --- /dev/null +++ b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/bld.bat @@ -0,0 +1 @@ +python %RECIPE_DIR%\write_binary_has_prefix.py diff --git a/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/build.sh b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/build.sh new file mode 100644 index 0000000000..3204039722 --- /dev/null +++ b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/build.sh @@ -0,0 +1 @@ +python $RECIPE_DIR/write_binary_has_prefix.py diff --git a/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/meta.yaml b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/meta.yaml new file mode 100644 index 0000000000..4be1deffea --- /dev/null +++ b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/meta.yaml @@ -0,0 +1,6 @@ +package: + name: conda-build-test-skip-detect-binary-files-with-prefix + version: 1.0 + +build: + detect_binary_files_with_prefix: false diff --git a/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/run_test.sh b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/run_test.sh new file mode 100644 index 0000000000..105dca0f58 --- /dev/null +++ b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/run_test.sh @@ -0,0 +1,3 @@ +cd $PREFIX +cat binary-has-prefix +cat binary-has-prefix | grep --invert-match $PREFIX diff --git a/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/write_binary_has_prefix.py b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/write_binary_has_prefix.py new file mode 100644 index 0000000000..ae01483920 --- /dev/null +++ b/tests/test-recipes/metadata/_skip_detect_binary_files_with_prefix/write_binary_has_prefix.py @@ -0,0 +1,10 @@ +import os + +prefix = os.environ['PREFIX'] +fn = '%s/binary-has-prefix' % prefix + +if not os.path.isdir(prefix): + os.makedirs(prefix) + +with open(fn, 'wb') as f: + f.write(prefix.encode('utf-8') + b'\x00') diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 588d342398..b2a73e2af9 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -672,6 +672,41 @@ def test_skip_compile_pyc(test_config): assert pyc_count == 2, "there should be 2 .pyc files, instead there were {}".format(pyc_count) +#@pytest.mark.skipif(on_win, reason="binary prefixes not supported on Windows") +def test_detect_binary_files_with_prefix(test_config): + recipe = os.path.join(metadata_dir, "_detect_binary_files_with_prefix") + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) + matches = [] + with tarfile.open(fn) as tf: + with tf.extractfile('info/has_prefix') as has_prefix: + contents = [p.strip().decode('utf-8') for p in + has_prefix.readlines()] + matches = [entry for entry in contents if entry.endswith('binary-has-prefix') + or entry.endswith('"binary-has-prefix"')] + assert len(matches) == 1, "binary-has-prefix not recorded in info/has_prefix" + assert ' binary ' in matches[0], "binary-has-prefix not recorded as binary in info/has_prefix" + + +@pytest.mark.xfail(reason="https://github.com/conda/conda-build/issues/1475") +def test_skip_detect_binary_files_with_prefix(test_config): + recipe = os.path.join(metadata_dir, "_skip_detect_binary_files_with_prefix") + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) + matches = [] + with tarfile.open(fn) as tf: + try: + with tf.extractfile('info/has_prefix') as has_prefix: + contents = [p.strip().decode('utf-8') for p in + has_prefix.readlines()] + matches = [entry for entry in contents if entry.endswith('binary-has-prefix') + or entry.endswith('"binary-has-prefix"')] + except: + pass + assert len(matches) == 0, "binary-has-prefix recorded in info/has_prefix despite:" \ + "build/detect_binary_files_with_prefix: false" + + def test_fix_permissions(test_config): recipe = os.path.join(metadata_dir, "fix_permissions") fn = api.get_output_file_path(recipe, config=test_config) From 6a37f1c8202fe7afedec8d5b0364542accb841b0 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Mon, 24 Oct 2016 13:23:27 +0100 Subject: [PATCH 076/156] Fix `build/detect_binary_files_with_prefix: false` --- conda_build/build.py | 15 +++++++++------ tests/test_api_build.py | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 6a589b96af..c4c5f1ec11 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -242,12 +242,15 @@ def detect_and_record_prefix_files(m, files, prefix, config): text_has_prefix_files = m.has_prefix_files() ignore_files = m.ignore_prefix_files() - if ignore_files: - # do we have a list of files, or just ignore everything? - if hasattr(ignore_files, "__iter__"): - files_with_prefix = [f for f in files_with_prefix if f[2] not in ignore_files] - else: - files_with_prefix = [] + ignore_types = set() + if not hasattr(ignore_files, "__iter__"): + if ignore_files == True: + ignore_types.update(('text', 'binary')) + ignore_files = [] + if not m.get_value('build/detect_binary_files_with_prefix', True): + ignore_types.update(('binary',)) + ignore_files.extend([f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files]) + files_with_prefix = [f for f in files_with_prefix if f[2] not in ignore_files] is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m) or m.get_value('build/noarch') diff --git a/tests/test_api_build.py b/tests/test_api_build.py index b2a73e2af9..ec2a944ebb 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -688,7 +688,6 @@ def test_detect_binary_files_with_prefix(test_config): assert ' binary ' in matches[0], "binary-has-prefix not recorded as binary in info/has_prefix" -@pytest.mark.xfail(reason="https://github.com/conda/conda-build/issues/1475") def test_skip_detect_binary_files_with_prefix(test_config): recipe = os.path.join(metadata_dir, "_skip_detect_binary_files_with_prefix") fn = api.get_output_file_path(recipe, config=test_config) From c37a475525ae34cfebd3814a845a27dd9682949e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 24 Oct 2016 19:58:53 -0500 Subject: [PATCH 077/156] extractfile doesn't have context management --- tests/test_api_build.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index ec2a944ebb..6699f3dc58 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -679,11 +679,12 @@ def test_detect_binary_files_with_prefix(test_config): api.build(recipe, config=test_config) matches = [] with tarfile.open(fn) as tf: - with tf.extractfile('info/has_prefix') as has_prefix: - contents = [p.strip().decode('utf-8') for p in - has_prefix.readlines()] - matches = [entry for entry in contents if entry.endswith('binary-has-prefix') - or entry.endswith('"binary-has-prefix"')] + has_prefix = tf.extractfile('info/has_prefix') + contents = [p.strip().decode('utf-8') for p in + has_prefix.readlines()] + has_prefix.close() + matches = [entry for entry in contents if entry.endswith('binary-has-prefix') or + entry.endswith('"binary-has-prefix"')] assert len(matches) == 1, "binary-has-prefix not recorded in info/has_prefix" assert ' binary ' in matches[0], "binary-has-prefix not recorded as binary in info/has_prefix" @@ -695,11 +696,12 @@ def test_skip_detect_binary_files_with_prefix(test_config): matches = [] with tarfile.open(fn) as tf: try: - with tf.extractfile('info/has_prefix') as has_prefix: - contents = [p.strip().decode('utf-8') for p in - has_prefix.readlines()] - matches = [entry for entry in contents if entry.endswith('binary-has-prefix') - or entry.endswith('"binary-has-prefix"')] + has_prefix = tf.extractfile('info/has_prefix') + contents = [p.strip().decode('utf-8') for p in + has_prefix.readlines()] + has_prefix.close() + matches = [entry for entry in contents if entry.endswith('binary-has-prefix') or + entry.endswith('"binary-has-prefix"')] except: pass assert len(matches) == 0, "binary-has-prefix recorded in info/has_prefix despite:" \ From 5c390339ac31afedd4b616fbba65a647b0457a64 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 24 Oct 2016 20:52:43 -0500 Subject: [PATCH 078/156] changelog 2.0.7 --- CHANGELOG.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 64c18350d8..d9ee90864e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,28 @@ +2016-10-24 2.0.7: +----------------- + +Enhancements: +------------- + +* populate SHLIB_EXT environment variable. For example, .so, .dll, .dylib file extensions use this for their native ending. #1478 + +Bug fixes: +---------- + +* fix metapackage not going through api, thus not showing output file path. #1470 +* restore script exe creation on Windows. These are for standalone scripts installed by distutils or setuptools in setup.py. #1471 +* fix noarch value propagation from meta.yaml to config. Was causing noarch to not be respected in some cases. #1472 +* fix calls to subprocess not converting unicode to str #1473 +* restore detect_binary_files_with_prefix behavior - in particular, respect it when false. # 1477 + +Contributors: +------------- + +* @jschueller +* @mingwandroid +* @msarahan + + 2016-10-13 2.0.6: ----------------- From bbc06e63abe14ad74e82efc51216fb7d67e448ac Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 25 Oct 2016 10:16:37 +0200 Subject: [PATCH 079/156] otool -h header changed otool -h output starts with "Mach header" not the file path Xcode 8, macOS 10.12 --- conda_build/os_utils/macho.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/conda_build/os_utils/macho.py b/conda_build/os_utils/macho.py index 49667fb6c2..2b11af7c90 100644 --- a/conda_build/os_utils/macho.py +++ b/conda_build/os_utils/macho.py @@ -46,8 +46,14 @@ def is_dylib(path): def human_filetype(path): - lines = check_output(['otool', '-h', path]).decode('utf-8').splitlines() - assert lines[0].startswith(path), path + output = check_output(['otool', '-h', path]).decode('utf-8') + lines = output.splitlines() + if not lines[0].startswith((path, 'Mach header')): + raise ValueError( + 'Expected `otool -h` output to start with' + ' Mach header or {0}, got:\n{1}'.format(path, output) + ) + assert lines[0].startswith((path, 'Mach header')), path for line in lines: if line.strip().startswith('0x'): From ee2f658a6a372dd290a64a193f27b434737f448b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 31 Oct 2016 09:40:16 -0500 Subject: [PATCH 080/156] fix lists of empty strings created by ensure_list --- conda_build/utils.py | 7 +++++-- .../test-recipes/metadata/empty_patch_section/meta.yaml | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 tests/test-recipes/metadata/empty_patch_section/meta.yaml diff --git a/conda_build/utils.py b/conda_build/utils.py index 882930380b..1cf9d045c1 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -598,6 +598,9 @@ def package_has_file(package_path, file_path): def ensure_list(arg): from .conda_interface import string_types - if isinstance(arg, string_types) or not hasattr(arg, '__iter__'): - arg = [arg] + if (isinstance(arg, string_types) or not hasattr(arg, '__iter__')): + if arg: + arg = [arg] + else: + arg = [] return arg diff --git a/tests/test-recipes/metadata/empty_patch_section/meta.yaml b/tests/test-recipes/metadata/empty_patch_section/meta.yaml new file mode 100644 index 0000000000..5387aa8066 --- /dev/null +++ b/tests/test-recipes/metadata/empty_patch_section/meta.yaml @@ -0,0 +1,9 @@ +package: + name: patch_section_empty + version: 1.0 + +source: + path: . + # the test here is that selectors can make this field empty. Make it empty here no matter what. + # https://github.com/conda/conda-build/issues/1490 + patches: From c43e011ae2aeef61dbad7d6992d6dc73fdd7a842 Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Mon, 31 Oct 2016 18:45:05 -0400 Subject: [PATCH 081/156] First commit - Move allowed_license_families to new package --- conda_build/license_family.py | 16 ++++++++++++++++ conda_build/metadata.py | 16 ---------------- conda_build/skeletons/cran.py | 4 ++-- conda_build/skeletons/pypi.py | 3 ++- 4 files changed, 20 insertions(+), 19 deletions(-) create mode 100644 conda_build/license_family.py diff --git a/conda_build/license_family.py b/conda_build/license_family.py new file mode 100644 index 0000000000..6e104544c0 --- /dev/null +++ b/conda_build/license_family.py @@ -0,0 +1,16 @@ +from __future__ import absolute_import, division, print_function + +allowed_license_families = set(""" +AGPL +Apache +BSD +GPL +GPL2 +GPL3 +LGPL +MIT +Other +PSF +Proprietary +Public-Domain +""".split()) \ No newline at end of file diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 3ef76ef5e1..6bc077b965 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -136,22 +136,6 @@ def yamlize(data): raise exceptions.UnableToParse(original=e) -allowed_license_families = set(""" -AGPL -Apache -BSD -GPL -GPL2 -GPL3 -LGPL -MIT -Other -PSF -Proprietary -Public-Domain -""".split()) - - def ensure_valid_license_family(meta): try: license_family = meta['about']['license_family'] diff --git a/conda_build/skeletons/cran.py b/conda_build/skeletons/cran.py index 708b2ca408..a6092c5d88 100644 --- a/conda_build/skeletons/cran.py +++ b/conda_build/skeletons/cran.py @@ -26,6 +26,7 @@ from conda_build.utils import rm_rf, guess_license_family from conda_build.conda_interface import text_type, iteritems from conda_build.conda_interface import Completer +from conda_build.license_family import allowed_license_families CRAN_META = """\ {{% set posix = 'm2-' if win else '' %}} @@ -582,8 +583,7 @@ def skeletonize(packages, output_dir=".", version=None, git_tag=None, # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") - d['license_family'] = guess_license_family(d['license'], - metadata.allowed_license_families) + d['license_family'] = guess_license_family(d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' diff --git a/conda_build/skeletons/pypi.py b/conda_build/skeletons/pypi.py index c542e22030..91c44b9cea 100644 --- a/conda_build/skeletons/pypi.py +++ b/conda_build/skeletons/pypi.py @@ -33,7 +33,8 @@ from conda_build.source import apply_patch from conda_build.build import create_env from conda_build.config import Config -from conda_build.metadata import MetaData, allowed_license_families +from conda_build.metadata import MetaData +from conda_build.license_family import allowed_license_families if PY3: try: From 4f302e767ac379510ddf6462ebf9389e072bdfc3 Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Mon, 31 Oct 2016 18:45:05 -0400 Subject: [PATCH 082/156] First commit - Move allowed_license_families to new module --- CHANGELOG.txt | 16 +++++ conda_build/license_family.py | 108 ++++++++++++++++++++++++++++++++++ conda_build/metadata.py | 17 +----- conda_build/skeletons/cran.py | 6 +- conda_build/skeletons/pypi.py | 5 +- conda_build/utils.py | 11 ---- tests/test_license_family.py | 45 ++++++++++++++ 7 files changed, 176 insertions(+), 32 deletions(-) create mode 100644 conda_build/license_family.py create mode 100644 tests/test_license_family.py diff --git a/CHANGELOG.txt b/CHANGELOG.txt index d9ee90864e..98732ccb83 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,19 @@ +2016-11-01 2.0.8: +----------------- + +Enhancements: +------------- + +* Improved logic to guess the appropriate license_family to add to package's index. This improves filtering. +* Logic for the license_family is now shared between open-source conda-build, and proprietary cas-mirror packages. + +Contributors: +------------- + +* @caseyclements +* @minrk +* @msarahan + 2016-10-24 2.0.7: ----------------- diff --git a/conda_build/license_family.py b/conda_build/license_family.py new file mode 100644 index 0000000000..d1710531f3 --- /dev/null +++ b/conda_build/license_family.py @@ -0,0 +1,108 @@ +from __future__ import absolute_import, division, print_function + +from difflib import get_close_matches +import re +import string + +allowed_license_families = """ +AGPL +LGPL +GPL3 +GPL2 +GPL +BSD +MIT +APACHE +PSF +PUBLICDOMAIN +PROPRIETARY +OTHER +NONE +""".split() + +# regular expressions +gpl2_regex = re.compile('GPL[^3]*2') # match GPL2 +gpl3_regex = re.compile('GPL[^2]*3') # match GPL3 +gpl23_regex = re.compile('GPL *>= *2') # match GPL >= 2 +punk_regex = re.compile('[%s]' % re.escape(string.punctuation)) # removes punks + + +def match_gpl3(family): + """True if family matches GPL3 and GPL >= 2, else False""" + return (gpl23_regex.search(family) or + gpl3_regex.search(family)) + + +def normalize(s): + """Set to ALL CAPS, replace common GPL patterns, and strip""" + s = s.upper() + s = re.sub('GENERAL PUBLIC LICENSE', 'GPL', s) + s = re.sub('LESSER *', 'L', s) + s = re.sub('AFFERO *', 'A', s) + return s.strip() + + +def remove_special_characters(s): + """Remove punctuation, spaces, tabs, and line feeds""" + s = punk_regex.sub(' ', s) + s = re.sub('\s+', '', s) + return s + + +def guess_license_family_from_index(index=None, + recognized=allowed_license_families): + """Return best guess of license_family from the conda package index. + + Note: Logic here is simple, and focuses on existing set of allowed families + """ + + if isinstance(index, dict): + license_name = index.get('license_family', index.get('license')) + else: # index argument is actually a string + license_name = index + + return guess_license_family(license_name, recognized) + + +def guess_license_family(license_name=None, + recognized=allowed_license_families): + """Return best guess of license_family from the conda package index. + + Note: Logic here is simple, and focuses on existing set of allowed families + """ + + if license_name is None: + return 'NONE' + + license_name = normalize(license_name) + + # Handle GPL families as special cases + # Remove AGPL and LGPL before looking for GPL2 and GPL3 + sans_lgpl = re.sub('[A,L]GPL', '', license_name) + if match_gpl3(sans_lgpl): + return 'GPL3' + elif gpl2_regex.search(sans_lgpl): + return 'GPL2' + + license_name = remove_special_characters(license_name) + for family in recognized: + if family in license_name: + return family + for family in recognized: # TODO - Necessary? What cases does this catch? + if license_name in family: + return family + return 'OTHER' + + +def deprecated_guess_license_family(license_name, recognized=allowed_license_families): + """Deprecated guess of license_family from license + + Use fuzzy_license_family instead + """ + # Tend towards the more clear GPL3 and away from the ambiguity of GPL2. + if 'GPL (>= 2)' in license_name or license_name == 'GPL': + return 'GPL3' + elif 'LGPL' in license_name: + return 'LGPL' + else: + return get_close_matches(license_name, recognized, 1, 0.0)[0] diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 3ef76ef5e1..3fbb7ca1e0 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -16,6 +16,7 @@ from conda_build.features import feature_list from conda_build.config import Config from conda_build.utils import rec_glob, ensure_list +from conda_build.license_family import allowed_license_families try: import yaml @@ -136,22 +137,6 @@ def yamlize(data): raise exceptions.UnableToParse(original=e) -allowed_license_families = set(""" -AGPL -Apache -BSD -GPL -GPL2 -GPL3 -LGPL -MIT -Other -PSF -Proprietary -Public-Domain -""".split()) - - def ensure_valid_license_family(meta): try: license_family = meta['about']['license_family'] diff --git a/conda_build/skeletons/cran.py b/conda_build/skeletons/cran.py index 708b2ca408..950861f34c 100644 --- a/conda_build/skeletons/cran.py +++ b/conda_build/skeletons/cran.py @@ -23,9 +23,10 @@ from conda_build import source, metadata from conda_build.config import Config -from conda_build.utils import rm_rf, guess_license_family +from conda_build.utils import rm_rf from conda_build.conda_interface import text_type, iteritems from conda_build.conda_interface import Completer +from conda_build.license_family import allowed_license_families, guess_license_family CRAN_META = """\ {{% set posix = 'm2-' if win else '' %}} @@ -582,8 +583,7 @@ def skeletonize(packages, output_dir=".", version=None, git_tag=None, # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") - d['license_family'] = guess_license_family(d['license'], - metadata.allowed_license_families) + d['license_family'] = guess_license_family(d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' diff --git a/conda_build/skeletons/pypi.py b/conda_build/skeletons/pypi.py index c542e22030..73519288e8 100644 --- a/conda_build/skeletons/pypi.py +++ b/conda_build/skeletons/pypi.py @@ -29,11 +29,12 @@ from conda_build.conda_interface import human_bytes, hashsum_file from conda_build.conda_interface import default_python -from conda_build.utils import tar_xf, unzip, rm_rf, guess_license_family +from conda_build.utils import tar_xf, unzip, rm_rf from conda_build.source import apply_patch from conda_build.build import create_env from conda_build.config import Config -from conda_build.metadata import MetaData, allowed_license_families +from conda_build.metadata import MetaData +from conda_build.license_family import allowed_license_families, guess_license_family if PY3: try: diff --git a/conda_build/utils.py b/conda_build/utils.py index 882930380b..f9c9dff01f 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -513,17 +513,6 @@ def create_entry_points(items, config): create_entry_point(join(bin_dir, cmd), module, func, config) -def guess_license_family(license_name, allowed_license_families): - # Tend towards the more clear GPL3 and away from the ambiguity of GPL2. - if 'GPL (>= 2)' in license_name or license_name == 'GPL': - return 'GPL3' - elif 'LGPL' in license_name: - return 'LGPL' - else: - return get_close_matches(license_name, - allowed_license_families, 1, 0.0)[0] - - # Return all files in dir, and all its subdirectories, ending in pattern def get_ext_files(start_path, pattern): for root, _, files in os.walk(start_path): diff --git a/tests/test_license_family.py b/tests/test_license_family.py new file mode 100644 index 0000000000..5213928568 --- /dev/null +++ b/tests/test_license_family.py @@ -0,0 +1,45 @@ +from conda_build.license_family import guess_license_family, deprecated_guess_license_family + + +def test_guess_license_family_matching(): + cens = "GPL" + fam = guess_license_family(cens) + assert fam == 'GPL' + prev = deprecated_guess_license_family(cens) + assert fam != prev, 'new and deprecated guesses are unexpectedly the same' + assert prev == 'GPL3' # bizarre when GPL is an allowed license family + + cens = "GPL (>= 3)" + fam = guess_license_family(cens) + assert fam == 'GPL3' + prev = deprecated_guess_license_family(cens) + assert fam == prev, 'new and deprecated guesses differ' + + cens = 'GNU Lesser General Public License' + fam = guess_license_family(cens) + assert fam == 'LGPL', 'guess_license_family({}) is {}'.format(cens, fam) + prev = deprecated_guess_license_family(cens) + assert fam == prev, 'new and deprecated guesses differ' + + cens = 'GNU General Public License some stuff then a 3 then stuff' + fam = guess_license_family(cens) + assert fam == 'GPL3', 'guess_license_family({}) is {}'.format(cens, fam) + prev = deprecated_guess_license_family(cens) + assert fam == prev, 'new and deprecated guesses differ' + + cens = 'Affero GPL' + fam = guess_license_family(cens) + assert fam == 'AGPL', 'guess_license_family({}) is {}'.format(cens, fam) + prev = deprecated_guess_license_family(cens) + assert fam == prev, 'new and deprecated guesses differ' + + cens = u'GPL-2 | GPL-3 | file LICENSE' + fam = guess_license_family(cens) + assert fam == 'GPL3', 'guess_license_family_from_index({}) is {}'.format(cens, fam) + prev = deprecated_guess_license_family(cens) + assert fam != prev, 'new and deprecated guesses are unexpectedly the same' + # Somehow PUBICDOMAIN is closer to cens than GPL2 or GPL3 + assert prev == 'PUBLICDOMAIN' + +if __name__ == '__main__': + test_guess_license_family_matching() From f88651dc9792bded75f95edfa87cce3df1e26d5a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Nov 2016 11:30:02 -0500 Subject: [PATCH 083/156] centralize locks for sharing within a process --- conda_build/build.py | 64 ++----- conda_build/index.py | 180 +++++++++--------- conda_build/render.py | 2 - conda_build/utils.py | 65 ++++--- .../fail/source_git_jinja2_oops/meta.yaml | 4 +- 5 files changed, 147 insertions(+), 168 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index c4c5f1ec11..a2285a4770 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function from collections import deque +from contextlib import nested import fnmatch from glob import glob import io @@ -53,7 +54,7 @@ from conda_build.utils import (rm_rf, _check_call, copy_into, on_win, get_build_folders, silence_loggers, path_prepended, create_entry_points, prepend_bin_path, codec, root_script_dir, print_skip_message, - ensure_list) + ensure_list, get_lock) from conda_build.index import update_index from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) @@ -485,23 +486,24 @@ def create_env(prefix, specs, config, clear_cache=True): for folder in locked_folders: if not os.path.isdir(folder): os.makedirs(folder) - lock = filelock.SoftFileLock(join(folder, '.conda_lock')) + lock = get_lock(join(folder, '.conda_lock'), + timeout=config.timeout) if not folder.endswith('pkgs'): update_index(folder, config=config, lock=lock, could_be_mirror=False) - lock.acquire(timeout=config.timeout) locks.append(lock) - index = get_build_index(config=config, clear_cache=True) + with nested(*locks): + index = get_build_index(config=config, clear_cache=True) - actions = plan.install_actions(prefix, index, specs) - if config.disable_pip: - actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('pip-')] # noqa - actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('setuptools-')] # noqa - plan.display_actions(actions, index) - if on_win: - for k, v in os.environ.items(): - os.environ[k] = str(v) - plan.execute_actions(actions, index, verbose=config.debug) + actions = plan.install_actions(prefix, index, specs) + if config.disable_pip: + actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('pip-')] # noqa + actions['LINK'] = [spec for spec in actions['LINK'] if not spec.startswith('setuptools-')] # noqa + plan.display_actions(actions, index) + if on_win: + for k, v in os.environ.items(): + os.environ[k] = str(v) + plan.execute_actions(actions, index, verbose=config.debug) except (SystemExit, PaddingError, LinkError) as exc: if (("too short in" in str(exc) or 'post-link failed for: openssl' in str(exc) or @@ -520,23 +522,8 @@ def create_env(prefix, specs, config, clear_cache=True): # Setting this here is important because we use it below (symlink) prefix = config.build_prefix - for lock in locks: - lock.release() - if os.path.isfile(lock._lock_file): - os.remove(lock._lock_file) create_env(prefix, specs, config=config, clear_cache=clear_cache) - else: - for lock in locks: - lock.release() - if os.path.isfile(lock._lock_file): - os.remove(lock._lock_file) - raise - finally: - for lock in locks: - lock.release() - if os.path.isfile(lock._lock_file): - os.remove(lock._lock_file) warn_on_old_conda_build(index=index) # ensure prefix exists, even if empty, i.e. when specs are empty @@ -683,7 +670,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F print("Package:", m.dist()) - with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"), + with get_lock(join(config.build_folder, ".conda_lock"), timeout=config.timeout): # get_dir here might be just work, or it might be one level deeper, # dependening on the source. @@ -844,14 +831,8 @@ def order(f): def clean_pkg_cache(dist, timeout): cc.pkgs_dirs = cc.pkgs_dirs[:1] - locks = [] - for folder in cc.pkgs_dirs: - locks.append(filelock.SoftFileLock(join(folder, ".conda_lock"))) - - for lock in locks: - lock.acquire(timeout=timeout) - - try: + locks = [get_lock(join(folder, ".conda_lock"), timeout=timeout) for folder in cc.pkgs_dirs] + with nested(*locks): rmplan = [ 'RM_EXTRACTED {0} local::{0}'.format(dist), 'RM_FETCHED {0} local::{0}'.format(dist), @@ -877,13 +858,6 @@ def clean_pkg_cache(dist, timeout): del cache[pkg_id] for entry in glob(os.path.join(folder, dist + '*')): rm_rf(entry) - except: - raise - finally: - for lock in locks: - lock.release() - if os.path.isfile(lock._lock_file): - os.remove(lock._lock_file) def test(m, config, move_broken=True): @@ -899,7 +873,7 @@ def test(m, config, move_broken=True): clean_pkg_cache(m.dist(), config.timeout) - with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"), timeout=config.timeout): + with get_lock(join(config.build_folder, ".conda_lock"), timeout=config.timeout): tmp_dir = config.test_dir if not isdir(tmp_dir): os.makedirs(tmp_dir) diff --git a/conda_build/index.py b/conda_build/index.py index cad56d179f..cece468eab 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function +from contextlib import nested import os import bz2 import sys @@ -11,9 +12,7 @@ import tarfile from os.path import isfile, join, getmtime -import filelock - -from conda_build.utils import file_info +from conda_build.utils import file_info, get_lock from .conda_interface import PY3, md5_file @@ -21,43 +20,37 @@ def read_index_tar(tar_path, config, lock=None): """ Returns the index.json dict inside the given package tarball. """ if not lock: - lock = filelock.SoftFileLock(join(os.path.dirname(tar_path), ".conda_lock")) - lock.acquire(timeout=config.timeout) - try: - with tarfile.open(tar_path) as t: - try: - return json.loads(t.extractfile('info/index.json').read().decode('utf-8')) - except EOFError: - raise RuntimeError("Could not extract %s. File probably corrupt." - % tar_path) - except OSError as e: - raise RuntimeError("Could not extract %s (%s)" % (tar_path, e)) - except tarfile.ReadError: - raise RuntimeError("Could not extract metadata from %s. " + lock = get_lock(join(os.path.dirname(tar_path), ".conda_lock"), + timeout=config.timeout) + with nested(lock, tarfile.open(tar_path)) as (_, t): + try: + return json.loads(t.extractfile('info/index.json').read().decode('utf-8')) + except EOFError: + raise RuntimeError("Could not extract %s. File probably corrupt." + % tar_path) + except OSError as e: + raise RuntimeError("Could not extract %s (%s)" % (tar_path, e)) + except tarfile.ReadError: + raise RuntimeError("Could not extract metadata from %s. " "File probably corrupt." % tar_path) - finally: - lock.release() -def write_repodata(repodata, dir_path, config=None, lock=None): +def write_repodata(repodata, dir_path, lock, config=None): """ Write updated repodata.json and repodata.json.bz2 """ if not config: import conda_build.config config = conda_build.config.config - if not lock: - lock = filelock.SoftFileLock(join(dir_path, ".conda_lock")) - lock.acquire(timeout=config.timeout) - data = json.dumps(repodata, indent=2, sort_keys=True) - # strip trailing whitespace - data = '\n'.join(line.rstrip() for line in data.splitlines()) - # make sure we have newline at the end - if not data.endswith('\n'): - data += '\n' - with open(join(dir_path, 'repodata.json'), 'w') as fo: - fo.write(data) - with open(join(dir_path, 'repodata.json.bz2'), 'wb') as fo: - fo.write(bz2.compress(data.encode('utf-8'))) - lock.release() + with lock: + data = json.dumps(repodata, indent=2, sort_keys=True) + # strip trailing whitespace + data = '\n'.join(line.rstrip() for line in data.splitlines()) + # make sure we have newline at the end + if not data.endswith('\n'): + data += '\n' + with open(join(dir_path, 'repodata.json'), 'w') as fo: + fo.write(data) + with open(join(dir_path, 'repodata.json.bz2'), 'wb') as fo: + fo.write(bz2.compress(data.encode('utf-8'))) def update_index(dir_path, config, force=False, check_md5=False, remove=True, lock=None, @@ -82,68 +75,67 @@ def update_index(dir_path, config, force=False, check_md5=False, remove=True, lo os.makedirs(dir_path) if not lock: - lock = filelock.SoftFileLock(join(dir_path, ".conda_lock")) - lock.acquire(timeout=config.timeout) + lock = get_lock(join(dir_path, ".conda_lock")) - if force: - index = {} - else: - try: - mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} - with open(index_path, **mode_dict) as fi: - index = json.load(fi) - except (IOError, ValueError): + with lock: + if force: index = {} - - files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) - if could_be_mirror and any(fn.startswith('_license-') for fn in files): - sys.exit("""\ -Error: - Indexing a copy of the Anaconda conda package channel is neither - necessary nor supported. If you wish to add your own packages, - you can do so by adding them to a separate channel. -""") - for fn in files: - path = join(dir_path, fn) - if fn in index: - if check_md5: - if index[fn]['md5'] == md5_file(path): + else: + try: + mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} + with open(index_path, **mode_dict) as fi: + index = json.load(fi) + except (IOError, ValueError): + index = {} + + files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) + if could_be_mirror and any(fn.startswith('_license-') for fn in files): + sys.exit("""\ + Error: + Indexing a copy of the Anaconda conda package channel is neither + necessary nor supported. If you wish to add your own packages, + you can do so by adding them to a separate channel. + """) + for fn in files: + path = join(dir_path, fn) + if fn in index: + if check_md5: + if index[fn]['md5'] == md5_file(path): + continue + elif index[fn]['mtime'] == getmtime(path): continue - elif index[fn]['mtime'] == getmtime(path): - continue - if config.verbose: - print('updating:', fn) - d = read_index_tar(path, config, lock=lock) - d.update(file_info(path)) - index[fn] = d - - for fn in files: - index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None - - if remove: - # remove files from the index which are not on disk - for fn in set(index) - files: if config.verbose: - print("removing:", fn) - del index[fn] - - # Deal with Python 2 and 3's different json module type reqs - mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} - with open(index_path, **mode_dict) as fo: - json.dump(index, fo, indent=2, sort_keys=True, default=str) - - # --- new repodata - for fn in index: - info = index[fn] - for varname in 'arch', 'platform', 'mtime', 'ucs': - try: - del info[varname] - except KeyError: - pass - - if 'requires' in info and 'depends' not in info: - info['depends'] = info['requires'] - - repodata = {'packages': index, 'info': {}} - write_repodata(repodata, dir_path, config, lock=lock) - lock.release() + print('updating:', fn) + d = read_index_tar(path, config, lock=lock) + d.update(file_info(path)) + index[fn] = d + + for fn in files: + index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None + + if remove: + # remove files from the index which are not on disk + for fn in set(index) - files: + if config.verbose: + print("removing:", fn) + del index[fn] + + # Deal with Python 2 and 3's different json module type reqs + mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} + with open(index_path, **mode_dict) as fo: + json.dump(index, fo, indent=2, sort_keys=True, default=str) + + # --- new repodata + for fn in index: + info = index[fn] + for varname in 'arch', 'platform', 'mtime', 'ucs': + try: + del info[varname] + except KeyError: + pass + + if 'requires' in info and 'depends' not in info: + info['depends'] = info['requires'] + + repodata = {'packages': index, 'info': {}} + write_repodata(repodata, dir_path, lock=lock, config=config) diff --git a/conda_build/render.py b/conda_build/render.py index 5a4f20250d..a2ea05ae9b 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -98,8 +98,6 @@ def parse_or_try_download(metadata, no_download_source, config, elif not metadata.get_section('source'): need_source_download = False - if not os.path.isdir(config.work_dir): - os.makedirs(config.work_dir) else: # we have not downloaded source in the render phase. Download it in # the build phase diff --git a/conda_build/utils.py b/conda_build/utils.py index 882930380b..f4fceeff19 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -85,10 +85,10 @@ def get_recipe_abspath(recipe): return recipe_dir, need_cleanup -def copy_into(src, dst, timeout=90, symlinks=False): +def copy_into(src, dst, timeout=90, symlinks=False, lock=None): "Copy all the files and directories in src to the directory dst" if isdir(src): - merge_tree(src, dst, symlinks, timeout=timeout) + merge_tree(src, dst, symlinks, timeout=timeout, lock=lock) else: if isdir(dst): @@ -96,15 +96,19 @@ def copy_into(src, dst, timeout=90, symlinks=False): else: dst_fn = dst - lock = None if os.path.isabs(src): src_folder = os.path.dirname(src) - lock = filelock.SoftFileLock(join(src_folder, ".conda_lock")) - try: - if os.path.sep in dst_fn and not os.path.isdir(os.path.dirname(dst_fn)): - os.makedirs(os.path.dirname(dst_fn)) - if lock: - lock.acquire(timeout=timeout) + else: + if os.path.sep in dst_fn: + src_folder = os.path.dirname(dst_fn) + if not os.path.isdir(src_folder): + os.makedirs(src_folder) + else: + src_folder = os.getcwd() + + if not lock: + lock = get_lock(join(src_folder, ".conda_lock"), timeout=timeout) + with lock: # with each of these, we are copying less metadata. This seems to be necessary # to cope with some shared filesystems with some virtual machine setups. # See https://github.com/conda/conda-build/issues/1426 @@ -115,11 +119,9 @@ def copy_into(src, dst, timeout=90, symlinks=False): shutil.copy(src, dst_fn) except OSError: shutil.copyfile(src, dst_fn) - except shutil.Error: - log.debug("skipping %s - already exists in %s", os.path.basename(src), dst) - finally: - if lock: - lock.release() + except shutil.Error: + log.debug("skipping %s - already exists in %s", + os.path.basename(src), dst) # http://stackoverflow.com/a/22331852/1170370 @@ -132,6 +134,10 @@ def copytree(src, dst, symlinks=False, ignore=None, dry_run=False): excl = ignore(src, lst) lst = [x for x in lst if x not in excl] + # do not copy lock files + if '.conda_lock' in lst: + lst.remove('.conda_lock') + dst_lst = [os.path.join(dst, item) for item in lst] if not dry_run: @@ -161,7 +167,7 @@ def copytree(src, dst, symlinks=False, ignore=None, dry_run=False): return dst_lst -def merge_tree(src, dst, symlinks=False, timeout=90): +def merge_tree(src, dst, symlinks=False, timeout=90, lock=None): """ Merge src into dst recursively by copying all files from src into dst. Return a list of all files copied. @@ -173,23 +179,32 @@ def merge_tree(src, dst, symlinks=False, timeout=90): "separate spaces for these things.") new_files = copytree(src, dst, symlinks=symlinks, dry_run=True) - # do not copy lock files - new_files = [f for f in new_files if not f.endswith('.conda_lock')] existing = [f for f in new_files if isfile(f)] if existing: raise IOError("Can't merge {0} into {1}: file exists: " "{2}".format(src, dst, existing[0])) - lock = filelock.SoftFileLock(join(src, ".conda_lock")) - lock.acquire(timeout=timeout) - try: + if not lock: + lock = get_lock(join(src, ".conda_lock"), timeout=timeout) + with lock: copytree(src, dst, symlinks=symlinks) - except: - raise - finally: - lock.release() - rm_rf(os.path.join(dst, '.conda_lock')) + + +# purpose here is that we want *one* lock per location on disk. It can be locked or unlocked +# at any time, but the lock within this process should all be tied to the same tracking +# mechanism. +_locations = {} + + +def get_lock(lock_file, timeout=90): + global _locations + location = os.path.abspath(os.path.normpath(lock_file)) + if not os.path.isdir(os.path.dirname(location)): + os.makedirs(os.path.dirname(location)) + if location not in _locations: + _locations[location] = filelock.SoftFileLock(location, timeout) + return _locations[location] def relative(f, d='lib'): diff --git a/tests/test-recipes/fail/source_git_jinja2_oops/meta.yaml b/tests/test-recipes/fail/source_git_jinja2_oops/meta.yaml index 1c4c482340..7213b08a6d 100644 --- a/tests/test-recipes/fail/source_git_jinja2_oops/meta.yaml +++ b/tests/test-recipes/fail/source_git_jinja2_oops/meta.yaml @@ -3,8 +3,8 @@ package: version: {{ GIT_DSECRIBE_TAG }} source: - git_url: https://github.com/conda/conda-build - git_tag: 1.8.1 + git_url: ../../../../../conda_build_test_recipe + git_tag: 1.20.2 requirements: build: From 33174c5a853ccd43ca1db01d4b0acb7c698a7793 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Nov 2016 12:46:16 -0500 Subject: [PATCH 084/156] use ExitStack, not nested for context managers (py3 compat) --- .travis.yml | 2 +- appveyor.yml | 2 +- conda.recipe/meta.yaml | 1 + conda_build/build.py | 13 +++++++------ conda_build/index.py | 8 +++++--- conda_build/utils.py | 4 ++++ 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93da447f55..8a39047801 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ install: fi fi - - conda install -q anaconda-client requests filelock jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 + - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 - pip install pkginfo - if [[ "$FLAKE8" == "true" ]]; then conda install -q flake8; diff --git a/appveyor.yml b/appveyor.yml index 47dcae5248..38feba5447 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,7 +50,7 @@ install: - python -c "import sys; print(sys.version)" - python -c "import sys; print(sys.executable)" - python -c "import sys; print(sys.prefix)" - - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests + - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests contextlib2 - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy - conda install -c conda-forge -q perl - conda update -q --all diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 8852b51ca3..c4d3c6abea 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -23,6 +23,7 @@ requirements: - python run: - conda >=4.1 + - contextlib2 [py2] - filelock - jinja2 - patchelf [linux] diff --git a/conda_build/build.py b/conda_build/build.py index a2285a4770..401dec612f 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function from collections import deque -from contextlib import nested import fnmatch from glob import glob import io @@ -26,8 +25,6 @@ # http://stackoverflow.com/a/13057751/1170370 import encodings.idna # NOQA -import filelock - # used to get version from .conda_interface import cc from .conda_interface import envs_dirs, root_dir @@ -54,7 +51,7 @@ from conda_build.utils import (rm_rf, _check_call, copy_into, on_win, get_build_folders, silence_loggers, path_prepended, create_entry_points, prepend_bin_path, codec, root_script_dir, print_skip_message, - ensure_list, get_lock) + ensure_list, get_lock, ExitStack) from conda_build.index import update_index from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) @@ -492,7 +489,9 @@ def create_env(prefix, specs, config, clear_cache=True): update_index(folder, config=config, lock=lock, could_be_mirror=False) locks.append(lock) - with nested(*locks): + with ExitStack() as stack: + for lock in locks: + stack.enter_context(lock) index = get_build_index(config=config, clear_cache=True) actions = plan.install_actions(prefix, index, specs) @@ -832,7 +831,9 @@ def order(f): def clean_pkg_cache(dist, timeout): cc.pkgs_dirs = cc.pkgs_dirs[:1] locks = [get_lock(join(folder, ".conda_lock"), timeout=timeout) for folder in cc.pkgs_dirs] - with nested(*locks): + with ExitStack() as stack: + for lock in locks: + stack.enter_context(lock) rmplan = [ 'RM_EXTRACTED {0} local::{0}'.format(dist), 'RM_FETCHED {0} local::{0}'.format(dist), diff --git a/conda_build/index.py b/conda_build/index.py index cece468eab..fa71df22cb 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function -from contextlib import nested import os import bz2 import sys @@ -12,7 +11,7 @@ import tarfile from os.path import isfile, join, getmtime -from conda_build.utils import file_info, get_lock +from conda_build.utils import file_info, get_lock, ExitStack from .conda_interface import PY3, md5_file @@ -22,7 +21,10 @@ def read_index_tar(tar_path, config, lock=None): if not lock: lock = get_lock(join(os.path.dirname(tar_path), ".conda_lock"), timeout=config.timeout) - with nested(lock, tarfile.open(tar_path)) as (_, t): + with ExitStack() as stack: + stack.enter_context(lock) + t = tarfile.open(tar_path) + stack.enter_context(t) try: return json.loads(t.extractfile('info/index.json').read().decode('utf-8')) except EOFError: diff --git a/conda_build/utils.py b/conda_build/utils.py index f4fceeff19..1b674a63b5 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -32,9 +32,13 @@ if PY3: import urllib.parse as urlparse import urllib.request as urllib + # NOQA because it is not used in this file. + from contextlib import ExitStack # NOQA else: import urlparse import urllib + # NOQA because it is not used in this file. + from contextlib2 import ExitStack # NOQA log = logging.getLogger(__file__) From 09632f628d6dcd3ba9503b8a2134e9a34d70e75b Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Tue, 1 Nov 2016 15:06:29 -0400 Subject: [PATCH 085/156] Split tests, added more which uncovered new special cases --- conda_build/license_family.py | 2 +- tests/test_license_family.py | 126 +++++++++++++++++++++++++++++++--- 2 files changed, 118 insertions(+), 10 deletions(-) diff --git a/conda_build/license_family.py b/conda_build/license_family.py index 33349a280d..d68398d124 100644 --- a/conda_build/license_family.py +++ b/conda_build/license_family.py @@ -23,7 +23,7 @@ # regular expressions gpl2_regex = re.compile('GPL[^3]*2') # match GPL2 gpl3_regex = re.compile('GPL[^2]*3') # match GPL3 -gpl23_regex = re.compile('GPL *>= *2') # match GPL >= 2 +gpl23_regex = re.compile('GPL[^2]*>= *2') # match GPL >= 2 punk_regex = re.compile('[%s]' % re.escape(string.punctuation)) # removes punks diff --git a/tests/test_license_family.py b/tests/test_license_family.py index 5213928568..d03f909c76 100644 --- a/tests/test_license_family.py +++ b/tests/test_license_family.py @@ -1,13 +1,9 @@ from conda_build.license_family import guess_license_family, deprecated_guess_license_family +from conda_build.license_family import allowed_license_families -def test_guess_license_family_matching(): - cens = "GPL" - fam = guess_license_family(cens) - assert fam == 'GPL' - prev = deprecated_guess_license_family(cens) - assert fam != prev, 'new and deprecated guesses are unexpectedly the same' - assert prev == 'GPL3' # bizarre when GPL is an allowed license family +def test_new_vs_previous_guesses_match(): + """Test cases where new and deprecated functions match""" cens = "GPL (>= 3)" fam = guess_license_family(cens) @@ -33,13 +29,125 @@ def test_guess_license_family_matching(): prev = deprecated_guess_license_family(cens) assert fam == prev, 'new and deprecated guesses differ' + +def test_new_vs_previous_guess_differ_gpl(): + """Test cases where new and deprecated functions differ + + license = 'GPL' + New guess is GPL, which is an allowed family, hence the most accurate. + Previously, GPL3 was chosen over GPL + """ + cens = "GPL" + fam = guess_license_family(cens) + assert fam == 'GPL' + prev = deprecated_guess_license_family(cens) + assert fam != prev, 'new and deprecated guesses are unexpectedly the same' + assert prev == 'GPL3' # bizarre when GPL is an allowed license family + + +def test_new_vs_previous_guess_differ_multiple_gpl(): + """Test cases where new and deprecated functions differ + + license = 'GPL-2 | GPL-3 | file LICENSE' + New guess is GPL-3, which is the most accurate. + Previously, somehow PUBLICDOMAIN is closer than GPL2 or GPL3! + """ cens = u'GPL-2 | GPL-3 | file LICENSE' fam = guess_license_family(cens) assert fam == 'GPL3', 'guess_license_family_from_index({}) is {}'.format(cens, fam) prev = deprecated_guess_license_family(cens) assert fam != prev, 'new and deprecated guesses are unexpectedly the same' - # Somehow PUBICDOMAIN is closer to cens than GPL2 or GPL3 assert prev == 'PUBLICDOMAIN' + +def test_old_warnings_no_longer_fail(): + # the following previously threw warnings. Came from r/linux-64 + warnings = {u'MIT License', u'GNU Lesser General Public License (LGPL)', + u'GPL-2 | GPL-3 | file LICENSE', u'GPL (>= 3) | file LICENCE', + u'BSL-1.0', u'GPL (>= 2)', u'file LICENSE (FOSS)', + u'Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)', + u'MIT + file LICENSE', u'GPL-2 | GPL-3', u'GPL (>= 2) | file LICENSE', + u'Unlimited', u'GPL-3 | file LICENSE', + u'GNU General Public License v2 or later (GPLv2+)', u'LGPL-2.1', + u'LGPL-2', u'LGPL-3', u'GPL', + u'zlib (http://zlib.net/zlib_license.html)', + u'Free software (X11 License)', u'Custom free software license', + u'Old MIT', u'GPL 3', u'Apache License (== 2.0)', u'GPL (>= 3)', None, + u'LGPL (>= 2)', u'BSD_2_clause + file LICENSE', u'GPL-3', u'GPL-2', + u'BSD License and GNU Library or Lesser General Public License (LGPL)', + u'GPL-2 | file LICENSE', u'BSD_3_clause + file LICENSE', u'CC0', + u'MIT + file LICENSE | Unlimited', u'Apache License 2.0', + u'BSD License', u'Lucent Public License'} + + for cens in warnings: + fam = guess_license_family(cens) + print('{}:{}'.format(cens, fam)) + assert fam in allowed_license_families + + +def test_gpl2(): + licenses = {u'GPL (>= 2)', u'GNU General Public License v2 or later (GPLv2+)' + u'GPL-2', u'GPL-2 | file LICENSE'} + for cens in licenses: + fam = guess_license_family(cens) + assert fam == u'GPL2' + + +def test_not_gpl2(): + licenses = {u'GPL (>= 2)', u'LGPL (>= 2)', u'GPL', + u'LGPL-3', u'GPL 3', u'GPL (>= 3)', + u'Apache License (== 2.0)'} + for cens in licenses: + fam = guess_license_family(cens) + assert fam != u'GPL2' + + +def test_gpl3(): + licenses = {u'GPL 3', u'GPL-3', u'GPL-3 | file LICENSE', + u'GPL-2 | GPL-3 | file LICENSE', u'GPL (>= 3) | file LICENCE', + u'GPL (>= 2)', u'GPL-2 | GPL-3', u'GPL (>= 2) | file LICENSE'} + for cens in licenses: + fam = guess_license_family(cens) + assert fam == u'GPL3' + + +def test_lgpl(): + licenses = {u'GNU Lesser General Public License (LGPL)', u'LGPL-2.1', + u'LGPL-2', u'LGPL-3', u'LGPL (>= 2)', + u'BSD License and GNU Library or Lesser General Public License (LGPL)'} + for cens in licenses: + fam = guess_license_family(cens) + assert fam == u'LGPL' + + +def test_mit(): + licenses = {u'MIT License', u'MIT + file LICENSE', u'Old MIT'} + for cens in licenses: + fam = guess_license_family(cens) + assert fam == u'MIT' + + +def test_unlimited(): + """The following is an unfortunate case where MIT is in UNLIMITED + + We could add words to filter out, but it would be hard to keep track of... + """ + cens = u'Unlimited' + assert guess_license_family(cens) == 'MIT' + + +def test_other(): + licenses = {u'file LICENSE (FOSS)', u'CC0', + u'Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)', + u'zlib (http://zlib.net/zlib_license.html)', + u'Free software (X11 License)', u'Custom free software license'} + for cens in licenses: + fam = guess_license_family(cens) + assert fam == u'OTHER' + if __name__ == '__main__': - test_guess_license_family_matching() + test_new_vs_previous_guesses_match() + #test_old_warnings_no_longer_fail() + #test_not_gpl2() + #test_other() + #test_gpl3() From b1a86487e84d4a610a6448184f95701ceee98f7d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Nov 2016 14:00:31 -0500 Subject: [PATCH 086/156] fix overly broad regex for hg in uses_vcs_in_build --- conda_build/metadata.py | 60 ++++++++++++++++++++++------------------- tests/test_metadata.py | 22 +++++++++++++++ 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 3ef76ef5e1..d9f97663c7 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -825,23 +825,6 @@ def __repr__(self): ''' return self.__str__() - @property - def uses_vcs_in_meta(self): - """returns name of vcs used if recipe contains metadata associated with version control systems. - If this metadata is present, a download/copy will be forced in parse_or_try_download. - """ - vcs_types = ["git", "svn", "hg"] - # We would get here if we use Jinja2 templating, but specify source with path. - with open(self.meta_path) as f: - metayaml = f.read() - for vcs in vcs_types: - matches = re.findall(r"{}_[^\.\s\'\"]+".format(vcs.upper()), metayaml) - if len(matches) > 0: - if vcs == "hg": - vcs = "mercurial" - return vcs - return None - @property def uses_setup_py_in_meta(self): with open(self.meta_path) as f: @@ -867,18 +850,41 @@ def uses_jinja(self): matches = re.findall(r"{{.*}}", metayaml) return len(matches) > 0 + @property + def uses_vcs_in_meta(self): + """returns name of vcs used if recipe contains metadata associated with version control systems. + If this metadata is present, a download/copy will be forced in parse_or_try_download. + """ + vcs_types = ["git", "svn", "hg"] + # We would get here if we use Jinja2 templating, but specify source with path. + with open(self.meta_path) as f: + metayaml = f.read() + for vcs in vcs_types: + matches = re.findall(r"{}_[^\.\s\'\"]+".format(vcs.upper()), metayaml) + if len(matches) > 0: + if vcs == "hg": + vcs = "mercurial" + return vcs + return None + @property def uses_vcs_in_build(self): build_script = "bld.bat" if on_win else "build.sh" build_script = os.path.join(os.path.dirname(self.meta_path), build_script) - if os.path.isfile(build_script): - vcs_types = ["git", "svn", "hg"] - with open(self.meta_path) as f: - build_script = f.read() - for vcs in vcs_types: - matches = re.findall(r"{}(?:\.exe)?".format(vcs), build_script) - if len(matches) > 0: - if vcs == "hg": - vcs = "mercurial" - return vcs + for recipe_file in (build_script, self.meta_path): + if os.path.isfile(recipe_file): + vcs_types = ["git", "svn", "hg"] + with open(recipe_file) as f: + build_script = f.read() + for vcs in vcs_types: + # commands are assumed to have 3 parts: + # 1. the vcs command, optionally with an exe extension + # 2. a subcommand - for example, "clone" + # 3. a target url or other argument + matches = re.findall(r"{}(?:\.exe)?(?:\s+\w+\s+[\w\/\.:@]+)".format(vcs), + build_script, flags=re.IGNORECASE) + if len(matches) > 0: + if vcs == "hg": + vcs = "mercurial" + return vcs return None diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 7b87665d75..a10659a274 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,8 +1,30 @@ +import os import unittest from conda_build.conda_interface import MatchSpec from conda_build.metadata import select_lines, handle_config_version +from .utils import testing_workdir, test_config, test_metadata + + +def test_uses_vcs_in_metadata(testing_workdir, test_metadata): + test_metadata.meta_path = os.path.join(testing_workdir, 'meta.yaml') + with open(test_metadata.meta_path, 'w') as f: + f.write('http://hg.something.com') + assert not test_metadata.uses_vcs_in_meta + assert not test_metadata.uses_vcs_in_build + with open(test_metadata.meta_path, 'w') as f: + f.write('hg something something') + assert not test_metadata.uses_vcs_in_meta + assert test_metadata.uses_vcs_in_build + with open(test_metadata.meta_path, 'w') as f: + f.write('hg.exe something something') + assert not test_metadata.uses_vcs_in_meta + assert test_metadata.uses_vcs_in_build + with open(test_metadata.meta_path, 'w') as f: + f.write('HG_WEEEEE') + assert test_metadata.uses_vcs_in_meta + assert not test_metadata.uses_vcs_in_build def test_select_lines(): From 995c9c3cd8c1e3229402df354271fdf473620666 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Nov 2016 14:40:32 -0500 Subject: [PATCH 087/156] fix recipe selector; fix url test --- conda.recipe/meta.yaml | 2 +- tests/test_api_build.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index c4d3c6abea..7860630c80 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -23,7 +23,7 @@ requirements: - python run: - conda >=4.1 - - contextlib2 [py2] + - contextlib2 [py<3] - filelock - jinja2 - patchelf [linux] diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 6699f3dc58..8ec98539f4 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -337,20 +337,23 @@ def test_skip_existing(testing_workdir, test_config, capfd): output, error = capfd.readouterr() assert "is already built" in output + def test_skip_existing_url(test_metadata, testing_workdir, capfd): # make sure that it is built output_file = api.get_output_file_path(test_metadata) api.build(test_metadata) # Copy our package into some new folder - platform = os.path.join(testing_workdir, test_metadata.config.subdir) + output_dir = os.path.join(testing_workdir, 'someoutput') + platform = os.path.join(output_dir, test_metadata.config.subdir) + os.makedirs(platform) copy_into(output_file, os.path.join(platform, os.path.basename(output_file))) # create the index so conda can find the file api.update_index(platform, config=test_metadata.config) test_metadata.config.skip_existing = True - test_metadata.config.channel_urls = [url_path(testing_workdir)] + test_metadata.config.channel_urls = [url_path(output_dir)] api.build(test_metadata) output, error = capfd.readouterr() From b858fa629cef95fb44a5bcb1476403fa076cd214 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Nov 2016 14:58:34 -0500 Subject: [PATCH 088/156] clarify error message when extracting egg dir fails due to existing files --- conda_build/post.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index c39db1580d..bba6f1df75 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -119,8 +119,17 @@ def remove_easy_install_pth(files, prefix, config, preserve_egg_dir=False): # so the package directory already exists # from another installed dependency if os.path.exists(join(sp_dir, fn)): - utils.copy_into(join(egg_path, fn), join(sp_dir, fn), config.timeout) - utils.rm_rf(join(egg_path, fn)) + try: + utils.copy_into(join(egg_path, fn), join(sp_dir, fn), config.timeout) + utils.rm_rf(join(egg_path, fn)) + except IOError as e: + fn = os.path.basename(str(e).split()[-1]) + raise IOError("Tried to merge folder {egg_path} into {sp_dir}, but {fn}" + " exists in both locations. Please either add " + "build/preserve_egg_dir: True to meta.yaml, or manually " + "remove the file during your install process to avoid " + "this conflict." + .format(egg_path=egg_path, sp_dir=sp_dir, fn=fn)) else: os.rename(join(egg_path, fn), join(sp_dir, fn)) From 8a244070d8cd6fd6667833b6a36cbcfa8d2816cf Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Wed, 2 Nov 2016 10:05:32 -0400 Subject: [PATCH 089/156] Fix failing test. Expected result was wrong.. --- tests/test_license_family.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_license_family.py b/tests/test_license_family.py index d03f909c76..167572a158 100644 --- a/tests/test_license_family.py +++ b/tests/test_license_family.py @@ -86,8 +86,8 @@ def test_old_warnings_no_longer_fail(): def test_gpl2(): - licenses = {u'GPL (>= 2)', u'GNU General Public License v2 or later (GPLv2+)' - u'GPL-2', u'GPL-2 | file LICENSE'} + licenses = {u'GPL-2', u'GPL-2 | file LICENSE', + u'GNU General Public License v2 or later (GPLv2+)' } for cens in licenses: fam = guess_license_family(cens) assert fam == u'GPL2' @@ -146,8 +146,9 @@ def test_other(): assert fam == u'OTHER' if __name__ == '__main__': - test_new_vs_previous_guesses_match() + #test_new_vs_previous_guesses_match() #test_old_warnings_no_longer_fail() #test_not_gpl2() #test_other() #test_gpl3() + test_gpl2() From 4338fd016ae3b1de894c71082346db147e128f83 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Nov 2016 14:45:25 -0500 Subject: [PATCH 090/156] simplify locks - always folder locks. disable lock on build folder. --- conda_build/build.py | 338 +++++++++++++++++++++---------------------- conda_build/index.py | 5 +- conda_build/utils.py | 15 +- 3 files changed, 177 insertions(+), 181 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 401dec612f..6a40b975eb 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -483,8 +483,7 @@ def create_env(prefix, specs, config, clear_cache=True): for folder in locked_folders: if not os.path.isdir(folder): os.makedirs(folder) - lock = get_lock(join(folder, '.conda_lock'), - timeout=config.timeout) + lock = get_lock(folder, timeout=config.timeout) if not folder.endswith('pkgs'): update_index(folder, config=config, lock=lock, could_be_mirror=False) locks.append(lock) @@ -669,80 +668,78 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F print("Package:", m.dist()) - with get_lock(join(config.build_folder, ".conda_lock"), - timeout=config.timeout): - # get_dir here might be just work, or it might be one level deeper, - # dependening on the source. - src_dir = config.work_dir - if isdir(src_dir): - print("source tree in:", src_dir) + # get_dir here might be just work, or it might be one level deeper, + # dependening on the source. + src_dir = config.work_dir + if isdir(src_dir): + print("source tree in:", src_dir) + else: + print("no source - creating empty work folder") + os.makedirs(src_dir) + + rm_rf(config.info_dir) + files1 = prefix_files(prefix=config.build_prefix) + for pat in m.always_include_files(): + has_matches = False + for f in set(files1): + if fnmatch.fnmatch(f, pat): + print("Including in package existing file", f) + files1.discard(f) + has_matches = True + if not has_matches: + log.warn("Glob %s from always_include_files does not match any files", pat) + # Save this for later + with open(join(config.croot, 'prefix_files.txt'), 'w') as f: + f.write(u'\n'.join(sorted(list(files1)))) + f.write(u'\n') + + # Use script from recipe? + script = m.get_value('build/script', None) + if script: + if isinstance(script, list): + script = '\n'.join(script) + + if isdir(src_dir): + if on_win: + build_file = join(m.path, 'bld.bat') + if script: + build_file = join(src_dir, 'bld.bat') + with open(build_file, 'w') as bf: + bf.write(script) + import conda_build.windows as windows + windows.build(m, build_file, config=config) else: - print("no source - creating empty work folder") - os.makedirs(src_dir) - - rm_rf(config.info_dir) - files1 = prefix_files(prefix=config.build_prefix) - for pat in m.always_include_files(): - has_matches = False - for f in set(files1): - if fnmatch.fnmatch(f, pat): - print("Including in package existing file", f) - files1.discard(f) - has_matches = True - if not has_matches: - log.warn("Glob %s from always_include_files does not match any files", pat) - # Save this for later - with open(join(config.croot, 'prefix_files.txt'), 'w') as f: - f.write(u'\n'.join(sorted(list(files1)))) - f.write(u'\n') - - # Use script from recipe? - script = m.get_value('build/script', None) - if script: - if isinstance(script, list): - script = '\n'.join(script) - - if isdir(src_dir): - if on_win: - build_file = join(m.path, 'bld.bat') + build_file = join(m.path, 'build.sh') + + # There is no sense in trying to run an empty build script. + if isfile(build_file) or script: + with path_prepended(config.build_prefix): + env = environ.get_dict(config=config, m=m) + env["CONDA_BUILD_STATE"] = "BUILD" + work_file = join(config.work_dir, 'conda_build.sh') if script: - build_file = join(src_dir, 'bld.bat') - with open(build_file, 'w') as bf: + with open(work_file, 'w') as bf: bf.write(script) - import conda_build.windows as windows - windows.build(m, build_file, config=config) - else: - build_file = join(m.path, 'build.sh') - - # There is no sense in trying to run an empty build script. - if isfile(build_file) or script: - with path_prepended(config.build_prefix): - env = environ.get_dict(config=config, m=m) - env["CONDA_BUILD_STATE"] = "BUILD" - work_file = join(config.work_dir, 'conda_build.sh') - if script: - with open(work_file, 'w') as bf: - bf.write(script) - if config.activate: - if isfile(build_file): - data = open(build_file).read() - else: - data = open(work_file).read() - with open(work_file, 'w') as bf: - bf.write('source "{conda_root}activate" "{build_prefix}" &> ' - '/dev/null\n'.format(conda_root=root_script_dir + - os.path.sep, - build_prefix=config.build_prefix)) - bf.write(data) + if config.activate: + if isfile(build_file): + data = open(build_file).read() else: - if not isfile(work_file): - copy_into(build_file, work_file, config.timeout) - os.chmod(work_file, 0o766) - - if isfile(work_file): - cmd = [shell_path, '-x', '-e', work_file] - # this should raise if any problems occur while building - _check_call(cmd, env=env, cwd=src_dir) + data = open(work_file).read() + with open(work_file, 'w') as bf: + bf.write('source "{conda_root}activate" "{build_prefix}" &> ' + '/dev/null\n'.format(conda_root=root_script_dir + + os.path.sep, + build_prefix=config.build_prefix)) + bf.write(data) + else: + if not isfile(work_file): + copy_into(build_file, work_file, config.timeout) + os.chmod(work_file, 0o766) + + if isfile(work_file): + cmd = [shell_path, '-x', '-e', work_file] + # this should raise if any problems occur while building + _check_call(cmd, env=env, cwd=src_dir) if post in [True, None]: if post: @@ -830,7 +827,7 @@ def order(f): def clean_pkg_cache(dist, timeout): cc.pkgs_dirs = cc.pkgs_dirs[:1] - locks = [get_lock(join(folder, ".conda_lock"), timeout=timeout) for folder in cc.pkgs_dirs] + locks = [get_lock(folder, timeout=timeout) for folder in cc.pkgs_dirs] with ExitStack() as stack: for lock in locks: stack.enter_context(lock) @@ -874,116 +871,115 @@ def test(m, config, move_broken=True): clean_pkg_cache(m.dist(), config.timeout) - with get_lock(join(config.build_folder, ".conda_lock"), timeout=config.timeout): - tmp_dir = config.test_dir - if not isdir(tmp_dir): - os.makedirs(tmp_dir) - create_files(tmp_dir, m, config) - # Make Perl or Python-specific test files - if m.name().startswith('perl-'): - pl_files = create_pl_files(tmp_dir, m) - py_files = False - lua_files = False - else: - py_files = create_py_files(tmp_dir, m) - pl_files = False - lua_files = False - shell_files = create_shell_files(tmp_dir, m, config) - if not (py_files or shell_files or pl_files or lua_files): - print("Nothing to test for:", m.dist()) - return + tmp_dir = config.test_dir + if not isdir(tmp_dir): + os.makedirs(tmp_dir) + create_files(tmp_dir, m, config) + # Make Perl or Python-specific test files + if m.name().startswith('perl-'): + pl_files = create_pl_files(tmp_dir, m) + py_files = False + lua_files = False + else: + py_files = create_py_files(tmp_dir, m) + pl_files = False + lua_files = False + shell_files = create_shell_files(tmp_dir, m, config) + if not (py_files or shell_files or pl_files or lua_files): + print("Nothing to test for:", m.dist()) + return - print("TEST START:", m.dist()) + print("TEST START:", m.dist()) - get_build_metadata(m, config=config) - specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())] + get_build_metadata(m, config=config) + specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())] - # add packages listed in the run environment and test/requires - specs.extend(ms.spec for ms in m.ms_depends('run')) - specs += ensure_list(m.get_value('test/requires', [])) + # add packages listed in the run environment and test/requires + specs.extend(ms.spec for ms in m.ms_depends('run')) + specs += ensure_list(m.get_value('test/requires', [])) - if py_files: - # as the tests are run by python, ensure that python is installed. - # (If they already provided python as a run or test requirement, - # this won't hurt anything.) - specs += ['python %s*' % environ.get_py_ver(config)] - if pl_files: - # as the tests are run by perl, we need to specify it - specs += ['perl %s*' % environ.get_perl_ver(config)] - if lua_files: - # not sure how this shakes out - specs += ['lua %s*' % environ.get_lua_ver(config)] + if py_files: + # as the tests are run by python, ensure that python is installed. + # (If they already provided python as a run or test requirement, + # this won't hurt anything.) + specs += ['python %s*' % environ.get_py_ver(config)] + if pl_files: + # as the tests are run by perl, we need to specify it + specs += ['perl %s*' % environ.get_perl_ver(config)] + if lua_files: + # not sure how this shakes out + specs += ['lua %s*' % environ.get_lua_ver(config)] - create_env(config.test_prefix, specs, config=config) + create_env(config.test_prefix, specs, config=config) - with path_prepended(config.test_prefix): - env = dict(os.environ.copy()) - env.update(environ.get_dict(config=config, m=m, prefix=config.test_prefix)) - env["CONDA_BUILD_STATE"] = "TEST" + with path_prepended(config.test_prefix): + env = dict(os.environ.copy()) + env.update(environ.get_dict(config=config, m=m, prefix=config.test_prefix)) + env["CONDA_BUILD_STATE"] = "TEST" - if not config.activate: - # prepend bin (or Scripts) directory - env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True) + if not config.activate: + # prepend bin (or Scripts) directory + env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True) + if on_win: + env['PATH'] = config.test_prefix + os.pathsep + env['PATH'] + + for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA': + env[varname] = str(getattr(config, varname) or '') + + # Python 2 Windows requires that envs variables be string, not unicode + env = {str(key): str(value) for key, value in env.items()} + suffix = "bat" if on_win else "sh" + test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix)) + + with open(test_script, 'w') as tf: + if config.activate: + ext = ".bat" if on_win else "" + tf.write('{source} "{conda_root}activate{ext}" "{test_env}" {squelch}\n'.format( + conda_root=root_script_dir + os.path.sep, + source="call" if on_win else "source", + ext=ext, + test_env=config.test_prefix, + squelch=">nul 2>&1" if on_win else "&> /dev/null")) if on_win: - env['PATH'] = config.test_prefix + os.pathsep + env['PATH'] - - for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA': - env[varname] = str(getattr(config, varname) or '') - - # Python 2 Windows requires that envs variables be string, not unicode - env = {str(key): str(value) for key, value in env.items()} - suffix = "bat" if on_win else "sh" - test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix)) - - with open(test_script, 'w') as tf: - if config.activate: - ext = ".bat" if on_win else "" - tf.write('{source} "{conda_root}activate{ext}" "{test_env}" {squelch}\n'.format( - conda_root=root_script_dir + os.path.sep, - source="call" if on_win else "source", - ext=ext, - test_env=config.test_prefix, - squelch=">nul 2>&1" if on_win else "&> /dev/null")) - if on_win: - tf.write("if errorlevel 1 exit 1\n") - if py_files: - tf.write("{python} -s {test_file}\n".format( - python=config.test_python, - test_file=join(tmp_dir, 'run_test.py'))) - if on_win: - tf.write("if errorlevel 1 exit 1\n") - if pl_files: - tf.write("{perl} {test_file}\n".format( - perl=config.test_perl, - test_file=join(tmp_dir, 'run_test.pl'))) - if on_win: - tf.write("if errorlevel 1 exit 1\n") - if lua_files: - tf.write("{lua} {test_file}\n".format( - lua=config.test_lua, - test_file=join(tmp_dir, 'run_test.lua'))) + tf.write("if errorlevel 1 exit 1\n") + if py_files: + tf.write("{python} -s {test_file}\n".format( + python=config.test_python, + test_file=join(tmp_dir, 'run_test.py'))) + if on_win: + tf.write("if errorlevel 1 exit 1\n") + if pl_files: + tf.write("{perl} {test_file}\n".format( + perl=config.test_perl, + test_file=join(tmp_dir, 'run_test.pl'))) + if on_win: + tf.write("if errorlevel 1 exit 1\n") + if lua_files: + tf.write("{lua} {test_file}\n".format( + lua=config.test_lua, + test_file=join(tmp_dir, 'run_test.lua'))) + if on_win: + tf.write("if errorlevel 1 exit 1\n") + if shell_files: + test_file = join(tmp_dir, 'run_test.' + suffix) + if on_win: + tf.write("call {test_file}\n".format(test_file=test_file)) if on_win: tf.write("if errorlevel 1 exit 1\n") - if shell_files: - test_file = join(tmp_dir, 'run_test.' + suffix) - if on_win: - tf.write("call {test_file}\n".format(test_file=test_file)) - if on_win: - tf.write("if errorlevel 1 exit 1\n") - else: - # TODO: Run the test/commands here instead of in run_test.py - tf.write("{shell_path} -x -e {test_file}\n".format(shell_path=shell_path, - test_file=test_file)) + else: + # TODO: Run the test/commands here instead of in run_test.py + tf.write("{shell_path} -x -e {test_file}\n".format(shell_path=shell_path, + test_file=test_file)) - if on_win: - cmd = ['cmd.exe', "/d", "/c", test_script] - else: - cmd = [shell_path, '-x', '-e', test_script] - try: - subprocess.check_call(cmd, env=env, cwd=tmp_dir) - except subprocess.CalledProcessError: - tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config) + if on_win: + cmd = ['cmd.exe', "/d", "/c", test_script] + else: + cmd = [shell_path, '-x', '-e', test_script] + try: + subprocess.check_call(cmd, env=env, cwd=tmp_dir) + except subprocess.CalledProcessError: + tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config) print("TEST END:", m.dist()) diff --git a/conda_build/index.py b/conda_build/index.py index fa71df22cb..240a318c6b 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -19,8 +19,7 @@ def read_index_tar(tar_path, config, lock=None): """ Returns the index.json dict inside the given package tarball. """ if not lock: - lock = get_lock(join(os.path.dirname(tar_path), ".conda_lock"), - timeout=config.timeout) + lock = get_lock(os.path.dirname(tar_path), timeout=config.timeout) with ExitStack() as stack: stack.enter_context(lock) t = tarfile.open(tar_path) @@ -77,7 +76,7 @@ def update_index(dir_path, config, force=False, check_md5=False, remove=True, lo os.makedirs(dir_path) if not lock: - lock = get_lock(join(dir_path, ".conda_lock")) + lock = get_lock(dir_path) with lock: if force: diff --git a/conda_build/utils.py b/conda_build/utils.py index 5e049b2316..a289ea02b5 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -111,7 +111,7 @@ def copy_into(src, dst, timeout=90, symlinks=False, lock=None): src_folder = os.getcwd() if not lock: - lock = get_lock(join(src_folder, ".conda_lock"), timeout=timeout) + lock = get_lock(src_folder, timeout=timeout) with lock: # with each of these, we are copying less metadata. This seems to be necessary # to cope with some shared filesystems with some virtual machine setups. @@ -190,7 +190,7 @@ def merge_tree(src, dst, symlinks=False, timeout=90, lock=None): "{2}".format(src, dst, existing[0])) if not lock: - lock = get_lock(join(src, ".conda_lock"), timeout=timeout) + lock = get_lock(src, timeout=timeout) with lock: copytree(src, dst, symlinks=symlinks) @@ -201,13 +201,14 @@ def merge_tree(src, dst, symlinks=False, timeout=90, lock=None): _locations = {} -def get_lock(lock_file, timeout=90): +def get_lock(folder, timeout=90, filename=".conda_lock"): global _locations - location = os.path.abspath(os.path.normpath(lock_file)) - if not os.path.isdir(os.path.dirname(location)): - os.makedirs(os.path.dirname(location)) + location = os.path.abspath(os.path.normpath(folder)) + if not os.path.isdir(location): + os.makedirs(location) if location not in _locations: - _locations[location] = filelock.SoftFileLock(location, timeout) + _locations[location] = filelock.SoftFileLock(os.path.join(location, filename), + timeout) return _locations[location] From 4ca2fda236051ccb07ff70a8468bea2639baae75 Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Wed, 2 Nov 2016 16:48:59 -0400 Subject: [PATCH 091/156] Update to ensure_valid_license_family so that existing recipes don't break --- conda_build/license_family.py | 14 ++++++++++++++ conda_build/metadata.py | 15 +-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/conda_build/license_family.py b/conda_build/license_family.py index d68398d124..cdd5251c11 100644 --- a/conda_build/license_family.py +++ b/conda_build/license_family.py @@ -3,6 +3,8 @@ from difflib import get_close_matches import re import string +from conda_build import exceptions +from conda_build.utils import comma_join allowed_license_families = """ AGPL @@ -94,6 +96,18 @@ def guess_license_family(license_name=None, return 'OTHER' +def ensure_valid_license_family(meta): + try: + license_family = meta['about']['license_family'] + except KeyError: + return + if (remove_special_characters(normalize(license_family)) + not in allowed_license_families): + raise RuntimeError(exceptions.indent( + "about/license_family '%s' not allowed. Allowed families are %s." % + (license_family, comma_join(sorted(allowed_license_families))))) + + def deprecated_guess_license_family(license_name, recognized=allowed_license_families): """Deprecated guess of license_family from license diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 3fbb7ca1e0..642a002f7c 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -16,7 +16,7 @@ from conda_build.features import feature_list from conda_build.config import Config from conda_build.utils import rec_glob, ensure_list -from conda_build.license_family import allowed_license_families +from conda_build.license_family import ensure_valid_license_family try: import yaml @@ -30,8 +30,6 @@ sys.exit('Error: could not import yaml (required to read meta.yaml ' 'files of conda recipes)') -from conda_build.utils import comma_join - on_win = (sys.platform == 'win32') log = logging.getLogger(__file__) @@ -137,17 +135,6 @@ def yamlize(data): raise exceptions.UnableToParse(original=e) -def ensure_valid_license_family(meta): - try: - license_family = meta['about']['license_family'] - except KeyError: - return - if license_family not in allowed_license_families: - raise RuntimeError(exceptions.indent( - "about/license_family '%s' not allowed. Allowed families are %s." % - (license_family, comma_join(sorted(allowed_license_families))))) - - def ensure_valid_fields(meta): try: pin_depends = meta['build']['pin_depends'] From 698147b4f679727e8135ef84ee0b460fc124757a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Nov 2016 18:29:17 -0500 Subject: [PATCH 092/156] fix bits being used where arch needed to be --- conda_build/config.py | 8 ++++---- conda_build/environ.py | 6 +++--- conda_build/utils.py | 5 +++-- tests/test_config.py | 14 +++++++------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/conda_build/config.py b/conda_build/config.py index d851c0abde..2980355173 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -12,7 +12,7 @@ import time from .conda_interface import string_types, binstar_upload -from .conda_interface import root_dir, root_writable, cc, bits, platform +from .conda_interface import root_dir, root_writable, cc, subdir, platform from .utils import get_build_folders, rm_rf @@ -105,7 +105,7 @@ def env(lang, default): Setting('verbose', False), Setting('debug', False), Setting('timeout', 90), - Setting('bits', bits), + Setting('arch', subdir.split('-')[-1]), Setting('platform', platform), Setting('set_build_id', True), Setting('disable_pip', False) @@ -124,14 +124,14 @@ def subdir(self): if self.platform == 'noarch': return self.platform else: - return "-".join([self.platform, str(self.bits)]) + return "-".join([self.platform, str(self.arch)]) @subdir.setter def subdir(self, value): values = value.split('-') self.platform = values[0] if len(values) > 1: - self.bits = values[1] + self.arch = values[1] @property def croot(self): diff --git a/conda_build/environ.py b/conda_build/environ.py index 710d9f13dc..14f114512a 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -235,7 +235,7 @@ def conda_build_vars(prefix, config): 'CONDA_BUILD': '1', 'PYTHONNOUSERSITE': '1', 'CONDA_DEFAULT_ENV': config.build_prefix, - 'ARCH': str(config.bits), + 'ARCH': str(config.arch), 'PREFIX': prefix, 'SYS_PREFIX': sys.prefix, 'SYS_PYTHON': sys.executable, @@ -383,7 +383,7 @@ def unix_vars(prefix): def osx_vars(compiler_vars, config): - OSX_ARCH = 'i386' if config.bits == 32 else 'x86_64' + OSX_ARCH = 'i386' if config.arch == 32 else 'x86_64' compiler_vars['CFLAGS'] += ' -arch {0}'.format(OSX_ARCH) compiler_vars['CXXFLAGS'] += ' -arch {0}'.format(OSX_ARCH) compiler_vars['LDFLAGS'] += ' -arch {0}'.format(OSX_ARCH) @@ -398,7 +398,7 @@ def osx_vars(compiler_vars, config): def linux_vars(compiler_vars, prefix, config): compiler_vars['LD_RUN_PATH'] = prefix + '/lib' - if config.bits == 32: + if config.arch == 32: compiler_vars['CFLAGS'] += ' -m32' compiler_vars['CXXFLAGS'] += ' -m32' return {} diff --git a/conda_build/utils.py b/conda_build/utils.py index a289ea02b5..60cfb24eee 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -25,7 +25,7 @@ from .conda_interface import md5_file, unix_path_to_win, win_path_to_unix from .conda_interface import PY3, iteritems from .conda_interface import linked -from .conda_interface import bits, root_dir +from .conda_interface import root_dir from conda_build.os_utils import external @@ -515,7 +515,8 @@ def create_entry_point(path, module, func, config): if 'debug' in packages_names: fo.write('#!python_d\n') fo.write(pyscript) - copy_into(join(dirname(__file__), 'cli-%d.exe' % bits), path + '.exe', config.timeout) + copy_into(join(dirname(__file__), 'cli-%d.exe' % config.arch), + path + '.exe', config.timeout) else: with open(path, 'w') as fo: fo.write('#!%s\n' % config.build_python) diff --git a/tests/test_config.py b/tests/test_config.py index 608c8abae6..1ce1d1051c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -53,23 +53,23 @@ def test_create_config_with_subdir(): def test_set_platform(config): config.platform = 'steve' - bits = config.bits - assert config.subdir == 'steve-' + str(bits) + arch = config.arch + assert config.subdir == 'steve-' + str(arch) def test_set_subdir(config): config.subdir = 'steve' - bits = config.bits - assert config.subdir == 'steve-' + str(bits) + arch = config.arch + assert config.subdir == 'steve-' + str(arch) assert config.platform == 'steve' config.subdir = 'steve-128' assert config.subdir == 'steve-128' assert config.platform == 'steve' - assert config.bits == '128' + assert config.arch == '128' def test_set_bits(config): - config.bits = 128 + config.arch = 128 assert config.subdir == config.platform + '-' + str(128) - assert config.bits == 128 + assert config.arch == 128 From 3d598dcb80defa6e5a9cfacf4b0898178daa9ac6 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 3 Nov 2016 08:18:47 -0500 Subject: [PATCH 093/156] fix win cli exe str format for arch --- conda_build/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index 60cfb24eee..0414bab65c 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -515,7 +515,7 @@ def create_entry_point(path, module, func, config): if 'debug' in packages_names: fo.write('#!python_d\n') fo.write(pyscript) - copy_into(join(dirname(__file__), 'cli-%d.exe' % config.arch), + copy_into(join(dirname(__file__), 'cli-{}.exe'.format(config.arch)), path + '.exe', config.timeout) else: with open(path, 'w') as fo: From ae1b136f4a873f3f5cb6f17f5a0390680c12ae20 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 3 Nov 2016 09:51:29 -0500 Subject: [PATCH 094/156] update changelog for 2.0.8 --- CHANGELOG.txt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 98732ccb83..f73af0a95f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,11 +1,21 @@ -2016-11-01 2.0.8: +2016-11-03 2.0.8: ----------------- Enhancements: ------------- -* Improved logic to guess the appropriate license_family to add to package's index. This improves filtering. -* Logic for the license_family is now shared between open-source conda-build, and proprietary cas-mirror packages. +* Support otool -h changes in MacOS 10.12 #1479 +* Fix lists of empty strings created by ensure_list (patches failing due to empty patch list) #1493 +* Improved logic to guess the appropriate license_family to add to package's index. This improves filtering. #1495 #1503 +* Logic for the license_family is now shared between open-source conda-build, and proprietary cas-mirror packages. #1495 #1503 + +Bug fixes: +---------- + +* Centralize locks in memory to avoid lock timeouts within a single process #1496 +* fix overly broad regex in detecting whether a recipe uses version control systems #1498 +* clarify error message when extracting egg fails due to overlapping file names #1500 +* fix regression where subdir was not respecting non-x86 arch (values other than 32 or 64) #1506 Contributors: ------------- @@ -14,6 +24,7 @@ Contributors: * @minrk * @msarahan + 2016-10-24 2.0.7: ----------------- From 04d996bc87985f8a76af234ffdebc566f8d5b7d6 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Prinz Date: Fri, 4 Nov 2016 17:13:19 +0100 Subject: [PATCH 095/156] add creation of intermediate folder for utils.copy_into --- conda_build/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conda_build/utils.py b/conda_build/utils.py index 0414bab65c..392e2a11cd 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -113,6 +113,11 @@ def copy_into(src, dst, timeout=90, symlinks=False, lock=None): if not lock: lock = get_lock(src_folder, timeout=timeout) with lock: + # if intermediate folders not not exist create them + dst_folder = os.path.dirname(dst) + if not os.path.exists(dst_folder): + os.makedirs(dst_folder) + # with each of these, we are copying less metadata. This seems to be necessary # to cope with some shared filesystems with some virtual machine setups. # See https://github.com/conda/conda-build/issues/1426 From bfe25468892d97396d9906ab92c13c95834c05e7 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Prinz Date: Fri, 4 Nov 2016 20:43:29 +0100 Subject: [PATCH 096/156] fix potential empty path --- conda_build/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index 392e2a11cd..eb2f458662 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -115,8 +115,11 @@ def copy_into(src, dst, timeout=90, symlinks=False, lock=None): with lock: # if intermediate folders not not exist create them dst_folder = os.path.dirname(dst) - if not os.path.exists(dst_folder): - os.makedirs(dst_folder) + if dst_folder and not os.path.exists(dst_folder): + try: + os.makedirs(dst_folder) + except OSError: + pass # with each of these, we are copying less metadata. This seems to be necessary # to cope with some shared filesystems with some virtual machine setups. From 07245d1d6b63085d449786d73c18cc1481391496 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 6 Nov 2016 14:44:22 -0600 Subject: [PATCH 097/156] break build string construction out into standalone function for external usage --- conda_build/metadata.py | 67 ++++++++++++++++++++++------------------- conda_build/render.py | 3 ++ 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 4b9a201f01..f632a55102 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -338,6 +338,39 @@ def handle_config_version(ms, ver, dep_type='run'): return MatchSpec('%s %s*' % (ms.name, ver)) +def build_string_from_metadata(metadata): + res = [] + version_pat = re.compile(r'(?:==)?(\d+)\.(\d+)') + for name, s in (('numpy', 'np'), ('python', 'py'), + ('perl', 'pl'), ('lua', 'lua'), + ('r', 'r'), ('r-base', 'r')): + for ms in metadata.ms_depends(): + if ms.name == name: + try: + v = ms.spec.split()[1] + except IndexError: + if name not in ['numpy']: + res.append(s) + break + if any(i in v for i in ',|>!<'): + break + if name not in ['perl', 'lua', 'r', 'r-base']: + match = version_pat.match(v) + if match: + res.append(s + match.group(1) + match.group(2)) + else: + res.append(s + v.strip('*')) + break + + features = ensure_list(metadata.get_value('build/features', [])) + if res: + res.append('_') + if features: + res.extend(('_'.join(features), '_')) + res.append('{0}'.format(metadata.build_number() if metadata.build_number() else 0)) + return "".join(res) + + def find_recipe(path): """recurse through a folder, locating meta.yaml. Raises error if more than one is found. @@ -582,37 +615,9 @@ def build_id(self): ret = self.get_value('build/string') if ret: check_bad_chrs(ret, 'build/string') - return ret - res = [] - version_pat = re.compile(r'(?:==)?(\d+)\.(\d+)') - for name, s in (('numpy', 'np'), ('python', 'py'), - ('perl', 'pl'), ('lua', 'lua'), - ('r', 'r'), ('r-base', 'r')): - for ms in self.ms_depends(): - if ms.name == name: - try: - v = ms.spec.split()[1] - except IndexError: - if name not in ['numpy']: - res.append(s) - break - if any(i in v for i in ',|>!<'): - break - if name not in ['perl', 'lua', 'r', 'r-base']: - match = version_pat.match(v) - if match: - res.append(s + match.group(1) + match.group(2)) - else: - res.append(s + v.strip('*')) - break - - features = ensure_list(self.get_value('build/features', [])) - if res: - res.append('_') - if features: - res.extend(('_'.join(features), '_')) - res.append('{0}'.format(self.build_number() if self.build_number() else 0)) - return ''.join(res) + else: + ret = build_string_from_metadata(self) + return ret def dist(self): return '%s-%s-%s' % (self.name(), self.version(), self.build_id()) diff --git a/conda_build/render.py b/conda_build/render.py index a2ea05ae9b..70364baeb8 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -132,6 +132,9 @@ def render_recipe(recipe_path, config, no_download_source=False): t.extractall(path=recipe_dir) t.close() need_cleanup = True + elif arg.endswith('.yaml'): + recipe_dir = os.path.dirname(arg) + need_cleanup = False else: print("Ignoring non-recipe: %s" % arg) return From 8b8950f7aea4b85cfed5de5898b8bbcfa65aacd2 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 7 Nov 2016 15:06:07 -0600 Subject: [PATCH 098/156] Use version info from config if exists --- conda_build/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/api.py b/conda_build/api.py index 9e4956f65f..e3914f4a86 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -159,6 +159,7 @@ def skeletonize(packages, repo, output_dir=".", version=None, recursive=False, """Generate a conda recipe from an external repo. Translates metadata from external sources into expected conda recipe format.""" + version = getattr(config, "version", version) if version: kwargs.update({'version': version}) if recursive: From ab5ba28613bf90ba156a88d583b379f8b701dcfe Mon Sep 17 00:00:00 2001 From: sophia Date: Sat, 22 Oct 2016 17:21:21 -0500 Subject: [PATCH 099/156] Enable users to verify recipes and packages when they build packages; - verify scripts (defined by the user) will be run by default but this can be disabled by using the `--no-verify` parameter - the location of the verify scripts is assumed to be `$PREFIX/verify but can be overridden by setting `conda_build_verify_scripts` in `.condarc` Inside the verify folder - the verify folder should have a `package` and `recipe` folder. The scripts in the folders will be run to verify packages and recipes respectively - a package verify script should have a function `verify` that takes a path to the package (.tar.bz2 file) - a recipe should have a function `verify` that takes the rendered `meta.yaml` and a path to the recipe dir --- conda_build/build.py | 6 ++++ conda_build/cli/main_build.py | 8 +++++- conda_build/conda_interface.py | 2 +- conda_build/config.py | 13 +++++++++ conda_build/exceptions.py | 8 ++++++ conda_build/verify.py | 51 ++++++++++++++++++++++++++++++++++ tests/utils.py | 3 +- 7 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 conda_build/verify.py diff --git a/conda_build/build.py b/conda_build/build.py index 6a40b975eb..7771462736 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -59,6 +59,7 @@ from conda_build.features import feature_list import conda_build.noarch_python as noarch_python +from .verify import verify_package, verify_recipe if 'bsd' in sys.platform: @@ -818,6 +819,9 @@ def order(f): copy_into(tmp_path, path, config.timeout) update_index(config.bldpkgs_dir, config, could_be_mirror=False) + if not getattr(config, "noverify", False): + verify_package(path, config) + else: print("STOPPING BUILD BEFORE POST:", m.dist()) @@ -1062,6 +1066,8 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, recipe_config.compute_build_id(os.path.basename(recipe), reset=True) metadata, need_source_download, need_reparse_in_env = render_recipe(recipe, config=recipe_config) + if not getattr(config, "noverify", False): + verify_recipe(metadata, config) try: with recipe_config: ok_to_test = build(metadata, post=post, diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 43264a3a1c..fe14f12eff 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -153,6 +153,12 @@ def parse_args(args): help=("Build root folder. Equivalent to CONDA_BLD_PATH, but applies only " "to this call of conda-build.") ) + p.add_argument( + "--no-verify", + action="store_true", + dest="verify", + help=("do not run verification on recipes or packages when building") + ) add_parser_channels(p) @@ -231,7 +237,7 @@ def execute(args): else: api.build(args.recipe, post=args.post, build_only=args.build_only, notest=args.notest, keep_old_work=args.keep_old_work, - already_built=None, config=config) + already_built=None, config=config, noverify=args.verify) if not args.output and len(build.get_build_folders(config.croot)) > 0: build.print_build_intermediate_warning(config) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index d565ef8173..d6af5429e9 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -22,7 +22,7 @@ from conda.signature import KEYS, KEYS_DIR, hash_file, verify # NOQA from conda.utils import human_bytes, hashsum_file, md5_file, memoized, unix_path_to_win, win_path_to_unix, url_path # NOQA import conda.config as cc # NOQA -from conda.config import sys_rc_path # NOQA +from conda.config import rc_path # NOQA from conda.version import VersionOrder # NOQA if parse_version(conda.__version__) >= parse_version("4.2"): diff --git a/conda_build/config.py b/conda_build/config.py index 2980355173..6ca125a1ea 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -10,9 +10,12 @@ from os.path import abspath, expanduser, join import sys import time +import yaml from .conda_interface import string_types, binstar_upload from .conda_interface import root_dir, root_writable, cc, subdir, platform +from .conda_interface import string_types, binstar_upload, rc_path +from .conda_interface import root_dir, root_writable, cc, subdir, bits, platform, from .utils import get_build_folders, rm_rf @@ -24,6 +27,7 @@ # changes. DEFAULT_PREFIX_LENGTH = 255 +DEFAULT_VERIFY_SCRIPTS_PATH = os.path.join(sys.prefix, 'verify') def _ensure_dir(path): @@ -364,6 +368,15 @@ def test_dir(self): _ensure_dir(path) return path + @property + def verify_scripts_path(self): + if os.path.isfile(rc_path): + with open(rc_path, 'r') as f: + condarc = yaml.load(f) + if condarc.get("conda_build_verify_scripts"): + return condarc.get("conda_build_verify_scripts") + return DEFAULT_VERIFY_SCRIPTS_PATH + def clean(self): # build folder is the whole burrito containing envs and source folders # It will only exist if we download source, or create a build or test environment diff --git a/conda_build/exceptions.py b/conda_build/exceptions.py index b2b99fccc0..6710d7feca 100644 --- a/conda_build/exceptions.py +++ b/conda_build/exceptions.py @@ -48,3 +48,11 @@ def error_body(self): class UnableToParseMissingSetuptoolsDependencies(CondaBuildException): pass + + +class VerifyError(RuntimeError): + def __init__(self, error, script, *args): + self.error = error + self.script = script + msg = "%s failed to verify\n%s" % (script, error) + super(VerifyError, self).__init__(msg) diff --git a/conda_build/verify.py b/conda_build/verify.py new file mode 100644 index 0000000000..d9a981a996 --- /dev/null +++ b/conda_build/verify.py @@ -0,0 +1,51 @@ +import os +from .exceptions import VerifyError +import sys + + +def can_import_importlib(): + return sys.version_info.major >= 3 and sys.version_info.minor >= 3 + + +if can_import_importlib(): + from importlib.machinery import SourceFileLoader +else: + # imp.load_source has been deprecated in python3.3. So importlib should be used for versions + # for versions of python greater than 3.3 + import imp + + +def list_verify_script(path_to_script): + import pkgutil + verify_modules = pkgutil.iter_modules([path_to_script]) + files = [] + for loader, name, _ in verify_modules: + files.append(os.path.join(loader.path, name)) + return files + + +def verify(verify_path, *args): + verify_scripts = list_verify_script(verify_path) + + for verify_script in verify_scripts: + script = "%s.py"%verify_script + print("Running script %s" % script) + if can_import_importlib(): + mod = SourceFileLoader("test", verify_script).load_module() + else: + mod = imp.load_source("test", script) + try: + mod.verify(*args) + except AttributeError as e: + raise VerifyError(e, script) + print("All scripts passed") + + +def verify_package(path_to_package, config): + verify_path = os.path.join(config.verify_scripts_path, "package") + verify(verify_path, path_to_package) + + +def verify_recipe(recipe, config): + verify_path = os.path.join(config.verify_scripts_path, "recipe") + verify(verify_path, recipe.meta, recipe.path) diff --git a/tests/utils.py b/tests/utils.py index 156702b288..b73c39593e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -55,7 +55,8 @@ def return_to_saved_path(): @pytest.fixture(scope='function') def test_config(testing_workdir, request): - return Config(croot=testing_workdir, anaconda_upload=False, verbose=True, activate=False) + return Config(croot=testing_workdir, anaconda_upload=False, verbose=True, activate=False, + noverify=True) @pytest.fixture(scope='function') From 4ab9e0da5bb9a570871e5d3e489d9eef7ae38fbb Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 31 Oct 2016 14:25:36 -0500 Subject: [PATCH 100/156] Test verify module --- conda_build/config.py | 5 ++-- conda_build/verify.py | 9 +++---- .../package/test_package.py | 2 ++ .../package/test_package_2.py | 2 ++ .../test-verify-scripts/recipe/test_recipe.py | 3 +++ tests/test_verify.py | 27 +++++++++++++++++++ tests/utils.py | 4 ++- 7 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 tests/test-verify-scripts/package/test_package.py create mode 100644 tests/test-verify-scripts/package/test_package_2.py create mode 100644 tests/test-verify-scripts/recipe/test_recipe.py create mode 100644 tests/test_verify.py diff --git a/conda_build/config.py b/conda_build/config.py index 6ca125a1ea..6e0c85e920 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -112,7 +112,8 @@ def env(lang, default): Setting('arch', subdir.split('-')[-1]), Setting('platform', platform), Setting('set_build_id', True), - Setting('disable_pip', False) + Setting('disable_pip', False), + Setting('default_verify_scripts_path', DEFAULT_VERIFY_SCRIPTS_PATH) ] # handle known values better than unknown (allow defaults) @@ -375,7 +376,7 @@ def verify_scripts_path(self): condarc = yaml.load(f) if condarc.get("conda_build_verify_scripts"): return condarc.get("conda_build_verify_scripts") - return DEFAULT_VERIFY_SCRIPTS_PATH + return self.default_verify_scripts_path def clean(self): # build folder is the whole burrito containing envs and source folders diff --git a/conda_build/verify.py b/conda_build/verify.py index d9a981a996..2a915eb79e 100644 --- a/conda_build/verify.py +++ b/conda_build/verify.py @@ -20,7 +20,7 @@ def list_verify_script(path_to_script): verify_modules = pkgutil.iter_modules([path_to_script]) files = [] for loader, name, _ in verify_modules: - files.append(os.path.join(loader.path, name)) + files.append("%s.py" % os.path.join(loader.path, name)) return files @@ -28,16 +28,15 @@ def verify(verify_path, *args): verify_scripts = list_verify_script(verify_path) for verify_script in verify_scripts: - script = "%s.py"%verify_script - print("Running script %s" % script) + print("Running script %s" % verify_script) if can_import_importlib(): mod = SourceFileLoader("test", verify_script).load_module() else: - mod = imp.load_source("test", script) + mod = imp.load_source("test", verify_script) try: mod.verify(*args) except AttributeError as e: - raise VerifyError(e, script) + raise VerifyError(e, verify_script) print("All scripts passed") diff --git a/tests/test-verify-scripts/package/test_package.py b/tests/test-verify-scripts/package/test_package.py new file mode 100644 index 0000000000..ecd9afeaf3 --- /dev/null +++ b/tests/test-verify-scripts/package/test_package.py @@ -0,0 +1,2 @@ +def verify(path_to_package): + print(path_to_package) diff --git a/tests/test-verify-scripts/package/test_package_2.py b/tests/test-verify-scripts/package/test_package_2.py new file mode 100644 index 0000000000..dc829898ac --- /dev/null +++ b/tests/test-verify-scripts/package/test_package_2.py @@ -0,0 +1,2 @@ +def verify(path_to_package): + print(path_to_package) \ No newline at end of file diff --git a/tests/test-verify-scripts/recipe/test_recipe.py b/tests/test-verify-scripts/recipe/test_recipe.py new file mode 100644 index 0000000000..ab4bae6746 --- /dev/null +++ b/tests/test-verify-scripts/recipe/test_recipe.py @@ -0,0 +1,3 @@ +def verify(recipe, path_to_recipe): + print(recipe) + print(path_to_recipe) diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100644 index 0000000000..c0872a0a9c --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,27 @@ +import unittest +from os.path import join, dirname +from conda_build import verify + + +class TestVerify(unittest.TestCase): + def test_list_verify_script(self): + path_to_scripts = join(dirname(__file__), "test-verify-scripts/package") + files = verify.list_verify_script(path_to_scripts) + expected_scripts = [join(path_to_scripts, "test_package.py"), + join(path_to_scripts, "test_package_2.py")] + self.assertListEqual(files, expected_scripts) + + path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") + files = verify.list_verify_script(path_to_scripts) + self.assertListEqual(files, [join(path_to_scripts, "test_recipe.py")]) + + def test_verify(self): + path_to_scripts = join(dirname(__file__), "test-verify-scripts/package") + verify.verify(path_to_scripts, "path_to_package_arg") + + path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") + verify.verify(path_to_scripts, "recipe_arg", "path_to_recipe_arg") + + def test_verify_no_script(self): + path_to_script = "non/sense" + verify.verify(path_to_script) diff --git a/tests/utils.py b/tests/utils.py index b73c39593e..743757cbbd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,7 @@ from collections import defaultdict import contextlib import os +from os.path import join, dirname import stat import subprocess import sys @@ -56,7 +57,8 @@ def return_to_saved_path(): @pytest.fixture(scope='function') def test_config(testing_workdir, request): return Config(croot=testing_workdir, anaconda_upload=False, verbose=True, activate=False, - noverify=True) + noverify=False, + default_verify_scripts_path=join(dirname(__file__), "test-verify-scripts")) @pytest.fixture(scope='function') From caded82e54ff7d0f6e41fb7acc8ea37f75eb7cbe Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 1 Nov 2016 14:18:33 -0500 Subject: [PATCH 101/156] Test verify error --- conda_build/cli/main_build.py | 3 +-- conda_build/exceptions.py | 6 +++--- conda_build/verify.py | 2 +- tests/test_verify.py | 12 +++++++++++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index fe14f12eff..4fadbd3780 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -156,7 +156,6 @@ def parse_args(args): p.add_argument( "--no-verify", action="store_true", - dest="verify", help=("do not run verification on recipes or packages when building") ) @@ -237,7 +236,7 @@ def execute(args): else: api.build(args.recipe, post=args.post, build_only=args.build_only, notest=args.notest, keep_old_work=args.keep_old_work, - already_built=None, config=config, noverify=args.verify) + already_built=None, config=config, noverify=args.no_verify) if not args.output and len(build.get_build_folders(config.croot)) > 0: build.print_build_intermediate_warning(config) diff --git a/conda_build/exceptions.py b/conda_build/exceptions.py index 6710d7feca..7328c7bc3c 100644 --- a/conda_build/exceptions.py +++ b/conda_build/exceptions.py @@ -50,9 +50,9 @@ class UnableToParseMissingSetuptoolsDependencies(CondaBuildException): pass -class VerifyError(RuntimeError): +class VerifyError(CondaBuildException): def __init__(self, error, script, *args): self.error = error self.script = script - msg = "%s failed to verify\n%s" % (script, error) - super(VerifyError, self).__init__(msg) + self.msg = "%s failed to verify\n%s" % (script, error) + super(VerifyError, self).__init__(self.msg) diff --git a/conda_build/verify.py b/conda_build/verify.py index 2a915eb79e..ed9c6343c2 100644 --- a/conda_build/verify.py +++ b/conda_build/verify.py @@ -35,7 +35,7 @@ def verify(verify_path, *args): mod = imp.load_source("test", verify_script) try: mod.verify(*args) - except AttributeError as e: + except TypeError as e: raise VerifyError(e, verify_script) print("All scripts passed") diff --git a/tests/test_verify.py b/tests/test_verify.py index c0872a0a9c..4f22dee47f 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -1,6 +1,7 @@ import unittest from os.path import join, dirname from conda_build import verify +from conda_build.exceptions import VerifyError class TestVerify(unittest.TestCase): @@ -13,7 +14,8 @@ def test_list_verify_script(self): path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") files = verify.list_verify_script(path_to_scripts) - self.assertListEqual(files, [join(path_to_scripts, "test_recipe.py")]) + expected_scripts = [join(path_to_scripts, "test_recipe.py")] + self.assertListEqual(files, expected_scripts) def test_verify(self): path_to_scripts = join(dirname(__file__), "test-verify-scripts/package") @@ -25,3 +27,11 @@ def test_verify(self): def test_verify_no_script(self): path_to_script = "non/sense" verify.verify(path_to_script) + + def test_verify_error_script(self): + path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") + try: + verify.verify(path_to_scripts, "recipe_arg") + except VerifyError as err: + self.assertTrue(err.error, TypeError) + self.assertEquals(err.script, join(path_to_scripts, "test_recipe.py")) From 2eb6873e1786112c49f8b619baeb1f1aa770b748 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 2 Nov 2016 10:55:47 -0500 Subject: [PATCH 102/156] Use conda context to ignore verify scripts --- conda_build/conda_interface.py | 2 ++ conda_build/config.py | 32 +++++++++++++++++++------------- conda_build/verify.py | 4 ++-- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index d6af5429e9..431bb55296 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -24,6 +24,8 @@ import conda.config as cc # NOQA from conda.config import rc_path # NOQA from conda.version import VersionOrder # NOQA +from conda.base.constants import SEARCH_PATH +from conda.common.configuration import Configuration, SequenceParameter if parse_version(conda.__version__) >= parse_version("4.2"): # conda 4.2.x diff --git a/conda_build/config.py b/conda_build/config.py index 6e0c85e920..296dfbc4e7 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -15,7 +15,9 @@ from .conda_interface import string_types, binstar_upload from .conda_interface import root_dir, root_writable, cc, subdir, platform from .conda_interface import string_types, binstar_upload, rc_path -from .conda_interface import root_dir, root_writable, cc, subdir, bits, platform, +from .conda_interface import root_dir, root_writable, cc, subdir, bits, platform +from .conda_interface import SEARCH_PATH +from .conda_interface import Configuration, SequenceParameter from .utils import get_build_folders, rm_rf @@ -27,7 +29,7 @@ # changes. DEFAULT_PREFIX_LENGTH = 255 -DEFAULT_VERIFY_SCRIPTS_PATH = os.path.join(sys.prefix, 'verify') +conda_build = "conda-build" def _ensure_dir(path): @@ -112,8 +114,7 @@ def env(lang, default): Setting('arch', subdir.split('-')[-1]), Setting('platform', platform), Setting('set_build_id', True), - Setting('disable_pip', False), - Setting('default_verify_scripts_path', DEFAULT_VERIFY_SCRIPTS_PATH) + Setting('disable_pip', False) ] # handle known values better than unknown (allow defaults) @@ -369,15 +370,6 @@ def test_dir(self): _ensure_dir(path) return path - @property - def verify_scripts_path(self): - if os.path.isfile(rc_path): - with open(rc_path, 'r') as f: - condarc = yaml.load(f) - if condarc.get("conda_build_verify_scripts"): - return condarc.get("conda_build_verify_scripts") - return self.default_verify_scripts_path - def clean(self): # build folder is the whole burrito containing envs and source folders # It will only exist if we download source, or create a build or test environment @@ -422,5 +414,19 @@ def show(config): # legacy exports for conda croot = Config().croot + +class Context(Configuration): + ignore_recipe_verify_scripts = SequenceParameter(string_types) + ignore_package_verify_scripts = SequenceParameter(string_types) + + +def reset_context(search_path=SEARCH_PATH, argparse_args=None): + context.__init__(search_path, conda_build, argparse_args) + return context + + +context = Context(SEARCH_PATH, conda_build, None) + + if __name__ == '__main__': show(Config()) diff --git a/conda_build/verify.py b/conda_build/verify.py index ed9c6343c2..b249e4537b 100644 --- a/conda_build/verify.py +++ b/conda_build/verify.py @@ -41,10 +41,10 @@ def verify(verify_path, *args): def verify_package(path_to_package, config): - verify_path = os.path.join(config.verify_scripts_path, "package") + verify_path = "" verify(verify_path, path_to_package) def verify_recipe(recipe, config): - verify_path = os.path.join(config.verify_scripts_path, "recipe") + verify_path = "" verify(verify_path, recipe.meta, recipe.path) From 71f1592278aee9dfc721793f5eedf6e7cf4a7ac9 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 2 Nov 2016 12:03:06 -0500 Subject: [PATCH 103/156] Use scripts for verify from verify package --- conda_build/verify.py | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/conda_build/verify.py b/conda_build/verify.py index b249e4537b..658d0eb5d0 100644 --- a/conda_build/verify.py +++ b/conda_build/verify.py @@ -3,18 +3,6 @@ import sys -def can_import_importlib(): - return sys.version_info.major >= 3 and sys.version_info.minor >= 3 - - -if can_import_importlib(): - from importlib.machinery import SourceFileLoader -else: - # imp.load_source has been deprecated in python3.3. So importlib should be used for versions - # for versions of python greater than 3.3 - import imp - - def list_verify_script(path_to_script): import pkgutil verify_modules = pkgutil.iter_modules([path_to_script]) @@ -29,22 +17,28 @@ def verify(verify_path, *args): for verify_script in verify_scripts: print("Running script %s" % verify_script) - if can_import_importlib(): - mod = SourceFileLoader("test", verify_script).load_module() - else: - mod = imp.load_source("test", verify_script) - try: - mod.verify(*args) - except TypeError as e: - raise VerifyError(e, verify_script) + + + # try: + # mod.verify(*args) + # except TypeError as e: + # raise VerifyError(e, verify_script) print("All scripts passed") def verify_package(path_to_package, config): - verify_path = "" - verify(verify_path, path_to_package) + try: + verify_module = __import__("verify.package") + except ImportError as e: + print("can't find verify.package module, skipping verification") + return False + verify(verify_module.__path__, path_to_package) def verify_recipe(recipe, config): - verify_path = "" - verify(verify_path, recipe.meta, recipe.path) + try: + verify_module = __import__("verify.recipe") + except ImportError as e: + print("can't find verify.recipe module, skipping verification") + return False + verify(verify_module.__path__, recipe.meta, recipe.path) From 0ff6038a7d2c0acb871e99acf8b992749324d97b Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 2 Nov 2016 13:20:18 -0500 Subject: [PATCH 104/156] Verify recipes --- conda_build/build.py | 4 ++-- conda_build/verify.py | 55 +++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 7771462736..318ec9053b 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -820,7 +820,7 @@ def order(f): update_index(config.bldpkgs_dir, config, could_be_mirror=False) if not getattr(config, "noverify", False): - verify_package(path, config) + verify_package(path) else: print("STOPPING BUILD BEFORE POST:", m.dist()) @@ -1067,7 +1067,7 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, metadata, need_source_download, need_reparse_in_env = render_recipe(recipe, config=recipe_config) if not getattr(config, "noverify", False): - verify_recipe(metadata, config) + verify_recipe(metadata) try: with recipe_config: ok_to_test = build(metadata, post=post, diff --git a/conda_build/verify.py b/conda_build/verify.py index 658d0eb5d0..fd965e8e27 100644 --- a/conda_build/verify.py +++ b/conda_build/verify.py @@ -1,44 +1,43 @@ -import os +from os.path import join +import pkgutil +from .config import context from .exceptions import VerifyError -import sys def list_verify_script(path_to_script): - import pkgutil verify_modules = pkgutil.iter_modules([path_to_script]) files = [] - for loader, name, _ in verify_modules: - files.append("%s.py" % os.path.join(loader.path, name)) + for _, name, _ in verify_modules: + files.append(name) return files -def verify(verify_path, *args): - verify_scripts = list_verify_script(verify_path) +def verify_package(path_to_package): + pass + # try: + # verify_module = __import__("verify.package") + # except ImportError as e: + # print("can't find verify.package module, skipping verification") + # return False + # package_dir = join(verify_module.__path__[0], "package") + # verify(package_dir, context.ignore_package_verify_scripts, path_to_package) - for verify_script in verify_scripts: - print("Running script %s" % verify_script) - - - # try: - # mod.verify(*args) - # except TypeError as e: - # raise VerifyError(e, verify_script) - print("All scripts passed") - -def verify_package(path_to_package, config): - try: - verify_module = __import__("verify.package") - except ImportError as e: - print("can't find verify.package module, skipping verification") - return False - verify(verify_module.__path__, path_to_package) - - -def verify_recipe(recipe, config): +def verify_recipe(recipe): try: verify_module = __import__("verify.recipe") except ImportError as e: print("can't find verify.recipe module, skipping verification") return False - verify(verify_module.__path__, recipe.meta, recipe.path) + recipe_dir = join(verify_module.__path__[0], "recipe") + + verify_scripts = list_verify_script(recipe_dir) + for verify_script in verify_scripts: + if verify_script not in context.ignore_recipe_verify_scripts: + mod = getattr(__import__("verify.recipe", fromlist=[verify_script]), verify_script) + print("Running script %s.py" % verify_script) + try: + mod.verify(recipe.meta, recipe.path) + except TypeError as e: + raise VerifyError(e, verify_script) + print("All scripts passed") From f2428ee29c3f51b7fae660a037e6710604771eb4 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 2 Nov 2016 14:05:00 -0500 Subject: [PATCH 105/156] Extract function to run verify scripts --- conda_build/verify.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/conda_build/verify.py b/conda_build/verify.py index fd965e8e27..79acb98abc 100644 --- a/conda_build/verify.py +++ b/conda_build/verify.py @@ -12,32 +12,30 @@ def list_verify_script(path_to_script): return files -def verify_package(path_to_package): - pass - # try: - # verify_module = __import__("verify.package") - # except ImportError as e: - # print("can't find verify.package module, skipping verification") - # return False - # package_dir = join(verify_module.__path__[0], "package") - # verify(package_dir, context.ignore_package_verify_scripts, path_to_package) - - -def verify_recipe(recipe): +def verify(verify_type, ignore_scripts, **kargs): + import_type = "verify.%s" % verify_type try: - verify_module = __import__("verify.recipe") + verify_module = __import__(import_type) except ImportError as e: print("can't find verify.recipe module, skipping verification") return False - recipe_dir = join(verify_module.__path__[0], "recipe") - - verify_scripts = list_verify_script(recipe_dir) + script_dir = join(verify_module.__path__[0], verify_type) + verify_scripts = list_verify_script(script_dir) for verify_script in verify_scripts: - if verify_script not in context.ignore_recipe_verify_scripts: - mod = getattr(__import__("verify.recipe", fromlist=[verify_script]), verify_script) + if verify_script not in ignore_scripts: + mod = getattr(__import__(import_type, fromlist=[verify_script]), verify_script) print("Running script %s.py" % verify_script) try: - mod.verify(recipe.meta, recipe.path) + mod.verify(**kargs) except TypeError as e: raise VerifyError(e, verify_script) print("All scripts passed") + + +def verify_recipe(recipe): + verify("recipe", context.ignore_recipe_verify_scripts, rendered_meta=recipe.meta, + recipe_dir=recipe.path) + + +def verify_package(path_to_package): + verify("package", context.ignore_package_verify_scripts, path_to_package=path_to_package) From 2d3191d0febbd9a1a4634dfd419fde5489ac2e8d Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 2 Nov 2016 14:18:17 -0500 Subject: [PATCH 106/156] Update tests --- tests/test_verify.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/tests/test_verify.py b/tests/test_verify.py index 4f22dee47f..17f1add22c 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -8,30 +8,13 @@ class TestVerify(unittest.TestCase): def test_list_verify_script(self): path_to_scripts = join(dirname(__file__), "test-verify-scripts/package") files = verify.list_verify_script(path_to_scripts) - expected_scripts = [join(path_to_scripts, "test_package.py"), - join(path_to_scripts, "test_package_2.py")] + expected_scripts = ["test_package", "test_package_2"] self.assertListEqual(files, expected_scripts) path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") files = verify.list_verify_script(path_to_scripts) - expected_scripts = [join(path_to_scripts, "test_recipe.py")] + expected_scripts = ["test_recipe"] self.assertListEqual(files, expected_scripts) - def test_verify(self): - path_to_scripts = join(dirname(__file__), "test-verify-scripts/package") - verify.verify(path_to_scripts, "path_to_package_arg") - - path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") - verify.verify(path_to_scripts, "recipe_arg", "path_to_recipe_arg") - - def test_verify_no_script(self): - path_to_script = "non/sense" - verify.verify(path_to_script) - - def test_verify_error_script(self): - path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") - try: - verify.verify(path_to_scripts, "recipe_arg") - except VerifyError as err: - self.assertTrue(err.error, TypeError) - self.assertEquals(err.script, join(path_to_scripts, "test_recipe.py")) + def test_no_verify_module(self): + self.assertFalse(verify.verify("nonsense", [])) From 7ae89da337ce3efa1e297875213d2514fcf64b73 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 3 Nov 2016 10:28:35 -0500 Subject: [PATCH 107/156] Offload verification steps to verify module Instead, use anaconda-verify library for verification --- conda_build/build.py | 10 +++++++--- conda_build/verify.py | 41 ----------------------------------------- tests/test_verify.py | 20 -------------------- tests/utils.py | 3 +-- 4 files changed, 8 insertions(+), 66 deletions(-) delete mode 100644 conda_build/verify.py delete mode 100644 tests/test_verify.py diff --git a/conda_build/build.py b/conda_build/build.py index 318ec9053b..9ec92bc84d 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -59,7 +59,8 @@ from conda_build.features import feature_list import conda_build.noarch_python as noarch_python -from .verify import verify_package, verify_recipe +from .config import context +from anaconda_verify.verify import Verify if 'bsd' in sys.platform: @@ -820,7 +821,8 @@ def order(f): update_index(config.bldpkgs_dir, config, could_be_mirror=False) if not getattr(config, "noverify", False): - verify_package(path) + verifier = Verify() + verifier.verify_package(context.ignore_package_verify_scripts, path_to_package=path) else: print("STOPPING BUILD BEFORE POST:", m.dist()) @@ -1067,7 +1069,9 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, metadata, need_source_download, need_reparse_in_env = render_recipe(recipe, config=recipe_config) if not getattr(config, "noverify", False): - verify_recipe(metadata) + verifier = Verify() + verifier.verify_recipe(context.ignore_recipe_verify_scripts, + rendered_meta=metadata.meta, recipe_dir=metadata.path) try: with recipe_config: ok_to_test = build(metadata, post=post, diff --git a/conda_build/verify.py b/conda_build/verify.py deleted file mode 100644 index 79acb98abc..0000000000 --- a/conda_build/verify.py +++ /dev/null @@ -1,41 +0,0 @@ -from os.path import join -import pkgutil -from .config import context -from .exceptions import VerifyError - - -def list_verify_script(path_to_script): - verify_modules = pkgutil.iter_modules([path_to_script]) - files = [] - for _, name, _ in verify_modules: - files.append(name) - return files - - -def verify(verify_type, ignore_scripts, **kargs): - import_type = "verify.%s" % verify_type - try: - verify_module = __import__(import_type) - except ImportError as e: - print("can't find verify.recipe module, skipping verification") - return False - script_dir = join(verify_module.__path__[0], verify_type) - verify_scripts = list_verify_script(script_dir) - for verify_script in verify_scripts: - if verify_script not in ignore_scripts: - mod = getattr(__import__(import_type, fromlist=[verify_script]), verify_script) - print("Running script %s.py" % verify_script) - try: - mod.verify(**kargs) - except TypeError as e: - raise VerifyError(e, verify_script) - print("All scripts passed") - - -def verify_recipe(recipe): - verify("recipe", context.ignore_recipe_verify_scripts, rendered_meta=recipe.meta, - recipe_dir=recipe.path) - - -def verify_package(path_to_package): - verify("package", context.ignore_package_verify_scripts, path_to_package=path_to_package) diff --git a/tests/test_verify.py b/tests/test_verify.py deleted file mode 100644 index 17f1add22c..0000000000 --- a/tests/test_verify.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest -from os.path import join, dirname -from conda_build import verify -from conda_build.exceptions import VerifyError - - -class TestVerify(unittest.TestCase): - def test_list_verify_script(self): - path_to_scripts = join(dirname(__file__), "test-verify-scripts/package") - files = verify.list_verify_script(path_to_scripts) - expected_scripts = ["test_package", "test_package_2"] - self.assertListEqual(files, expected_scripts) - - path_to_scripts = join(dirname(__file__), "test-verify-scripts/recipe") - files = verify.list_verify_script(path_to_scripts) - expected_scripts = ["test_recipe"] - self.assertListEqual(files, expected_scripts) - - def test_no_verify_module(self): - self.assertFalse(verify.verify("nonsense", [])) diff --git a/tests/utils.py b/tests/utils.py index 743757cbbd..c9778c780a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -57,8 +57,7 @@ def return_to_saved_path(): @pytest.fixture(scope='function') def test_config(testing_workdir, request): return Config(croot=testing_workdir, anaconda_upload=False, verbose=True, activate=False, - noverify=False, - default_verify_scripts_path=join(dirname(__file__), "test-verify-scripts")) + noverify=False) @pytest.fixture(scope='function') From 617d51d3b1521f5ae97900d7f147991ac6454a49 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 3 Nov 2016 16:20:51 -0500 Subject: [PATCH 108/156] Add anaconda-verify as a dependency --- .travis.yml | 2 +- appveyor.yml | 2 +- conda.recipe/meta.yaml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a39047801..e809b18d07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ install: fi fi - - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 + - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 anaconda-verify - pip install pkginfo - if [[ "$FLAKE8" == "true" ]]; then conda install -q flake8; diff --git a/appveyor.yml b/appveyor.yml index 38feba5447..30ccabc1be 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,7 +51,7 @@ install: - python -c "import sys; print(sys.executable)" - python -c "import sys; print(sys.prefix)" - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests contextlib2 - - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy + - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy anaconda-verify - conda install -c conda-forge -q perl - conda update -q --all # this is to ensure dependencies diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 7860630c80..9669f7e11a 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -22,6 +22,7 @@ requirements: build: - python run: + - anaconda-verify - conda >=4.1 - contextlib2 [py<3] - filelock From cf026bbb1f730347cb1be0d2e07636eaa3f12851 Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 4 Nov 2016 12:03:30 -0500 Subject: [PATCH 109/156] Add config for whitelisting tests --- conda_build/build.py | 13 +++++++++++-- conda_build/config.py | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 9ec92bc84d..226bf7c208 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -822,7 +822,12 @@ def order(f): if not getattr(config, "noverify", False): verifier = Verify() - verifier.verify_package(context.ignore_package_verify_scripts, path_to_package=path) + ignore_scripts = context.ignore_package_verify_scripts if \ + context.ignore_package_verify_scripts else None + run_scripts = context.run_package_verify_scripts if \ + context.run_package_verify_scripts else None + verifier.verify_package(ignore_scripts=ignore_scripts, run_scripts=run_scripts, + path_to_package=path) else: print("STOPPING BUILD BEFORE POST:", m.dist()) @@ -1070,7 +1075,11 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, config=recipe_config) if not getattr(config, "noverify", False): verifier = Verify() - verifier.verify_recipe(context.ignore_recipe_verify_scripts, + ignore_scripts = context.ignore_recipe_verify_scripts if \ + context.ignore_recipe_verify_scripts else None + run_scripts = context.run_recipe_verify_scripts if \ + context.run_recipe_verify_scripts else None + verifier.verify_recipe(ignore_scripts=ignore_scripts, run_scripts=run_scripts, rendered_meta=metadata.meta, recipe_dir=metadata.path) try: with recipe_config: diff --git a/conda_build/config.py b/conda_build/config.py index 296dfbc4e7..239a7924db 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -418,6 +418,8 @@ def show(config): class Context(Configuration): ignore_recipe_verify_scripts = SequenceParameter(string_types) ignore_package_verify_scripts = SequenceParameter(string_types) + run_recipe_verify_scripts = SequenceParameter(string_types) + run_package_verify_scripts = SequenceParameter(string_types) def reset_context(search_path=SEARCH_PATH, argparse_args=None): From b4f8faa976ab76c1a781d4846891e61144788764 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 7 Nov 2016 12:34:32 -0600 Subject: [PATCH 110/156] Depend on conda_verify for verification --- .travis.yml | 2 +- appveyor.yml | 2 +- conda.recipe/meta.yaml | 2 +- conda_build/build.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e809b18d07..19fdcf2e1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ install: fi fi - - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 anaconda-verify + - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 conda-verify - pip install pkginfo - if [[ "$FLAKE8" == "true" ]]; then conda install -q flake8; diff --git a/appveyor.yml b/appveyor.yml index 30ccabc1be..b36b05a0c2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,7 +51,7 @@ install: - python -c "import sys; print(sys.executable)" - python -c "import sys; print(sys.prefix)" - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests contextlib2 - - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy anaconda-verify + - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy conda-verify - conda install -c conda-forge -q perl - conda update -q --all # this is to ensure dependencies diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 9669f7e11a..8105b7bf10 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -22,7 +22,7 @@ requirements: build: - python run: - - anaconda-verify + - conda-verify - conda >=4.1 - contextlib2 [py<3] - filelock diff --git a/conda_build/build.py b/conda_build/build.py index 226bf7c208..ae4cd4b0d0 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -60,7 +60,7 @@ import conda_build.noarch_python as noarch_python from .config import context -from anaconda_verify.verify import Verify +from conda_verify.verify import Verify if 'bsd' in sys.platform: From fd3743aeda0594b550ce7db76d572438826f5f51 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 8 Nov 2016 16:55:07 -0600 Subject: [PATCH 111/156] Delete unused test Currently verify will run for every test taht builds a package --- tests/test-verify-scripts/package/test_package.py | 2 -- tests/test-verify-scripts/package/test_package_2.py | 2 -- tests/test-verify-scripts/recipe/test_recipe.py | 3 --- tests/utils.py | 3 +-- 4 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 tests/test-verify-scripts/package/test_package.py delete mode 100644 tests/test-verify-scripts/package/test_package_2.py delete mode 100644 tests/test-verify-scripts/recipe/test_recipe.py diff --git a/tests/test-verify-scripts/package/test_package.py b/tests/test-verify-scripts/package/test_package.py deleted file mode 100644 index ecd9afeaf3..0000000000 --- a/tests/test-verify-scripts/package/test_package.py +++ /dev/null @@ -1,2 +0,0 @@ -def verify(path_to_package): - print(path_to_package) diff --git a/tests/test-verify-scripts/package/test_package_2.py b/tests/test-verify-scripts/package/test_package_2.py deleted file mode 100644 index dc829898ac..0000000000 --- a/tests/test-verify-scripts/package/test_package_2.py +++ /dev/null @@ -1,2 +0,0 @@ -def verify(path_to_package): - print(path_to_package) \ No newline at end of file diff --git a/tests/test-verify-scripts/recipe/test_recipe.py b/tests/test-verify-scripts/recipe/test_recipe.py deleted file mode 100644 index ab4bae6746..0000000000 --- a/tests/test-verify-scripts/recipe/test_recipe.py +++ /dev/null @@ -1,3 +0,0 @@ -def verify(recipe, path_to_recipe): - print(recipe) - print(path_to_recipe) diff --git a/tests/utils.py b/tests/utils.py index c9778c780a..11353d69f9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -56,8 +56,7 @@ def return_to_saved_path(): @pytest.fixture(scope='function') def test_config(testing_workdir, request): - return Config(croot=testing_workdir, anaconda_upload=False, verbose=True, activate=False, - noverify=False) + return Config(croot=testing_workdir, anaconda_upload=False, verbose=True, activate=False) @pytest.fixture(scope='function') From 6a0e558e39c1fe78aecb210daa1d0938c5ab6fa1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 10 Nov 2016 20:17:17 -0600 Subject: [PATCH 112/156] break symlink and copy file if symlink points to executable --- conda_build/post.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/conda_build/post.py b/conda_build/post.py index bba6f1df75..b82f6098e5 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -452,7 +452,13 @@ def check_symlinks(files, prefix, croot): if islink(path): link_path = readlink(path) real_link_path = realpath(path) - if real_link_path.startswith(real_build_prefix): + # symlinks to binaries outside of the same dir don't work. RPATH stuff gets confused. + # If condition exists, then copy the file rather than symlink it. + if (not os.path.dirname(link_path) == os.path.dirname(real_link_path) and + is_obj(f)): + os.remove(path) + utils.copy_into(real_link_path, path) + elif real_link_path.startswith(real_build_prefix): # If the path is in the build prefix, this is fine, but # the link needs to be relative if not link_path.startswith('.'): From ed361cc72774f6d97faa0edcade236e0214316f6 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 10 Nov 2016 20:20:23 -0600 Subject: [PATCH 113/156] shuffle symlink check above rpath replacement --- conda_build/post.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index b82f6098e5..818a96c3d7 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -432,6 +432,8 @@ def post_build(m, files, prefix, build_python, croot): print("Skipping binary relocation logic") osx_is_app = bool(m.get_value('build/osx_is_app', False)) + check_symlinks(files, prefix, croot) + for f in files: if f.startswith('bin/'): fix_shebang(f, prefix=prefix, build_python=build_python, osx_is_app=osx_is_app) @@ -439,8 +441,6 @@ def post_build(m, files, prefix, build_python, croot): mk_relative(m, f, prefix) make_hardlink_copy(f, prefix) - check_symlinks(files, prefix, croot) - def check_symlinks(files, prefix, croot): if readlink is False: From 75b0e180b3500b24b5afd3537b4618f162651248 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 11 Nov 2016 08:59:18 -0600 Subject: [PATCH 114/156] update symlink copy comment per @mingwandroid --- conda_build/post.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_build/post.py b/conda_build/post.py index 818a96c3d7..c0276880e5 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -452,7 +452,8 @@ def check_symlinks(files, prefix, croot): if islink(path): link_path = readlink(path) real_link_path = realpath(path) - # symlinks to binaries outside of the same dir don't work. RPATH stuff gets confused. + # symlinks to binaries outside of the same dir don't work. RPATH stuff gets confused + # because ld.so follows symlinks in RPATHS # If condition exists, then copy the file rather than symlink it. if (not os.path.dirname(link_path) == os.path.dirname(real_link_path) and is_obj(f)): From e01eca8da5237499ac9a7dd2097a71c484e54928 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 11 Nov 2016 09:38:54 -0600 Subject: [PATCH 115/156] fix perl version comparison sometimes being str-int uncomparable --- conda_build/skeletons/cpan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/skeletons/cpan.py b/conda_build/skeletons/cpan.py index 0ab2418327..cb09341be3 100644 --- a/conda_build/skeletons/cpan.py +++ b/conda_build/skeletons/cpan.py @@ -600,7 +600,7 @@ def get_release_info(cpan_url, package, version, perl_version, config, # If the latest isn't the version we're looking for, we have to do another # request version_str = str(version) - if (version is not None) and (version != LooseVersion('0') and + if (version is not None) and (LooseVersion('0') != version_str and (rel_dict['version'] != version_str)): author = rel_dict['author'] try: From 2082be757d7755a4e8bee404921551db2fb3369c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 11 Nov 2016 10:14:28 -0600 Subject: [PATCH 116/156] ignore broken symlinks in copying files --- conda_build/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda_build/utils.py b/conda_build/utils.py index eb2f458662..d92ea0848b 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -110,6 +110,10 @@ def copy_into(src, dst, timeout=90, symlinks=False, lock=None): else: src_folder = os.getcwd() + if os.path.islink(src) and not os.path.exists(os.path.realpath(src)): + log.warn('path %s is a broken symlink - ignoring copy', src) + return + if not lock: lock = get_lock(src_folder, timeout=timeout) with lock: From 482e3cef693dee1d0a12a95a733352b61353e619 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 11 Nov 2016 16:11:40 -0600 Subject: [PATCH 117/156] specify conda exe more specifically for Windows; it was not always being found --- conda_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index 6a40b975eb..4c82f6b3f5 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -297,7 +297,7 @@ def write_about_json(m, config): if value: d[key] = value - bin_path = os.path.join(sys.prefix, "Scripts" if on_win else "bin", 'conda') + bin_path = os.path.join(sys.prefix, "Scripts\\conda.exe" if on_win else "bin/conda") # for sake of reproducibility, record some conda info conda_info = subprocess.check_output([bin_path, 'info', '--json', '-s']) From 7016ee81371a26970583cf292f2eb331d51362c1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 11 Nov 2016 16:14:40 -0600 Subject: [PATCH 118/156] changelog 2.0.9 --- CHANGELOG.txt | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index f73af0a95f..3f7430818a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,29 @@ +2016-11-11 2.0.9: +----------------- + +Enhancements: +------------- + +* break build string construction out into standalone function for external usage (Concourse CI project) #1513 +* add conda-verify support. Defaults to enabled. Adds conda-verify as runtime requirement. +* + +Bug fixes: +---------- + +* handle creation of intermediate folders when filenames provided as build/source_files arguments #1511 +* Fix passing of version argument to pypi skeleton arguments #1516 +* break symlinks and copy files if symlinks point to executable outside of same path (fix RPATH misbehavior on linux/mac, because ld.so follows symlinks) #1521 +* specify conda executable name more specifically when getting about.json info. It was not being found in some cases without the file extension. #1525 + +Contributors: +------------- + +* @jhprinz +* @msarahan +* @soapy1 + + 2016-11-03 2.0.8: ----------------- From 84c398573935c2c89b1f7233e306f17ea388d1a5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 12 Nov 2016 15:06:16 -0600 Subject: [PATCH 119/156] don't update all in travis yml, to avoid updating conda --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 19fdcf2e1a..d32c9d36eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,6 @@ install: else conda install -c conda-forge -q perl; conda install -q pytest pip pytest-cov numpy mock; - conda update -q --all; $HOME/miniconda/bin/pip install pytest-xdist pytest-capturelog; pushd .. && git clone https://github.com/conda/conda_build_test_recipe && popd; fi From bf4f8bbe912e2e6402508e25cbf99cf432576359 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 12 Nov 2016 16:10:18 -0600 Subject: [PATCH 120/156] make conda verify config use condarc and be more back compat --- conda_build/build.py | 17 ++++++++--------- conda_build/conda_interface.py | 2 -- conda_build/config.py | 32 ++++++++++---------------------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 4f5151ff0f..61c3ed2bf4 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -59,7 +59,6 @@ from conda_build.features import feature_list import conda_build.noarch_python as noarch_python -from .config import context from conda_verify.verify import Verify @@ -822,10 +821,10 @@ def order(f): if not getattr(config, "noverify", False): verifier = Verify() - ignore_scripts = context.ignore_package_verify_scripts if \ - context.ignore_package_verify_scripts else None - run_scripts = context.run_package_verify_scripts if \ - context.run_package_verify_scripts else None + ignore_scripts = config.ignore_package_verify_scripts if \ + config.ignore_package_verify_scripts else None + run_scripts = config.run_package_verify_scripts if \ + config.run_package_verify_scripts else None verifier.verify_package(ignore_scripts=ignore_scripts, run_scripts=run_scripts, path_to_package=path) @@ -1075,10 +1074,10 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, config=recipe_config) if not getattr(config, "noverify", False): verifier = Verify() - ignore_scripts = context.ignore_recipe_verify_scripts if \ - context.ignore_recipe_verify_scripts else None - run_scripts = context.run_recipe_verify_scripts if \ - context.run_recipe_verify_scripts else None + ignore_scripts = config.ignore_recipe_verify_scripts if \ + config.ignore_recipe_verify_scripts else None + run_scripts = config.run_recipe_verify_scripts if \ + config.run_recipe_verify_scripts else None verifier.verify_recipe(ignore_scripts=ignore_scripts, run_scripts=run_scripts, rendered_meta=metadata.meta, recipe_dir=metadata.path) try: diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 431bb55296..d6af5429e9 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -24,8 +24,6 @@ import conda.config as cc # NOQA from conda.config import rc_path # NOQA from conda.version import VersionOrder # NOQA -from conda.base.constants import SEARCH_PATH -from conda.common.configuration import Configuration, SequenceParameter if parse_version(conda.__version__) >= parse_version("4.2"): # conda 4.2.x diff --git a/conda_build/config.py b/conda_build/config.py index 239a7924db..bdaf0300b8 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -10,14 +10,9 @@ from os.path import abspath, expanduser, join import sys import time -import yaml -from .conda_interface import string_types, binstar_upload from .conda_interface import root_dir, root_writable, cc, subdir, platform -from .conda_interface import string_types, binstar_upload, rc_path -from .conda_interface import root_dir, root_writable, cc, subdir, bits, platform -from .conda_interface import SEARCH_PATH -from .conda_interface import Configuration, SequenceParameter +from .conda_interface import string_types, binstar_upload from .utils import get_build_folders, rm_rf @@ -114,7 +109,15 @@ def env(lang, default): Setting('arch', subdir.split('-')[-1]), Setting('platform', platform), Setting('set_build_id', True), - Setting('disable_pip', False) + Setting('disable_pip', False), + Setting('ignore_recipe_verify_scripts', + cc.rc.get('conda-build', {}).get('ignore_recipe_verify_scripts', [])), + Setting('ignore_package_verify_scripts', + cc.rc.get('conda-build', {}).get('ignore_package_verify_scripts', [])), + Setting('run_recipe_verify_scripts', + cc.rc.get('conda-build', {}).get('run_package_verify_scripts', [])), + Setting('run_package_verify_scripts', + cc.rc.get('conda-build', {}).get('run_package_verify_scripts', [])), ] # handle known values better than unknown (allow defaults) @@ -415,20 +418,5 @@ def show(config): croot = Config().croot -class Context(Configuration): - ignore_recipe_verify_scripts = SequenceParameter(string_types) - ignore_package_verify_scripts = SequenceParameter(string_types) - run_recipe_verify_scripts = SequenceParameter(string_types) - run_package_verify_scripts = SequenceParameter(string_types) - - -def reset_context(search_path=SEARCH_PATH, argparse_args=None): - context.__init__(search_path, conda_build, argparse_args) - return context - - -context = Context(SEARCH_PATH, conda_build, None) - - if __name__ == '__main__': show(Config()) From 29db96a07364243d6da8a67fab3dc7ad0b96f467 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 14 Nov 2016 08:43:52 -0600 Subject: [PATCH 121/156] changelog 2.0.10 --- CHANGELOG.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3f7430818a..e813377459 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,17 @@ +2016-11-14 2.0.10: +------------------ + +Bug fixes: +---------- + +* Fix backwards incompatibility with conda 4.1 #1528 + +Contributors: +------------- + +* @msarahan + + 2016-11-11 2.0.9: ----------------- From f127925971b8229968c752eeb2fb73c6a0937762 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 26 Oct 2016 19:15:57 -0500 Subject: [PATCH 122/156] Extract function for creating info/files.json ref: https://github.com/conda/conda-build/issues/1486 --- conda.recipe/meta.yaml | 1 + conda_build/build.py | 156 +++++++++++++++++++++++++++++++++++----- tests/test_api_build.py | 26 +++++++ 3 files changed, 165 insertions(+), 18 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 8105b7bf10..205b3b644e 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -32,6 +32,7 @@ requirements: - python - pyyaml - pkginfo + - enum34 [py<34] test: requires: diff --git a/conda_build/build.py b/conda_build/build.py index 61c3ed2bf4..ec16534fed 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -18,6 +18,7 @@ import subprocess import sys import tarfile +import hashlib # this is to compensate for a requests idna encoding error. Conda is a better place to fix, # eventually @@ -60,6 +61,14 @@ import conda_build.noarch_python as noarch_python from conda_verify.verify import Verify +from enum import Enum + + +class FileType(Enum): + regular = "regular" + softlink = "softlink" + hardlink = "hardlink" + directory = "directory" if 'bsd' in sys.platform: @@ -235,22 +244,27 @@ def copy_license(m, config): join(config.info_dir, 'LICENSE.txt'), config.timeout) -def detect_and_record_prefix_files(m, files, prefix, config): +def get_files_with_prefix(m, files, prefix): files_with_prefix = sorted(have_prefix_files(files, prefix)) - binary_has_prefix_files = m.binary_has_prefix_files() - text_has_prefix_files = m.has_prefix_files() ignore_files = m.ignore_prefix_files() ignore_types = set() if not hasattr(ignore_files, "__iter__"): - if ignore_files == True: + if ignore_files is True: ignore_types.update(('text', 'binary')) ignore_files = [] if not m.get_value('build/detect_binary_files_with_prefix', True): ignore_types.update(('binary',)) - ignore_files.extend([f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files]) + ignore_files.extend( + [f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files]) files_with_prefix = [f for f in files_with_prefix if f[2] not in ignore_files] + return files_with_prefix + +def detect_and_record_prefix_files(m, files, prefix, config): + files_with_prefix = get_files_with_prefix(m, files, prefix) + binary_has_prefix_files = m.binary_has_prefix_files() + text_has_prefix_files = m.has_prefix_files() is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m) or m.get_value('build/noarch') if files_with_prefix and not is_noarch: @@ -410,20 +424,35 @@ def create_info_files(m, files, config, prefix): # make sure we use '/' path separators in metadata files = [_f.replace('\\', '/') for _f in files] + short_paths = files + if is_noarch_python(m): + for index, short_path in enumerate(short_paths): + if short_path.find("site-packages") > 0: + short_paths[index] = short_path[short_path.find("site-packages"):] + elif short_path.startswith("bin") and (short_path not in entry_point_script_names): + short_paths[index] = short_path.replace("bin", "python-scripts") + elif short_path.startswith("Scripts") and (short_path not in entry_point_script_names): + short_paths[index] = short_path.replace("Scripts", "python-scripts") + elif m.get_value('build/noarch_python'): + short_paths = [] + with open(join(config.info_dir, 'files'), **mode_dict) as fo: - if m.get_value('build/noarch_python'): - fo.write('\n') - elif is_noarch_python(m): - for f in files: - if f.find("site-packages") > 0: - fo.write(f[f.find("site-packages"):] + '\n') - elif f.startswith("bin") and (f not in entry_point_script_names): - fo.write(f.replace("bin", "python-scripts") + '\n') - elif f.startswith("Scripts") and (f not in entry_point_script_names): - fo.write(f.replace("Scripts", "python-scripts") + '\n') - else: - for f in files: - fo.write(f + '\n') + for f in short_paths: + fo.write(f + "\n") + + no_link = m.get_value('build/no_link') + if no_link: + if not isinstance(no_link, list): + no_link = [no_link] + files_with_prefix = get_files_with_prefix(m, files, prefix) + + no_link = m.get_value('build/no_link') + if no_link: + if not isinstance(no_link, list): + no_link = [no_link] + files_with_prefix = get_files_with_prefix(m, files, prefix) + create_info_files_json(m, config.info_dir, prefix, files, files_with_prefix) + detect_and_record_prefix_files(m, files, prefix, config) write_no_link(m, config, files) @@ -438,6 +467,97 @@ def create_info_files(m, files, config, prefix): config.timeout) +def get_short_path(m, target_file): + entry_point_script_names = get_entry_point_script_names(m.get_value('build/entry_points')) + if is_noarch_python(m): + if target_file.find("site-packages") > 0: + return target_file[target_file.find("site-packages"):] + elif target_file.startswith("bin") and (target_file not in entry_point_script_names): + return target_file.replace("bin", "python-scripts") + elif target_file.startswith("Scripts") and (target_file not in entry_point_script_names): + return target_file.replace("Scripts", "python-scripts") + elif m.get_value('build/noarch_python'): + return None + else: + return target_file + + +def sha256_checksum(filename, buffersize=65536): + sha256 = hashlib.sha256() + with open(filename, 'rb') as f: + for block in iter(lambda: f.read(buffersize), b''): + sha256.update(block) + return sha256.hexdigest() + + +def has_prefix(short_path, files_with_prefix): + for prefix, mode, filename in files_with_prefix: + if short_path == filename: + return prefix, mode + return None, None + + +def is_no_link(no_link, short_path): + if no_link is not None and short_path in no_link: + return True + + +def get_sorted_inode_first_path(short_paths, target_short_path, prefix): + target_short_path_inode = os.stat(join(prefix, target_short_path)).st_ino + hardlinked_files = [sp for sp in short_paths + if os.stat(join(prefix, sp)).st_ino == target_short_path_inode] + return sorted(hardlinked_files) + + +def file_type(path): + if isdir(path): + return FileType.directory + elif islink(path): + return FileType.softlink + elif os.stat(path).st_nlink >= 2: + return FileType.hardlink + return FileType.regular + + +def build_info_files_json(m, prefix, files, files_with_prefix): + no_link = m.get_value('build/no_link') + files_json = [] + for fi in files: + prefix_placeholder, file_mode = has_prefix(fi, files_with_prefix) + path = os.path.join(prefix, fi) + file_info = { + "short_path": get_short_path(m, fi), + "sha256": sha256_checksum(path), + "size_in_bytes": os.path.getsize(path), + "file_type": getattr(file_type(path), "name"), + "prefix_placeholder": prefix_placeholder, + "file_mode": file_mode, + "no_link": is_no_link(no_link, fi), + } + if file_info.get("file_type") == "hardlink": + inode_first_path = get_sorted_inode_first_path(fi, fi, prefix) + file_info["inode_first_path"] = inode_first_path[0] + files_json.append(file_info) + return files_json + + +def get_files_version(): + return 1 + + +def create_info_files_json(m, info_dir, prefix, files, files_with_prefix): + files_json_fields = ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "prefix_placeholder", "no_link", "inode_first_path"] + files_json_files = build_info_files_json(m, prefix, files, files_with_prefix) + files_json_info = { + "version": get_files_version(), + "fields": files_json_fields, + "files": files_json_files, + } + with open(join(info_dir, 'files.json'), "w") as files_json: + json.dump(files_json_info, files_json) + + def get_build_index(config, clear_cache=True): # priority: local by croot (can vary), then channels passed as args, # then channels from config. diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 8ec98539f4..940256ef58 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -727,3 +727,29 @@ def test_script_win_creates_exe(test_config): api.build(recipe, config=test_config) assert package_has_file(fn, 'Scripts/test-script.exe') assert package_has_file(fn, 'Scripts/test-script-script.py') + + +def test_info_json(test_config): + recipe = os.path.join(metadata_dir, "ignore_some_prefix_files") + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) + assert package_has_file(fn, "info/files.json") + with tarfile.open(fn) as tf: + data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8')) + fields = ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link", + "prefix_placeholder", "inode_first_path"] + for key in data.keys(): + assert key in ['files', 'fields', 'version'] + for field in data.get('fields'): + assert field in fields + assert len(data.get('files')) == 2 + for file in data.get('files'): + for key in file.keys(): + assert key in fields + short_path = file.get("short_path") + if short_path == "test.sh" or short_path == "test.bat": + assert file.get("prefix_placeholder") is not None + assert file.get("file_mode") is not None + else: + assert file.get("prefix_placeholder") is None + assert file.get("file_mode") is None From 8eaf372bdf901547be544531acf1e2235dcd5c4b Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 4 Nov 2016 16:12:11 -0500 Subject: [PATCH 123/156] Build files.json using the right file paths --- conda_build/build.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index ec16534fed..16e6a0d70b 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -424,21 +424,20 @@ def create_info_files(m, files, config, prefix): # make sure we use '/' path separators in metadata files = [_f.replace('\\', '/') for _f in files] - short_paths = files - if is_noarch_python(m): - for index, short_path in enumerate(short_paths): - if short_path.find("site-packages") > 0: - short_paths[index] = short_path[short_path.find("site-packages"):] - elif short_path.startswith("bin") and (short_path not in entry_point_script_names): - short_paths[index] = short_path.replace("bin", "python-scripts") - elif short_path.startswith("Scripts") and (short_path not in entry_point_script_names): - short_paths[index] = short_path.replace("Scripts", "python-scripts") - elif m.get_value('build/noarch_python'): - short_paths = [] - with open(join(config.info_dir, 'files'), **mode_dict) as fo: - for f in short_paths: - fo.write(f + "\n") + if m.get_value('build/noarch_python'): + fo.write('\n') + elif is_noarch_python(m): + for f in files: + if f.find("site-packages") > 0: + fo.write(f[f.find("site-packages"):] + '\n') + elif f.startswith("bin") and (f not in entry_point_script_names): + fo.write(f.replace("bin", "python-scripts") + '\n') + elif f.startswith("Scripts") and (f not in entry_point_script_names): + fo.write(f.replace("Scripts", "python-scripts") + '\n') + else: + for f in files: + fo.write(f + '\n') no_link = m.get_value('build/no_link') if no_link: @@ -453,7 +452,6 @@ def create_info_files(m, files, config, prefix): files_with_prefix = get_files_with_prefix(m, files, prefix) create_info_files_json(m, config.info_dir, prefix, files, files_with_prefix) - detect_and_record_prefix_files(m, files, prefix, config) write_no_link(m, config, files) From b854bedccea58601b64b32d44668e8437094ab3d Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 10 Nov 2016 11:22:53 -0600 Subject: [PATCH 124/156] More testing for files.json For win32 py27: os.link is not available here, so skip tests that require that for setup --- conda_build/build.py | 7 ++- tests/test_api_build.py | 2 +- tests/test_build.py | 119 +++++++++++++++++++++++++++++++++++++++- tests/utils.py | 6 ++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 16e6a0d70b..3fecb5267d 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -500,9 +500,10 @@ def is_no_link(no_link, short_path): return True -def get_sorted_inode_first_path(short_paths, target_short_path, prefix): +def get_sorted_inode_first_path(files, target_short_path, prefix): + ensure_list(files) target_short_path_inode = os.stat(join(prefix, target_short_path)).st_ino - hardlinked_files = [sp for sp in short_paths + hardlinked_files = [sp for sp in files if os.stat(join(prefix, sp)).st_ino == target_short_path_inode] return sorted(hardlinked_files) @@ -533,7 +534,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): "no_link": is_no_link(no_link, fi), } if file_info.get("file_type") == "hardlink": - inode_first_path = get_sorted_inode_first_path(fi, fi, prefix) + inode_first_path = get_sorted_inode_first_path(files, fi, prefix) file_info["inode_first_path"] = inode_first_path[0] files_json.append(file_info) return files_json diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 940256ef58..00c1f530d6 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -729,7 +729,7 @@ def test_script_win_creates_exe(test_config): assert package_has_file(fn, 'Scripts/test-script-script.py') -def test_info_json(test_config): +def test_info_files_json(test_config): recipe = os.path.join(metadata_dir, "ignore_some_prefix_files") fn = api.get_output_file_path(recipe, config=test_config) api.build(recipe, config=test_config) diff --git a/tests/test_build.py b/tests/test_build.py index 3301755bb8..d8b77b3da5 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -15,7 +15,8 @@ from conda_build.metadata import MetaData from conda_build.utils import rm_rf, on_win -from .utils import testing_workdir, test_config, test_metadata, metadata_dir, put_bad_conda_on_path +from .utils import (testing_workdir, test_config, test_metadata, metadata_dir, + get_noarch_python_meta, put_bad_conda_on_path) prefix_tests = {"normal": os.path.sep} if sys.platform == "win32": @@ -159,3 +160,119 @@ def test_write_about_json_without_conda_on_path(testing_workdir, test_metadata): about = json.load(f) assert 'conda_version' in about assert 'conda_build_version' in about + + +def test_get_short_path(test_metadata): + # Test for regular package + assert build.get_short_path(test_metadata, "test/file") == "test/file" + + # Test for noarch: python + meta = get_noarch_python_meta(test_metadata) + assert build.get_short_path(meta, "lib/site-packages/test") == "site-packages/test" + assert build.get_short_path(meta, "bin/test") == "python-scripts/test" + assert build.get_short_path(meta, "Scripts/test") == "python-scripts/test" + + +def test_has_prefix(): + files_with_prefix = [("prefix/path", "text", "short/path/1"), + ("prefix/path", "text", "short/path/2")] + assert build.has_prefix("short/path/1", files_with_prefix) == ("prefix/path", "text") + assert build.has_prefix("short/path/nope", files_with_prefix) == (None, None) + + +def test_is_no_link(): + no_link = ["path/1", "path/2"] + assert build.is_no_link(no_link, "path/1") is True + assert build.is_no_link(no_link, "path/nope") is None + + +@pytest.mark.skipif(on_win and sys.version[:3] == "2.7", + reason="os.link is not available so can't setup test") +def test_sorted_inode_first_path(testing_workdir): + path_one = os.path.join(testing_workdir, "one") + path_two = os.path.join(testing_workdir, "two") + path_one_hardlink = os.path.join(testing_workdir, "one_hl") + open(path_one, "a").close() + open(path_two, "a").close() + + os.link(path_one, path_one_hardlink) + + files = ["one", "two", "one_hl"] + assert build.get_sorted_inode_first_path(files, "one", testing_workdir) == ["one", "one_hl"] + assert build.get_sorted_inode_first_path(files, "one_hl", testing_workdir) == ["one", "one_hl"] + assert build.get_sorted_inode_first_path(files, "two", testing_workdir) == ["two"] + + +def test_create_info_files_json(testing_workdir, test_metadata): + info_dir = os.path.join(testing_workdir, "info") + os.mkdir(info_dir) + path_one = os.path.join(testing_workdir, "one") + path_two = os.path.join(testing_workdir, "two") + path_foo = os.path.join(testing_workdir, "foo") + open(path_one, "a").close() + open(path_two, "a").close() + open(path_foo, "a").close() + files_with_prefix = [("prefix/path", "text", "foo")] + files = ["one", "two", "foo"] + + build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) + files_json_path = os.path.join(info_dir, "files.json") + expected_output = { + "files": [{"file_mode": None, "no_link": None, "file_type": "regular", "short_path": "one", + "prefix_placeholder": None, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"file_mode": None, "no_link": None, "file_type": "regular", "short_path": "two", + "prefix_placeholder": None, "size_in_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {"file_mode": "text", "no_link": None, "file_type": "regular", + "short_path": "foo", "prefix_placeholder": "prefix/path", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}], + "fields": ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "prefix_placeholder", "no_link", "inode_first_path"], + "version": 1} + with open(files_json_path, "r") as files_json: + output = json.load(files_json) + assert output == expected_output + + +@pytest.mark.skipif(on_win and sys.version[:3] == "2.7", + reason="os.link is not available so can't setup test") +def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): + info_dir = os.path.join(testing_workdir, "info") + os.mkdir(info_dir) + path_one = os.path.join(testing_workdir, "one") + path_two = os.path.join(testing_workdir, "two") + path_foo = os.path.join(testing_workdir, "foo") + path_one_hardlink = os.path.join(testing_workdir, "one_hl") + open(path_one, "a").close() + open(path_two, "a").close() + open(path_foo, "a").close() + os.link(path_one, path_one_hardlink) + files_with_prefix = [("prefix/path", "text", "foo")] + files = ["one", "two", "one_hl", "foo"] + + build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) + files_json_path = os.path.join(info_dir, "files.json") + expected_output = { + "files": [{"inode_first_path": "one", "file_mode": None, "no_link": None, + "file_type": "hardlink", "short_path": "one", "prefix_placeholder": None, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"file_mode": None, "no_link": None, "file_type": "regular", "short_path": "two", + "prefix_placeholder": None, "size_in_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {"inode_first_path": "one", "file_mode": None, "no_link": None, + "file_type": "hardlink", "short_path": "one_hl", "prefix_placeholder": None, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"file_mode": "text", "no_link": None, "file_type": "regular", + "short_path": "foo", "prefix_placeholder": "prefix/path", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}], + "fields": ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "prefix_placeholder", "no_link", "inode_first_path"], + "version": 1} + with open(files_json_path, "r") as files_json: + assert json.load(files_json) == expected_output diff --git a/tests/utils.py b/tests/utils.py index 11353d69f9..496b5af900 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -122,3 +122,9 @@ def put_bad_conda_on_path(testing_workdir): raise finally: os.environ['PATH'] = path_backup + + +def get_noarch_python_meta(meta): + d = meta.meta + d['build']['noarch'] = "python" + return MetaData.fromdict(d, config=meta.config) From 2e3c308fa01df78bce613f6ea835b1ee7692be36 Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 11 Nov 2016 15:36:23 -0600 Subject: [PATCH 125/156] Update the contents of files.json New spec says: - prefix_placeholder, no_link, and file_mode should only exist if they have vaules - inode_first_path should instead be inode_paths that is a list of inodes that the file shares inodes with --- conda_build/build.py | 22 +++++++++++----------- tests/test_build.py | 22 +++++++++------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 3fecb5267d..5f1f667575 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -65,7 +65,6 @@ class FileType(Enum): - regular = "regular" softlink = "softlink" hardlink = "hardlink" directory = "directory" @@ -500,7 +499,7 @@ def is_no_link(no_link, short_path): return True -def get_sorted_inode_first_path(files, target_short_path, prefix): +def get_inode_paths(files, target_short_path, prefix): ensure_list(files) target_short_path_inode = os.stat(join(prefix, target_short_path)).st_ino hardlinked_files = [sp for sp in files @@ -513,9 +512,7 @@ def file_type(path): return FileType.directory elif islink(path): return FileType.softlink - elif os.stat(path).st_nlink >= 2: - return FileType.hardlink - return FileType.regular + return FileType.hardlink def build_info_files_json(m, prefix, files, files_with_prefix): @@ -529,13 +526,16 @@ def build_info_files_json(m, prefix, files, files_with_prefix): "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), "file_type": getattr(file_type(path), "name"), - "prefix_placeholder": prefix_placeholder, - "file_mode": file_mode, - "no_link": is_no_link(no_link, fi), } - if file_info.get("file_type") == "hardlink": - inode_first_path = get_sorted_inode_first_path(files, fi, prefix) - file_info["inode_first_path"] = inode_first_path[0] + no_link = is_no_link(no_link, fi) + if no_link: + file_info["no_link"] = no_link + if prefix_placeholder and file_mode: + file_info["prefix_placeholder"] = prefix_placeholder + file_info["file_mode"] = file_mode + if file_info.get("file_type") == "hardlink" and os.stat(fi).st_nlink > 1: + inode_paths = get_inode_paths(files, fi, prefix) + file_info["inode_paths"] = inode_paths files_json.append(file_info) return files_json diff --git a/tests/test_build.py b/tests/test_build.py index d8b77b3da5..eb1c96642c 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -218,14 +218,12 @@ def test_create_info_files_json(testing_workdir, test_metadata): build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) files_json_path = os.path.join(info_dir, "files.json") expected_output = { - "files": [{"file_mode": None, "no_link": None, "file_type": "regular", "short_path": "one", - "prefix_placeholder": None, + "files": [{"file_type": "hardlink", "short_path": "one", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_mode": None, "no_link": None, "file_type": "regular", "short_path": "two", - "prefix_placeholder": None, "size_in_bytes": 0, + {"file_type": "hardlink", "short_path": "two", "size_in_bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, - {"file_mode": "text", "no_link": None, "file_type": "regular", + {"file_mode": "text", "file_type": "hardlink", "short_path": "foo", "prefix_placeholder": "prefix/path", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], @@ -256,19 +254,17 @@ def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) files_json_path = os.path.join(info_dir, "files.json") expected_output = { - "files": [{"inode_first_path": "one", "file_mode": None, "no_link": None, - "file_type": "hardlink", "short_path": "one", "prefix_placeholder": None, + "files": [{"inode_paths": ["one", "one_hl"], "file_type": "hardlink", "short_path": "one", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_mode": None, "no_link": None, "file_type": "regular", "short_path": "two", - "prefix_placeholder": None, "size_in_bytes": 0, + {"file_type": "hardlink", "short_path": "two", "size_in_bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, - {"inode_first_path": "one", "file_mode": None, "no_link": None, - "file_type": "hardlink", "short_path": "one_hl", "prefix_placeholder": None, + {"inode_paths": ["one", "one_hl"], "file_type": "hardlink", + "short_path": "one_hl", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_mode": "text", "no_link": None, "file_type": "regular", - "short_path": "foo", "prefix_placeholder": "prefix/path", + {"file_mode": "text", "file_type": "hardlink", "short_path": "foo", + "prefix_placeholder": "prefix/path", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], "fields": ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", From 36a3ce10ea94547690b7af22a52b593137c53068 Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 11 Nov 2016 16:44:24 -0600 Subject: [PATCH 126/156] Fix tests --- conda_build/build.py | 12 +----------- tests/test_build.py | 6 +++--- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 5f1f667575..25af32b89f 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -438,16 +438,6 @@ def create_info_files(m, files, config, prefix): for f in files: fo.write(f + '\n') - no_link = m.get_value('build/no_link') - if no_link: - if not isinstance(no_link, list): - no_link = [no_link] - files_with_prefix = get_files_with_prefix(m, files, prefix) - - no_link = m.get_value('build/no_link') - if no_link: - if not isinstance(no_link, list): - no_link = [no_link] files_with_prefix = get_files_with_prefix(m, files, prefix) create_info_files_json(m, config.info_dir, prefix, files, files_with_prefix) @@ -533,7 +523,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): if prefix_placeholder and file_mode: file_info["prefix_placeholder"] = prefix_placeholder file_info["file_mode"] = file_mode - if file_info.get("file_type") == "hardlink" and os.stat(fi).st_nlink > 1: + if file_info.get("file_type") == "hardlink" and os.stat(join(prefix, fi)).st_nlink > 1: inode_paths = get_inode_paths(files, fi, prefix) file_info["inode_paths"] = inode_paths files_json.append(file_info) diff --git a/tests/test_build.py b/tests/test_build.py index eb1c96642c..ac112f0064 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -198,9 +198,9 @@ def test_sorted_inode_first_path(testing_workdir): os.link(path_one, path_one_hardlink) files = ["one", "two", "one_hl"] - assert build.get_sorted_inode_first_path(files, "one", testing_workdir) == ["one", "one_hl"] - assert build.get_sorted_inode_first_path(files, "one_hl", testing_workdir) == ["one", "one_hl"] - assert build.get_sorted_inode_first_path(files, "two", testing_workdir) == ["two"] + assert build.get_inode_paths(files, "one", testing_workdir) == ["one", "one_hl"] + assert build.get_inode_paths(files, "one_hl", testing_workdir) == ["one", "one_hl"] + assert build.get_inode_paths(files, "two", testing_workdir) == ["two"] def test_create_info_files_json(testing_workdir, test_metadata): From c6a526028910dbecba2570cc66d9f817e07578b8 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 14 Nov 2016 15:19:41 -0600 Subject: [PATCH 127/156] Rename short_path to path in files.json --- conda_build/build.py | 4 ++-- tests/test_api_build.py | 4 ++-- tests/test_build.py | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 25af32b89f..57fc439851 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -512,7 +512,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): prefix_placeholder, file_mode = has_prefix(fi, files_with_prefix) path = os.path.join(prefix, fi) file_info = { - "short_path": get_short_path(m, fi), + "path": get_short_path(m, fi), "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), "file_type": getattr(file_type(path), "name"), @@ -535,7 +535,7 @@ def get_files_version(): def create_info_files_json(m, info_dir, prefix, files, files_with_prefix): - files_json_fields = ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + files_json_fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "prefix_placeholder", "no_link", "inode_first_path"] files_json_files = build_info_files_json(m, prefix, files, files_with_prefix) files_json_info = { diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 00c1f530d6..8a8cce73f0 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -736,7 +736,7 @@ def test_info_files_json(test_config): assert package_has_file(fn, "info/files.json") with tarfile.open(fn) as tf: data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8')) - fields = ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link", + fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link", "prefix_placeholder", "inode_first_path"] for key in data.keys(): assert key in ['files', 'fields', 'version'] @@ -746,7 +746,7 @@ def test_info_files_json(test_config): for file in data.get('files'): for key in file.keys(): assert key in fields - short_path = file.get("short_path") + short_path = file.get("path") if short_path == "test.sh" or short_path == "test.bat": assert file.get("prefix_placeholder") is not None assert file.get("file_mode") is not None diff --git a/tests/test_build.py b/tests/test_build.py index ac112f0064..577e758d60 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -218,16 +218,16 @@ def test_create_info_files_json(testing_workdir, test_metadata): build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) files_json_path = os.path.join(info_dir, "files.json") expected_output = { - "files": [{"file_type": "hardlink", "short_path": "one", + "files": [{"file_type": "hardlink", "path": "one", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_type": "hardlink", "short_path": "two", "size_in_bytes": 0, + {"file_type": "hardlink", "path": "two", "size_in_bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, {"file_mode": "text", "file_type": "hardlink", - "short_path": "foo", "prefix_placeholder": "prefix/path", + "path": "foo", "prefix_placeholder": "prefix/path", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], - "fields": ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "fields": ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "prefix_placeholder", "no_link", "inode_first_path"], "version": 1} with open(files_json_path, "r") as files_json: @@ -254,20 +254,20 @@ def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) files_json_path = os.path.join(info_dir, "files.json") expected_output = { - "files": [{"inode_paths": ["one", "one_hl"], "file_type": "hardlink", "short_path": "one", + "files": [{"inode_paths": ["one", "one_hl"], "file_type": "hardlink", "path": "one", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_type": "hardlink", "short_path": "two", "size_in_bytes": 0, + {"file_type": "hardlink", "path": "two", "size_in_bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, {"inode_paths": ["one", "one_hl"], "file_type": "hardlink", - "short_path": "one_hl", + "path": "one_hl", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_mode": "text", "file_type": "hardlink", "short_path": "foo", + {"file_mode": "text", "file_type": "hardlink", "path": "foo", "prefix_placeholder": "prefix/path", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], - "fields": ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "fields": ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "prefix_placeholder", "no_link", "inode_first_path"], "version": 1} with open(files_json_path, "r") as files_json: From 2fcded8c67259ad3c2cd5b5f4acdc8241a1c9122 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 16 Nov 2016 14:10:17 -0600 Subject: [PATCH 128/156] support output-folder option; fix test using built-package --- conda_build/api.py | 49 +++++++++++++++++++++------- conda_build/build.py | 43 ++++++++++++++++++++++--- conda_build/cli/main_build.py | 29 ++++++++--------- conda_build/cli/main_render.py | 2 +- conda_build/config.py | 2 +- conda_build/environ.py | 58 ++++++++++++++++++++++++++++++++++ conda_build/render.py | 4 +-- conda_build/utils.py | 2 ++ tests/test_api_build.py | 16 ++++++++++ 9 files changed, 168 insertions(+), 37 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index e3914f4a86..f71bca4f0a 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -39,13 +39,11 @@ def get_output_file_path(recipe_path_or_metadata, no_download_source=False, conf config = get_or_merge_config(config, **kwargs) if hasattr(recipe_path_or_metadata, 'config'): metadata = recipe_path_or_metadata - recipe_config = metadata.config else: metadata, _, _ = render_recipe(recipe_path_or_metadata, no_download_source=no_download_source, config=config) - recipe_config = config - return bldpkg_path(metadata, recipe_config) + return bldpkg_path(metadata) def check(recipe_path, no_download_source=False, config=None, **kwargs): @@ -79,34 +77,61 @@ def build(recipe_paths_or_metadata, post=None, need_source_download=True, def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwargs): import os + from conda_build.conda_interface import url_path from conda_build.build import test from conda_build.render import render_recipe + from conda_build.utils import get_recipe_abspath, rm_rf + from conda_build import source config = get_or_merge_config(config, **kwargs) + # we want to know if we're dealing with package input. If so, we can move the input on success. + is_package = False + if hasattr(recipedir_or_package_or_metadata, 'config'): metadata = recipedir_or_package_or_metadata recipe_config = metadata.config - elif os.path.isdir(recipedir_or_package_or_metadata): + else: + recipe_dir, need_cleanup = get_recipe_abspath(recipedir_or_package_or_metadata) + config.need_cleanup = need_cleanup + # This will create a new local build folder if and only if config doesn't already have one. # What this means is that if we're running a test immediately after build, we use the one # that the build already provided - config.compute_build_id(recipedir_or_package_or_metadata) - metadata, _, _ = render_recipe(recipedir_or_package_or_metadata, config=config) - recipe_config = config - else: - # fall back to old way (use recipe, rather than package) - metadata, _, _ = render_recipe(recipedir_or_package_or_metadata, no_download_source=False, - config=config, **kwargs) + metadata, _, _ = render_recipe(recipe_dir, config=config) recipe_config = config + # this recipe came from an extracted tarball. + if need_cleanup: + # ensure that the local location of the package is indexed, so that conda can find the + # local package + local_location = os.path.dirname(recipedir_or_package_or_metadata) + # strip off extra subdir folders + for platform in ('win', 'linux', 'osx'): + if os.path.basename(local_location).startswith(platform + "-"): + local_location = os.path.dirname(local_location) + update_index(local_location, config=config) + local_location = url_path(local_location) + # channel_urls is an iterable, but we don't know if it's a tuple or list. Don't know + # how to add elements. + recipe_config.channel_urls = list(recipe_config.channel_urls) + recipe_config.channel_urls.insert(0, local_location) + is_package = True + if metadata.meta.get('test') and metadata.meta['test'].get('source_files'): + source.provide(metadata.path, metadata.get_section('source'), config=config) + rm_rf(recipe_dir) with recipe_config: # This will create a new local build folder if and only if config doesn't already have one. # What this means is that if we're running a test immediately after build, we use the one # that the build already provided - config.compute_build_id(metadata.name()) + recipe_config.compute_build_id(metadata.name()) test_result = test(metadata, config=recipe_config, move_broken=move_broken) + + if test_result and is_package and hasattr(recipe_config, 'output_folder'): + os.rename(recipedir_or_package_or_metadata, + os.path.join(recipe_config.output_folder, + os.path.basename(recipedir_or_package_or_metadata))) return test_result diff --git a/conda_build/build.py b/conda_build/build.py index 61c3ed2bf4..c69a42ee30 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function from collections import deque +import copy import fnmatch from glob import glob import io @@ -25,6 +26,8 @@ # http://stackoverflow.com/a/13057751/1170370 import encodings.idna # NOQA +from conda_verify.verify import Verify + # used to get version from .conda_interface import cc from .conda_interface import envs_dirs, root_dir @@ -59,7 +62,6 @@ from conda_build.features import feature_list import conda_build.noarch_python as noarch_python -from conda_verify.verify import Verify if 'bsd' in sys.platform: @@ -201,7 +203,32 @@ def copy_recipe(m, config): else: original_recipe = "" - rendered = output_yaml(m) + rendered_metadata = copy.deepcopy(m) + # fill in build versions used + build_deps = [] + # we only care if we actually have build deps. Otherwise, the environment will not be + # valid for inspection. + if m.meta.get('requirements') and m.meta['requirements'].get('build'): + build_deps = environ.Environment(m.config.build_prefix).package_specs + + # hard-code build string so that any future "renderings" can't go wrong based on user env + rendered_metadata.meta['build']['string'] = m.build_id() + + rendered_metadata.meta['requirements'] = rendered_metadata.meta.get('requirements', {}) + rendered_metadata.meta['requirements']['build'] = build_deps + + # if source/path is relative, then the output package makes no sense at all. The next + # best thing is to hard-code the absolute path. This probably won't exist on any + # system other than the original build machine, but at least it will work there. + if m.meta.get('source'): + if 'path' in m.meta['source'] and not os.path.isabs(m.meta['source']['path']): + rendered_metadata.meta['source']['path'] = os.path.normpath( + os.path.join(m.path, m.meta['source']['path'])) + elif ('git_url' in m.meta['source'] and not os.path.isabs(m.meta['source']['git_url'])): + rendered_metadata.meta['source']['git_url'] = os.path.normpath( + os.path.join(m.path, m.meta['source']['git_url'])) + + rendered = output_yaml(rendered_metadata) if not original_recipe or not open(original_recipe).read() == rendered: with open(join(recipe_dir, "meta.yaml"), 'w') as f: f.write("# This file created by conda-build {}\n".format(__version__)) @@ -791,7 +818,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F files3 = prefix_files(prefix=config.build_prefix) fix_permissions(files3 - files1, config.build_prefix) - path = bldpkg_path(m, config) + path = bldpkg_path(m) # lock the output directory while we build this file # create the tarball in a temporary directory to minimize lock time @@ -1005,7 +1032,7 @@ def tests_failed(m, move_broken, broken_dir, config): os.makedirs(broken_dir) if move_broken: - shutil.move(bldpkg_path(m, config), join(broken_dir, "%s.tar.bz2" % m.dist())) + shutil.move(bldpkg_path(m), join(broken_dir, "%s.tar.bz2" % m.dist())) sys.exit("TESTS FAILED: " + m.dist()) @@ -1126,10 +1153,16 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, # outputs message, or does upload, depending on value of args.anaconda_upload if post in [True, None]: - output_file = bldpkg_path(metadata, config=recipe_config) + output_file = bldpkg_path(metadata) handle_anaconda_upload(output_file, config=recipe_config) already_built.add(output_file) + if hasattr(recipe_config, 'output_folder') and recipe_config.output_folder: + destination = os.path.join(recipe_config.output_folder, os.path.basename(output_file)) + if os.path.exists(destination): + os.remove(destination) + os.rename(output_file, destination) + def handle_anaconda_upload(path, config): from conda_build.os_utils.external import find_executable diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 4fadbd3780..34e978a63e 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -158,6 +158,11 @@ def parse_args(args): action="store_true", help=("do not run verification on recipes or packages when building") ) + p.add_argument( + "--output-folder", + help=("folder to dump output package to. Package are moved here if build or test succeeds." + " Destination folder must exist prior to using this.") + ) add_parser_channels(p) @@ -165,12 +170,13 @@ def parse_args(args): return p, args -def output_action(metadata, config): +def output_action(recipe, config): silence_loggers(show_warnings_and_errors=False) + metadata, _, _ = api.render(recipe, config=config) if metadata.skip(): print_skip_message(metadata) else: - print(bldpkg_path(metadata, config)) + print(bldpkg_path(metadata)) def source_action(metadata, config): @@ -178,12 +184,12 @@ def source_action(metadata, config): print('Source tree in:', config.work_dir) -def test_action(metadata, config): - return api.test(metadata.path, move_broken=False, config=config) +def test_action(recipe, config): + return api.test(recipe, move_broken=False, config=config) -def check_action(metadata, config): - return api.check(metadata.path, config=config) +def check_action(recipe, config): + return api.check(recipe, config=config) def execute(args): @@ -222,17 +228,8 @@ def execute(args): if action: for recipe in args.recipe: - recipe_dir, need_cleanup = get_recipe_abspath(recipe) - - if not isdir(recipe_dir): - sys.exit("Error: no such directory: %s" % recipe_dir) - - # this fully renders any jinja templating, throwing an error if any data is missing - m, _, _ = render_recipe(recipe_dir, no_download_source=False, config=config) - action(m, config) + action(recipe, config) - if need_cleanup: - rm_rf(recipe_dir) else: api.build(args.recipe, post=args.post, build_only=args.build_only, notest=args.notest, keep_old_work=args.keep_old_work, diff --git a/conda_build/cli/main_render.py b/conda_build/cli/main_render.py index 8887e88078..b35fcb0276 100644 --- a/conda_build/cli/main_render.py +++ b/conda_build/cli/main_render.py @@ -131,7 +131,7 @@ def execute(args): if args.output: logging.basicConfig(level=logging.ERROR) silence_loggers(show_warnings_and_errors=False) - print(bldpkg_path(metadata, config=config)) + print(bldpkg_path(metadata)) else: logging.basicConfig(level=logging.INFO) print(output_yaml(metadata, args.file)) diff --git a/conda_build/config.py b/conda_build/config.py index bdaf0300b8..1cc8f27a14 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -93,7 +93,7 @@ def env(lang, default): Setting = namedtuple("ConfigSetting", "name, default") values = [Setting('activate', True), Setting('anaconda_upload', binstar_upload), - Setting('channel_urls', ()), + Setting('channel_urls', []), Setting('dirty', False), Setting('include_recipe', True), Setting('keep_old_work', False), diff --git a/conda_build/environ.py b/conda_build/environ.py index 14f114512a..c04c920f63 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, print_function +import json import logging import multiprocessing import os @@ -443,6 +444,63 @@ def system_vars(env_dict, prefix, config): return d +class InvalidEnvironment(Exception): + pass + + +# Stripped-down Environment class from conda-tools ( https://github.com/groutr/conda-tools ) +# Vendored here to avoid the whole dependency for just this bit. +def _load_json(path): + with open(path, 'r') as fin: + x = json.load(fin) + return x + + +def _load_all_json(path): + """ + Load all json files in a directory. Return dictionary with filenames mapped to json + dictionaries. + """ + root, _, files = next(os.walk(path)) + result = {} + for f in files: + if f.endswith('.json'): + result[f] = _load_json(join(root, f)) + return result + + +class Environment(object): + def __init__(self, path): + """ + Initialize an Environment object. + + To reflect changes in the underlying environment, a new Environment object should be + created. + """ + self.path = path + self._meta = join(path, 'conda-meta') + if os.path.isdir(path) and os.path.isdir(self._meta): + self._packages = {} + else: + raise InvalidEnvironment('Unable to load environment {}'.format(path)) + + def _read_package_json(self): + if not self._packages: + self._packages = _load_all_json(self._meta) + + def package_specs(self): + """ + List all package specs in the environment. + """ + self._read_package_json() + json_objs = self._packages.values() + specs = [] + for i in json_objs: + p, v, b = i['name'], i['version'], i['build'] + specs.append('{} {} {}'.format(p, v, b)) + return specs + + if __name__ == '__main__': e = get_dict(cc) for k in sorted(e): diff --git a/conda_build/render.py b/conda_build/render.py index 70364baeb8..f26cd52de6 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -65,12 +65,12 @@ def set_language_env_vars(args, parser, config, execute=None): os.environ[var] = str(getattr(config, var)) -def bldpkg_path(m, config): +def bldpkg_path(m): ''' Returns path to built package's tarball given its ``Metadata``. ''' output_dir = m.info_index()['subdir'] - return os.path.join(os.path.dirname(config.bldpkgs_dir), output_dir, '%s.tar.bz2' % m.dist()) + return os.path.join(os.path.dirname(m.config.bldpkgs_dir), output_dir, '%s.tar.bz2' % m.dist()) def parse_or_try_download(metadata, no_download_source, config, diff --git a/conda_build/utils.py b/conda_build/utils.py index d92ea0848b..18c6213f8a 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -86,6 +86,8 @@ def get_recipe_abspath(recipe): else: recipe_dir = abspath(recipe) need_cleanup = False + if not os.path.exists(recipe_dir): + raise ValueError("Package or recipe at path {0} does not exist".format(recipe_dir)) return recipe_dir, need_cleanup diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 8ec98539f4..a3e61dd94f 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -727,3 +727,19 @@ def test_script_win_creates_exe(test_config): api.build(recipe, config=test_config) assert package_has_file(fn, 'Scripts/test-script.exe') assert package_has_file(fn, 'Scripts/test-script-script.py') + + +def test_build_output_folder_moves_file(test_metadata, testing_workdir): + output_path = api.get_output_file_path(test_metadata) + test_metadata.config.output_folder = testing_workdir + api.build(test_metadata, no_test=True) + assert not os.path.exists(output_path) + assert os.path.isfile(os.path.join(testing_workdir, os.path.basename(output_path))) + + +def test_test_output_folder_moves_file(test_metadata, testing_workdir): + output_path = api.get_output_file_path(test_metadata) + api.build(test_metadata, no_test=True) + api.test(output_path, output_folder=testing_workdir) + assert not os.path.exists(output_path) + assert os.path.isfile(os.path.join(testing_workdir, os.path.basename(output_path))) From bcec1af6069584007e7767f1f4c0d9d32f7092d9 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 16 Nov 2016 15:23:24 -0600 Subject: [PATCH 129/156] fix test return values --- conda_build/api.py | 7 ++++--- conda_build/build.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index f71bca4f0a..2962687d77 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -110,11 +110,11 @@ def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwar if os.path.basename(local_location).startswith(platform + "-"): local_location = os.path.dirname(local_location) update_index(local_location, config=config) - local_location = url_path(local_location) + local_url = url_path(local_location) # channel_urls is an iterable, but we don't know if it's a tuple or list. Don't know # how to add elements. recipe_config.channel_urls = list(recipe_config.channel_urls) - recipe_config.channel_urls.insert(0, local_location) + recipe_config.channel_urls.insert(0, local_url) is_package = True if metadata.meta.get('test') and metadata.meta['test'].get('source_files'): source.provide(metadata.path, metadata.get_section('source'), config=config) @@ -128,7 +128,8 @@ def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwar recipe_config.compute_build_id(metadata.name()) test_result = test(metadata, config=recipe_config, move_broken=move_broken) - if test_result and is_package and hasattr(recipe_config, 'output_folder'): + if (test_result and is_package and hasattr(recipe_config, 'output_folder') and + recipe_config.output_folder): os.rename(recipedir_or_package_or_metadata, os.path.join(recipe_config.output_folder, os.path.basename(recipedir_or_package_or_metadata))) diff --git a/conda_build/build.py b/conda_build/build.py index 5a43de4f53..9790cea923 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1033,7 +1033,7 @@ def test(m, config, move_broken=True): shell_files = create_shell_files(tmp_dir, m, config) if not (py_files or shell_files or pl_files or lua_files): print("Nothing to test for:", m.dist()) - return + return True print("TEST START:", m.dist()) @@ -1128,6 +1128,7 @@ def test(m, config, move_broken=True): tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config) print("TEST END:", m.dist()) + return True def tests_failed(m, move_broken, broken_dir, config): From 3499abc69023f015b7bec70d5113235c3aabe393 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Mon, 21 Nov 2016 12:28:21 +0000 Subject: [PATCH 130/156] Allow `!` in package version It is used to represent 'epoch'. --- conda_build/metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index f632a55102..367c752e1f 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -295,9 +295,11 @@ def _git_clean(source_meta): def check_bad_chrs(s, field): - bad_chrs = '=!@#$%^&*:;"\'\\|<>?/ ' + bad_chrs = '=@#$%^&*:;"\'\\|<>?/ ' if field in ('package/version', 'build/string'): bad_chrs += '-' + if field != 'package/version': + bad_chrs += '!' for c in bad_chrs: if c in s: sys.exit("Error: bad character '%s' in %s: %s" % (c, field, s)) From 1847dd29e7b375ab7ff8cd7d666ec7d2a8b1173f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 21 Nov 2016 08:58:26 -0600 Subject: [PATCH 131/156] fix merge check for dst starting with src --- conda_build/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index d92ea0848b..aa69bb16e9 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -191,8 +191,10 @@ def merge_tree(src, dst, symlinks=False, timeout=90, lock=None): Like copytree(src, dst), but raises an error if merging the two trees would overwrite any files. """ - assert src not in dst, ("Can't merge/copy source into subdirectory of itself. Please create " - "separate spaces for these things.") + dst = os.path.normpath(os.path.normcase(dst)) + src = os.path.normpath(os.path.normcase(src)) + assert not dst.startswith(src), ("Can't merge/copy source into subdirectory of itself. " + "Please create separate spaces for these things.") new_files = copytree(src, dst, symlinks=symlinks, dry_run=True) existing = [f for f in new_files if isfile(f)] From 0470e1d672db9abb8329a6ae5cb3ba6e386c8895 Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 21 Nov 2016 09:33:34 -0600 Subject: [PATCH 132/156] Don't compute sha256 for folders --- conda_build/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_build/build.py b/conda_build/build.py index 57fc439851..6da7170e0c 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -470,6 +470,8 @@ def get_short_path(m, target_file): def sha256_checksum(filename, buffersize=65536): + if not isfile(filename): + return None sha256 = hashlib.sha256() with open(filename, 'rb') as f: for block in iter(lambda: f.read(buffersize), b''): From ead9f881005fb67f02ef1ca6c2d39a7bc43354a4 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 21 Nov 2016 17:00:40 -0600 Subject: [PATCH 133/156] pin requests to 2.11.1 --- .travis.yml | 4 ++-- appveyor.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d32c9d36eb..e19c3fc021 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,11 +33,11 @@ install: conda config --set always_yes yes; else if [ -n "$CONDA" ]; then - conda install -y --no-deps conda=${CONDA} requests; + conda install -y --no-deps conda=${CONDA}; fi fi - - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 conda-verify + - conda install -q anaconda-client requests=2.11.1 filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 conda-verify - pip install pkginfo - if [[ "$FLAKE8" == "true" ]]; then conda install -q flake8; diff --git a/appveyor.yml b/appveyor.yml index b36b05a0c2..635fd43580 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,7 +50,7 @@ install: - python -c "import sys; print(sys.version)" - python -c "import sys; print(sys.executable)" - python -c "import sys; print(sys.prefix)" - - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests contextlib2 + - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests=2.11.1 contextlib2 - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy conda-verify - conda install -c conda-forge -q perl - conda update -q --all From cc21f9823af516ce7db4bedc5e70f25dd88ca17f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 21 Nov 2016 18:05:27 -0600 Subject: [PATCH 134/156] use normpath when comparing utils.relative --- conda_build/post.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index c0276880e5..a6e92f9ebc 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -364,8 +364,8 @@ def mk_relative_linux(f, prefix, rpaths=('lib',)): # IMHO utils.relative shouldn't exist, but I am too paranoid to remove # it, so instead, make sure that what I think it should be replaced by # gives the same result and assert if not. Yeah, I am a chicken. - rel_ours = utils.relative(f, rpath) - rel_stdlib = os.path.relpath(rpath, os.path.dirname(f)) + rel_ours = os.path.normpath(utils.relative(f, rpath)) + rel_stdlib = os.path.normpath(os.path.relpath(rpath, os.path.dirname(f))) assert rel_ours == rel_stdlib, \ 'utils.relative {0} and relpath {1} disagree for {2}, {3}'.format( rel_ours, rel_stdlib, f, rpath) From a46965cd2e46ff7d81799feb27fb4895d9f58c86 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 21 Nov 2016 18:34:33 -0600 Subject: [PATCH 135/156] disallow softlinks for conda in conda-build --- conda_build/conda_interface.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index d6af5429e9..1e8facd6df 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -55,6 +55,9 @@ NoPackagesFoundError = conda.exceptions.NoPackagesFoundError CondaValueError = conda.exceptions.CondaValueError + # disallow softlinks. This avoids a lot of dumb issues, at the potential cost of disk space. + conda.base.context.context.allow_softlinks = False + else: from conda.config import get_default_urls, non_x86_linux_machines, load_condarc # NOQA from conda.cli.common import get_prefix # NOQA @@ -74,6 +77,8 @@ get_rc_urls = cc.get_rc_urls get_local_urls = cc.get_local_urls + cc.allow_softlinks = False + class PaddingError(Exception): pass From ac9d969c0cadc208c257805e259457f178dbf789 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 28 Nov 2016 11:16:28 -0600 Subject: [PATCH 136/156] support globs in prefix file functions --- conda_build/metadata.py | 20 +++++++++++++++----- tests/test_metadata.py | 11 ++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 367c752e1f..52c6e7456e 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, print_function +import glob import logging import os import re @@ -175,6 +176,13 @@ def parse(data, config, path=None): return sanitize(res) +def expand_globs(path_list): + files = [] + for path in path_list: + files.extend(glob.glob(path)) + return files + + trues = {'y', 'on', 'true', 'yes'} falses = {'n', 'no', 'false', 'off'} @@ -685,17 +693,18 @@ def has_prefix_files(self): if any('\\' in i for i in ret): raise RuntimeError("build/has_prefix_files paths must use / " "as the path delimiter on Windows") - return ret + return expand_globs(ret) def ignore_prefix_files(self): ret = self.get_value('build/ignore_prefix_files', False) if type(ret) not in (list, bool): - raise RuntimeError('build/ignore_prefix_files should be boolean or a list of paths') + raise RuntimeError('build/ignore_prefix_files should be boolean or a list of paths ' + '(optionally globs)') if sys.platform == 'win32': if type(ret) is list and any('\\' in i for i in ret): raise RuntimeError("build/ignore_prefix_files paths must use / " "as the path delimiter on Windows") - return ret + return expand_globs(ret) if type(ret) is list else ret def always_include_files(self): files = ensure_list(self.get_value('build/always_include_files', [])) @@ -704,7 +713,8 @@ def always_include_files(self): "as the path delimiter on Windows") if on_win: files = [f.replace("/", "\\") for f in files] - return files + + return expand_globs(files) def include_recipe(self): return self.get_value('build/include_recipe', True) @@ -717,7 +727,7 @@ def binary_has_prefix_files(self): if any('\\' in i for i in ret): raise RuntimeError("build/binary_has_prefix_files paths must use / " "as the path delimiter on Windows") - return ret + return expand_globs(ret) def skip(self): return self.get_value('build/skip', False) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index a10659a274..f643f90dec 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -3,7 +3,7 @@ from conda_build.conda_interface import MatchSpec -from conda_build.metadata import select_lines, handle_config_version +from conda_build.metadata import select_lines, handle_config_version, expand_globs from .utils import testing_workdir, test_config, test_metadata @@ -107,3 +107,12 @@ def test_numpy(self): self.assertRaises(RuntimeError, handle_config_version, MatchSpec('numpy x.x'), None) + + +def test_expand_globs(testing_workdir): + files = ['abc', 'acb'] + for f in files: + with open(f, 'w') as _f: + _f.write('weee') + assert expand_globs(files) == files + assert expand_globs(['a*']) == files From 363d0823f7208920fb2c7c491d085aa0ce8988ac Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 28 Nov 2016 11:51:33 -0600 Subject: [PATCH 137/156] fix relative path issues with prefix globs --- conda_build/metadata.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 52c6e7456e..82452adfef 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -176,11 +176,12 @@ def parse(data, config, path=None): return sanitize(res) -def expand_globs(path_list): +def expand_globs(path_list, root_dir): files = [] for path in path_list: - files.extend(glob.glob(path)) - return files + files.extend(glob.glob(os.path.join(root_dir, path))) + # list comp is getting rid of absolute prefix, to match relative paths used in file list + return [f.replace(root_dir + os.path.sep, '') for f in files] trues = {'y', 'on', 'true', 'yes'} @@ -693,7 +694,7 @@ def has_prefix_files(self): if any('\\' in i for i in ret): raise RuntimeError("build/has_prefix_files paths must use / " "as the path delimiter on Windows") - return expand_globs(ret) + return expand_globs(ret, self.config.build_prefix) def ignore_prefix_files(self): ret = self.get_value('build/ignore_prefix_files', False) @@ -704,7 +705,7 @@ def ignore_prefix_files(self): if type(ret) is list and any('\\' in i for i in ret): raise RuntimeError("build/ignore_prefix_files paths must use / " "as the path delimiter on Windows") - return expand_globs(ret) if type(ret) is list else ret + return expand_globs(ret, self.config.build_prefix) if type(ret) is list else ret def always_include_files(self): files = ensure_list(self.get_value('build/always_include_files', [])) @@ -714,7 +715,7 @@ def always_include_files(self): if on_win: files = [f.replace("/", "\\") for f in files] - return expand_globs(files) + return expand_globs(files, self.config.build_prefix) def include_recipe(self): return self.get_value('build/include_recipe', True) @@ -727,7 +728,7 @@ def binary_has_prefix_files(self): if any('\\' in i for i in ret): raise RuntimeError("build/binary_has_prefix_files paths must use / " "as the path delimiter on Windows") - return expand_globs(ret) + return expand_globs(ret, self.config.build_prefix) def skip(self): return self.get_value('build/skip', False) From 3f44d032fcfdc0457da7c0891d2487079efce3c1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 28 Nov 2016 12:06:29 -0600 Subject: [PATCH 138/156] fix expand globs test --- tests/test_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index f643f90dec..6b990a398c 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -114,5 +114,5 @@ def test_expand_globs(testing_workdir): for f in files: with open(f, 'w') as _f: _f.write('weee') - assert expand_globs(files) == files - assert expand_globs(['a*']) == files + assert expand_globs(files, testing_workdir) == files + assert expand_globs(['a*'], testing_workdir) == files From 95bae038250486c165f2736bb23982eaf0e9cd20 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 28 Nov 2016 11:52:48 -0600 Subject: [PATCH 139/156] decouple ignore_prefix_files from binary_relocation. Make latter accept list of files/globs --- conda_build/metadata.py | 11 +++++++++++ conda_build/post.py | 10 ++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 82452adfef..f72ab783e3 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -717,6 +717,17 @@ def always_include_files(self): return expand_globs(files, self.config.build_prefix) + def binary_relocation(self): + ret = self.get_value('build/binary_relocation', True) + if type(ret) not in (list, bool): + raise RuntimeError('build/ignore_prefix_files should be boolean or a list of paths ' + '(optionally globs)') + if sys.platform == 'win32': + if type(ret) is list and any('\\' in i for i in ret): + raise RuntimeError("build/ignore_prefix_files paths must use / " + "as the path delimiter on Windows") + return expand_globs(ret, self.config.build_prefix) if type(ret) is list else ret + def include_recipe(self): return self.get_value('build/include_recipe', True) diff --git a/conda_build/post.py b/conda_build/post.py index a6e92f9ebc..ca0b62ad34 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -388,12 +388,6 @@ def mk_relative(m, f, prefix): if not is_obj(path): return - # skip over this file - if (m.ignore_prefix_files() and (type(m.ignore_prefix_files()) is bool or - f in m.ignore_prefix_files())): - print("Skipping relocation path patch for " + f) - return - if sys.platform.startswith('linux'): mk_relative_linux(f, prefix=prefix, rpaths=m.get_value('build/rpaths', ['lib'])) elif sys.platform == 'darwin': @@ -427,7 +421,7 @@ def post_build(m, files, prefix, build_python, croot): if sys.platform == 'win32': return - binary_relocation = bool(m.get_value('build/binary_relocation', True)) + binary_relocation = m.binary_relocation() if not binary_relocation: print("Skipping binary relocation logic") osx_is_app = bool(m.get_value('build/osx_is_app', False)) @@ -437,7 +431,7 @@ def post_build(m, files, prefix, build_python, croot): for f in files: if f.startswith('bin/'): fix_shebang(f, prefix=prefix, build_python=build_python, osx_is_app=osx_is_app) - if binary_relocation: + if binary_relocation is True or (isinstance(f, list) and f in binary_relocation): mk_relative(m, f, prefix) make_hardlink_copy(f, prefix) From 2105fa07d25421bd519cbbae84d022862e6d4c07 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 28 Nov 2016 12:07:00 -0600 Subject: [PATCH 140/156] changelog 2.0.11 --- CHANGELOG.txt | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e813377459..723537a48a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,31 @@ +2016-11-28 2.0.11: +------------------ + +Enhancements: +------------- + +* Further develop and update files.json #1501 +* New command line option: ``--output-folder`` allows moving artifact after build (to facilitate CI) #1538 +* support globs in `ignore_prefix_files`, `has_prefix_files`, `always_include_files`, `binary_has_prefix_files` #1554 +* decouple `ignore_prefix_files` from `binary_relocation`; make `binary_relocation` also accept a list of files or globs #1555 + +Bug fixes: +---------- + +* rename `short_path` key in files.json to `path` #1501 +* allow `!` in package version (used in epoch) #1542 +* don't compute SHA256 for folders #1544 +* fix merge check for dst starting with src #1546 +* use normpath when comparing utils.relative (fixes git clone issue) #1547 +* disallow softlinks for conda (>=v.4.2) in conda-build created environments #1548 + +Contributors: +------------- + +* @mingwandroid +* @msarahan +* @soapy1 + 2016-11-14 2.0.10: ------------------ From 122daf57df7e73088621abea6951257b70be1e9c Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Mon, 28 Nov 2016 12:48:45 -0600 Subject: [PATCH 141/156] document extra package install for testing fixes #1552 --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index f4b6e80637..65b2abbb80 100644 --- a/README.rst +++ b/README.rst @@ -98,6 +98,12 @@ Running our test suite requires cloning one other repo at the same level as cond https://github.com/conda/conda_build_test_recipe - this is necessary for relative path tests outside of conda build's build tree. +Additionally, you need to install a few extra packages: + +.. code-block:: bash + + conda install pytest pytest-cov mock + The test suite runs with py.test. Some useful commands to run select tests, assuming you are in the conda-build root folder: From 39dfb9bb61f151082969b765485aa64015164ae1 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 29 Nov 2016 16:03:02 -0500 Subject: [PATCH 142/156] Whitelist the environment variable `MACOSX_DEPLOYMENT_TARGET`. --- conda_build/environ.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index c04c920f63..959eb1f71e 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -385,6 +385,8 @@ def unix_vars(prefix): def osx_vars(compiler_vars, config): OSX_ARCH = 'i386' if config.arch == 32 else 'x86_64' + MACOSX_DEPLOYMENT_TARGET = os.environ.get('MACOSX_DEPLOYMENT_TARGET', '10.7') + compiler_vars['CFLAGS'] += ' -arch {0}'.format(OSX_ARCH) compiler_vars['CXXFLAGS'] += ' -arch {0}'.format(OSX_ARCH) compiler_vars['LDFLAGS'] += ' -arch {0}'.format(OSX_ARCH) @@ -393,7 +395,7 @@ def osx_vars(compiler_vars, config): # d['LDFLAGS'] = ldflags + rpath + ' -arch %(OSX_ARCH)s' % d return { 'OSX_ARCH': OSX_ARCH, - 'MACOSX_DEPLOYMENT_TARGET': '10.7', + 'MACOSX_DEPLOYMENT_TARGET': MACOSX_DEPLOYMENT_TARGET, } From a41ea6085e66a061c8d353bda064f1f350a35217 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Tue, 29 Nov 2016 19:22:43 +0000 Subject: [PATCH 143/156] Copy CONDA_PATH_BACKUP into build/test env. Since https://github.com/conda/conda/pull/3175 is in flux* we must detect whether this environment variable is being used by the conda coda-base on this platform. Since this is on the interface between conda and conda-build this check is performed in conda_interface.py. * one of the nice things about that PR is that, AFAIK, it makes the behaviour in-line with Windows since https://github.com/conda/conda/pull/2917 --- conda_build/build.py | 6 +++++- conda_build/conda_interface.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index 8aef7e2239..b9bcec7b01 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -32,7 +32,7 @@ # used to get version from .conda_interface import cc -from .conda_interface import envs_dirs, root_dir +from .conda_interface import envs_dirs, env_path_backup_var_exists, root_dir from .conda_interface import plan from .conda_interface import get_index from .conda_interface import PY3 @@ -855,6 +855,8 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F with path_prepended(config.build_prefix): env = environ.get_dict(config=config, m=m) env["CONDA_BUILD_STATE"] = "BUILD" + if env_path_backup_var_exists: + env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] work_file = join(config.work_dir, 'conda_build.sh') if script: with open(work_file, 'w') as bf: @@ -1064,6 +1066,8 @@ def test(m, config, move_broken=True): env = dict(os.environ.copy()) env.update(environ.get_dict(config=config, m=m, prefix=config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" + if env_path_backup_var_exists: + env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] if not config.activate: # prepend bin (or Scripts) directory diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 1e8facd6df..f12d92e675 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -25,6 +25,8 @@ from conda.config import rc_path # NOQA from conda.version import VersionOrder # NOQA +import os + if parse_version(conda.__version__) >= parse_version("4.2"): # conda 4.2.x import conda.base.context @@ -58,6 +60,10 @@ # disallow softlinks. This avoids a lot of dumb issues, at the potential cost of disk space. conda.base.context.context.allow_softlinks = False + # when deactivating envs (e.g. switching from root to build/test) this env var is used, + # except the PR that removed this has been reverted (for now) and Windows doesnt need it. + env_path_backup_var_exists = os.environ.get('CONDA_PATH_BACKUP', None) + else: from conda.config import get_default_urls, non_x86_linux_machines, load_condarc # NOQA from conda.cli.common import get_prefix # NOQA @@ -91,6 +97,7 @@ class NoPackagesFoundError(Exception): class CondaValueError(Exception): pass + env_path_backup_var_exists = os.environ.get('CONDA_PATH_BACKUP', None) class SignatureError(Exception): pass From b80a66e9e9b84802518438bbe324f929cd6518b5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 30 Nov 2016 11:44:33 -0600 Subject: [PATCH 144/156] define nomkl selector when FEATURE_NOMKL env var not set --- conda_build/metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index f72ab783e3..b0f0b8e902 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -69,6 +69,7 @@ def ns_cfg(config): np=np, os=os, environ=os.environ, + nomkl=bool(int(os.environ.get('FEATURE_NOMKL', False))) ) for machine in non_x86_linux_machines: d[machine] = bool(plat == 'linux-%s' % machine) From 6ae13ba7364c4339f59dd43e305c3b021edfcdc7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 2 Dec 2016 21:49:38 -0600 Subject: [PATCH 145/156] move test removal of recipe until after test completes --- conda_build/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/api.py b/conda_build/api.py index 2962687d77..7ae7ff5404 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -87,6 +87,7 @@ def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwar # we want to know if we're dealing with package input. If so, we can move the input on success. is_package = False + need_cleanup = False if hasattr(recipedir_or_package_or_metadata, 'config'): metadata = recipedir_or_package_or_metadata @@ -118,7 +119,6 @@ def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwar is_package = True if metadata.meta.get('test') and metadata.meta['test'].get('source_files'): source.provide(metadata.path, metadata.get_section('source'), config=config) - rm_rf(recipe_dir) with recipe_config: # This will create a new local build folder if and only if config doesn't already have one. @@ -133,6 +133,8 @@ def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwar os.rename(recipedir_or_package_or_metadata, os.path.join(recipe_config.output_folder, os.path.basename(recipedir_or_package_or_metadata))) + if need_cleanup: + rm_rf(recipe_dir) return test_result From 37dca7eabf59d3fde341208814ada7d63735224b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 4 Dec 2016 17:41:51 -0600 Subject: [PATCH 146/156] fix contextlib2 python 3 requirement --- conda.recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 205b3b644e..03e062049d 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -24,7 +24,7 @@ requirements: run: - conda-verify - conda >=4.1 - - contextlib2 [py<3] + - contextlib2 [py<34] - filelock - jinja2 - patchelf [linux] From cee4b476dfa2488ba76f91120f42d7955cfeff69 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 4 Dec 2016 17:42:28 -0600 Subject: [PATCH 147/156] allow relative local paths as channels --- conda_build/api.py | 3 +++ conda_build/cli/main_build.py | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index 7ae7ff5404..d4af2ff658 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -111,6 +111,9 @@ def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwar if os.path.basename(local_location).startswith(platform + "-"): local_location = os.path.dirname(local_location) update_index(local_location, config=config) + if not os.path.abspath(local_location): + local_location = os.path.normpath(os.path.abspath( + os.path.join(os.getcwd(), local_location))) local_url = url_path(local_location) # channel_urls is an iterable, but we don't know if it's a tuple or list. Don't know # how to add elements. diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 34e978a63e..2beb4201f6 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -8,7 +8,7 @@ import argparse import logging -from os.path import isdir +import os import sys import filelock @@ -16,11 +16,10 @@ import conda_build.api as api import conda_build.build as build from conda_build.cli.main_render import (set_language_env_vars, RecipeCompleter, - render_recipe, get_render_parser, bldpkg_path) -from conda_build.conda_interface import cc -from conda_build.conda_interface import add_parser_channels + get_render_parser, bldpkg_path) +from conda_build.conda_interface import cc, add_parser_channels, url_path import conda_build.source as source -from conda_build.utils import get_recipe_abspath, silence_loggers, rm_rf, print_skip_message +from conda_build.utils import silence_loggers, print_skip_message from conda_build.config import Config on_win = (sys.platform == 'win32') @@ -198,7 +197,19 @@ def execute(args): build.check_external() # change globals in build module, see comment there as well - config.channel_urls = args.channel or () + channel_urls = args.channel or () + config.channel_urls = [] + + for url in channel_urls: + # allow people to specify relative or absolute paths to local channels + # These channels still must follow conda rules - they must have the + # appropriate platform-specific subdir (e.g. win-64) + if os.path.isdir(url): + if not os.path.isabs(url): + url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url))) + url = url_path(url) + config.channel_urls.append(url) + config.override_channels = args.override_channels config.verbose = not args.quiet or args.debug From 67515a645d8a7e0ae39b99b0869223abf975d2c2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 11 Dec 2016 10:06:03 -0600 Subject: [PATCH 148/156] allow source_files in recipe check --- conda_build/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index b0f0b8e902..a090885ee2 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -296,7 +296,7 @@ def _git_clean(source_meta): 'requirements': ['build', 'run', 'conflicts'], 'app': ['entry', 'icon', 'summary', 'type', 'cli_opts', 'own_environment'], - 'test': ['requires', 'commands', 'files', 'imports'], + 'test': ['requires', 'commands', 'files', 'imports', 'source_files'], 'about': ['home', 'dev_url', 'doc_url', 'license_url', # these are URLs 'license', 'summary', 'description', 'license_family', # text 'license_file', 'readme', # paths in source tree From 0f9b1a6ea7d7193cc58993c1ddbe84fb6d4e356f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 12 Dec 2016 09:56:28 -0600 Subject: [PATCH 149/156] fix appveyor requests and git checkout errors --- appveyor.yml | 2 +- conda_build/build.py | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 635fd43580..718b9c3711 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,10 +50,10 @@ install: - python -c "import sys; print(sys.version)" - python -c "import sys; print(sys.executable)" - python -c "import sys; print(sys.prefix)" + - conda update -q --all - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests=2.11.1 contextlib2 - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy conda-verify - conda install -c conda-forge -q perl - - conda update -q --all # this is to ensure dependencies - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" diff --git a/conda_build/build.py b/conda_build/build.py index b9bcec7b01..118f618dfb 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1021,20 +1021,19 @@ def test(m, config, move_broken=True): clean_pkg_cache(m.dist(), config.timeout) - tmp_dir = config.test_dir - if not isdir(tmp_dir): - os.makedirs(tmp_dir) - create_files(tmp_dir, m, config) + if not isdir(config.test_dir): + os.makedirs(config.test_dir) + create_files(config.test_dir, m, config) # Make Perl or Python-specific test files if m.name().startswith('perl-'): - pl_files = create_pl_files(tmp_dir, m) + pl_files = create_pl_files(config.test_dir, m) py_files = False lua_files = False else: - py_files = create_py_files(tmp_dir, m) + py_files = create_py_files(config.test_dir, m) pl_files = False lua_files = False - shell_files = create_shell_files(tmp_dir, m, config) + shell_files = create_shell_files(config.test_dir, m, config) if not (py_files or shell_files or pl_files or lua_files): print("Nothing to test for:", m.dist()) return True @@ -1082,7 +1081,7 @@ def test(m, config, move_broken=True): # Python 2 Windows requires that envs variables be string, not unicode env = {str(key): str(value) for key, value in env.items()} suffix = "bat" if on_win else "sh" - test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix)) + test_script = join(config.test_dir, "conda_test_runner.{suffix}".format(suffix=suffix)) with open(test_script, 'w') as tf: if config.activate: @@ -1098,23 +1097,23 @@ def test(m, config, move_broken=True): if py_files: tf.write("{python} -s {test_file}\n".format( python=config.test_python, - test_file=join(tmp_dir, 'run_test.py'))) + test_file=join(config.test_dir, 'run_test.py'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if pl_files: tf.write("{perl} {test_file}\n".format( perl=config.test_perl, - test_file=join(tmp_dir, 'run_test.pl'))) + test_file=join(config.test_dir, 'run_test.pl'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if lua_files: tf.write("{lua} {test_file}\n".format( lua=config.test_lua, - test_file=join(tmp_dir, 'run_test.lua'))) + test_file=join(config.test_dir, 'run_test.lua'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if shell_files: - test_file = join(tmp_dir, 'run_test.' + suffix) + test_file = join(config.test_dir, 'run_test.' + suffix) if on_win: tf.write("call {test_file}\n".format(test_file=test_file)) if on_win: @@ -1129,7 +1128,7 @@ def test(m, config, move_broken=True): else: cmd = [shell_path, '-x', '-e', test_script] try: - subprocess.check_call(cmd, env=env, cwd=tmp_dir) + subprocess.check_call(cmd, env=env, cwd=config.test_dir) except subprocess.CalledProcessError: tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config) From 289037df3285ff15ef1222aa3ba6d2d66ec1f18d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 12 Dec 2016 16:21:20 -0600 Subject: [PATCH 150/156] update changelog for 2.0.12 --- CHANGELOG.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 723537a48a..e4d45e7c96 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,27 @@ +2016-12-12 2.0.12: + +Enhancements: +------------- + +* Whitelist, rather than hardcode, MACOSX_DEPLOYMENT_TARGET. Default to 10.7 #1561 +* Allow local relative paths to be passed as channel arguments #1565 + +Bug fixes: +---------- + +* Keep CONDA_PATH_BACKUP as allowed variable in build/test env activation. Necessary to make deactivation work correctly. #1560 +* Define nomkl selector when FEATURE_NOMKL environment variable is not set #1562 +* Move test removal of packaged recipe until after test completes #1563 +* Allow source_files in recognized meta.yaml fields #1572 + +Contributors: +------------- + +* @jakirkham +* @mingwandroid +* @msarahan + + 2016-11-28 2.0.11: ------------------ From 7342cbcce018eeb4fd9d9fd912b216af2b225377 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 27 Sep 2016 20:19:21 -0500 Subject: [PATCH 151/156] add initial test files to guide development --- .../alternate_type_wheel/meta.yaml | 16 +++++++++ .../meta.yaml | 21 ++++++++++++ .../run_test.py | 2 ++ .../split-packages/copying_files/meta.yaml | 33 +++++++++++++++++++ .../split-packages/copying_files/run_test.py | 1 + .../copying_files/test_subpackage1.py | 18 ++++++++++ .../jinja2_subpackage_name/meta.yaml | 10 ++++++ .../noarch_subpackage/meta.yaml | 14 ++++++++ .../overlapping_files/meta.yaml | 20 +++++++++++ .../script_autodetect_interpreter/meta.yaml | 19 +++++++++++ .../subpackage1.py | 4 +++ .../subpackage2.sh | 2 ++ .../subpackage3.unrecognized | 0 .../script_install_files/meta.yaml | 15 +++++++++ .../script_install_files/subpackage1.py | 4 +++ .../script_install_files/test_subpackage1.py | 6 ++++ tests/test_subpackages.py | 20 +++++++++++ tests/utils.py | 1 + 18 files changed, 206 insertions(+) create mode 100644 tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml create mode 100644 tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml create mode 100644 tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py create mode 100644 tests/test-recipes/split-packages/copying_files/meta.yaml create mode 120000 tests/test-recipes/split-packages/copying_files/run_test.py create mode 100644 tests/test-recipes/split-packages/copying_files/test_subpackage1.py create mode 100644 tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml create mode 100644 tests/test-recipes/split-packages/noarch_subpackage/meta.yaml create mode 100644 tests/test-recipes/split-packages/overlapping_files/meta.yaml create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh create mode 100644 tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized create mode 100644 tests/test-recipes/split-packages/script_install_files/meta.yaml create mode 100644 tests/test-recipes/split-packages/script_install_files/subpackage1.py create mode 100644 tests/test-recipes/split-packages/script_install_files/test_subpackage1.py create mode 100644 tests/test_subpackages.py diff --git a/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml b/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml new file mode 100644 index 0000000000..d149b3dcc5 --- /dev/null +++ b/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml @@ -0,0 +1,16 @@ +package: + name: split_packages_alternate_type_wheel + version: 1.0 + +requirements: + run: + - my_script_subpackage + +outputs: + - name: my_script_subpackage + script: subpackage1.py + script_interpreter: python + type: wheel + test: + script: test_subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml new file mode 100644 index 0000000000..e0bf52eb60 --- /dev/null +++ b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/meta.yaml @@ -0,0 +1,21 @@ +# Test that our composite run requirements for the parent package are the union of subpackage +# requirements + +package: + name: split_packages_compose_run_requirements_from_subpackages + version: 1.0 + +requirements: + run: + - my_script_subpackage + - my_script_subpackage_2 + +outputs: + - name: my_script_subpackage + requirements: + - cython + - name: my_script_subpackage_2 + requirements: + - click + +# tests are in run_test.py, and they check that the packages above are both installed diff --git a/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py new file mode 100644 index 0000000000..5088a1f4b2 --- /dev/null +++ b/tests/test-recipes/split-packages/compose_run_requirements_from_subpackages/run_test.py @@ -0,0 +1,2 @@ +import cython +import click diff --git a/tests/test-recipes/split-packages/copying_files/meta.yaml b/tests/test-recipes/split-packages/copying_files/meta.yaml new file mode 100644 index 0000000000..c400a4b9cd --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/meta.yaml @@ -0,0 +1,33 @@ +package: + name: split_packages_file_list + version: 1.0 + +build: + script: + # test copying filename + - echo weee > $PREFIX/subpackage_file1 # [unix] + - echo weee > %PREFIX%/subpackage_file1 # [win] + # test copying by folder name + - mkdir $PREFIX/somedir # [unix] + - mkdir %PREFIX%/somedir # [win] + - echo weee > $PREFIX/somedir/subpackage_file1 # [unix] + - echo weee > %PREFIX%/somedir/subpackage_file1 # [win] + # test glob patterns + - echo weee > $PREFIX/subpackage_file1.ext # [unix] + - echo weee > $PREFIX/subpackage_file2.ext # [unix] + - echo weee > %PREFIX%/subpackage_file1.ext # [win] + - echo weee > %PREFIX%/subpackage_file2.ext # [win] + +requirements: + run: + - my_script_subpackage + +outputs: + - name: my_script_subpackage + files: + - subpackage_file1 + - somedir + - *.ext + test: + script: test_subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/copying_files/run_test.py b/tests/test-recipes/split-packages/copying_files/run_test.py new file mode 120000 index 0000000000..829b62d71c --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/run_test.py @@ -0,0 +1 @@ +test_subpackage1.py \ No newline at end of file diff --git a/tests/test-recipes/split-packages/copying_files/test_subpackage1.py b/tests/test-recipes/split-packages/copying_files/test_subpackage1.py new file mode 100644 index 0000000000..e096bc0c90 --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/test_subpackage1.py @@ -0,0 +1,18 @@ +import os + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" + +filename = os.path.join(os.environ['PREFIX'], 'subdir', 'subpackage_file1') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1.ext') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" + + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file2.ext') +assert os.path.isfile(filename) +assert open(filename).read() == "weee" diff --git a/tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml b/tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml new file mode 100644 index 0000000000..da5ae9c950 --- /dev/null +++ b/tests/test-recipes/split-packages/jinja2_subpackage_name/meta.yaml @@ -0,0 +1,10 @@ +package: + name: split_packages_jinja2_subpackage_name + version: 1.0 + +requirements: + run: + - {{ PKG_NAME }}_subpackage + +outputs: + - name: {{ PKG_NAME }}_subpackage diff --git a/tests/test-recipes/split-packages/noarch_subpackage/meta.yaml b/tests/test-recipes/split-packages/noarch_subpackage/meta.yaml new file mode 100644 index 0000000000..9b307273ee --- /dev/null +++ b/tests/test-recipes/split-packages/noarch_subpackage/meta.yaml @@ -0,0 +1,14 @@ +package: + name: split_packages_jinja2_subpackage_name + version: 1.0 + +requirements: + run: + - pkg_subpackage + - pkg_subpackage_python_noarch + +outputs: + - name: pkg_subpackage + noarch: True + - name: pkg_subpackage_python_noarch + noarch: python diff --git a/tests/test-recipes/split-packages/overlapping_files/meta.yaml b/tests/test-recipes/split-packages/overlapping_files/meta.yaml new file mode 100644 index 0000000000..0cdcc36de0 --- /dev/null +++ b/tests/test-recipes/split-packages/overlapping_files/meta.yaml @@ -0,0 +1,20 @@ +# this test is to make sure that we raise an error when more than one subpackage +# contains the same file. This is important to avoid, as conda does nothing smart +# about keeping files that are installed by two packages when one is removed. + +package: + name: split_packages_script_overlapping_files + version: 1.0 + +requirements: + run: + - my_script_subpackage + - my_script_subpackage_2 + +outputs: + - name: my_script_subpackage + script: subpackage1.py + script_interpreter: python + - name: my_script_subpackage_2 + script: subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml b/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml new file mode 100644 index 0000000000..08d7a25776 --- /dev/null +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml @@ -0,0 +1,19 @@ +package: + name: split_packages_autodetect_interpreter + version: 1.0 + +requirements: + run: + - my_script_subpackage + - my_script_subpackage_shell + - my_script_subpackage_unrecognized + +outputs: + - name: my_script_subpackage + script: subpackage1.py + # Assume that on Windows, we have bash available here + - name: my_script_subpackage_shell + script: subpackage2.sh + # what happens when we have an unrecognized script type? + - name: my_script_subpackage_unrecognized + script: subpackage3.unrecognized diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py new file mode 100644 index 0000000000..09ec8a7540 --- /dev/null +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py @@ -0,0 +1,4 @@ +import os + +with open(os.path.join(os.environ['PREFIX'], 'subpackage_file_1'), 'w') as f: + f.write("weeee") diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh new file mode 100644 index 0000000000..ce59ebf7a5 --- /dev/null +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo weeee > $PREFIX/subpackage_file_2 diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test-recipes/split-packages/script_install_files/meta.yaml b/tests/test-recipes/split-packages/script_install_files/meta.yaml new file mode 100644 index 0000000000..af39bc4b15 --- /dev/null +++ b/tests/test-recipes/split-packages/script_install_files/meta.yaml @@ -0,0 +1,15 @@ +package: + name: split_packages_script_install_files + version: 1.0 + +requirements: + run: + - my_script_subpackage + +outputs: + - name: my_script_subpackage + script: subpackage1.py + script_interpreter: python + test: + script: test_subpackage1.py + script_interpreter: python diff --git a/tests/test-recipes/split-packages/script_install_files/subpackage1.py b/tests/test-recipes/split-packages/script_install_files/subpackage1.py new file mode 100644 index 0000000000..09ec8a7540 --- /dev/null +++ b/tests/test-recipes/split-packages/script_install_files/subpackage1.py @@ -0,0 +1,4 @@ +import os + +with open(os.path.join(os.environ['PREFIX'], 'subpackage_file_1'), 'w') as f: + f.write("weeee") diff --git a/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py b/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py new file mode 100644 index 0000000000..96ed1deba1 --- /dev/null +++ b/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py @@ -0,0 +1,6 @@ +import os + +filename = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') + +assert os.path.isfile(filename) +assert open(filename).read() == "weeee" diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py new file mode 100644 index 0000000000..65ba50d7ab --- /dev/null +++ b/tests/test_subpackages.py @@ -0,0 +1,20 @@ +import os +import pytest + +from conda_build import api + +from .utils import test_config, testing_workdir, is_valid_dir, subpackage_dir + +@pytest.fixture(params=[dirname for dirname in os.listdir(subpackage_dir) + if is_valid_dir(subpackage_dir, dirname)]) +def recipe(request): + return os.path.join(subpackage_dir, request.param) + + +def test_subpackage_recipes(recipe, test_config): + api.build(recipe, config=test_config) + pass + + +def test_all_subpackages_uploaded(): + pass diff --git a/tests/utils.py b/tests/utils.py index 496b5af900..c39dae7f85 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,6 +16,7 @@ thisdir = os.path.dirname(os.path.realpath(__file__)) metadata_dir = os.path.join(thisdir, "test-recipes/metadata") +subpackage_dir = os.path.join(thisdir, "test-recipes/split-packages") fail_dir = os.path.join(thisdir, "test-recipes/fail") From 36857b5310d2cf90b4f38e67140f28953a337e3a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 13 Dec 2016 12:17:55 -0600 Subject: [PATCH 152/156] implement package testing and uploading for split packages --- conda_build/api.py | 54 +--- conda_build/build.py | 268 ++++++++++++------ .../split-packages/copying_files/meta.yaml | 2 +- .../subpackage1.py | 6 +- tests/test_subpackages.py | 3 +- 5 files changed, 199 insertions(+), 134 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index d4af2ff658..5870e39a8c 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -76,68 +76,20 @@ def build(recipe_paths_or_metadata, post=None, need_source_download=True, def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwargs): - import os - from conda_build.conda_interface import url_path from conda_build.build import test - from conda_build.render import render_recipe - from conda_build.utils import get_recipe_abspath, rm_rf - from conda_build import source config = get_or_merge_config(config, **kwargs) - - # we want to know if we're dealing with package input. If so, we can move the input on success. - is_package = False - need_cleanup = False - if hasattr(recipedir_or_package_or_metadata, 'config'): - metadata = recipedir_or_package_or_metadata - recipe_config = metadata.config - else: - recipe_dir, need_cleanup = get_recipe_abspath(recipedir_or_package_or_metadata) - config.need_cleanup = need_cleanup - - # This will create a new local build folder if and only if config doesn't already have one. - # What this means is that if we're running a test immediately after build, we use the one - # that the build already provided - metadata, _, _ = render_recipe(recipe_dir, config=config) - recipe_config = config - # this recipe came from an extracted tarball. - if need_cleanup: - # ensure that the local location of the package is indexed, so that conda can find the - # local package - local_location = os.path.dirname(recipedir_or_package_or_metadata) - # strip off extra subdir folders - for platform in ('win', 'linux', 'osx'): - if os.path.basename(local_location).startswith(platform + "-"): - local_location = os.path.dirname(local_location) - update_index(local_location, config=config) - if not os.path.abspath(local_location): - local_location = os.path.normpath(os.path.abspath( - os.path.join(os.getcwd(), local_location))) - local_url = url_path(local_location) - # channel_urls is an iterable, but we don't know if it's a tuple or list. Don't know - # how to add elements. - recipe_config.channel_urls = list(recipe_config.channel_urls) - recipe_config.channel_urls.insert(0, local_url) - is_package = True - if metadata.meta.get('test') and metadata.meta['test'].get('source_files'): - source.provide(metadata.path, metadata.get_section('source'), config=config) + recipe_config = recipedir_or_package_or_metadata.config with recipe_config: # This will create a new local build folder if and only if config doesn't already have one. # What this means is that if we're running a test immediately after build, we use the one # that the build already provided - recipe_config.compute_build_id(metadata.name()) - test_result = test(metadata, config=recipe_config, move_broken=move_broken) + test_result = test(recipedir_or_package_or_metadata, config=recipe_config, + move_broken=move_broken) - if (test_result and is_package and hasattr(recipe_config, 'output_folder') and - recipe_config.output_folder): - os.rename(recipedir_or_package_or_metadata, - os.path.join(recipe_config.output_folder, - os.path.basename(recipedir_or_package_or_metadata))) - if need_cleanup: - rm_rf(recipe_dir) return test_result diff --git a/conda_build/build.py b/conda_build/build.py index 118f618dfb..fbfecfee0d 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -56,7 +56,8 @@ from conda_build.utils import (rm_rf, _check_call, copy_into, on_win, get_build_folders, silence_loggers, path_prepended, create_entry_points, prepend_bin_path, codec, root_script_dir, print_skip_message, - ensure_list, get_lock, ExitStack) + ensure_list, get_lock, ExitStack, get_recipe_abspath) +from conda_build.metadata import MetaData, build_string_from_metadata, expand_globs from conda_build.index import update_index from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) @@ -195,7 +196,10 @@ def get_run_dists(m, config): def copy_recipe(m, config): if config.include_recipe and m.include_recipe(): recipe_dir = join(config.info_dir, 'recipe') - os.makedirs(recipe_dir) + try: + os.makedirs(recipe_dir) + except: + pass if os.path.isdir(m.path): for fn in os.listdir(m.path): @@ -219,6 +223,8 @@ def copy_recipe(m, config): if m.meta.get('requirements') and m.meta['requirements'].get('build'): build_deps = environ.Environment(m.config.build_prefix).package_specs + if not rendered_metadata.meta.get('build'): + rendered_metadata.meta['build'] = {} # hard-code build string so that any future "renderings" can't go wrong based on user env rendered_metadata.meta['build']['string'] = m.build_id() @@ -479,6 +485,8 @@ def create_info_files(m, files, config, prefix): copy_into(join(m.path, m.get_value('app/icon')), join(config.info_dir, 'icon.png'), config.timeout) + return [f.replace(config.build_prefix + '/', '') for root, _, _ in os.walk(config.info_dir) + for f in glob(os.path.join(root, '*'))] def get_short_path(m, target_file): @@ -726,6 +734,38 @@ def warn_on_old_conda_build(index=None, installed_version=None, available_packag """ % (installed_version, available_packages[-1]), file=sys.stderr) +def bundle_files(pkg_files, metadata, config, output_filename): + info_files = create_info_files(metadata, pkg_files, config=config, prefix=config.build_prefix) + for f in info_files: + if f not in pkg_files: + pkg_files.append(f) + + # lock the output directory while we build this file + # create the tarball in a temporary directory to minimize lock time + with TemporaryDirectory() as tmp: + tmp_path = os.path.join(tmp, os.path.basename(output_filename)) + t = tarfile.open(tmp_path, 'w:bz2') + + def order(f): + # we don't care about empty files so send them back via 100000 + fsize = os.stat(join(config.build_prefix, f)).st_size or 100000 + # info/* records will be False == 0, others will be 1. + info_order = int(os.path.dirname(f) != 'info') + return info_order, fsize + + # add files in order of a) in info directory, b) increasing size so + # we can access small manifest or json files without decompressing + # possible large binary or data files + for f in sorted(pkg_files, key=order): + t.add(join(config.build_prefix, f), f) + t.close() + + # we're done building, perform some checks + tarcheck.check_all(tmp_path) + copy_into(tmp_path, config.bldpkgs_dir, config.timeout) + return os.path.join(config.bldpkgs_dir, output_filename) + + def build(m, config, post=None, need_source_download=True, need_reparse_in_env=False): ''' Build the package with the specified metadata. @@ -741,13 +781,21 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F if m.skip(): print_skip_message(m) - return False + return [] + + with path_prepended(config.build_prefix): + env = environ.get_dict(config=config, m=m) + env["CONDA_BUILD_STATE"] = "BUILD" + if env_path_backup_var_exists: + env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] if config.skip_existing: package_exists = is_package_built(m, config) if package_exists: print(m.dist(), "is already built in {0}, skipping.".format(package_exists)) - return False + return [] + + built_packages = [] if post in [False, None]: print("BUILD START:", m.dist()) @@ -852,11 +900,6 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F # There is no sense in trying to run an empty build script. if isfile(build_file) or script: - with path_prepended(config.build_prefix): - env = environ.get_dict(config=config, m=m) - env["CONDA_BUILD_STATE"] = "BUILD" - if env_path_backup_var_exists: - env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] work_file = join(config.work_dir, 'conda_build.sh') if script: with open(work_file, 'w') as bf: @@ -920,8 +963,6 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F else: pkg_files = sorted(files2 - files1) - create_info_files(m, pkg_files, config=config, prefix=config.build_prefix) - if m.get_value('build/noarch_python'): noarch_python.transform(m, sorted(files2 - files1), config.build_prefix) elif is_noarch_python(m): @@ -931,48 +972,73 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F files3 = prefix_files(prefix=config.build_prefix) fix_permissions(files3 - files1, config.build_prefix) - path = bldpkg_path(m) - - # lock the output directory while we build this file - # create the tarball in a temporary directory to minimize lock time - with TemporaryDirectory() as tmp: - tmp_path = os.path.join(tmp, os.path.basename(path)) - t = tarfile.open(tmp_path, 'w:bz2') - - def order(f): - # we don't care about empty files so send them back via 100000 - fsize = os.stat(join(config.build_prefix, f)).st_size or 100000 - # info/* records will be False == 0, others will be 1. - info_order = int(os.path.dirname(f) != 'info') - return info_order, fsize + outputs = m.get_section('outputs') + if not outputs: + outputs = [{'name': m.name(), + 'files': files3 - files1}] + else: + # make a metapackage if the recipe depends on any of its subpackages as runtime reqs + requirements = m.get_value('requirements/run') + for out in outputs: + if out['name'] in requirements: + requirements.extend(out.get('requirements', [])) + outputs.append({'name': m.name(), + 'requirements': m.get_value('requirements/run')}) + + for output in outputs: + files = expand_globs(output.get('files', []), config.build_prefix) + if not files and output.get('script'): + interpreter = output.get('script_interpreter') + if not interpreter: + interpreter = guess_interpreter(output['script']) + files = subprocess.check_output(interpreter.split(' ') + + [os.path.join(m.path, output['script'])], + cwd=config.build_prefix, env=env).split('\n') + tmp_metadata = MetaData.fromdict({'package': {'name': output['name'], + 'version': m.version()}, + 'requirements': {'run': output.get('requirements', + [])}}) + + output_filename = ('-'.join([output['name'], m.version(), + build_string_from_metadata(tmp_metadata)]) + + '.tar.bz2') + + output_package = bundle_files(files, tmp_metadata, config, output_filename) + + if not getattr(config, "noverify", False): + verifier = Verify() + ignore_scripts = config.ignore_package_verify_scripts if \ + config.ignore_package_verify_scripts else None + run_scripts = config.run_package_verify_scripts if \ + config.run_package_verify_scripts else None + verifier.verify_package(ignore_scripts=ignore_scripts, run_scripts=run_scripts, + path_to_package=output_package) + built_packages.append(output_package) + update_index(config.bldpkgs_dir, config, could_be_mirror=False) - # add files in order of a) in info directory, b) increasing size so - # we can access small manifest or json files without decompressing - # possible large binary or data files - for f in sorted(files3 - files1, key=order): - t.add(join(config.build_prefix, f), f) - t.close() + else: + print("STOPPING BUILD BEFORE POST:", m.dist()) - # we're done building, perform some checks - tarcheck.check_all(tmp_path) + # return list of all package files emitted by this build + return built_packages - copy_into(tmp_path, path, config.timeout) - update_index(config.bldpkgs_dir, config, could_be_mirror=False) - - if not getattr(config, "noverify", False): - verifier = Verify() - ignore_scripts = config.ignore_package_verify_scripts if \ - config.ignore_package_verify_scripts else None - run_scripts = config.run_package_verify_scripts if \ - config.run_package_verify_scripts else None - verifier.verify_package(ignore_scripts=ignore_scripts, run_scripts=run_scripts, - path_to_package=path) +def guess_interpreter(script_filename): + extensions_to_run_commands = {'.sh': 'sh', + '.bat': 'cmd', + '.ps1': 'powershell -executionpolicy bypass -File', + '.py': 'python'} + file_ext = os.path.splitext(script_filename)[1] + for ext, command in extensions_to_run_commands.items(): + if file_ext.lower().startswith(ext): + interpreter_command = command + break else: - print("STOPPING BUILD BEFORE POST:", m.dist()) + raise NotImplementedError("Don't know how to run {0} file. Please specify " + "script_interpreter for {1} output".format(file_ext, + script_filename)) + return interpreter_command - # returning true here says package is OK to test - return True def clean_pkg_cache(dist, timeout): @@ -1008,44 +1074,79 @@ def clean_pkg_cache(dist, timeout): rm_rf(entry) -def test(m, config, move_broken=True): +def test(recipedir_or_package_or_metadata, config, move_broken=True): ''' Execute any test scripts for the given package. :param m: Package's metadata. :type m: Metadata ''' + # we want to know if we're dealing with package input. If so, we can move the input on success. + is_package = False + need_cleanup = False - if not os.path.isdir(config.build_folder): - os.makedirs(config.build_folder) - - clean_pkg_cache(m.dist(), config.timeout) + if hasattr(recipedir_or_package_or_metadata, 'config'): + metadata = recipedir_or_package_or_metadata + config = metadata.config + else: + recipe_dir, need_cleanup = get_recipe_abspath(recipedir_or_package_or_metadata) + config.need_cleanup = need_cleanup + + # This will create a new local build folder if and only if config doesn't already have one. + # What this means is that if we're running a test immediately after build, we use the one + # that the build already provided + metadata, _, _ = render_recipe(recipe_dir, config=config) + # this recipe came from an extracted tarball. + if need_cleanup: + # ensure that the local location of the package is indexed, so that conda can find the + # local package + local_location = os.path.dirname(recipedir_or_package_or_metadata) + # strip off extra subdir folders + for platform in ('win', 'linux', 'osx'): + if os.path.basename(local_location).startswith(platform + "-"): + local_location = os.path.dirname(local_location) + update_index(local_location, config=config) + if not os.path.abspath(local_location): + local_location = os.path.normpath(os.path.abspath( + os.path.join(os.getcwd(), local_location))) + local_url = url_path(local_location) + # channel_urls is an iterable, but we don't know if it's a tuple or list. Don't know + # how to add elements. + config.channel_urls = list(config.channel_urls) + config.channel_urls.insert(0, local_url) + is_package = True + if metadata.meta.get('test') and metadata.meta['test'].get('source_files'): + source.provide(metadata.path, metadata.get_section('source'), config=config) + + config.compute_build_id(metadata.name()) + + clean_pkg_cache(metadata.dist(), config.timeout) if not isdir(config.test_dir): os.makedirs(config.test_dir) - create_files(config.test_dir, m, config) + create_files(config.test_dir, metadata, config) # Make Perl or Python-specific test files - if m.name().startswith('perl-'): - pl_files = create_pl_files(config.test_dir, m) + if metadata.name().startswith('perl-'): + pl_files = create_pl_files(config.test_dir, metadata) py_files = False lua_files = False else: - py_files = create_py_files(config.test_dir, m) + py_files = create_py_files(config.test_dir, metadata) pl_files = False lua_files = False - shell_files = create_shell_files(config.test_dir, m, config) + shell_files = create_shell_files(config.test_dir, metadata, config) if not (py_files or shell_files or pl_files or lua_files): - print("Nothing to test for:", m.dist()) + print("Nothing to test for:", metadata.dist()) return True - print("TEST START:", m.dist()) + print("TEST START:", metadata.dist()) - get_build_metadata(m, config=config) - specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())] + get_build_metadata(metadata, config=config) + specs = ['%s %s %s' % (metadata.name(), metadata.version(), metadata.build_id())] # add packages listed in the run environment and test/requires - specs.extend(ms.spec for ms in m.ms_depends('run')) - specs += ensure_list(m.get_value('test/requires', [])) + specs.extend(ms.spec for ms in metadata.ms_depends('run')) + specs += ensure_list(metadata.get_value('test/requires', [])) if py_files: # as the tests are run by python, ensure that python is installed. @@ -1063,7 +1164,7 @@ def test(m, config, move_broken=True): with path_prepended(config.test_prefix): env = dict(os.environ.copy()) - env.update(environ.get_dict(config=config, m=m, prefix=config.test_prefix)) + env.update(environ.get_dict(config=config, m=metadata, prefix=config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" if env_path_backup_var_exists: env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] @@ -1130,9 +1231,16 @@ def test(m, config, move_broken=True): try: subprocess.check_call(cmd, env=env, cwd=config.test_dir) except subprocess.CalledProcessError: - tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config) + tests_failed(metadata, move_broken=move_broken, broken_dir=config.broken_dir, config=config) + + if (is_package and hasattr(config, 'output_folder') and config.output_folder): + os.rename(recipedir_or_package_or_metadata, + os.path.join(config.output_folder, + os.path.basename(recipedir_or_package_or_metadata))) + if need_cleanup: + rm_rf(recipe_dir) - print("TEST END:", m.dist()) + print("TEST END:", metadata.dist()) return True @@ -1224,12 +1332,13 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, rendered_meta=metadata.meta, recipe_dir=metadata.path) try: with recipe_config: - ok_to_test = build(metadata, post=post, - need_source_download=need_source_download, - need_reparse_in_env=need_reparse_in_env, - config=recipe_config) - if not notest and ok_to_test: - test(metadata, config=recipe_config) + built_packages = build(metadata, post=post, + need_source_download=need_source_download, + need_reparse_in_env=need_reparse_in_env, + config=recipe_config) + if not notest and built_packages: + for pkg in built_packages: + test(pkg, config=recipe_config) except (NoPackagesFound, NoPackagesFoundError, Unsatisfiable, CondaValueError) as e: error_str = str(e) skip_names = ['python', 'r'] @@ -1268,15 +1377,16 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, # outputs message, or does upload, depending on value of args.anaconda_upload if post in [True, None]: - output_file = bldpkg_path(metadata) - handle_anaconda_upload(output_file, config=recipe_config) - already_built.add(output_file) + for f in built_packages: + handle_anaconda_upload(f, config=recipe_config) + already_built.add(f) if hasattr(recipe_config, 'output_folder') and recipe_config.output_folder: - destination = os.path.join(recipe_config.output_folder, os.path.basename(output_file)) - if os.path.exists(destination): - os.remove(destination) - os.rename(output_file, destination) + for f in built_packages: + destination = os.path.join(recipe_config.output_folder, os.path.basename(f)) + if os.path.exists(destination): + os.remove(destination) + os.rename(f, destination) def handle_anaconda_upload(path, config): diff --git a/tests/test-recipes/split-packages/copying_files/meta.yaml b/tests/test-recipes/split-packages/copying_files/meta.yaml index c400a4b9cd..b2cb03c465 100644 --- a/tests/test-recipes/split-packages/copying_files/meta.yaml +++ b/tests/test-recipes/split-packages/copying_files/meta.yaml @@ -27,7 +27,7 @@ outputs: files: - subpackage_file1 - somedir - - *.ext + - "*.ext" test: script: test_subpackage1.py script_interpreter: python diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py index 09ec8a7540..a73b5b8fc7 100644 --- a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py @@ -1,4 +1,8 @@ import os -with open(os.path.join(os.environ['PREFIX'], 'subpackage_file_1'), 'w') as f: +out_file = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') + +with open(out_file, 'w') as f: f.write("weeee") + +print(out_file) diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py index 65ba50d7ab..0c71114cc2 100644 --- a/tests/test_subpackages.py +++ b/tests/test_subpackages.py @@ -13,8 +13,7 @@ def recipe(request): def test_subpackage_recipes(recipe, test_config): api.build(recipe, config=test_config) - pass def test_all_subpackages_uploaded(): - pass + raise NotImplementedError From c303293fa946410af83ea75e881048757d0d416a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 13 Dec 2016 13:12:36 -0600 Subject: [PATCH 153/156] fix glob expansion to expand dirs. Fix script interpreter autodetect test. --- conda_build/build.py | 4 ++-- conda_build/metadata.py | 7 ++++++- .../_invalid_script_extension/meta.yaml | 12 ++++++++++++ .../subpackage3.unrecognized | 0 .../split-packages/overlapping_files/subpackage1.py | 8 ++++++++ .../script_autodetect_interpreter/meta.yaml | 4 ---- tests/test_subpackages.py | 4 ++++ 7 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 tests/test-recipes/split-packages/_invalid_script_extension/meta.yaml rename tests/test-recipes/split-packages/{script_autodetect_interpreter => _invalid_script_extension}/subpackage3.unrecognized (100%) create mode 100644 tests/test-recipes/split-packages/overlapping_files/subpackage1.py diff --git a/conda_build/build.py b/conda_build/build.py index fbfecfee0d..f894c97db2 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -986,7 +986,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F 'requirements': m.get_value('requirements/run')}) for output in outputs: - files = expand_globs(output.get('files', []), config.build_prefix) + files = output.get('files', []) if not files and output.get('script'): interpreter = output.get('script_interpreter') if not interpreter: @@ -1002,7 +1002,7 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F output_filename = ('-'.join([output['name'], m.version(), build_string_from_metadata(tmp_metadata)]) + '.tar.bz2') - + files = expand_globs(files, config.build_prefix) output_package = bundle_files(files, tmp_metadata, config, output_filename) if not getattr(config, "noverify", False): diff --git a/conda_build/metadata.py b/conda_build/metadata.py index a090885ee2..a4ab353737 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -180,7 +180,12 @@ def parse(data, config, path=None): def expand_globs(path_list, root_dir): files = [] for path in path_list: - files.extend(glob.glob(os.path.join(root_dir, path))) + if os.path.isdir(os.path.join(root_dir, path)): + files.extend([os.path.join(root, f) for root, _, _ in os.walk(path) + for f in glob(os.path.join(root, '*'))]) + else: + files.extend(glob.glob(os.path.join(root_dir, path))) + # list comp is getting rid of absolute prefix, to match relative paths used in file list return [f.replace(root_dir + os.path.sep, '') for f in files] diff --git a/tests/test-recipes/split-packages/_invalid_script_extension/meta.yaml b/tests/test-recipes/split-packages/_invalid_script_extension/meta.yaml new file mode 100644 index 0000000000..4472488db0 --- /dev/null +++ b/tests/test-recipes/split-packages/_invalid_script_extension/meta.yaml @@ -0,0 +1,12 @@ +package: + name: split_packages_unrecognized_script_type + version: 1.0 + +requirements: + run: + - my_script_subpackage_unrecognized + +outputs: + # what happens when we have an unrecognized script type? + - name: my_script_subpackage_unrecognized + script: subpackage3.unrecognized diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized b/tests/test-recipes/split-packages/_invalid_script_extension/subpackage3.unrecognized similarity index 100% rename from tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage3.unrecognized rename to tests/test-recipes/split-packages/_invalid_script_extension/subpackage3.unrecognized diff --git a/tests/test-recipes/split-packages/overlapping_files/subpackage1.py b/tests/test-recipes/split-packages/overlapping_files/subpackage1.py new file mode 100644 index 0000000000..a73b5b8fc7 --- /dev/null +++ b/tests/test-recipes/split-packages/overlapping_files/subpackage1.py @@ -0,0 +1,8 @@ +import os + +out_file = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') + +with open(out_file, 'w') as f: + f.write("weeee") + +print(out_file) diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml b/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml index 08d7a25776..69187cfbbc 100644 --- a/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/meta.yaml @@ -6,7 +6,6 @@ requirements: run: - my_script_subpackage - my_script_subpackage_shell - - my_script_subpackage_unrecognized outputs: - name: my_script_subpackage @@ -14,6 +13,3 @@ outputs: # Assume that on Windows, we have bash available here - name: my_script_subpackage_shell script: subpackage2.sh - # what happens when we have an unrecognized script type? - - name: my_script_subpackage_unrecognized - script: subpackage3.unrecognized diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py index 0c71114cc2..9e991edcf1 100644 --- a/tests/test_subpackages.py +++ b/tests/test_subpackages.py @@ -15,5 +15,9 @@ def test_subpackage_recipes(recipe, test_config): api.build(recipe, config=test_config) +def test_autodetect_raises_on_invalid_extension(test_config): + with pytest.raises(NotImplementedError): + api.build(os.path.join(subpackage_dir, '_invalid_script_extension'), config=test_config) + def test_all_subpackages_uploaded(): raise NotImplementedError From 6d2e79f9b9d602859836d194f6b72e69cd3e6967 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 13 Dec 2016 22:54:31 -0600 Subject: [PATCH 154/156] implement wheel bundling, fix up tests --- conda_build/api.py | 9 +- conda_build/build.py | 182 ++++++++++++------ conda_build/cli/main_build.py | 27 ++- conda_build/config.py | 9 + conda_build/create_test.py | 28 ++- conda_build/metadata.py | 8 +- conda_build/render.py | 2 +- .../alternate_type_wheel/meta.yaml | 13 +- .../split-packages/copying_files/bld.bat | 5 + .../split-packages/copying_files/build.sh | 8 + .../split-packages/copying_files/meta.yaml | 16 -- .../copying_files/test_subpackage1.py | 14 +- .../overlapping_files/subpackage1.py | 9 +- .../subpackage1.py | 9 +- .../subpackage2.sh | 4 +- .../script_install_files/subpackage1.py | 9 +- .../script_install_files/test_subpackage1.py | 3 +- tests/test_subpackages.py | 5 +- 18 files changed, 241 insertions(+), 119 deletions(-) create mode 100644 tests/test-recipes/split-packages/copying_files/bld.bat create mode 100644 tests/test-recipes/split-packages/copying_files/build.sh diff --git a/conda_build/api.py b/conda_build/api.py index 5870e39a8c..0f282d9bb4 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -78,16 +78,17 @@ def build(recipe_paths_or_metadata, post=None, need_source_download=True, def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwargs): from conda_build.build import test - config = get_or_merge_config(config, **kwargs) if hasattr(recipedir_or_package_or_metadata, 'config'): - recipe_config = recipedir_or_package_or_metadata.config + config = recipedir_or_package_or_metadata.config + else: + config = get_or_merge_config(config, **kwargs) - with recipe_config: + with config: # This will create a new local build folder if and only if config doesn't already have one. # What this means is that if we're running a test immediately after build, we use the one # that the build already provided - test_result = test(recipedir_or_package_or_metadata, config=recipe_config, + test_result = test(recipedir_or_package_or_metadata, config=config, move_broken=move_broken) return test_result diff --git a/conda_build/build.py b/conda_build/build.py index f894c97db2..a8f1e00c73 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -734,11 +734,34 @@ def warn_on_old_conda_build(index=None, installed_version=None, available_packag """ % (installed_version, available_packages[-1]), file=sys.stderr) -def bundle_files(pkg_files, metadata, config, output_filename): - info_files = create_info_files(metadata, pkg_files, config=config, prefix=config.build_prefix) +def bundle_conda(output, metadata, config, env, **kw): + files = output.get('files', []) + if not files and output.get('script'): + interpreter = output.get('script_interpreter') + if not interpreter: + interpreter = guess_interpreter(output['script']) + subprocess.check_output(interpreter.split(' ') + + [os.path.join(metadata.path, output['script'])], + cwd=config.build_prefix, env=env) + files_txt = os.path.splitext(os.path.join(config.build_prefix, output['script']))[0] + '.txt' # NOQA + try: + files = open(files_txt).read().splitlines() + except OSError: + raise ValueError("output script {0} specified, but script does not create " + "/{1} - please rectify by making your script output the " + "list of files (one per line) to that file.".format(output['script'], + os.path.basename(files_txt))) + tmp_metadata = copy.deepcopy(metadata) + tmp_metadata.meta['package']['name'] = output['name'] + tmp_metadata.meta['requirements'] = {'run': output.get('requirements', [])} + + output_filename = ('-'.join([output['name'], metadata.version(), + build_string_from_metadata(tmp_metadata)]) + '.tar.bz2') + files = list(set(expand_globs(files, config.build_prefix))) + info_files = create_info_files(tmp_metadata, files, config=config, prefix=config.build_prefix) for f in info_files: - if f not in pkg_files: - pkg_files.append(f) + if f not in files: + files.append(f) # lock the output directory while we build this file # create the tarball in a temporary directory to minimize lock time @@ -756,16 +779,39 @@ def order(f): # add files in order of a) in info directory, b) increasing size so # we can access small manifest or json files without decompressing # possible large binary or data files - for f in sorted(pkg_files, key=order): + for f in sorted(files, key=order): t.add(join(config.build_prefix, f), f) t.close() # we're done building, perform some checks tarcheck.check_all(tmp_path) + if not getattr(config, "noverify", False): + verifier = Verify() + ignore_scripts = config.ignore_package_verify_scripts if \ + config.ignore_package_verify_scripts else None + run_scripts = config.run_package_verify_scripts if \ + config.run_package_verify_scripts else None + verifier.verify_package(ignore_scripts=ignore_scripts, run_scripts=run_scripts, + path_to_package=tmp_path) copy_into(tmp_path, config.bldpkgs_dir, config.timeout) return os.path.join(config.bldpkgs_dir, output_filename) +def bundle_wheel(output, metadata, config, env): + with TemporaryDirectory() as tmpdir: + subprocess.check_call(['pip', 'wheel', '--wheel-dir', tmpdir, '--no-deps', '.'], + env=env, cwd=config.work_dir) + wheel_file = glob(os.path.join(tmpdir, "*.whl"))[0] + copy_into(wheel_file, config.bldpkgs_dir) + return os.path.join(config.bldpkgs_dir, os.path.basename(wheel_file)) + + +bundlers = { + 'conda': bundle_conda, + 'wheel': bundle_wheel, +} + + def build(m, config, post=None, need_source_download=True, need_reparse_in_env=False): ''' Build the package with the specified metadata. @@ -803,6 +849,8 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F print(" (actual version deferred until further download or env creation)") specs = [ms.spec for ms in m.ms_depends('build')] + if any(out.get('type') == 'wheel' for out in m.meta.get('outputs', [])): + specs.extend(['pip', 'wheel']) create_env(config.build_prefix, specs, config=config) vcs_source = m.uses_vcs_in_build if vcs_source and vcs_source not in specs: @@ -973,47 +1021,26 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F fix_permissions(files3 - files1, config.build_prefix) outputs = m.get_section('outputs') + # this is the old, default behavior: conda package, with difference between start + # set of files and end set of files + requirements = m.get_value('requirements/run') if not outputs: outputs = [{'name': m.name(), - 'files': files3 - files1}] + 'files': files3 - files1, + 'requirements': requirements}] else: - # make a metapackage if the recipe depends on any of its subpackages as runtime reqs - requirements = m.get_value('requirements/run') + made_meta = False for out in outputs: - if out['name'] in requirements: + if out.get('name') in requirements: requirements.extend(out.get('requirements', [])) - outputs.append({'name': m.name(), - 'requirements': m.get_value('requirements/run')}) + made_meta = True + else: + if made_meta: + outputs.append({'name': m.name(), 'requirements': requirements}) for output in outputs: - files = output.get('files', []) - if not files and output.get('script'): - interpreter = output.get('script_interpreter') - if not interpreter: - interpreter = guess_interpreter(output['script']) - files = subprocess.check_output(interpreter.split(' ') + - [os.path.join(m.path, output['script'])], - cwd=config.build_prefix, env=env).split('\n') - tmp_metadata = MetaData.fromdict({'package': {'name': output['name'], - 'version': m.version()}, - 'requirements': {'run': output.get('requirements', - [])}}) - - output_filename = ('-'.join([output['name'], m.version(), - build_string_from_metadata(tmp_metadata)]) + - '.tar.bz2') - files = expand_globs(files, config.build_prefix) - output_package = bundle_files(files, tmp_metadata, config, output_filename) - - if not getattr(config, "noverify", False): - verifier = Verify() - ignore_scripts = config.ignore_package_verify_scripts if \ - config.ignore_package_verify_scripts else None - run_scripts = config.run_package_verify_scripts if \ - config.run_package_verify_scripts else None - verifier.verify_package(ignore_scripts=ignore_scripts, run_scripts=run_scripts, - path_to_package=output_package) - built_packages.append(output_package) + built_packages.append(bundlers[output.get('type', 'conda')](output, m, config, env)) + update_index(config.bldpkgs_dir, config, could_be_mirror=False) else: @@ -1040,7 +1067,6 @@ def guess_interpreter(script_filename): return interpreter_command - def clean_pkg_cache(dist, timeout): cc.pkgs_dirs = cc.pkgs_dirs[:1] locks = [get_lock(folder, timeout=timeout) for folder in cc.pkgs_dirs] @@ -1122,8 +1148,6 @@ def test(recipedir_or_package_or_metadata, config, move_broken=True): clean_pkg_cache(metadata.dist(), config.timeout) - if not isdir(config.test_dir): - os.makedirs(config.test_dir) create_files(config.test_dir, metadata, config) # Make Perl or Python-specific test files if metadata.name().startswith('perl-'): @@ -1288,6 +1312,8 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, already_built = set() extra_help = "" + built_packages = [] + while recipe_list: # This loop recursively builds dependencies if recipes exist if build_only: @@ -1304,24 +1330,23 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, recipe = recipe_list.popleft() if hasattr(recipe, 'config'): metadata = recipe - recipe_config = metadata.config + config = metadata.config # this code is duplicated below because we need to be sure that the build id is set # before downloading happens - or else we lose where downloads are - if recipe_config.set_build_id: - recipe_config.compute_build_id(metadata.name(), reset=True) + if config.set_build_id: + config.compute_build_id(metadata.name(), reset=True) recipe_parent_dir = "" to_build_recursive.append(metadata.name()) else: recipe_parent_dir = os.path.dirname(recipe) recipe = recipe.rstrip("/").rstrip("\\") - recipe_config = config to_build_recursive.append(os.path.basename(recipe)) # before downloading happens - or else we lose where downloads are - if recipe_config.set_build_id: - recipe_config.compute_build_id(os.path.basename(recipe), reset=True) + if config.set_build_id: + config.compute_build_id(os.path.basename(recipe), reset=True) metadata, need_source_download, need_reparse_in_env = render_recipe(recipe, - config=recipe_config) + config=config) if not getattr(config, "noverify", False): verifier = Verify() ignore_scripts = config.ignore_recipe_verify_scripts if \ @@ -1331,14 +1356,17 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, verifier.verify_recipe(ignore_scripts=ignore_scripts, run_scripts=run_scripts, rendered_meta=metadata.meta, recipe_dir=metadata.path) try: - with recipe_config: - built_packages = build(metadata, post=post, - need_source_download=need_source_download, - need_reparse_in_env=need_reparse_in_env, - config=recipe_config) - if not notest and built_packages: - for pkg in built_packages: - test(pkg, config=recipe_config) + with config: + packages_from_this = build(metadata, post=post, + need_source_download=need_source_download, + need_reparse_in_env=need_reparse_in_env, + config=config) + if not notest and packages_from_this: + for pkg in packages_from_this: + if pkg.endswith('.tar.bz2'): + # we only know how to test conda packages + test(pkg, config=config) + built_packages.append(pkg) except (NoPackagesFound, NoPackagesFoundError, Unsatisfiable, CondaValueError) as e: error_str = str(e) skip_names = ['python', 'r'] @@ -1378,15 +1406,21 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, # outputs message, or does upload, depending on value of args.anaconda_upload if post in [True, None]: for f in built_packages: - handle_anaconda_upload(f, config=recipe_config) + # TODO: could probably use a better check for pkg type than this... + if f.endswith('.tar.bz2'): + handle_anaconda_upload(f, config=config) + elif f.endswith('.whl'): + handle_pypi_upload(f, config=config) already_built.add(f) - if hasattr(recipe_config, 'output_folder') and recipe_config.output_folder: + if hasattr(config, 'output_folder') and config.output_folder: for f in built_packages: - destination = os.path.join(recipe_config.output_folder, os.path.basename(f)) - if os.path.exists(destination): - os.remove(destination) - os.rename(f, destination) + # may have already been moved during testing + destination = os.path.join(config.output_folder, os.path.basename(f)) + if os.path.isfile(f): + if os.path.exists(destination): + os.remove(destination) + os.rename(f, destination) def handle_anaconda_upload(path, config): @@ -1442,6 +1476,28 @@ def handle_anaconda_upload(path, config): raise +def handle_pypi_upload(f, config): + args = ['twine', 'upload', '--sign-with', config.sign_with, '--repository', config.repository] + if config.user: + args.extend(['--user', config.user]) + if config.password: + args.extend(['--password', config.password]) + if config.sign: + args.extend(['--sign']) + if config.identity: + args.extend(['--identity', config.identity]) + if config.config_file: + args.extend(['--config-file', config.config_file]) + if config.repository: + args.extend(['--repository', config.repository]) + + args.append(f) + try: + subprocess.check_call() + except: + log.warn("wheel upload failed - is twine installed? Is this package registered?") + + def print_build_intermediate_warning(config): print("\n\n") print('#' * 84) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 2beb4201f6..44eaea0da7 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -132,7 +132,32 @@ def parse_args(args): ) p.add_argument( '--user', - help="User/organization to upload packages to on anaconda.org" + help="User/organization to upload packages to on anaconda.org or pypi" + ) + pypi_grp = p.add_argument_group("PyPI upload parameters (twine)") + pypi_grp.add_argument( + '--password', + help="password to use when uploading packages to pypi" + ) + pypi_grp.add_argument( + '--sign', default=False, + help="sign files when uploading to pypi" + ) + pypi_grp.add_argument( + '--sign-with', default='gpg', dest='sign_with', + help="program to use to sign files when uploading to pypi" + ) + pypi_grp.add_argument( + '--identity', + help="GPG identity to use to sign files when uploading to pypi" + ) + pypi_grp.add_argument( + '--config-file', + help="path to .pypirc file to use when uploading to pypi" + ) + pypi_grp.add_argument( + '--repository', default='pypi', + help="PyPI repository to upload to" ) p.add_argument( "--no-activate", diff --git a/conda_build/config.py b/conda_build/config.py index 1cc8f27a14..9d014f4bf2 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -110,6 +110,15 @@ def env(lang, default): Setting('platform', platform), Setting('set_build_id', True), Setting('disable_pip', False), + + # pypi upload settings (twine) + Setting('password', None), + Setting('sign', False), + Setting('sign_with', 'gpg'), + Setting('identity', None), + Setting('config_file', None), + Setting('repository', 'pypi'), + Setting('ignore_recipe_verify_scripts', cc.rc.get('conda-build', {}).get('ignore_recipe_verify_scripts', [])), Setting('ignore_package_verify_scripts', diff --git a/conda_build/create_test.py b/conda_build/create_test.py index cdd7bc1a5d..d158d6832c 100644 --- a/conda_build/create_test.py +++ b/conda_build/create_test.py @@ -72,10 +72,17 @@ def create_files(dir_path, m, config): def create_shell_files(dir_path, m, config): has_tests = False - if sys.platform == 'win32': - name = 'run_test.bat' + ext = '.bat' if sys.platform == 'win32' else '.sh' + name = 'no-file' + + for out in m.meta.get('outputs', []): + if m.name() == out['name']: + out_test_script = out.get('test', {}).get('script', 'no-file') + if os.path.splitext(out_test_script)[1].lower() == ext: + name = out_test_script + break else: - name = 'run_test.sh' + name = "run_test{}".format(ext) if exists(join(m.path, name)): copy_into(join(m.path, name), dir_path, config.timeout) @@ -107,7 +114,13 @@ def create_py_files(dir_path, m): has_tests = True try: - with open(join(m.path, 'run_test.py')) as fi: + name = 'run_test.py' + for out in m.meta.get('outputs', []): + if m.name() == out['name']: + out_test_script = out.get('test', {}).get('script', 'no-file') + name = out_test_script if out_test_script.endswith('.py') else 'no-file' + + with open(join(m.path, name)) as fi: fo.write("print('running run_test.py')\n") fo.write("# --- run_test.py (begin) ---\n") fo.write(fi.read()) @@ -146,7 +159,12 @@ def create_pl_files(dir_path, m): has_tests = True try: - with open(join(m.path, 'run_test.pl')) as fi: + name = 'run_test.pl' + for out in m.meta.get('outputs', []): + if m.name() == out['name']: + out_test_script = out.get('test', {}).get('script', 'no-file') + name = out_test_script if out_test_script[-3:].lower() == '.pl' else 'no-file' + with open(join(m.path, name)) as fi: print("# --- run_test.pl (begin) ---", file=fo) fo.write(fi.read()) print("# --- run_test.pl (end) ---", file=fo) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index a4ab353737..918b115968 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -180,9 +180,9 @@ def parse(data, config, path=None): def expand_globs(path_list, root_dir): files = [] for path in path_list: - if os.path.isdir(os.path.join(root_dir, path)): - files.extend([os.path.join(root, f) for root, _, _ in os.walk(path) - for f in glob(os.path.join(root, '*'))]) + fullpath = os.path.join(root_dir, path) + if os.path.isdir(fullpath): + files.extend([os.path.join(root, f) for root, _, fs in os.walk(fullpath) for f in fs]) else: files.extend(glob.glob(os.path.join(root_dir, path))) @@ -356,6 +356,8 @@ def handle_config_version(ms, ver, dep_type='run'): def build_string_from_metadata(metadata): + if metadata.meta.get('build', {}).get('string'): + return metadata.get_value('build/string') res = [] version_pat = re.compile(r'(?:==)?(\d+)\.(\d+)') for name, s in (('numpy', 'np'), ('python', 'py'), diff --git a/conda_build/render.py b/conda_build/render.py index f26cd52de6..b2353125a6 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -168,7 +168,7 @@ def render_recipe(recipe_path, config, no_download_source=False): # Next bit of stuff is to support YAML output in the order we expect. # http://stackoverflow.com/a/17310199/1170370 class _MetaYaml(dict): - fields = ["package", "source", "build", "requirements", "test", "about", "extra"] + fields = ["package", "source", "build", "requirements", "test", "outputs", "about", "extra"] def to_omap(self): return [(field, self[field]) for field in _MetaYaml.fields if field in self] diff --git a/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml b/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml index d149b3dcc5..dda10e6be8 100644 --- a/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml +++ b/tests/test-recipes/split-packages/alternate_type_wheel/meta.yaml @@ -2,15 +2,8 @@ package: name: split_packages_alternate_type_wheel version: 1.0 -requirements: - run: - - my_script_subpackage +source: + path: ../../../../../conda_build_test_recipe outputs: - - name: my_script_subpackage - script: subpackage1.py - script_interpreter: python - type: wheel - test: - script: test_subpackage1.py - script_interpreter: python + - type: wheel diff --git a/tests/test-recipes/split-packages/copying_files/bld.bat b/tests/test-recipes/split-packages/copying_files/bld.bat new file mode 100644 index 0000000000..e1084a2a6f --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/bld.bat @@ -0,0 +1,5 @@ +echo "weee" > %PREFIX%\subpackage_file1 +mkdir %PREFIX%\somedir +echo "weee" > %PREFIX%\somedir\subpackage_file1 +echo "weee" > %PREFIX%\subpackage_file1.ext +echo "weee" > %PREFIX%\subpackage_file2.ext \ No newline at end of file diff --git a/tests/test-recipes/split-packages/copying_files/build.sh b/tests/test-recipes/split-packages/copying_files/build.sh new file mode 100644 index 0000000000..529dc13092 --- /dev/null +++ b/tests/test-recipes/split-packages/copying_files/build.sh @@ -0,0 +1,8 @@ +# test copying filename +echo "weee" > $PREFIX/subpackage_file1 +# test copying by folder name +mkdir $PREFIX/somedir +echo "weee" > $PREFIX/somedir/subpackage_file1 +# test glob patterns +echo "weee" > $PREFIX/subpackage_file1.ext +echo "weee" > $PREFIX/subpackage_file2.ext diff --git a/tests/test-recipes/split-packages/copying_files/meta.yaml b/tests/test-recipes/split-packages/copying_files/meta.yaml index b2cb03c465..944ab63497 100644 --- a/tests/test-recipes/split-packages/copying_files/meta.yaml +++ b/tests/test-recipes/split-packages/copying_files/meta.yaml @@ -2,22 +2,6 @@ package: name: split_packages_file_list version: 1.0 -build: - script: - # test copying filename - - echo weee > $PREFIX/subpackage_file1 # [unix] - - echo weee > %PREFIX%/subpackage_file1 # [win] - # test copying by folder name - - mkdir $PREFIX/somedir # [unix] - - mkdir %PREFIX%/somedir # [win] - - echo weee > $PREFIX/somedir/subpackage_file1 # [unix] - - echo weee > %PREFIX%/somedir/subpackage_file1 # [win] - # test glob patterns - - echo weee > $PREFIX/subpackage_file1.ext # [unix] - - echo weee > $PREFIX/subpackage_file2.ext # [unix] - - echo weee > %PREFIX%/subpackage_file1.ext # [win] - - echo weee > %PREFIX%/subpackage_file2.ext # [win] - requirements: run: - my_script_subpackage diff --git a/tests/test-recipes/split-packages/copying_files/test_subpackage1.py b/tests/test-recipes/split-packages/copying_files/test_subpackage1.py index e096bc0c90..cf188e2175 100644 --- a/tests/test-recipes/split-packages/copying_files/test_subpackage1.py +++ b/tests/test-recipes/split-packages/copying_files/test_subpackage1.py @@ -1,18 +1,22 @@ import os +print(os.getenv('PREFIX')) filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1') assert os.path.isfile(filename) -assert open(filename).read() == "weee" +assert open(filename).read().rstrip() == "weee" +print("plain file OK") -filename = os.path.join(os.environ['PREFIX'], 'subdir', 'subpackage_file1') +filename = os.path.join(os.environ['PREFIX'], 'somedir', 'subpackage_file1') assert os.path.isfile(filename) -assert open(filename).read() == "weee" +assert open(filename).read().rstrip() == "weee" +print("subfolder file OK") filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1.ext') assert os.path.isfile(filename) -assert open(filename).read() == "weee" +assert open(filename).read().rstrip() == "weee" filename = os.path.join(os.environ['PREFIX'], 'subpackage_file2.ext') assert os.path.isfile(filename) -assert open(filename).read() == "weee" +assert open(filename).read().rstrip() == "weee" +print("glob OK") diff --git a/tests/test-recipes/split-packages/overlapping_files/subpackage1.py b/tests/test-recipes/split-packages/overlapping_files/subpackage1.py index a73b5b8fc7..22cf26111b 100644 --- a/tests/test-recipes/split-packages/overlapping_files/subpackage1.py +++ b/tests/test-recipes/split-packages/overlapping_files/subpackage1.py @@ -1,8 +1,11 @@ import os -out_file = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') +out_path = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') -with open(out_file, 'w') as f: +with open(out_path, 'w') as f: f.write("weeee") -print(out_file) +# need to write output files to a file. Hokey, but only cross-language way to collect this. +# One file per line. Make sure this filename is right - conda-build looks for it. +with open(os.path.basename(__file__).replace('.py', '.txt'), 'a') as f: + f.write(out_path + "\n") diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py index a73b5b8fc7..22cf26111b 100644 --- a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage1.py @@ -1,8 +1,11 @@ import os -out_file = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') +out_path = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') -with open(out_file, 'w') as f: +with open(out_path, 'w') as f: f.write("weeee") -print(out_file) +# need to write output files to a file. Hokey, but only cross-language way to collect this. +# One file per line. Make sure this filename is right - conda-build looks for it. +with open(os.path.basename(__file__).replace('.py', '.txt'), 'a') as f: + f.write(out_path + "\n") diff --git a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh index ce59ebf7a5..90bafd3f98 100644 --- a/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh +++ b/tests/test-recipes/split-packages/script_autodetect_interpreter/subpackage2.sh @@ -1,2 +1,4 @@ #!/bin/sh -echo weeee > $PREFIX/subpackage_file_2 +echo "weeee" > $PREFIX/subpackage_file_2 + +echo "$PREFIX/subpackage_file_2" > $PREFIX/subpackage2.txt diff --git a/tests/test-recipes/split-packages/script_install_files/subpackage1.py b/tests/test-recipes/split-packages/script_install_files/subpackage1.py index 09ec8a7540..22cf26111b 100644 --- a/tests/test-recipes/split-packages/script_install_files/subpackage1.py +++ b/tests/test-recipes/split-packages/script_install_files/subpackage1.py @@ -1,4 +1,11 @@ import os -with open(os.path.join(os.environ['PREFIX'], 'subpackage_file_1'), 'w') as f: +out_path = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') + +with open(out_path, 'w') as f: f.write("weeee") + +# need to write output files to a file. Hokey, but only cross-language way to collect this. +# One file per line. Make sure this filename is right - conda-build looks for it. +with open(os.path.basename(__file__).replace('.py', '.txt'), 'a') as f: + f.write(out_path + "\n") diff --git a/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py b/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py index 96ed1deba1..6981cf4087 100644 --- a/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py +++ b/tests/test-recipes/split-packages/script_install_files/test_subpackage1.py @@ -1,6 +1,7 @@ import os -filename = os.path.join(os.environ['PREFIX'], 'subpackage_file_1') +print(os.getenv('PREFIX')) +filename = os.path.join(os.getenv('PREFIX'), 'subpackage_file_1') assert os.path.isfile(filename) assert open(filename).read() == "weeee" diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py index 9e991edcf1..763c7e610b 100644 --- a/tests/test_subpackages.py +++ b/tests/test_subpackages.py @@ -19,5 +19,6 @@ def test_autodetect_raises_on_invalid_extension(test_config): with pytest.raises(NotImplementedError): api.build(os.path.join(subpackage_dir, '_invalid_script_extension'), config=test_config) -def test_all_subpackages_uploaded(): - raise NotImplementedError + +# def test_all_subpackages_uploaded(): +# raise NotImplementedError From 1f532f16112926baef45ce85911731b25130f123 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 14 Dec 2016 07:55:30 -0600 Subject: [PATCH 155/156] compute build prefix files difference for script mode of outputs --- conda_build/build.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index a8f1e00c73..585197d7ef 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -740,17 +740,11 @@ def bundle_conda(output, metadata, config, env, **kw): interpreter = output.get('script_interpreter') if not interpreter: interpreter = guess_interpreter(output['script']) + initial_files_snapshot = prefix_files(config.build_prefix) subprocess.check_output(interpreter.split(' ') + [os.path.join(metadata.path, output['script'])], cwd=config.build_prefix, env=env) - files_txt = os.path.splitext(os.path.join(config.build_prefix, output['script']))[0] + '.txt' # NOQA - try: - files = open(files_txt).read().splitlines() - except OSError: - raise ValueError("output script {0} specified, but script does not create " - "/{1} - please rectify by making your script output the " - "list of files (one per line) to that file.".format(output['script'], - os.path.basename(files_txt))) + files = prefix_files(config.build_prefix) - initial_files_snapshot tmp_metadata = copy.deepcopy(metadata) tmp_metadata.meta['package']['name'] = output['name'] tmp_metadata.meta['requirements'] = {'run': output.get('requirements', [])} From d3536c8c77a13e2eb0393d5a4ad43fe3a4425b59 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 14 Dec 2016 12:00:05 -0600 Subject: [PATCH 156/156] fix api build returning outputs; fix testing when recipe not included --- conda_build/build.py | 12 +++++- tests/test_api_build.py | 81 +++++++++++++++++------------------------ 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 585197d7ef..2fc15f1f3b 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -622,6 +622,9 @@ def create_env(prefix, specs, config, clear_cache=True): specs.append('%s@' % feature) if specs: # Don't waste time if there is nothing to do + log.debug("Creating environment in %s", prefix) + log.debug(str(specs)) + with path_prepended(prefix): locks = [] try: @@ -1005,8 +1008,10 @@ def build(m, config, post=None, need_source_download=True, need_reparse_in_env=F else: pkg_files = sorted(files2 - files1) + # the legacy noarch if m.get_value('build/noarch_python'): noarch_python.transform(m, sorted(files2 - files1), config.build_prefix) + # new way: build/noarch: python elif is_noarch_python(m): noarch_python.populate_files( m, pkg_files, config.build_prefix, entry_point_script_names) @@ -1359,7 +1364,11 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, for pkg in packages_from_this: if pkg.endswith('.tar.bz2'): # we only know how to test conda packages - test(pkg, config=config) + try: + test(pkg, config=config) + # IOError means recipe was not included with package. metadata instead + except IOError: + test(metadata, config=config) built_packages.append(pkg) except (NoPackagesFound, NoPackagesFoundError, Unsatisfiable, CondaValueError) as e: error_str = str(e) @@ -1415,6 +1424,7 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False, if os.path.exists(destination): os.remove(destination) os.rename(f, destination) + return built_packages def handle_anaconda_upload(path, config): diff --git a/tests/test_api_build.py b/tests/test_api_build.py index bbb2c76869..7fb21ff310 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -80,9 +80,7 @@ def test_recipe_builds(recipe, test_config, testing_workdir): # so they can be checked within build scripts os.environ["CONDA_TEST_VAR"] = "conda_test" os.environ["CONDA_TEST_VAR_2"] = "conda_test_2" - ok_to_test = api.build(recipe, config=test_config) - if ok_to_test: - api.test(recipe, config=test_config) + outputs = api.build(recipe, config=test_config) def test_token_upload(testing_workdir): @@ -131,9 +129,8 @@ def test_git_describe_info_on_branch(test_config): def test_no_include_recipe_config_arg(test_metadata): """Two ways to not include recipe: build/include_recipe: False in meta.yaml; or this. Former is tested with specific recipe.""" - output_file = api.get_output_file_path(test_metadata) - api.build(test_metadata) - assert package_has_file(output_file, "info/recipe/meta.yaml") + outputs = api.build(test_metadata) + assert package_has_file(outputs[0], "info/recipe/meta.yaml") # make sure that it is not there when the command line flag is passed test_metadata.config.include_recipe = False @@ -146,9 +143,8 @@ def test_no_include_recipe_config_arg(test_metadata): def test_no_include_recipe_meta_yaml(test_metadata, test_config): # first, make sure that the recipe is there by default. This test copied from above, but copied # as a sanity check here. - output_file = api.get_output_file_path(test_metadata) - api.build(test_metadata) - assert package_has_file(output_file, "info/recipe/meta.yaml") + outputs = api.build(test_metadata) + assert package_has_file(outputs[0], "info/recipe/meta.yaml") output_file = api.get_output_file_path(os.path.join(metadata_dir, '_no_include_recipe'), config=test_config) @@ -245,7 +241,6 @@ def test_checkout_tool_as_dependency(testing_workdir, test_config): test_config.channel_urls = ('conda_build_test', ) # "hide" svn by putting a known bad one on PATH exename = dummy_executable(testing_workdir, "svn") - old_path = os.environ["PATH"] os.environ["PATH"] = os.pathsep.join([testing_workdir, os.environ["PATH"]]) FNULL = open(os.devnull, 'w') with pytest.raises(subprocess.CalledProcessError, message="Dummy svn was not executed"): @@ -340,14 +335,13 @@ def test_skip_existing(testing_workdir, test_config, capfd): def test_skip_existing_url(test_metadata, testing_workdir, capfd): # make sure that it is built - output_file = api.get_output_file_path(test_metadata) - api.build(test_metadata) + outputs = api.build(test_metadata) # Copy our package into some new folder output_dir = os.path.join(testing_workdir, 'someoutput') platform = os.path.join(output_dir, test_metadata.config.subdir) os.makedirs(platform) - copy_into(output_file, os.path.join(platform, os.path.basename(output_file))) + copy_into(outputs[0], os.path.join(platform, os.path.basename(outputs[0]))) # create the index so conda can find the file api.update_index(platform, config=test_metadata.config) @@ -649,19 +643,16 @@ def test_noarch_python_with_tests(test_config): def test_noarch_python(test_config): recipe = os.path.join(metadata_dir, "_noarch_python") - fn = api.get_output_file_path(recipe, config=test_config) - api.build(recipe, config=test_config) - assert package_has_file(fn, 'info/files') is not '' - noarch = json.loads(package_has_file(fn, 'info/noarch.json').decode()) + outputs = api.build(recipe, config=test_config) + noarch = json.loads(package_has_file(outputs[0], 'info/noarch.json').decode()) assert 'entry_points' in noarch assert 'type' in noarch def test_skip_compile_pyc(test_config): recipe = os.path.join(metadata_dir, "skip_compile_pyc") - fn = api.get_output_file_path(recipe, config=test_config) - api.build(recipe, config=test_config) - tf = tarfile.open(fn) + outputs = api.build(recipe, config=test_config) + tf = tarfile.open(outputs[0]) pyc_count = 0 for f in tf.getmembers(): filename = os.path.basename(f.name) @@ -678,10 +669,9 @@ def test_skip_compile_pyc(test_config): #@pytest.mark.skipif(on_win, reason="binary prefixes not supported on Windows") def test_detect_binary_files_with_prefix(test_config): recipe = os.path.join(metadata_dir, "_detect_binary_files_with_prefix") - fn = api.get_output_file_path(recipe, config=test_config) - api.build(recipe, config=test_config) + outputs = api.build(recipe, config=test_config) matches = [] - with tarfile.open(fn) as tf: + with tarfile.open(outputs[0]) as tf: has_prefix = tf.extractfile('info/has_prefix') contents = [p.strip().decode('utf-8') for p in has_prefix.readlines()] @@ -694,10 +684,9 @@ def test_detect_binary_files_with_prefix(test_config): def test_skip_detect_binary_files_with_prefix(test_config): recipe = os.path.join(metadata_dir, "_skip_detect_binary_files_with_prefix") - fn = api.get_output_file_path(recipe, config=test_config) - api.build(recipe, config=test_config) + outputs = api.build(recipe, config=test_config) matches = [] - with tarfile.open(fn) as tf: + with tarfile.open(outputs[0]) as tf: try: has_prefix = tf.extractfile('info/has_prefix') contents = [p.strip().decode('utf-8') for p in @@ -713,44 +702,40 @@ def test_skip_detect_binary_files_with_prefix(test_config): def test_fix_permissions(test_config): recipe = os.path.join(metadata_dir, "fix_permissions") - fn = api.get_output_file_path(recipe, config=test_config) - api.build(recipe, config=test_config) - tf = tarfile.open(fn) - for f in tf.getmembers(): - assert f.mode & 0o444 == 0o444, "tar member '{}' has invalid (read) mode".format(f.name) + outputs = api.build(recipe, config=test_config) + with tarfile.open(outputs[0]) as tf: + for f in tf.getmembers(): + assert f.mode & 0o444 == 0o444, "tar member '{}' has invalid (read) mode".format(f.name) @pytest.mark.skipif(not on_win, reason="windows-only functionality") def test_script_win_creates_exe(test_config): recipe = os.path.join(metadata_dir, "_script_win_creates_exe") - fn = api.get_output_file_path(recipe, config=test_config) - api.build(recipe, config=test_config) - assert package_has_file(fn, 'Scripts/test-script.exe') - assert package_has_file(fn, 'Scripts/test-script-script.py') + outputs = api.build(recipe, config=test_config) + assert package_has_file(outputs[0], 'Scripts/test-script.exe') + assert package_has_file(outputs[0], 'Scripts/test-script-script.py') def test_build_output_folder_moves_file(test_metadata, testing_workdir): - output_path = api.get_output_file_path(test_metadata) test_metadata.config.output_folder = testing_workdir - api.build(test_metadata, no_test=True) - assert not os.path.exists(output_path) - assert os.path.isfile(os.path.join(testing_workdir, os.path.basename(output_path))) + outputs = api.build(test_metadata, no_test=True) + assert not os.path.exists(outputs[0]) + assert os.path.isfile(os.path.join(testing_workdir, os.path.basename(outputs[0]))) def test_test_output_folder_moves_file(test_metadata, testing_workdir): - output_path = api.get_output_file_path(test_metadata) - api.build(test_metadata, no_test=True) - api.test(output_path, output_folder=testing_workdir) - assert not os.path.exists(output_path) - assert os.path.isfile(os.path.join(testing_workdir, os.path.basename(output_path))) + outputs = api.build(test_metadata, no_test=True) + assert os.path.exists(outputs[0]) + api.test(outputs[0], output_folder=testing_workdir) + assert not os.path.exists(outputs[0]) + assert os.path.isfile(os.path.join(testing_workdir, os.path.basename(outputs[0]))) def test_info_files_json(test_config): recipe = os.path.join(metadata_dir, "ignore_some_prefix_files") - fn = api.get_output_file_path(recipe, config=test_config) - api.build(recipe, config=test_config) - assert package_has_file(fn, "info/files.json") - with tarfile.open(fn) as tf: + outputs = api.build(recipe, config=test_config) + assert package_has_file(outputs[0], "info/files.json") + with tarfile.open(outputs[0]) as tf: data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8')) fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link", "prefix_placeholder", "inode_first_path"]