Master osx allow rpaths

conda_build/build.py

@@ -983,6 +983,7 @@ def bundle_conda(output, metadata, env, stats, **kw):
 
     basename = '-'.join([output['name'], metadata.version(), metadata.build_id()])
     tmp_archives = []
+    final_outputs = []
     with TemporaryDirectory() as tmp:
         tmp_path = os.path.join(tmp, basename)
 
@@ -1009,7 +1010,6 @@ def order(f):
             # add files in order of a) in info directory, b) increasing size so
             # we can access small manifest or json files without decompressing
             # possible large binary or data files
-            print("Compressing to {}".format(tmp_path + ext))
             with tmp_chdir(metadata.config.host_prefix):
                 with libarchive.file_writer(tmp_path + ext, 'gnutar', filter_name=filter, options=opts) as archive:
                     archive.add_files(*files_list)
@@ -1052,14 +1052,15 @@ def order(f):
             #    a major bottleneck.
             utils.copy_into(tmp_path, final_output, metadata.config.timeout,
                             locking=False)
+            final_outputs.append(final_output)
     update_index(os.path.dirname(output_folder), verbose=metadata.config.debug)
 
     # clean out host prefix so that this output's files don't interfere with other outputs
     #   We have a backup of how things were before any output scripts ran.  That's
     #   restored elsewhere.
     utils.rm_rf(metadata.config.host_prefix)
 
-    return final_output
+    return final_outputs
 
 
 def bundle_wheel(output, metadata, env, stats):
@@ -1622,7 +1623,7 @@ def build(m, stats, post=None, need_source_download=True, need_reparse_in_env=Fa
                     #    can be different from the env for the top level build.
                     with utils.path_prepended(m.config.build_prefix):
                         env = environ.get_dict(m=m)
-                    built_package = bundlers[output_d.get('type', 'conda')](output_d, m, env, stats)
+                    newly_built_packages = bundlers[output_d.get('type', 'conda')](output_d, m, env, stats)
                     # warn about overlapping files.
                     if 'checksums' in output_d:
                         for file, csum in output_d['checksums'].items():
@@ -1634,7 +1635,8 @@ def build(m, stats, post=None, need_source_download=True, need_reparse_in_env=Fa
                                     log.warn("{} overlap between {} in packages {} and {}"
                                              .format(nature, file, output_d['name'],
                                                      prev_output_d['name']))
-                    new_pkgs[built_package] = (output_d, m)
+                    for built_package in newly_built_packages:
+                        new_pkgs[built_package] = (output_d, m)
 
                     # must rebuild index because conda has no way to incrementally add our last
                     #    package to the index.

conda_build/post.py

@@ -233,21 +233,42 @@ def compile_missing_pyc(files, cwd, python_exe, skip_compile_pyc=()):
             print('compiling .pyc files... failed as no python interpreter was found')
         else:
             print('compiling .pyc files...')
+            # We avoid command lines longer than 8190
+            if sys.platform == 'win32':
+                limit = 8190
+            else:
+                limit = 32760
+            limit -= len(compile_files) * 2
+            lower_limit = len(max(compile_files, key=len)) + 1
+            if limit < lower_limit:
+                limit = lower_limit
+            groups = [[]]
+            args = [python_exe, '-Wi', '-m', 'py_compile']
+            args_len = length = len(' '.join(args)) + 1
             for f in compile_files:
-                call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd)
+                length_this = len(f) + 1
+                if length_this + length > limit:
+                    groups.append([])
+                    length = args_len
+                else:
+                    length += length_this
+                groups[len(groups) - 1].append(f)
+            for group in groups:
+                call(args + group, cwd=cwd)
 
 
 def check_dist_info_version(name, version, files):
     for f in files:
-        if '.dist-info' in f:
-            f_lower = f.lower()
-            f_lower, _, _ = f_lower.rpartition('.dist-info')
-            _, _, f_lower = f_lower.rpartition(name + '-')
-            if version != f_lower:
-                print("ERROR: dist-info version incorrect (is {}, should be {})".format(f_lower, version))
-                sys.exit(1)
-            else:
-                return
+        if f.endswith('.dist-info' + os.sep + 'METADATA'):
+            f_lower = os.path.basename(os.path.dirname(f).lower())
+            if f_lower.startswith(name + '-'):
+                f_lower, _, _ = f_lower.rpartition('.dist-info')
+                _, distname, f_lower = f_lower.rpartition(name + '-')
+                if distname == name and version != f_lower:
+                    print("ERROR: Top level dist-info version incorrect (is {}, should be {})".format(f_lower, version))
+                    sys.exit(1)
+                else:
+                    return
 
 
 def post_process(name, version, files, prefix, config, preserve_egg_dir=False, noarch=False, skip_compile_pyc=()):
@@ -354,7 +375,7 @@ def osx_ch_link(path, link_dict, host_prefix, build_prefix, files):
     return ret
 
 
-def mk_relative_osx(path, host_prefix, build_prefix, files):
+def mk_relative_osx(path, host_prefix, build_prefix, files, rpaths=('lib',)):
     assert sys.platform == 'darwin'
 
     names = macho.otool(path)
@@ -368,10 +389,11 @@ def mk_relative_osx(path, host_prefix, build_prefix, files):
     if names:
         # Add an rpath to every executable to increase the chances of it
         # being found.
-        rpath = os.path.join('@loader_path',
-                     os.path.relpath(os.path.join(host_prefix, 'lib'),
-                             os.path.dirname(path)), '').replace('/./', '/')
-        macho.add_rpath(path, rpath, verbose=True)
+        for rpath in rpaths:
+            rpath_new = os.path.join('@loader_path',
+                            os.path.relpath(os.path.join(host_prefix, rpath),
+                                os.path.dirname(path)), '').replace('/./', '/')
+            macho.add_rpath(path, rpath_new, verbose=True)
 
         # 10.7 install_name_tool -delete_rpath causes broken dylibs, I will revisit this ASAP.
         # .. and remove config.build_prefix/lib which was added in-place of
@@ -609,10 +631,11 @@ def post_process_shared_lib(m, f, files):
     codefile_t = codefile_type(path)
     if not codefile_t:
         return
+    rpaths = m.get_value('build/rpaths', ['lib'])
     if sys.platform.startswith('linux') and codefile_t == 'elffile':
-        mk_relative_linux(f, m.config.host_prefix, rpaths=m.get_value('build/rpaths', ['lib']))
+        mk_relative_linux(f, m.config.host_prefix, rpaths=rpaths)
     elif sys.platform == 'darwin' and codefile_t == 'machofile':
-        mk_relative_osx(path, m.config.host_prefix, m.config.build_prefix, files=files)
+        mk_relative_osx(path, m.config.host_prefix, m.config.build_prefix, files=files, rpaths=rpaths)
 
 
 def fix_permissions(files, prefix):
@@ -686,14 +709,15 @@ def check_symlinks(files, prefix, croot):
             elif real_link_path.startswith(real_build_prefix):
                 # If the path is in the build prefix, this is fine, but
                 # the link needs to be relative
-                if not link_path.startswith('.'):
+                relative_path = os.path.relpath(real_link_path, os.path.dirname(path))
+                if not link_path.startswith('.') and link_path != relative_path:
                     # Don't change the link structure if it is already a
                     # relative link. It's possible that ..'s later in the path
                     # can result in a broken link still, but we'll assume that
                     # such crazy things don't happen.
                     print("Making absolute symlink %s -> %s relative" % (f, link_path))
                     os.unlink(path)
-                    os.symlink(os.path.relpath(real_link_path, os.path.dirname(path)), path)
+                    os.symlink(relative_path, path)
             else:
                 # Symlinks to absolute paths on the system (like /usr) are fine.
                 if real_link_path.startswith(croot):

conda_build/render.py

@@ -39,6 +39,12 @@
 from conda_build.index import get_build_index
 # from conda_build.jinja_context import pin_subpackage_against_outputs
 
+try:
+    from conda.base.constants import CONDA_TARBALL_EXTENSIONS
+except Exception:
+    from conda.base.constants import CONDA_TARBALL_EXTENSION
+    CONDA_TARBALL_EXTENSIONS = (CONDA_TARBALL_EXTENSION,)
+
 
 def odict_representer(dumper, data):
     return dumper.represent_dict(data.items())
@@ -54,7 +60,7 @@ def bldpkg_path(m):
     Returns path to built package's tarball given its ``Metadata``.
     '''
     subdir = 'noarch' if m.noarch or m.noarch_python else m.config.host_subdir
-    return os.path.join(m.config.output_folder, subdir, '%s.tar.bz2' % m.dist())
+    return os.path.join(m.config.output_folder, subdir, '%s%s' % (m.dist(), CONDA_TARBALL_EXTENSIONS[0]))
 
 
 def actions_to_pins(actions):
@@ -186,7 +192,7 @@ def find_pkg_dir_or_file_in_pkgs_dirs(pkg_dist, m, files_only=False):
     pkg_loc = None
     for pkgs_dir in _pkgs_dirs:
         pkg_dir = os.path.join(pkgs_dir, pkg_dist)
-        pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2')
+        pkg_file = os.path.join(pkgs_dir, pkg_dist + CONDA_TARBALL_EXTENSIONS[0])
         if not files_only and os.path.isdir(pkg_dir):
             pkg_loc = pkg_dir
             break
@@ -536,16 +542,19 @@ def finalize_metadata(m, permit_unsatisfiable_variants=False):
         #   best thing is to hard-code the absolute path.  This probably won't exist on any
         #   system other than the original build machine, but at least it will work there.
         if m.meta.get('source'):
-            if 'path' in m.meta['source'] and not os.path.isabs(m.meta['source']['path']):
-                rendered_metadata.meta['source']['path'] = os.path.normpath(
-                    os.path.join(m.path, m.meta['source']['path']))
-            elif ('git_url' in m.meta['source'] and not (
-                    # absolute paths are not relative paths
-                    os.path.isabs(m.meta['source']['git_url']) or
-                    # real urls are not relative paths
-                    ":" in m.meta['source']['git_url'])):
-                rendered_metadata.meta['source']['git_url'] = os.path.normpath(
-                    os.path.join(m.path, m.meta['source']['git_url']))
+            if 'path' in m.meta['source']:
+                source_path = m.meta['source']['path']
+                os.path.expanduser(source_path)
+                if not os.path.isabs(source_path):
+                    rendered_metadata.meta['source']['path'] = os.path.normpath(
+                        os.path.join(m.path, source_path))
+                elif ('git_url' in m.meta['source'] and not (
+                        # absolute paths are not relative paths
+                        os.path.isabs(m.meta['source']['git_url']) or
+                        # real urls are not relative paths
+                        ":" in m.meta['source']['git_url'])):
+                    rendered_metadata.meta['source']['git_url'] = os.path.normpath(
+                        os.path.join(m.path, m.meta['source']['git_url']))
 
         if not rendered_metadata.meta.get('build'):
             rendered_metadata.meta['build'] = {}

conda_build/skeletons/rpm.py

@@ -293,7 +293,7 @@ def dictify(r, root=True):
     return d
 
 
-def dictify_pickled(xml_file, dict_massager=None, cdt=None):
+def dictify_pickled(xml_file, src_cache, dict_massager=None, cdt=None):
     pickled = xml_file + '.p'
     if exists(pickled):
         return pickle.load(open(pickled, 'rb'))
@@ -306,7 +306,7 @@ def dictify_pickled(xml_file, dict_massager=None, cdt=None):
         root = ET.fromstring(xmlstring.encode('utf-8'))
         result = dictify(root)
         if dict_massager:
-            result = dict_massager(result, cdt)
+            result = dict_massager(result, src_cache, cdt)
         pickle.dump(result, open(pickled, 'wb'))
         return result
 
@@ -339,7 +339,7 @@ def get_repo_dict(repomd_url, data_type, dict_massager, cdt, src_cache):
                         xml.write(xml_content)
                 else:
                     print("ERROR: Checksum of uncompressed file {} does not match".format(xmlgz_file))  # noqa
-        return dictify_pickled(xml_file, dict_massager, cdt)
+        return dictify_pickled(xml_file, src_cache, dict_massager, cdt)
     return dict({})
 
 
@@ -361,7 +361,7 @@ def massage_primary_requires(requires, cdt):
     return requires
 
 
-def massage_primary(repo_primary, cdt):
+def massage_primary(repo_primary, src_cache, cdt):
     """
     Massages the result of dictify() into a less cumbersome form.
     In particular:

conda_build/source.py

@@ -645,7 +645,8 @@ def provide(metadata):
                         verbose=metadata.config.verbose, timeout=metadata.config.timeout,
                         locking=metadata.config.locking)
             elif 'path' in source_dict:
-                path = normpath(abspath(join(metadata.path, source_dict['path'])))
+                source_path = os.path.expanduser(source_dict['path'])
+                path = normpath(abspath(join(metadata.path, source_path)))
                 if metadata.config.verbose:
                     print("Copying %s to %s" % (path, src_dir))
                 # careful here: we set test path to be outside of conda-build root in setup.cfg.