From cf368eca03b29eedd99703eedd26418d2395ef5b Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 3 Sep 2020 16:55:01 +0200 Subject: [PATCH 1/6] overlinking: Add prefix/DLLs to the RPATH search dir for pyd or things in DLLs .. arguably we should just add the directory of the DLL/pyd itself here and arguably we should just do that globally. Still, for now this is OK. There could be an argument made for adding support for allowing packages to modify PATHs somehow, and for conda-build to be able to know about that too --- conda_build/os_utils/liefldd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda_build/os_utils/liefldd.py b/conda_build/os_utils/liefldd.py index 1336703ebc..5c9ddb1173 100644 --- a/conda_build/os_utils/liefldd.py +++ b/conda_build/os_utils/liefldd.py @@ -433,6 +433,9 @@ def inspect_linkages_lief(filename, resolve_filenames=True, recurse=True, tmp_filename = parents_by_filename[tmp_filename] else: parent_exe_dirname = exedir + # This is a hack for Python on Windows. Sorry. + if '.pyd' in filename2 or (os.sep + 'DLLs' + os.sep) in filename2: + parent_exe_dirname = envroot.replace(os.sep, '/') + '/DLLs' rpaths_by_binary[filename2] = get_rpaths(binary, parent_exe_dirname, envroot.replace(os.sep, '/'), From fbdc2ca5cb9803a0fce8dfa06bc9474bc048fc42 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 3 Sep 2020 16:55:34 +0200 Subject: [PATCH 2/6] overlinking: We search for files with unix slashes --- conda_build/post.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_build/post.py b/conda_build/post.py index d671ee7375..8754fce8a4 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -1030,11 +1030,13 @@ def check_overlinking_impl(pkg_name, pkg_version, build_str, build_number, subdi errors = [] files_to_inspect = [] + filesu = [] for f in files: path = join(run_prefix, f) filetype = codefile_type(path) if filetype and filetype in filetypes_for_platform[subdir.split('-')[0]]: files_to_inspect.append(f) + filesu.append(f.replace('\\', '/')) if not files_to_inspect: return dict() From fb1ccd4ae872ac1ce5fde8c87c050617cbf9cfd8 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 3 Sep 2020 16:56:22 +0200 Subject: [PATCH 3/6] overlinking: Remove invalid if clause checking error_overlinking .. We always run all of the logic here (at this level, if we want to disable it globally that should happen higher up the callstack) --- conda_build/post.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index 8754fce8a4..30d8a7e216 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -1146,12 +1146,11 @@ def check_overlinking_impl(pkg_name, pkg_version, build_str, build_number, subdi orig = needed_dso resolved = needed_dso_info['resolved'] rpaths = needed_dso_info['rpaths'] - if (error_overlinking and - not resolved.startswith('/') and - not resolved.startswith(sysroot_substitution) and - not resolved.startswith(build_prefix_substitution) and - resolved.lower() not in [o.lower() for o in prefix_owners[run_prefix]] and - resolved not in files): + if (not resolved.startswith('/') and + not resolved.startswith(sysroot_substitution) and + not resolved.startswith(build_prefix_substitution) and + resolved.lower() not in [o.lower() for o in prefix_owners[run_prefix]] and + resolved not in filesu): in_whitelist = False if not build_is_host: in_whitelist = any([caseless_sepless_fnmatch([orig], w) for w in whitelist]) From e7ba96bb4cb58748a83669295ad2b24a88530797 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 3 Sep 2020 19:21:01 +0200 Subject: [PATCH 4/6] overlinking: Another Windows os.sep fix --- conda_build/post.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/post.py b/conda_build/post.py index 30d8a7e216..07d122afd4 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -789,7 +789,7 @@ def _map_file_to_package(files, run_prefix, build_prefix, all_needed_dsos, pkg_v # Looking at all the files is very slow. if not dynamic_lib and not static_lib: continue - rp = normpath(relpath(fp, prefix)) + rp = normpath(relpath(fp, prefix)).replace('\\', '/') if dynamic_lib and not any(rp.lower() == w for w in all_needed_dsos_lower): continue if any(rp == normpath(w) for w in all_lib_exports[prefix]): From 22667798cd121ff87629340a99d6c8262a91163d Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 3 Sep 2020 22:06:35 +0200 Subject: [PATCH 5/6] overlinking: Comment out OTT warning This warning complains about things in the whitelist too. It either needs to filter those out or (as I've done) removed. Soon we'll also check for files deleted from the prefix and that will cover the files that this check would have found anyway. --- conda_build/post.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index 07d122afd4..25c2d1f97b 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -1164,9 +1164,10 @@ def check_overlinking_impl(pkg_name, pkg_version, build_str, build_number, subdi # e.g. an elf will have a DT_NEEDED of just 'zlib.so.1' and to standardize # processing across platforms I prefixed them all with $RPATH. That should be # un-done so that this error message is more clearly related to the consumer.. - print("WARNING :: For consumer: '{}' with rpaths: '{}'\n" - "WARNING :: .. the package containing '{}' could not be found in the run prefix".format( - f, rpaths, needed_dso)) + # print("WARNING :: For consumer: '{}' with rpaths: '{}'\n" + # "WARNING :: .. the package containing '{}' could not be found in the run prefix".format( + # f, rpaths, needed_dso)) + pass _show_linking_messages(files, errors, needed_dsos_for_file, build_prefix, run_prefix, pkg_name, error_overlinking, runpath_whitelist, verbose, requirements_run, lib_packages, From dd82df719c8fe2e12b4a873cf01808a770487b67 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 3 Sep 2020 22:29:25 +0200 Subject: [PATCH 6/6] flake8 --- conda_build/post.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index 25c2d1f97b..e466073963 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -1145,12 +1145,11 @@ def check_overlinking_impl(pkg_name, pkg_version, build_str, build_number, subdi for needed_dso, needed_dso_info in needed.items(): orig = needed_dso resolved = needed_dso_info['resolved'] - rpaths = needed_dso_info['rpaths'] if (not resolved.startswith('/') and - not resolved.startswith(sysroot_substitution) and - not resolved.startswith(build_prefix_substitution) and - resolved.lower() not in [o.lower() for o in prefix_owners[run_prefix]] and - resolved not in filesu): + not resolved.startswith(sysroot_substitution) and + not resolved.startswith(build_prefix_substitution) and + resolved.lower() not in [o.lower() for o in prefix_owners[run_prefix]] and + resolved not in filesu): in_whitelist = False if not build_is_host: in_whitelist = any([caseless_sepless_fnmatch([orig], w) for w in whitelist])