diff --git a/CHANGELOG.md b/CHANGELOG.md index a04ed04ffa..9950e662df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,9 @@ END_UNRELEASED_TEMPLATE * (venvs) {obj}`--venvs_site_packages=yes` no longer errors when packages with overlapping files or directories are used together. ([#3204](https://github.com/bazel-contrib/rules_python/issues/3204)). +* (venvs) {obj}`--venvs_site_packages=yes` works for packages that dynamically + link to shared libraries + ([#3228](https://github.com/bazel-contrib/rules_python/issues/3228)). * (uv) {obj}`//python/uv:lock.bzl%lock` now works with a local platform runtime. * (toolchains) WORKSPACE builds now correctly register musl and freethreaded diff --git a/docs/pyproject.toml b/docs/pyproject.toml index 9a089df59c..f4bbbaf35a 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -13,5 +13,7 @@ dependencies = [ "absl-py", "typing-extensions", "sphinx-reredirects", - "pefile" + "pefile", + "pyelftools", + "macholib", ] diff --git a/docs/requirements.txt b/docs/requirements.txt index 290113c1b9..c5a5feaae0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -14,6 +14,10 @@ alabaster==1.0.0 ; python_full_version >= '3.10' \ --hash=sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e \ --hash=sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b # via sphinx +altgraph==0.17.4 \ + --hash=sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406 \ + --hash=sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff + # via macholib astroid==3.3.11 \ --hash=sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce \ --hash=sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec @@ -111,9 +115,9 @@ colorama==0.4.6 ; sys_platform == 'win32' \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via sphinx -docutils==0.22.2 \ - --hash=sha256:9fdb771707c8784c8f2728b67cb2c691305933d68137ef95a75db5f4dfbc213d \ - --hash=sha256:b0e98d679283fc3bb0ead8a5da7f501baa632654e7056e9c5846842213d674d8 +docutils==0.21.2 \ + --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \ + --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 # via # myst-parser # sphinx @@ -137,9 +141,13 @@ jinja2==3.1.6 \ # myst-parser # readthedocs-sphinx-ext # sphinx -markdown-it-py==4.0.0 \ - --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \ - --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3 +macholib==1.16.3 \ + --hash=sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30 \ + --hash=sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c + # via rules-python-docs (docs/pyproject.toml) +markdown-it-py==3.0.0 \ + --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ + --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb # via # mdit-py-plugins # myst-parser @@ -264,6 +272,10 @@ pefile==2024.8.26 \ --hash=sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632 \ --hash=sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f # via rules-python-docs (docs/pyproject.toml) +pyelftools==0.32 \ + --hash=sha256:013df952a006db5e138b1edf6d8a68ecc50630adbd0d83a2d41e7f846163d738 \ + --hash=sha256:6de90ee7b8263e740c8715a925382d4099b354f29ac48ea40d840cf7aa14ace5 + # via rules-python-docs (docs/pyproject.toml) pygments==2.19.2 \ --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 \ --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b @@ -432,39 +444,49 @@ sphinxcontrib-serializinghtml==2.0.0 \ --hash=sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 \ --hash=sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d # via sphinx -tomli==2.2.1 ; python_full_version < '3.11' \ - --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ - --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \ - --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \ - --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \ - --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \ - --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \ - --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \ - --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \ - --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \ - --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \ - --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \ - --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \ - --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \ - --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \ - --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \ - --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \ - --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \ - --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \ - --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \ - --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \ - --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \ - --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \ - --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \ - --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \ - --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \ - --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \ - --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \ - --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \ - --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \ - --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \ - --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ - --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 +tomli==2.3.0 ; python_full_version < '3.11' \ + --hash=sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456 \ + --hash=sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845 \ + --hash=sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999 \ + --hash=sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0 \ + --hash=sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878 \ + --hash=sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf \ + --hash=sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3 \ + --hash=sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be \ + --hash=sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52 \ + --hash=sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b \ + --hash=sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67 \ + --hash=sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549 \ + --hash=sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba \ + --hash=sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22 \ + --hash=sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c \ + --hash=sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f \ + --hash=sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6 \ + --hash=sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba \ + --hash=sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45 \ + --hash=sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f \ + --hash=sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77 \ + --hash=sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606 \ + --hash=sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441 \ + --hash=sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0 \ + --hash=sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f \ + --hash=sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530 \ + --hash=sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05 \ + --hash=sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8 \ + --hash=sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005 \ + --hash=sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879 \ + --hash=sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae \ + --hash=sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc \ + --hash=sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b \ + --hash=sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b \ + --hash=sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e \ + --hash=sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf \ + --hash=sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac \ + --hash=sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8 \ + --hash=sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b \ + --hash=sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf \ + --hash=sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463 \ + --hash=sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876 # via # sphinx # sphinx-autodoc2 diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl index 9318347819..95b739dff2 100644 --- a/python/private/py_info.bzl +++ b/python/private/py_info.bzl @@ -47,12 +47,17 @@ VenvSymlinkKind = struct( INCLUDE = "INCLUDE", ) +def _VenvSymlinkEntry_init(**kwargs): + kwargs.setdefault("link_to_file", None) + return kwargs + # A provider is used for memory efficiency. # buildifier: disable=name-conventions -VenvSymlinkEntry = provider( +VenvSymlinkEntry, _ = provider( doc = """ An entry in `PyInfo.venv_symlinks` """, + init = _VenvSymlinkEntry_init, fields = { "files": """ :type: depset[File] @@ -67,12 +72,21 @@ if one adds files to `venv_path=a/` and another adds files to `venv_path=a/b/`. One of the {obj}`VenvSymlinkKind` values. It represents which directory within the venv to create the path under. +""", + "link_to_file": """ +:type: File | None + +A file that `venv_path` should point to. The file to link to should also be in +`files`. + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, "link_to_path": """ :type: str | None -A runfiles-root relative path that `venv_path` will symlink to. If `None`, -it means to not create a symlink. +A runfiles-root relative path that `venv_path` will symlink to (if +`link_to_file` is `None`). If `None`, it means to not create it in the venv. """, "package": """ :type: str | None diff --git a/python/private/venv_runfiles.bzl b/python/private/venv_runfiles.bzl index 9fbe97a52e..9bdacf833e 100644 --- a/python/private/venv_runfiles.bzl +++ b/python/private/venv_runfiles.bzl @@ -81,7 +81,7 @@ def build_link_map(ctx, entries): Returns: {type}`dict[str, dict[str, str|File]]` Mappings of venv paths to their backing files. The first key is a `VenvSymlinkKind` value. - The inner dict keys are venv paths relative to the kind's diretory. The + The inner dict keys are venv paths relative to the kind's directory. The inner dict values are strings or Files to link to. """ @@ -116,7 +116,10 @@ def build_link_map(ctx, entries): # If there's just one group, we can symlink to the directory if len(group) == 1: entry = group[0] - keep_kind_link_map[entry.venv_path] = entry.link_to_path + if entry.link_to_file: + keep_kind_link_map[entry.venv_path] = entry.link_to_file + else: + keep_kind_link_map[entry.venv_path] = entry.link_to_path else: # Merge a group of overlapping prefixes _merge_venv_path_group(ctx, group, keep_kind_link_map) @@ -172,7 +175,9 @@ def _merge_venv_path_group(ctx, group, keep_map): # TODO: Compute the minimum number of entries to create. This can't avoid # flattening the files depset, but can lower the number of materialized # files significantly. Usually overlaps are limited to a small number - # of directories. + # of directories. Note that, when doing so, shared libraries need to + # be symlinked directly, not the directory containing them, due to + # dynamic linker symlink resolution semantics on Linux. for entry in group: prefix = entry.venv_path for file in entry.files.to_list(): @@ -249,13 +254,26 @@ def get_venv_symlinks(ctx, files, package, version_str, site_packages_root): continue path = path.removeprefix(site_packages_root) dir_name, _, filename = path.rpartition("/") + runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/") + + if _is_linker_loaded_library(filename): + entry = VenvSymlinkEntry( + kind = VenvSymlinkKind.LIB, + link_to_path = paths.join(runfiles_dir_name, site_packages_root, filename), + link_to_file = src, + package = package, + version = version_str, + venv_path = path, + files = depset([src]), + ) + venv_symlinks.append(entry) + continue if dir_name in dir_symlinks: # we already have this dir, this allows us to short-circuit since most of the # ctx.files.data might share the same directories as ctx.files.srcs continue - runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/") if dir_name: # This can be either: # * a directory with libs (e.g. numpy.libs, created by auditwheel) @@ -312,6 +330,24 @@ def get_venv_symlinks(ctx, files, package, version_str, site_packages_root): return venv_symlinks +def _is_linker_loaded_library(filename): + """Tells if a filename is one that `dlopen()` or the runtime linker handles. + + This should return true for regular C libraries, but false for Python + C extension modules. + + Python extensions: .so (linux, mac), .pyd (windows) + + C libraries: lib*.so (linux), lib*.so.* (linux), lib*.dylib (mac), .dll (windows) + """ + if filename.endswith(".dll"): + return True + if filename.startswith("lib") and ( + filename.endswith((".so", ".dylib")) or ".so." in filename + ): + return True + return False + def _repo_relative_short_path(short_path): # Convert `../+pypi+foo/some/file.py` to `some/file.py` if short_path.startswith("../"): diff --git a/tests/support/copy_file.bzl b/tests/support/copy_file.bzl new file mode 100644 index 0000000000..bd9bb218f3 --- /dev/null +++ b/tests/support/copy_file.bzl @@ -0,0 +1,33 @@ +"""Copies a file to a directory.""" + +def _copy_file_to_dir_impl(ctx): + out_file = ctx.actions.declare_file( + "{}/{}".format(ctx.attr.out_dir, ctx.file.src.basename), + ) + ctx.actions.run_shell( + inputs = [ctx.file.src], + outputs = [out_file], + arguments = [ctx.file.src.path, out_file.path], + # Perform a copy to better match how a file install from + # a repo-phase (e.g. whl extraction) looks. + command = 'cp -f "$1" "$2"', + progress_message = "Copying %{input} to %{output}", + ) + return [DefaultInfo(files = depset([out_file]))] + +copy_file_to_dir = rule( + implementation = _copy_file_to_dir_impl, + doc = """ +This allows copying a file whose name is platform-dependent to a directory. + +While bazel_skylib has a copy_file rule, you must statically specify the +output file name. +""", + attrs = { + "out_dir": attr.string(mandatory = True), + "src": attr.label( + allow_single_file = True, + mandatory = True, + ), + }, +) diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel index 92d5dec6d3..2ce4ad9d22 100644 --- a/tests/venv_site_packages_libs/BUILD.bazel +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -1,6 +1,10 @@ load("//python:py_library.bzl", "py_library") load("//tests/support:py_reconfig.bzl", "py_reconfig_test") -load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") +load( + "//tests/support:support.bzl", + "NOT_WINDOWS", + "SUPPORTS_BOOTSTRAP_SCRIPT", +) py_library( name = "user_lib", @@ -34,3 +38,17 @@ py_reconfig_test( "@other//with_external_data", ], ) + +py_reconfig_test( + name = "shared_lib_loading_test", + srcs = ["shared_lib_loading_test.py"], + bootstrap_impl = "script", + main = "shared_lib_loading_test.py", + target_compatible_with = NOT_WINDOWS, + venvs_site_packages = "yes", + deps = [ + "//tests/venv_site_packages_libs/ext_with_libs", + "@dev_pip//macholib", + "@dev_pip//pyelftools", + ], +) diff --git a/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl b/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl index 68e17160e7..31c720a986 100644 --- a/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl +++ b/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl @@ -4,7 +4,25 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:py_info.bzl", "VenvSymlinkEntry", "VenvSymlinkKind") # buildifier: disable=bzl-visibility -load("//python/private:venv_runfiles.bzl", "build_link_map") # buildifier: disable=bzl-visibility +load("//python/private:venv_runfiles.bzl", "build_link_map", "get_venv_symlinks") # buildifier: disable=bzl-visibility + +def _empty_files_impl(ctx): + files = [] + for p in ctx.attr.paths: + f = ctx.actions.declare_file(p) + ctx.actions.write(output = f, content = "") + files.append(f) + return [DefaultInfo(files = depset(files))] + +empty_files = rule( + implementation = _empty_files_impl, + attrs = { + "paths": attr.string_list( + doc = "A list of paths to create as files.", + mandatory = True, + ), + }, +) _tests = [] @@ -242,6 +260,63 @@ def _test_multiple_venv_symlink_kinds_impl(env, _): VenvSymlinkKind.INCLUDE, ]) +def _test_shared_library_symlinking(name): + empty_files( + name = name + "_files", + # NOTE: Test relies upon order + paths = [ + "site-packages/bar/libs/liby.so", + "site-packages/bar/x.py", + "site-packages/bar/y.so", + "site-packages/foo.libs/libx.so", + "site-packages/foo/a.py", + "site-packages/foo/b.so", + ], + ) + analysis_test( + name = name, + impl = _test_shared_library_symlinking_impl, + target = name + "_files", + ) + +_tests.append(_test_shared_library_symlinking) + +def _test_shared_library_symlinking_impl(env, target): + srcs = target.files.to_list() + actual_entries = get_venv_symlinks( + _ctx(), + srcs, + package = "foo", + version_str = "1.0", + site_packages_root = env.ctx.label.package + "/site-packages", + ) + + actual = [e for e in actual_entries if e.venv_path == "foo.libs/libx.so"] + if not actual: + fail("Did not find VenvSymlinkEntry with venv_path equal to foo.libs/libx.so. " + + "Found: {}".format(actual_entries)) + elif len(actual) > 1: + fail("Found multiple entries with venv_path=foo.libs/libx.so. " + + "Found: {}".format(actual_entries)) + actual = actual[0] + + actual_files = actual.files.to_list() + expected_lib_dso = [f for f in srcs if f.basename == "libx.so"] + env.expect.that_collection(actual_files).contains_exactly(expected_lib_dso) + + entries = actual_entries + actual = build_link_map(_ctx(), entries) + + # The important condition is that each lib*.so file is linked directly. + expected_libs = { + "bar/libs/liby.so": srcs[0], + "bar/x.py": srcs[1], + "bar/y.so": srcs[2], + "foo": "_main/tests/venv_site_packages_libs/app_files_building/site-packages/foo", + "foo.libs/libx.so": srcs[3], + } + env.expect.that_dict(actual[VenvSymlinkKind.LIB]).contains_exactly(expected_libs) + def app_files_building_test_suite(name): test_suite( name = name, diff --git a/tests/venv_site_packages_libs/ext_with_libs/BUILD.bazel b/tests/venv_site_packages_libs/ext_with_libs/BUILD.bazel new file mode 100644 index 0000000000..8f161ee17c --- /dev/null +++ b/tests/venv_site_packages_libs/ext_with_libs/BUILD.bazel @@ -0,0 +1,94 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library") +load("//python:py_library.bzl", "py_library") +load("//tests/support:copy_file.bzl", "copy_file_to_dir") + +package( + default_visibility = ["//visibility:public"], +) + +cc_library( + name = "increment_impl", + srcs = ["increment.c"], + deps = [":increment_headers"], +) + +cc_library( + name = "increment_headers", + hdrs = ["increment.h"], +) + +cc_shared_library( + name = "increment", + user_link_flags = select({ + "@platforms//os:osx": [ + # Needed so that DT_NEEDED=libincrement.dylib can find + # this shared library + "-Wl,-install_name,@rpath/libincrement.dylib", + ], + "//conditions:default": [], + }), + deps = [":increment_impl"], +) + +cc_library( + name = "adder_impl", + srcs = ["adder.c"], + deps = [ + ":increment_headers", + "@rules_python//python/cc:current_py_cc_headers", + ], +) + +cc_shared_library( + name = "adder", + # Necessary for several reasons: + # 1. Ensures the output doesn't include increment itself (avoids ODRs) + # 2. Adds -lincrement (DT_NEEDED for libincrement.so) + # 3. Ensures libincrement.so is available at link time to satisfy (2) + dynamic_deps = [":increment"], + shared_lib_name = "adder.so", + tags = ["manual"], + # NOTE: cc_shared_library adds Bazelized rpath entries, too. + user_link_flags = [ + ] + select({ + "@platforms//os:osx": [ + "-Wl,-rpath,@loader_path/libs", + "-undefined", + "dynamic_lookup", + "-Wl,-exported_symbol", + "-Wl,_PyInit_adder", + ], + # Assume linux default + "//conditions:default": [ + "-Wl,-rpath,$ORIGIN/libs", + ], + }), + deps = [":adder_impl"], +) + +copy_file_to_dir( + name = "relocate_adder", + src = ":adder", + out_dir = "site-packages/ext_with_libs", + tags = ["manual"], +) + +copy_file_to_dir( + name = "relocate_increment", + src = ":increment", + out_dir = "site-packages/ext_with_libs/libs", + tags = ["manual"], +) + +py_library( + name = "ext_with_libs", + srcs = glob(["site-packages/**/*.py"]), + data = [ + ":relocate_adder", + ":relocate_increment", + ], + experimental_venvs_site_packages = "//python/config_settings:venvs_site_packages", + imports = [package_name() + "/site-packages"], + tags = ["manual"], +) diff --git a/tests/venv_site_packages_libs/ext_with_libs/adder.c b/tests/venv_site_packages_libs/ext_with_libs/adder.c new file mode 100644 index 0000000000..8b04b1721f --- /dev/null +++ b/tests/venv_site_packages_libs/ext_with_libs/adder.c @@ -0,0 +1,15 @@ +#include + +#include "increment.h" + +static PyObject *do_add(PyObject *self, PyObject *Py_UNUSED(args)) { + return PyLong_FromLong(increment(1)); +} + +static PyMethodDef AdderMethods[] = { + {"do_add", do_add, METH_NOARGS, "Add one"}, {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef addermodule = {PyModuleDef_HEAD_INIT, "adder", NULL, + -1, AdderMethods}; + +PyMODINIT_FUNC PyInit_adder(void) { return PyModule_Create(&addermodule); } diff --git a/tests/venv_site_packages_libs/ext_with_libs/increment.c b/tests/venv_site_packages_libs/ext_with_libs/increment.c new file mode 100644 index 0000000000..b194325ac7 --- /dev/null +++ b/tests/venv_site_packages_libs/ext_with_libs/increment.c @@ -0,0 +1,3 @@ +#include "increment.h" + +int increment(int val) { return val + 1; } diff --git a/tests/venv_site_packages_libs/ext_with_libs/increment.h b/tests/venv_site_packages_libs/ext_with_libs/increment.h new file mode 100644 index 0000000000..8a13bf5621 --- /dev/null +++ b/tests/venv_site_packages_libs/ext_with_libs/increment.h @@ -0,0 +1,6 @@ +#ifndef TESTS_VENV_SITE_PACKAGES_LIBS_EXT_WITH_LIBS_INCREMENT_H_ +#define TESTS_VENV_SITE_PACKAGES_LIBS_EXT_WITH_LIBS_INCREMENT_H_ + +int increment(int); + +#endif // TESTS_VENV_SITE_PACKAGES_LIBS_EXT_WITH_LIBS_INCREMENT_H_ diff --git a/tests/venv_site_packages_libs/ext_with_libs/site-packages/ext_with_libs/__init__.py b/tests/venv_site_packages_libs/ext_with_libs/site-packages/ext_with_libs/__init__.py new file mode 100644 index 0000000000..ea0485cb1b --- /dev/null +++ b/tests/venv_site_packages_libs/ext_with_libs/site-packages/ext_with_libs/__init__.py @@ -0,0 +1,2 @@ +# This just marks the directory as a Pyton package. Python C extension modules +# and C libraries are populated in this directory at build time. diff --git a/tests/venv_site_packages_libs/shared_lib_loading_test.py b/tests/venv_site_packages_libs/shared_lib_loading_test.py new file mode 100644 index 0000000000..2b58f8571c --- /dev/null +++ b/tests/venv_site_packages_libs/shared_lib_loading_test.py @@ -0,0 +1,117 @@ +import importlib.util +import os +import unittest + +from elftools.elf.elffile import ELFFile +from macholib import mach_o +from macholib.MachO import MachO + +ELF_MAGIC = b"\x7fELF" +MACHO_MAGICS = ( + b"\xce\xfa\xed\xfe", # 32-bit big-endian + b"\xcf\xfa\xed\xfe", # 64-bit big-endian + b"\xfe\xed\xfa\xce", # 32-bit little-endian + b"\xfe\xed\xfa\xcf", # 64-bit little-endian +) + + +class SharedLibLoadingTest(unittest.TestCase): + def test_shared_library_linking(self): + try: + import ext_with_libs.adder + except ImportError as e: + spec = importlib.util.find_spec("ext_with_libs.adder") + if not spec or not spec.origin: + self.fail(f"Import failed and could not find module spec: {e}") + + info = self._get_linking_info(spec.origin) + + # Give a useful error message for debugging. + self.fail( + f"Failed to import adder extension.\n" + f"Original error: {e}\n" + f"Linking info for {spec.origin}:\n" + f" RPATHs: {info.get('rpaths', 'N/A')}\n" + f" Needed libs: {info.get('needed', 'N/A')}" + ) + + # Check that the module was loaded from the venv. + self.assertIn(".venv/", ext_with_libs.adder.__file__) + + adder_path = os.path.realpath(ext_with_libs.adder.__file__) + + with open(adder_path, "rb") as f: + magic_bytes = f.read(4) + + if magic_bytes == ELF_MAGIC: + self._assert_elf_linking(adder_path) + elif magic_bytes in MACHO_MAGICS: + self._assert_macho_linking(adder_path) + else: + self.fail(f"Unsupported file format for adder: magic bytes {magic_bytes!r}") + + # Check the function works regardless of format. + self.assertEqual(ext_with_libs.adder.do_add(), 2) + + def _get_linking_info(self, path): + """Parses a shared library and returns its rpaths and dependencies.""" + path = os.path.realpath(path) + with open(path, "rb") as f: + magic_bytes = f.read(4) + + if magic_bytes == ELF_MAGIC: + return self._get_elf_info(path) + elif magic_bytes in MACHO_MAGICS: + return self._get_macho_info(path) + return {} + + def _get_elf_info(self, path): + """Extracts linking information from an ELF file.""" + info = {"rpaths": [], "needed": [], "undefined_symbols": []} + with open(path, "rb") as f: + elf = ELFFile(f) + dynamic = elf.get_section_by_name(".dynamic") + if dynamic: + for tag in dynamic.iter_tags(): + if tag.entry.d_tag == "DT_NEEDED": + info["needed"].append(tag.needed) + elif tag.entry.d_tag == "DT_RPATH": + info["rpaths"].append(tag.rpath) + elif tag.entry.d_tag == "DT_RUNPATH": + info["rpaths"].append(tag.runpath) + + dynsym = elf.get_section_by_name(".dynsym") + if dynsym: + info["undefined_symbols"] = [ + s.name + for s in dynsym.iter_symbols() + if s.entry["st_shndx"] == "SHN_UNDEF" + ] + return info + + def _get_macho_info(self, path): + """Extracts linking information from a Mach-O file.""" + info = {"rpaths": [], "needed": []} + macho = MachO(path) + for header in macho.headers: + for cmd_load, cmd, data in header.commands: + if cmd_load.cmd == mach_o.LC_LOAD_DYLIB: + info["needed"].append(data.decode().strip("\x00")) + elif cmd_load.cmd == mach_o.LC_RPATH: + info["rpaths"].append(data.decode().strip("\x00")) + return info + + def _assert_elf_linking(self, path): + """Asserts dynamic linking properties for an ELF file.""" + info = self._get_elf_info(path) + self.assertIn("libincrement.so", info["needed"]) + self.assertIn("increment", info["undefined_symbols"]) + + def _assert_macho_linking(self, path): + """Asserts dynamic linking properties for a Mach-O file.""" + info = self._get_macho_info(path) + self.assertIn("@rpath/libincrement.dylib", info["needed"]) + + +if __name__ == "__main__": + unittest.main()