fix(venv): symlink shared libraries directly (bazel-contrib#3331)

It seems `$ORIGIN` resolves prior to symlink resolution. This makes it
resolve
differently depending on if the directory or file itself is symlinked.

To fix, special case shared libraries and have them symlinked directly.
Since an
explicit file is the target, `VenvSymlinkEntry.link_to_file` is added to
hold the
File object that will be linked to.

An unfortunate side-effect of this logic is any package with `lib*.so`
files
will be more expensive to build (depset flattened at analysis time, more
files
symlinked), but it beats not working at all. Optimizing that can be done
in
another change.

Tests added to generate libraries that look like what something from
PyPI
does. Manually verified a case using jax and jax plugins.

Fixes bazel-contrib#3228

Author: rickeylev

CHANGELOG.md

@@ -102,6 +102,9 @@ END_UNRELEASED_TEMPLATE
 * (venvs) {obj}`--venvs_site_packages=yes` no longer errors when packages with
   overlapping files or directories are used together.
   ([#3204](https://github.com/bazel-contrib/rules_python/issues/3204)).
+* (venvs) {obj}`--venvs_site_packages=yes` works for packages that dynamically
+  link to shared libraries
+  ([#3228](https://github.com/bazel-contrib/rules_python/issues/3228)).
 * (uv) {obj}`//python/uv:lock.bzl%lock` now works with a local platform
   runtime.
 * (toolchains) WORKSPACE builds now correctly register musl and freethreaded

docs/pyproject.toml

@@ -13,5 +13,7 @@ dependencies = [
     "absl-py",
     "typing-extensions",
     "sphinx-reredirects",
-    "pefile"
+    "pefile",
+    "pyelftools",
+    "macholib",
 ]

docs/requirements.txt

@@ -14,6 +14,10 @@ alabaster==1.0.0 ; python_full_version >= '3.10' \
     --hash=sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e \
     --hash=sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b
     # via sphinx
+altgraph==0.17.4 \
+    --hash=sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406 \
+    --hash=sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff
+    # via macholib
 astroid==3.3.11 \
     --hash=sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce \
     --hash=sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec
@@ -111,9 +115,9 @@ colorama==0.4.6 ; sys_platform == 'win32' \
     --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
     --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
     # via sphinx
-docutils==0.22.2 \
-    --hash=sha256:9fdb771707c8784c8f2728b67cb2c691305933d68137ef95a75db5f4dfbc213d \
-    --hash=sha256:b0e98d679283fc3bb0ead8a5da7f501baa632654e7056e9c5846842213d674d8
+docutils==0.21.2 \
+    --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \
+    --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2
     # via
     #   myst-parser
     #   sphinx
@@ -137,9 +141,13 @@ jinja2==3.1.6 \
     #   myst-parser
     #   readthedocs-sphinx-ext
     #   sphinx
-markdown-it-py==4.0.0 \
-    --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \
-    --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3
+macholib==1.16.3 \
+    --hash=sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30 \
+    --hash=sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c
+    # via rules-python-docs (docs/pyproject.toml)
+markdown-it-py==3.0.0 \
+    --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \
+    --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb
     # via
     #   mdit-py-plugins
     #   myst-parser
@@ -264,6 +272,10 @@ pefile==2024.8.26 \
     --hash=sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632 \
     --hash=sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f
     # via rules-python-docs (docs/pyproject.toml)
+pyelftools==0.32 \
+    --hash=sha256:013df952a006db5e138b1edf6d8a68ecc50630adbd0d83a2d41e7f846163d738 \
+    --hash=sha256:6de90ee7b8263e740c8715a925382d4099b354f29ac48ea40d840cf7aa14ace5
+    # via rules-python-docs (docs/pyproject.toml)
 pygments==2.19.2 \
     --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 \
     --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b
@@ -432,39 +444,49 @@ sphinxcontrib-serializinghtml==2.0.0 \
     --hash=sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 \
     --hash=sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d
     # via sphinx
-tomli==2.2.1 ; python_full_version < '3.11' \
-    --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \
-    --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \
-    --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \
-    --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \
-    --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \
-    --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \
-    --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \
-    --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \
-    --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \
-    --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \
-    --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \
-    --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \
-    --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \
-    --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \
-    --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \
-    --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \
-    --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \
-    --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \
-    --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \
-    --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \
-    --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \
-    --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \
-    --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \
-    --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \
-    --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \
-    --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \
-    --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \
-    --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \
-    --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \
-    --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \
-    --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \
-    --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7
+tomli==2.3.0 ; python_full_version < '3.11' \
+    --hash=sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456 \
+    --hash=sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845 \
+    --hash=sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999 \
+    --hash=sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0 \
+    --hash=sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878 \
+    --hash=sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf \
+    --hash=sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3 \
+    --hash=sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be \
+    --hash=sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52 \
+    --hash=sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b \
+    --hash=sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67 \
+    --hash=sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549 \
+    --hash=sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba \
+    --hash=sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22 \
+    --hash=sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c \
+    --hash=sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f \
+    --hash=sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6 \
+    --hash=sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba \
+    --hash=sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45 \
+    --hash=sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f \
+    --hash=sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77 \
+    --hash=sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606 \
+    --hash=sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441 \
+    --hash=sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0 \
+    --hash=sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f \
+    --hash=sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530 \
+    --hash=sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05 \
+    --hash=sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8 \
+    --hash=sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005 \
+    --hash=sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879 \
+    --hash=sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae \
+    --hash=sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc \
+    --hash=sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b \
+    --hash=sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b \
+    --hash=sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e \
+    --hash=sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf \
+    --hash=sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac \
+    --hash=sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8 \
+    --hash=sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b \
+    --hash=sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf \
+    --hash=sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463 \
+    --hash=sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876
     # via
     #   sphinx
     #   sphinx-autodoc2

python/private/py_info.bzl

@@ -47,12 +47,17 @@ VenvSymlinkKind = struct(
     INCLUDE = "INCLUDE",
 )
 
+def _VenvSymlinkEntry_init(**kwargs):
+    kwargs.setdefault("link_to_file", None)
+    return kwargs
+
 # A provider is used for memory efficiency.
 # buildifier: disable=name-conventions
-VenvSymlinkEntry = provider(
+VenvSymlinkEntry, _ = provider(
     doc = """
 An entry in `PyInfo.venv_symlinks`
 """,
+    init = _VenvSymlinkEntry_init,
     fields = {
         "files": """
 :type: depset[File]
@@ -67,12 +72,21 @@ if one adds files to `venv_path=a/` and another adds files to `venv_path=a/b/`.
 
 One of the {obj}`VenvSymlinkKind` values. It represents which directory within
 the venv to create the path under.
+""",
+        "link_to_file": """
+:type: File | None
+
+A file that `venv_path` should point to. The file to link to should also be in
+`files`.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
         "link_to_path": """
 :type: str | None
 
-A runfiles-root relative path that `venv_path` will symlink to. If `None`,
-it means to not create a symlink.
+A runfiles-root relative path that `venv_path` will symlink to (if
+`link_to_file` is `None`). If `None`, it means to not create it in the venv.
 """,
         "package": """
 :type: str | None

python/private/venv_runfiles.bzl

@@ -81,7 +81,7 @@ def build_link_map(ctx, entries):
     Returns:
         {type}`dict[str, dict[str, str|File]]` Mappings of venv paths to their
         backing files. The first key is a `VenvSymlinkKind` value.
-        The inner dict keys are venv paths relative to the kind's diretory. The
+        The inner dict keys are venv paths relative to the kind's directory. The
         inner dict values are strings or Files to link to.
     """
 
@@ -116,7 +116,10 @@ def build_link_map(ctx, entries):
             # If there's just one group, we can symlink to the directory
             if len(group) == 1:
                 entry = group[0]
-                keep_kind_link_map[entry.venv_path] = entry.link_to_path
+                if entry.link_to_file:
+                    keep_kind_link_map[entry.venv_path] = entry.link_to_file
+                else:
+                    keep_kind_link_map[entry.venv_path] = entry.link_to_path
             else:
                 # Merge a group of overlapping prefixes
                 _merge_venv_path_group(ctx, group, keep_kind_link_map)
@@ -172,7 +175,9 @@ def _merge_venv_path_group(ctx, group, keep_map):
     # TODO: Compute the minimum number of entries to create. This can't avoid
     # flattening the files depset, but can lower the number of materialized
     # files significantly. Usually overlaps are limited to a small number
-    # of directories.
+    # of directories. Note that, when doing so, shared libraries need to
+    # be symlinked directly, not the directory containing them, due to
+    # dynamic linker symlink resolution semantics on Linux.
     for entry in group:
         prefix = entry.venv_path
         for file in entry.files.to_list():
@@ -249,13 +254,26 @@ def get_venv_symlinks(ctx, files, package, version_str, site_packages_root):
             continue
         path = path.removeprefix(site_packages_root)
         dir_name, _, filename = path.rpartition("/")
+        runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/")
+
+        if _is_linker_loaded_library(filename):
+            entry = VenvSymlinkEntry(
+                kind = VenvSymlinkKind.LIB,
+                link_to_path = paths.join(runfiles_dir_name, site_packages_root, filename),
+                link_to_file = src,
+                package = package,
+                version = version_str,
+                venv_path = path,
+                files = depset([src]),
+            )
+            venv_symlinks.append(entry)
+            continue
 
         if dir_name in dir_symlinks:
             # we already have this dir, this allows us to short-circuit since most of the
             # ctx.files.data might share the same directories as ctx.files.srcs
             continue
 
-        runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/")
         if dir_name:
             # This can be either:
             # * a directory with libs (e.g. numpy.libs, created by auditwheel)
@@ -312,6 +330,24 @@ def get_venv_symlinks(ctx, files, package, version_str, site_packages_root):
 
     return venv_symlinks
 
+def _is_linker_loaded_library(filename):
+    """Tells if a filename is one that `dlopen()` or the runtime linker handles.
+
+    This should return true for regular C libraries, but false for Python
+    C extension modules.
+
+    Python extensions: .so (linux, mac), .pyd (windows)
+
+    C libraries: lib*.so (linux), lib*.so.* (linux), lib*.dylib (mac), .dll (windows)
+    """
+    if filename.endswith(".dll"):
+        return True
+    if filename.startswith("lib") and (
+        filename.endswith((".so", ".dylib")) or ".so." in filename
+    ):
+        return True
+    return False
+
 def _repo_relative_short_path(short_path):
     # Convert `../+pypi+foo/some/file.py` to `some/file.py`
     if short_path.startswith("../"):

tests/support/copy_file.bzl

@@ -0,0 +1,33 @@
+"""Copies a file to a directory."""
+
+def _copy_file_to_dir_impl(ctx):
+    out_file = ctx.actions.declare_file(
+        "{}/{}".format(ctx.attr.out_dir, ctx.file.src.basename),
+    )
+    ctx.actions.run_shell(
+        inputs = [ctx.file.src],
+        outputs = [out_file],
+        arguments = [ctx.file.src.path, out_file.path],
+        # Perform a copy to better match how a file install from
+        # a repo-phase (e.g. whl extraction) looks.
+        command = 'cp -f "$1" "$2"',
+        progress_message = "Copying %{input} to %{output}",
+    )
+    return [DefaultInfo(files = depset([out_file]))]
+
+copy_file_to_dir = rule(
+    implementation = _copy_file_to_dir_impl,
+    doc = """
+This allows copying a file whose name is platform-dependent to a directory.
+
+While bazel_skylib has a copy_file rule, you must statically specify the
+output file name.
+""",
+    attrs = {
+        "out_dir": attr.string(mandatory = True),
+        "src": attr.label(
+            allow_single_file = True,
+            mandatory = True,
+        ),
+    },
+)

tests/venv_site_packages_libs/BUILD.bazel

@@ -1,6 +1,10 @@
 load("//python:py_library.bzl", "py_library")
 load("//tests/support:py_reconfig.bzl", "py_reconfig_test")
-load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT")
+load(
+    "//tests/support:support.bzl",
+    "NOT_WINDOWS",
+    "SUPPORTS_BOOTSTRAP_SCRIPT",
+)
 
 py_library(
     name = "user_lib",
@@ -34,3 +38,17 @@ py_reconfig_test(
         "@other//with_external_data",
     ],
 )
+
+py_reconfig_test(
+    name = "shared_lib_loading_test",
+    srcs = ["shared_lib_loading_test.py"],
+    bootstrap_impl = "script",
+    main = "shared_lib_loading_test.py",
+    target_compatible_with = NOT_WINDOWS,
+    venvs_site_packages = "yes",
+    deps = [
+        "//tests/venv_site_packages_libs/ext_with_libs",
+        "@dev_pip//macholib",
+        "@dev_pip//pyelftools",
+    ],
+)