ENH: simplify and extend rpath handling

Author: dnicolodi

meson.build

@@ -9,9 +9,8 @@ py = import('python').find_installation()
 py.install_sources(
     'mesonpy/__init__.py',
     'mesonpy/_compat.py',
-    'mesonpy/_dylib.py',
     'mesonpy/_editable.py',
-    'mesonpy/_elf.py',
+    'mesonpy/_rpath.py',
     'mesonpy/_tags.py',
     'mesonpy/_util.py',
     'mesonpy/_wheelfile.py',

mesonpy/__init__.py

@@ -46,8 +46,7 @@
 import pyproject_metadata
 
 import mesonpy._compat
-import mesonpy._dylib
-import mesonpy._elf
+import mesonpy._rpath
 import mesonpy._tags
 import mesonpy._util
 import mesonpy._wheelfile
@@ -285,8 +284,6 @@ def __init__(
         self._build_dir = build_dir
         self._sources = sources
 
-        self._libs_build_dir = self._build_dir / 'mesonpy-wheel-libs'
-
     @cached_property
     def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]:
         return _map_to_wheel(self._sources)
@@ -438,49 +435,22 @@ def _is_native(self, file: Union[str, pathlib.Path]) -> bool:
             return True
         return False
 
-    def _install_path(
-        self,
-        wheel_file: mesonpy._wheelfile.WheelFile,
-        origin: Path,
-        destination: pathlib.Path,
-    ) -> None:
-        """"Install" file or directory into the wheel
-        and do the necessary processing before doing so.
-
-        Some files might need to be fixed up to set the RPATH to the internal
-        library directory on Linux wheels for eg.
-        """
-        location = destination.as_posix()
+    def _install_path(self, wheel_file: mesonpy._wheelfile.WheelFile, origin: Path, destination: pathlib.Path) -> None:
+        """Add a file to the wheel."""
 
         if self._has_internal_libs:
-            if platform.system() == 'Linux' or platform.system() == 'Darwin':
-                # add .mesonpy.libs to the RPATH of ELF files
-                if self._is_native(os.fspath(origin)):
-                    # copy ELF to our working directory to avoid Meson having to regenerate the file
-                    new_origin = self._libs_build_dir / pathlib.Path(origin).relative_to(self._build_dir)
-                    os.makedirs(new_origin.parent, exist_ok=True)
-                    shutil.copy2(origin, new_origin)
-                    origin = new_origin
-                    # add our in-wheel libs folder to the RPATH
-                    if platform.system() == 'Linux':
-                        elf = mesonpy._elf.ELF(origin)
-                        libdir_path = \
-                            f'$ORIGIN/{os.path.relpath(f".{self._project.name}.mesonpy.libs", destination.parent)}'
-                        if libdir_path not in elf.rpath:
-                            elf.rpath = [*elf.rpath, libdir_path]
-                    elif platform.system() == 'Darwin':
-                        dylib = mesonpy._dylib.Dylib(origin)
-                        libdir_path = \
-                            f'@loader_path/{os.path.relpath(f".{self._project.name}.mesonpy.libs", destination.parent)}'
-                        if libdir_path not in dylib.rpath:
-                            dylib.rpath = [*dylib.rpath, libdir_path]
-                    else:
-                        # Internal libraries are currently unsupported on this platform
-                        raise NotImplementedError("Bundling libraries in wheel is not supported on platform '{}'"
-                                                  .format(platform.system()))
+            if self._is_native(os.fspath(origin)):
+                # When an executable, libray, or Python extension module is
+                # dynamically linked to a library built as part of the project,
+                # Meson adds a library load path to it pointing to the build
+                # directory, in the form of a relative RPATH entry. meson-python
+                # relocates the shared libraries to the $project.mesonpy.libs
+                # folder. Rewrite the RPATH to point to that folder instead.
+                libspath = os.path.relpath(f'.{self._project.name}.mesonpy.libs', destination.parent)
+                mesonpy._rpath.fix_rpath(origin, libspath)
 
         try:
-            wheel_file.write(origin, location)
+            wheel_file.write(origin, destination.as_posix())
         except FileNotFoundError:
             # work around for Meson bug, see https://github.com/mesonbuild/meson/pull/11655
             if not os.fspath(origin).endswith('.pdb'):

mesonpy/_dylib.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: 2023 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import typing
+
+
+if typing.TYPE_CHECKING:
+    from typing import List
+
+    from mesonpy._compat import Iterable, Path
+
+
+if sys.platform == 'linux':
+
+    def _get_rpath(filepath: Path) -> List[str]:
+        r = subprocess.run(['patchelf', '--print-rpath', os.fspath(filepath)], capture_output=True, text=True)
+        return r.stdout.strip().split(':')
+
+    def _set_rpath(filepath: Path, rpath: Iterable[str]) -> None:
+        subprocess.run(['patchelf','--set-rpath', ':'.join(rpath), os.fspath(filepath)], check=True)
+
+    def fix_rpath(filepath: Path, libs_relative_path: str) -> None:
+        rpath = _get_rpath(filepath)
+        if '$ORIGIN/' in rpath:
+            rpath = [('$ORIGIN/' + libs_relative_path if path == '$ORIGIN/' else path) for path in rpath]
+            _set_rpath(filepath, rpath)
+
+
+elif sys.platform == 'darwin':
+
+    def _get_rpath(filepath: Path) -> List[str]:
+        rpath = []
+        r = subprocess.run(['otool', '-l', os.fspath(filepath)], capture_output=True, text=True)
+        rpath_tag = False
+        for line in [x.split() for x in r.stdout.split('\n')]:
+            if line == ['cmd', 'LC_RPATH']:
+                rpath_tag = True
+            elif len(line) >= 2 and line[0] == 'path' and rpath_tag:
+                rpath.append(line[1])
+                rpath_tag = False
+        return rpath
+
+    def _replace_rpath(filepath: Path, old: str, new: str) -> None:
+        subprocess.run(['install_name_tool', '-rpath', old, new, os.fspath(filepath)], check=True)
+
+    def fix_rpath(filepath: Path, libs_relative_path: str) -> None:
+        rpath = _get_rpath(filepath)
+        if '@loader_path/' in rpath:
+            _replace_rpath(filepath, '@loader_path/', '@loader_path/' + libs_relative_path)
+
+else:
+
+    def fix_rpath(filepath: Path, libs_relative_path: str) -> None:
+        raise NotImplementedError(f'Bundling libraries in wheel is not supported on {sys.platform}')

mesonpy/_elf.py

@@ -16,5 +16,6 @@ py.extension_module(
     'example',
     'examplemod.c',
     link_with: example_lib,
+    link_args: ['-Wl,-rpath,custom-rpath'],
     install: true,
 )

mesonpy/_rpath.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: MIT
 
 import os
-import platform
 import re
 import shutil
 import stat
@@ -166,28 +165,25 @@ def test_rpath(wheel_link_against_local_lib, tmp_path):
     artifact = wheel.wheelfile.WheelFile(wheel_link_against_local_lib)
     artifact.extractall(tmp_path)
 
-    if platform.system() == 'Linux':
-        elf = mesonpy._elf.ELF(tmp_path / f'example{EXT_SUFFIX}')
-        assert '$ORIGIN/.link_against_local_lib.mesonpy.libs' in elf.rpath
-    else:  # 'Darwin'
-        dylib = mesonpy._dylib.Dylib(tmp_path / f'example{EXT_SUFFIX}')
-        assert '@loader_path/.link_against_local_lib.mesonpy.libs' in dylib.rpath
+    origin = {'linux': '$ORIGIN', 'darwin': '@loader_path'}[sys.platform]
+    expected = {f'{origin}/.link_against_local_lib.mesonpy.libs', 'custom-rpath',}
+
+    rpath = set(mesonpy._rpath._get_rpath(tmp_path / f'example{EXT_SUFFIX}'))
+    # Verify that rpath is a superset of the expected one: linking to
+    # the Python runtime may require additional rpath entries.
+    assert rpath >= expected
 
 
 @pytest.mark.skipif(sys.platform not in {'linux', 'darwin'}, reason='Not supported on this platform')
 def test_uneeded_rpath(wheel_purelib_and_platlib, tmp_path):
     artifact = wheel.wheelfile.WheelFile(wheel_purelib_and_platlib)
     artifact.extractall(tmp_path)
 
-    if platform.system() == 'Linux':
-        shared_lib = mesonpy._elf.ELF(tmp_path / f'plat{EXT_SUFFIX}')
-    else:  # 'Darwin'
-        shared_lib = mesonpy._dylib.Dylib(tmp_path / f'plat{EXT_SUFFIX}')
-    if shared_lib.rpath:
-        # shared_lib.rpath is a frozenset, so iterate over it. An rpath may be
-        # present, e.g. when conda is used (rpath will be <conda-prefix>/lib/)
-        for rpath in shared_lib.rpath:
-            assert 'mesonpy.libs' not in rpath
+    origin = {'linux': '$ORIGIN', 'darwin': '@loader_path'}[sys.platform]
+
+    rpath = mesonpy._rpath._get_rpath(tmp_path / f'plat{EXT_SUFFIX}')
+    for path in rpath:
+        assert origin not in path
 
 
 @pytest.mark.skipif(sys.platform not in {'linux', 'darwin'}, reason='Not supported on this platform')