diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index b20c159cfc..c892da0c90 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -13,16 +13,27 @@ # limitations under the License. """Returns information about the local Python runtime as JSON.""" +import glob import json import os import sys import sysconfig +from typing import Any _IS_WINDOWS = sys.platform == "win32" _IS_DARWIN = sys.platform == "darwin" -def _search_directories(get_config, base_executable): +def _get_abi_flags(get_config) -> str: + """Returns the ABI flags for the Python runtime.""" + # sys.abiflags may not exist, but it still may be set in the config. + abi_flags = getattr(sys, "abiflags", None) + if abi_flags is None: + abi_flags = get_config("ABIFLAGS") or get_config("abiflags") or "" + return abi_flags + + +def _search_directories(get_config, base_executable) -> list[str]: """Returns a list of library directories to search for shared libraries.""" # There's several types of libraries with different names and a plethora # of settings, and many different config variables to check: @@ -73,23 +84,31 @@ def _search_directories(get_config, base_executable): lib_dirs.append(os.path.join(os.path.dirname(exec_dir), "lib")) # Dedup and remove empty values, keeping the order. - lib_dirs = [v for v in lib_dirs if v] - return {k: None for k in lib_dirs}.keys() + return list(dict.fromkeys(d for d in lib_dirs if d)) -def _get_shlib_suffix(get_config) -> str: - """Returns the suffix for shared libraries.""" - if _IS_DARWIN: - return ".dylib" +def _default_library_names(version, abi_flags) -> tuple[str, ...]: + """Returns a list of default library files to search for shared libraries.""" if _IS_WINDOWS: - return ".dll" - suffix = get_config("SHLIB_SUFFIX") - if not suffix: - suffix = ".so" - return suffix + return ( + f"python{version}{abi_flags}.dll", + f"python{version}.dll", + ) + elif _IS_DARWIN: + return ( + f"libpython{version}{abi_flags}.dylib", + f"libpython{version}.dylib", + ) + else: + return ( + f"libpython{version}{abi_flags}.so", + f"libpython{version}.so", + f"libpython{version}{abi_flags}.so.1.0", + f"libpython{version}.so.1.0", + ) -def _search_library_names(get_config, shlib_suffix): +def _search_library_names(get_config, version, abi_flags) -> list[str]: """Returns a list of library files to search for shared libraries.""" # Quoting configure.ac in the cpython code base: # "INSTSONAME is the name of the shared library that will be use to install @@ -112,71 +131,74 @@ def _search_library_names(get_config, shlib_suffix): ) ] - # Set the prefix and suffix to construct the library name used for linking. - # The suffix and version are set here to the default values for the OS, - # since they are used below to construct "default" library names. - if _IS_DARWIN: - prefix = "lib" - elif _IS_WINDOWS: - prefix = "" - else: - prefix = "lib" - - version = get_config("VERSION") - - # Ensure that the pythonXY.dll files are included in the search. - lib_names.append(f"{prefix}python{version}{shlib_suffix}") + # Include the default libraries for the system. + lib_names.extend(_default_library_names(version, abi_flags)) - # If there are ABIFLAGS, also add them to the python version lib search. - abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" - if abiflags: - lib_names.append(f"{prefix}python{version}{abiflags}{shlib_suffix}") + # Also include the abi3 libraries for the system. + lib_names.extend(_default_library_names(sys.version_info.major, abi_flags)) - # Add the abi-version includes to the search list. - lib_names.append(f"{prefix}python{sys.version_info.major}{shlib_suffix}") - - # Dedup and remove empty values, keeping the order. - lib_names = [v for v in lib_names if v] - return {k: None for k in lib_names}.keys() + return list(dict.fromkeys(k for k in lib_names if k)) -def _get_python_library_info(base_executable): +def _get_python_library_info(base_executable) -> dict[str, Any]: """Returns a dictionary with the static and dynamic python libraries.""" config_vars = sysconfig.get_config_vars() # VERSION is X.Y in Linux/macOS and XY in Windows. This is used to # construct library paths such as python3.12, so ensure it exists. - if not config_vars.get("VERSION"): - if sys.platform == "win32": - config_vars["VERSION"] = ( - f"{sys.version_info.major}{sys.version_info.minor}") + version = config_vars.get("VERSION") + if not version: + if _IS_WINDOWS: + version = f"{sys.version_info.major}{sys.version_info.minor}" else: - config_vars["VERSION"] = ( - f"{sys.version_info.major}.{sys.version_info.minor}") + version = f"{sys.version_info.major}.{sys.version_info.minor}" + + defines = [] + if config_vars.get("Py_GIL_DISABLED", "0") == "1": + defines.append("Py_GIL_DISABLED") + + # Avoid automatically linking the libraries on windows via pydefine.h + # pragma comment(lib ...) + if _IS_WINDOWS: + defines.append("Py_NO_LINK_LIB") + + # sys.abiflags may not exist, but it still may be set in the config. + abi_flags = _get_abi_flags(config_vars.get) - shlib_suffix = _get_shlib_suffix(config_vars.get) search_directories = _search_directories(config_vars.get, base_executable) - search_libnames = _search_library_names(config_vars.get, shlib_suffix) + search_libnames = _search_library_names(config_vars.get, version, + abi_flags) + + # Used to test whether the library is an abi3 library or a full api library. + abi3_libraries = _default_library_names(sys.version_info.major, abi_flags) - interface_libraries = {} - dynamic_libraries = {} - static_libraries = {} + # Found libraries + static_libraries: dict[str, None] = {} + dynamic_libraries: dict[str, None] = {} + interface_libraries: dict[str, None] = {} + abi_dynamic_libraries: dict[str, None] = {} + abi_interface_libraries: dict[str, None] = {} for root_dir in search_directories: for libname in search_libnames: - # Check whether the library exists. composed_path = os.path.join(root_dir, libname) + is_abi3_file = os.path.basename(composed_path) in abi3_libraries + + # Check whether the library exists and add it to the appropriate list. if os.path.exists(composed_path) or os.path.isdir(composed_path): - if libname.endswith(".a"): + if is_abi3_file: + if not libname.endswith(".a"): + abi_dynamic_libraries[composed_path] = None + elif libname.endswith(".a"): static_libraries[composed_path] = None else: dynamic_libraries[composed_path] = None interface_path = None if libname.endswith(".dll"): - # On windows a .lib file may be an "import library" or a static library. - # The file could be inspected to determine which it is; typically python - # is used as a shared library. + # On windows a .lib file may be an "import library" or a static + # library. The file could be inspected to determine which it is; + # typically python is used as a shared library. # # On Windows, extensions should link with the pythonXY.lib interface # libraries. @@ -190,39 +212,51 @@ def _get_python_library_info(base_executable): # Check whether an interface library exists. if interface_path and os.path.exists(interface_path): - interface_libraries[interface_path] = None + if is_abi3_file: + abi_interface_libraries[interface_path] = None + else: + interface_libraries[interface_path] = None - # Non-windows typically has abiflags. - if hasattr(sys, "abiflags"): - abiflags = sys.abiflags - else: - abiflags = "" + # Additional DLLs are needed on Windows to link properly. + dlls = [] + if _IS_WINDOWS: + dlls.extend( + glob.glob(os.path.join(os.path.dirname(base_executable), "*.dll"))) + dlls = [ + x for x in dlls + if x not in dynamic_libraries and x not in abi_dynamic_libraries + ] + + def _unique_basenames(inputs: dict[str, None]) -> list[str]: + """Returns a list of paths, keeping only the first path for each basename.""" + result = [] + seen = set() + for k in inputs: + b = os.path.basename(k) + if b not in seen: + seen.add(b) + result.append(k) + return result # When no libraries are found it's likely that the python interpreter is not # configured to use shared or static libraries (minilinux). If this seems # suspicious try running `uv tool run find_libpython --list-all -v` return { - "dynamic_libraries": list(dynamic_libraries.keys()), - "static_libraries": list(static_libraries.keys()), - "interface_libraries": list(interface_libraries.keys()), - "shlib_suffix": "" if _IS_WINDOWS else shlib_suffix, - "abi_flags": abiflags, + "dynamic_libraries": _unique_basenames(dynamic_libraries), + "static_libraries": _unique_basenames(static_libraries), + "interface_libraries": _unique_basenames(interface_libraries), + "abi_dynamic_libraries": _unique_basenames(abi_dynamic_libraries), + "abi_interface_libraries": _unique_basenames(abi_interface_libraries), + "abi_flags": abi_flags, + "shlib_suffix": ".dylib" if _IS_DARWIN else "", + "additional_dlls": dlls, + "defines": defines, } -def _get_base_executable(): +def _get_base_executable() -> str: """Returns the base executable path.""" - try: - if sys._base_executable: # pylint: disable=protected-access - return sys._base_executable # pylint: disable=protected-access - except AttributeError: - # Bug reports indicate sys._base_executable doesn't exist in some cases, - # but it's not clear why. - # See https://github.com/bazel-contrib/rules_python/issues/3172 - pass - # The normal sys.executable is the next-best guess if sys._base_executable - # is missing. - return sys.executable + return getattr(sys, "_base_executable", None) or sys.executable data = { diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 024f7c5e8a..df27c74950 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -34,15 +34,36 @@ define_local_runtime_toolchain_impl( major = "{major}", minor = "{minor}", micro = "{micro}", + abi_flags = "{abi_flags}", + os = "{os}", + implementation_name = "{implementation_name}", interpreter_path = "{interpreter_path}", interface_library = {interface_library}, libraries = {libraries}, - implementation_name = "{implementation_name}", - os = "{os}", - abi_flags = "{abi_flags}", + defines = {defines}, + abi3_interface_library = {abi3_interface_library}, + abi3_libraries = {abi3_libraries}, + additional_dlls = {additional_dlls}, ) """ +def _expand_incompatible_template(): + return _TOOLCHAIN_IMPL_TEMPLATE.format( + major = "0", + minor = "0", + micro = "0", + abi_flags = "", + os = "@platforms//:incompatible", + implementation_name = "incompatible", + interpreter_path = "/incompatible", + interface_library = "None", + libraries = "[]", + defines = "[]", + abi3_interface_library = "None", + abi3_libraries = "[]", + additional_dlls = "[]", + ) + def _norm_path(path): """Returns a path using '/' separators and no trailing slash.""" path = path.replace("\\", "/") @@ -50,33 +71,39 @@ def _norm_path(path): path = path[:-1] return path -def _symlink_first_library(rctx, logger, libraries, shlib_suffix): +def _symlink_libraries(rctx, logger, libraries, shlib_suffix): """Symlinks the shared libraries into the lib/ directory. Args: rctx: A repository_ctx object logger: A repo_utils.logger object - libraries: A list of static library paths to potentially symlink. - shlib_suffix: A suffix only provided for shared libraries to ensure - that the srcs restriction of cc_library targets are met. + libraries: paths to libraries to attempt to symlink. + shlib_suffix: Optional. Ensure that the generated symlinks end with this suffix. Returns: - A single library path linked by the action. + A list of library paths (under lib/) linked by the action. + + Individual files are symlinked instead of the whole directory because + shared_lib_dirs contains multiple search paths for the shared libraries, + and the python files may be missing from any of those directories, and + any of those directories may include non-python runtime libraries, + as would be the case if LIBDIR were, for example, /usr/lib. """ - for target in libraries: - origin = rctx.path(target) + result = [] + for source in libraries: + origin = rctx.path(source) if not origin.exists: # The reported names don't always exist; it depends on the particulars # of the runtime installation. continue - if shlib_suffix and not target.endswith(shlib_suffix): - linked = "lib/{}{}".format(origin.basename, shlib_suffix) + if shlib_suffix and not origin.basename.endswith(shlib_suffix): + target = "lib/{}{}".format(origin.basename, shlib_suffix) else: - linked = "lib/{}".format(origin.basename) - logger.debug("Symlinking {} to {}".format(origin, linked)) + target = "lib/{}".format(origin.basename) + logger.debug(lambda: "Symlinking {} to {}".format(origin, target)) rctx.watch(origin) - rctx.symlink(origin, linked) - return linked - return None + rctx.symlink(origin, target) + result.append(target) + return result def _local_runtime_repo_impl(rctx): logger = repo_utils.logger(rctx) @@ -150,31 +177,48 @@ def _local_runtime_repo_impl(rctx): # The cc_library.includes values have to be non-absolute paths, otherwise # the toolchain will give an error. Work around this error by making them # appear as part of this repo. + logger.debug(lambda: "Symlinking {} to include".format(include_path)) rctx.symlink(include_path, "include") rctx.report_progress("Symlinking external Python shared libraries") - interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"], None) - shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"], info["shlib_suffix"]) - static_library = _symlink_first_library(rctx, logger, info["static_libraries"], None) - - libraries = [] - if shared_library: - libraries.append(shared_library) - elif static_library: - libraries.append(static_library) + + interface_library = None + if info["dynamic_libraries"]: + libraries = _symlink_libraries(rctx, logger, info["dynamic_libraries"][:1], info["shlib_suffix"]) + symlinked = _symlink_libraries(rctx, logger, info["interface_libraries"][:1], None) + if symlinked: + interface_library = symlinked[0] else: - logger.warn("No external python libraries found.") + libraries = _symlink_libraries(rctx, logger, info["static_libraries"], None) + if not libraries: + logger.info("No python libraries found.") + + abi3_interface_library = None + if info["abi_dynamic_libraries"]: + abi3_libraries = _symlink_libraries(rctx, logger, info["abi_dynamic_libraries"][:1], info["shlib_suffix"]) + symlinked = _symlink_libraries(rctx, logger, info["abi_interface_libraries"][:1], None) + if symlinked: + abi3_interface_library = symlinked[0] + else: + abi3_libraries = [] + logger.info("No abi3 python libraries found.") + + additional_dlls = _symlink_libraries(rctx, logger, info["additional_dlls"], None) build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format( major = info["major"], minor = info["minor"], micro = info["micro"], + abi_flags = info["abi_flags"], + os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), + implementation_name = info["implementation_name"], interpreter_path = _norm_path(interpreter_path), interface_library = repr(interface_library), libraries = repr(libraries), - implementation_name = info["implementation_name"], - os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), - abi_flags = info["abi_flags"], + defines = repr(info["defines"]), + abi3_interface_library = repr(abi3_interface_library), + abi3_libraries = repr(abi3_libraries), + additional_dlls = repr(additional_dlls), ) logger.debug(lambda: "BUILD.bazel\n{}".format(build_bazel)) @@ -261,19 +305,6 @@ How to handle errors when trying to automatically determine settings. environ = ["PATH", REPO_DEBUG_ENV_VAR, "DEVELOPER_DIR", "XCODE_VERSION"], ) -def _expand_incompatible_template(): - return _TOOLCHAIN_IMPL_TEMPLATE.format( - interpreter_path = "/incompatible", - implementation_name = "incompatible", - interface_library = "None", - libraries = "[]", - major = "0", - minor = "0", - micro = "0", - os = "@platforms//:incompatible", - abi_flags = "", - ) - def _find_python_exe_from_target(rctx): base_path = rctx.path(rctx.attr.interpreter_target) if base_path.exists: diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 0ce1d4d764..0922181ffe 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Setup code called by the code generated by `local_runtime_repo`.""" load("@bazel_skylib//lib:selects.bzl", "selects") @@ -29,12 +28,16 @@ def define_local_runtime_toolchain_impl( major, minor, micro, + abi_flags, + os, + implementation_name, interpreter_path, interface_library, libraries, - implementation_name, - os, - abi_flags): + defines, + abi3_interface_library, + abi3_libraries, + additional_dlls): """Defines a toolchain implementation for a local Python runtime. Generates public targets: @@ -51,16 +54,23 @@ def define_local_runtime_toolchain_impl( major: `str` The major Python version, e.g. `3` of `3.9.1`. minor: `str` The minor Python version, e.g. `9` of `3.9.1`. micro: `str` The micro Python version, e.g. "1" of `3.9.1`. + abi_flags: `str` The abi flags, as returned by `sys.abiflags`. + os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for + this runtime. + implementation_name: `str` The implementation name, as returned by + `sys.implementation.name`. interpreter_path: `str` Absolute path to the interpreter. interface_library: `str` Path to the interface library. e.g. "lib/python312.lib" libraries: `list[str]` Path[s] to the python libraries. e.g. ["lib/python312.dll"] or ["lib/python312.so"] - implementation_name: `str` The implementation name, as returned by - `sys.implementation.name`. - os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for - this runtime. - abi_flags: `str` Str. Flags provided by sys.abiflags for the runtime. + defines: `list[str]` List of additional defines. + abi3_interface_library: `str` Path to the interface library. + e.g. "lib/python3.lib" + abi3_libraries: `list[str]` Path[s] to the python libraries. + e.g. ["lib/python3.dll"] or ["lib/python3.so"] + additional_dlls: `list[str]` Path[s] to additional DLLs. + e.g. ["lib/msvcrt123.dll"] """ major_minor = "{}.{}".format(major, minor) major_minor_micro = "{}.{}".format(major_minor, micro) @@ -69,44 +79,70 @@ def define_local_runtime_toolchain_impl( # See https://docs.python.org/3/extending/windows.html # However not all python installations (such as manylinux) include shared or static libraries, # so only create the import library when interface_library is set. - full_abi_deps = [] - abi3_deps = [] if interface_library: cc_import( - name = "_python_interface_library", + name = "interface", interface_library = interface_library, - system_provided = 1, + system_provided = True, ) - if interface_library.endswith("{}.lib".format(major)): - abi3_deps = [":_python_interface_library"] - else: - full_abi_deps = [":_python_interface_library"] - cc_library( - name = "_python_headers_abi3", - # NOTE: Keep in sync with watch_tree() called in local_runtime_repo + if abi3_interface_library: + cc_import( + name = "abi3_interface", + interface_library = abi3_interface_library, + system_provided = True, + ) + + native.filegroup( + name = "includes", srcs = native.glob( include = ["include/**/*.h"], exclude = ["include/numpy/**"], # numpy headers are handled separately allow_empty = True, # A Python install may not have C headers ), - deps = abi3_deps, + ) + + # header libraries. + cc_library( + name = "python_headers_abi3", + hdrs = [":includes"], + includes = ["include"], + defines = defines, # NOTE: Users should define Py_LIMITED_API=3 + deps = select({ + "@bazel_tools//src/conditions:windows": [":abi3_interface"], + "//conditions:default": [], + }), + ) + + cc_library( + name = "python_headers", + hdrs = [":includes"], includes = ["include"], + defines = defines, + deps = select({ + "@bazel_tools//src/conditions:windows": [":interface"], + "//conditions:default": [], + }), ) + + # python libraries cc_library( - name = "_python_headers", - deps = [":_python_headers_abi3"] + full_abi_deps, + name = "libpython_abi3", + hdrs = [":includes"], + defines = defines, # NOTE: Users should define Py_LIMITED_API=3 + srcs = abi3_libraries + additional_dlls, ) cc_library( - name = "_libpython", - hdrs = [":_python_headers"], - srcs = libraries, - deps = [], + name = "libpython", + hdrs = [":includes"], + defines = defines, + srcs = libraries + additional_dlls, ) + # runtime configuration py_runtime( - name = "_py3_runtime", + name = "py3_runtime", interpreter_path = interpreter_path, python_version = "PY3", interpreter_version_info = { @@ -116,12 +152,13 @@ def define_local_runtime_toolchain_impl( }, implementation_name = implementation_name, abi_flags = abi_flags, + pyc_tag = "{}-{}{}{}".format(implementation_name, major, minor, abi_flags), ) py_runtime_pair( name = "python_runtimes", py2_runtime = None, - py3_runtime = ":_py3_runtime", + py3_runtime = ":py3_runtime", visibility = ["//visibility:public"], ) @@ -133,9 +170,9 @@ def define_local_runtime_toolchain_impl( py_cc_toolchain( name = "py_cc_toolchain", - headers = ":_python_headers", - headers_abi3 = ":_python_headers_abi3", - libs = ":_libpython", + headers = ":python_headers", + headers_abi3 = ":python_headers_abi3", + libs = ":libpython", python_version = major_minor_micro, visibility = ["//visibility:public"], )