Skip to content

Commit 9b3a200

Browse files
tttapahenryiii
tttapa
and
henryiii
authored
fix(cmake): improved cross-compilation support (#5083)
* fix(cmake): do not use Python::Interpreter when cross-compiling * chore: apply cmake-format to pybind11NewTools.cmake * fix(cmake): do not look for Python Interpreter component when cross-compiling * feat(cmake): guess Python extension suffix * fix: add pybind11GuessPythonExtSuffix.cmake to packaging test * Use PYBIND11_CROSSCOMPILING instead of CMAKE_CROSSCOMPILING * refactor: require PYBIND11_USE_CROSSCOMPILING Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> --------- Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Henry Schreiner <henryschreineriii@gmail.com>
1 parent b9794be commit 9b3a200

File tree

7 files changed

+352
-62
lines changed

7 files changed

+352
-62
lines changed

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY
116116
set(PYBIND11_INTERNALS_VERSION
117117
""
118118
CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
119+
option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)
119120

120121
if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
121122
add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
@@ -299,6 +300,7 @@ if(PYBIND11_INSTALL)
299300
tools/pybind11Common.cmake
300301
tools/pybind11Tools.cmake
301302
tools/pybind11NewTools.cmake
303+
tools/pybind11GuessPythonExtSuffix.cmake
302304
DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
303305

304306
if(NOT PYBIND11_EXPORT_NAME)

‎tests/extra_python_package/test_files.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
"share/cmake/pybind11/pybind11Common.cmake",
7474
"share/cmake/pybind11/pybind11Config.cmake",
7575
"share/cmake/pybind11/pybind11ConfigVersion.cmake",
76+
"share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake",
7677
"share/cmake/pybind11/pybind11NewTools.cmake",
7778
"share/cmake/pybind11/pybind11Targets.cmake",
7879
"share/cmake/pybind11/pybind11Tools.cmake",

‎tools/FindPythonLibsNew.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ endif()
205205
# Make sure the Python has the same pointer-size as the chosen compiler
206206
# Skip if CMAKE_SIZEOF_VOID_P is not defined
207207
# This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN)
208-
if(NOT CMAKE_CROSSCOMPILING
208+
if(NOT _PYBIND11_CROSSCOMPILING
209209
AND CMAKE_SIZEOF_VOID_P
210210
AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
211211
if(PythonLibsNew_FIND_REQUIRED)

‎tools/pybind11Common.cmake‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,16 @@ set(pybind11_INCLUDE_DIRS
4242
"${pybind11_INCLUDE_DIR}"
4343
CACHE INTERNAL "Include directory for pybind11 (Python not requested)")
4444

45+
if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING)
46+
set(_PYBIND11_CROSSCOMPILING
47+
ON
48+
CACHE INTERNAL "")
49+
else()
50+
set(_PYBIND11_CROSSCOMPILING
51+
OFF
52+
CACHE INTERNAL "")
53+
endif()
54+
4555
# --------------------- Shared targets ----------------------------
4656

4757
# Build an interface library target:
@@ -195,7 +205,7 @@ endif()
195205

196206
# --------------------- pybind11_find_import -------------------------------
197207

198-
if(NOT _pybind11_nopython)
208+
if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING)
199209
# Check to see if modules are importable. Use REQUIRED to force an error if
200210
# one of the modules is not found. <package_name>_FOUND will be set if the
201211
# package was found (underscores replace dashes if present). QUIET will hide

‎tools/pybind11GuessPythonExtSuffix.cmake‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
cmake_minimum_required(VERSION 3.5)
2+
3+
function(pybind11_guess_python_module_extension python)
4+
5+
# The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence:
6+
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX})
7+
message(
8+
STATUS
9+
"Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}"
10+
)
11+
set(PYTHON_MODULE_EXT_SUFFIX
12+
"$ENV{SETUPTOOLS_EXT_SUFFIX}"
13+
CACHE
14+
STRING
15+
"Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)")
16+
endif()
17+
# If that didn't work, use the Python_SOABI variable:
18+
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI)
19+
message(
20+
STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}")
21+
# The final extension depends on the system
22+
set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}")
23+
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
24+
set(_PY_BUILD_EXTENSION ".pyd")
25+
endif()
26+
# If the SOABI already has an extension, use it as the full suffix
27+
# (used for debug versions of Python on Windows)
28+
if(${python}_SOABI MATCHES "\\.")
29+
set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}")
30+
# If the SOABI is empty, this is usually a bug, but we generate a
31+
# correct extension anyway, which is the best we can do
32+
elseif("${${python}_SOABI}" STREQUAL "")
33+
message(
34+
WARNING
35+
"${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly."
36+
)
37+
set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}")
38+
# Otherwise, add the system-dependent extension to it
39+
else()
40+
set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}")
41+
endif()
42+
endif()
43+
44+
# If we could not deduce the extension suffix, unset the results:
45+
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX)
46+
unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE)
47+
unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE)
48+
unset(PYTHON_IS_DEBUG PARENT_SCOPE)
49+
return()
50+
endif()
51+
52+
# Sanity checks:
53+
if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI
54+
OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\."))
55+
message(
56+
WARNING
57+
"Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})."
58+
)
59+
endif()
60+
61+
# Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699)
62+
get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
63+
get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT)
64+
65+
# Try to deduce the debug ABI from the extension suffix:
66+
if(NOT DEFINED _PYTHON_IS_DEBUG)
67+
if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-"
68+
OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d")
69+
set(_PYTHON_IS_DEBUG On)
70+
else()
71+
set(_PYTHON_IS_DEBUG Off)
72+
endif()
73+
endif()
74+
75+
# Return results
76+
set(PYTHON_MODULE_DEBUG_POSTFIX
77+
"${_PYTHON_MODULE_DEBUG_POSTFIX}"
78+
PARENT_SCOPE)
79+
set(PYTHON_MODULE_EXTENSION
80+
"${_PYTHON_MODULE_EXTENSION}"
81+
PARENT_SCOPE)
82+
set(PYTHON_IS_DEBUG
83+
"${_PYTHON_IS_DEBUG}"
84+
PARENT_SCOPE)
85+
86+
endfunction()

‎tools/pybind11NewTools.cmake‎

Lines changed: 90 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
3232
set(Python_ROOT_DIR "$ENV{pythonLocation}")
3333
endif()
3434

35+
# Interpreter should not be found when cross-compiling
36+
if(_PYBIND11_CROSSCOMPILING)
37+
set(_pybind11_interp_component "")
38+
else()
39+
set(_pybind11_interp_component Interpreter)
40+
endif()
41+
3542
# Development.Module support (required for manylinux) started in 3.18
3643
if(CMAKE_VERSION VERSION_LESS 3.18)
3744
set(_pybind11_dev_component Development)
@@ -48,8 +55,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
4855
endif()
4956
endif()
5057

51-
find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component}
52-
${_pybind11_quiet} ${_pybind11_global_keyword})
58+
find_package(
59+
Python 3.6 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component}
60+
${_pybind11_quiet} ${_pybind11_global_keyword})
5361

5462
# If we are in submodule mode, export the Python targets to global targets.
5563
# If this behavior is not desired, FindPython _before_ pybind11.
@@ -59,7 +67,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
5967
if(TARGET Python::Python)
6068
set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE)
6169
endif()
62-
set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
70+
if(TARGET Python::Interpreter)
71+
set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
72+
endif()
6373
if(TARGET Python::Module)
6474
set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE)
6575
endif()
@@ -100,69 +110,89 @@ if(PYBIND11_MASTER_PROJECT)
100110
endif()
101111
endif()
102112

103-
# If a user finds Python, they may forget to include the Interpreter component
104-
# and the following two steps require it. It is highly recommended by CMake
105-
# when finding development libraries anyway, so we will require it.
106-
if(NOT DEFINED ${_Python}_EXECUTABLE)
107-
message(
108-
FATAL_ERROR
109-
"${_Python} was found without the Interpreter component. Pybind11 requires this component.")
110-
111-
endif()
112-
113-
if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
114-
PYBIND11_PYTHON_EXECUTABLE_LAST)
115-
# Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
116-
unset(PYTHON_IS_DEBUG CACHE)
117-
unset(PYTHON_MODULE_EXTENSION CACHE)
118-
endif()
119-
120-
set(PYBIND11_PYTHON_EXECUTABLE_LAST
121-
"${${_Python}_EXECUTABLE}"
122-
CACHE INTERNAL "Python executable during the last CMake run")
123-
124-
if(NOT DEFINED PYTHON_IS_DEBUG)
125-
# Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
126-
execute_process(
127-
COMMAND "${${_Python}_EXECUTABLE}" "-c"
128-
"import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
129-
RESULT_VARIABLE _PYTHON_IS_DEBUG)
130-
set(PYTHON_IS_DEBUG
131-
"${_PYTHON_IS_DEBUG}"
132-
CACHE INTERNAL "Python debug status")
133-
endif()
134-
135-
# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
136-
# required for PyPy3 (as of 7.3.1)
137-
if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
138-
execute_process(
139-
COMMAND
140-
"${${_Python}_EXECUTABLE}" "-c"
141-
"import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
142-
OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
143-
ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
144-
OUTPUT_STRIP_TRAILING_WHITESPACE)
145-
146-
if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
113+
if(NOT _PYBIND11_CROSSCOMPILING)
114+
# If a user finds Python, they may forget to include the Interpreter component
115+
# and the following two steps require it. It is highly recommended by CMake
116+
# when finding development libraries anyway, so we will require it.
117+
if(NOT DEFINED ${_Python}_EXECUTABLE)
147118
message(
148-
FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'"
149-
"package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}"
119+
FATAL_ERROR
120+
"${_Python} was found without the Interpreter component. Pybind11 requires this component."
150121
)
122+
151123
endif()
152124

153-
# This needs to be available for the pybind11_extension function
154-
if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
155-
get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
156-
set(PYTHON_MODULE_DEBUG_POSTFIX
157-
"${_PYTHON_MODULE_DEBUG_POSTFIX}"
158-
CACHE INTERNAL "")
125+
if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
126+
PYBIND11_PYTHON_EXECUTABLE_LAST)
127+
# Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
128+
unset(PYTHON_IS_DEBUG CACHE)
129+
unset(PYTHON_MODULE_EXTENSION CACHE)
159130
endif()
160131

161-
if(NOT DEFINED PYTHON_MODULE_EXTENSION)
162-
get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
163-
set(PYTHON_MODULE_EXTENSION
164-
"${_PYTHON_MODULE_EXTENSION}"
165-
CACHE INTERNAL "")
132+
set(PYBIND11_PYTHON_EXECUTABLE_LAST
133+
"${${_Python}_EXECUTABLE}"
134+
CACHE INTERNAL "Python executable during the last CMake run")
135+
136+
if(NOT DEFINED PYTHON_IS_DEBUG)
137+
# Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
138+
execute_process(
139+
COMMAND "${${_Python}_EXECUTABLE}" "-c"
140+
"import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
141+
RESULT_VARIABLE _PYTHON_IS_DEBUG)
142+
set(PYTHON_IS_DEBUG
143+
"${_PYTHON_IS_DEBUG}"
144+
CACHE INTERNAL "Python debug status")
145+
endif()
146+
147+
# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
148+
# required for PyPy3 (as of 7.3.1)
149+
if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
150+
execute_process(
151+
COMMAND
152+
"${${_Python}_EXECUTABLE}" "-c"
153+
"import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
154+
OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
155+
ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
156+
OUTPUT_STRIP_TRAILING_WHITESPACE)
157+
158+
if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
159+
message(
160+
FATAL_ERROR
161+
"pybind11 could not query the module file extension, likely the 'distutils'"
162+
"package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}")
163+
endif()
164+
165+
# This needs to be available for the pybind11_extension function
166+
if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
167+
get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
168+
set(PYTHON_MODULE_DEBUG_POSTFIX
169+
"${_PYTHON_MODULE_DEBUG_POSTFIX}"
170+
CACHE INTERNAL "")
171+
endif()
172+
173+
if(NOT DEFINED PYTHON_MODULE_EXTENSION)
174+
get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
175+
set(PYTHON_MODULE_EXTENSION
176+
"${_PYTHON_MODULE_EXTENSION}"
177+
CACHE INTERNAL "")
178+
endif()
179+
endif()
180+
else()
181+
if(NOT DEFINED PYTHON_IS_DEBUG
182+
OR NOT DEFINED PYTHON_MODULE_EXTENSION
183+
OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
184+
include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake")
185+
pybind11_guess_python_module_extension("${_Python}")
186+
endif()
187+
# When cross-compiling, we cannot query the Python interpreter, so we require
188+
# the user to set these variables explicitly.
189+
if(NOT DEFINED PYTHON_IS_DEBUG
190+
OR NOT DEFINED PYTHON_MODULE_EXTENSION
191+
OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
192+
message(
193+
FATAL_ERROR
194+
"When cross-compiling, you should set the PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX \
195+
variables appropriately before loading pybind11 (e.g. in your CMake toolchain file)")
166196
endif()
167197
endif()
168198

0 commit comments

Comments
 (0)