Add test_multi_acquire_release_cross_module, while also forcing unique PYBIND11_INTERNALS_VERSION for cross_module_gil_utils.cpp

Author: rwgk

tests/cross_module_gil_utils.cpp

@@ -6,9 +6,14 @@
     All rights reserved. Use of this source code is governed by a
     BSD-style license that can be found in the LICENSE file.
 */
+#if defined(PYBIND11_INTERNALS_VERSION)
+#    undef PYBIND11_INTERNALS_VERSION
+#endif
+#define PYBIND11_INTERNALS_VERSION 21814642 // Ensure this module has its own `internals` instance.
 #include <pybind11/pybind11.h>
 
 #include <cstdint>
+#include <string>
 #include <thread>
 
 // This file mimics a DSO that makes pybind11 calls but does not define a
@@ -22,9 +27,24 @@
 namespace {
 
 namespace py = pybind11;
+
 void gil_acquire() { py::gil_scoped_acquire gil; }
 
-constexpr char kModuleName[] = "cross_module_gil_utils";
+std::string gil_multi_acquire_release(unsigned bits) {
+    if (bits & 0x1u) {
+        py::gil_scoped_acquire gil;
+    }
+    if (bits & 0x2u) {
+        py::gil_scoped_release gil;
+    }
+    if (bits & 0x4u) {
+        py::gil_scoped_acquire gil;
+    }
+    if (bits & 0x8u) {
+        py::gil_scoped_release gil;
+    }
+    return PYBIND11_INTERNALS_ID;
+}
 
 struct CustomAutoGIL {
     CustomAutoGIL() : gstate(PyGILState_Ensure()) {}
@@ -55,6 +75,8 @@ void gil_acquire_nested() {
     thread.join();
 }
 
+constexpr char kModuleName[] = "cross_module_gil_utils";
+
 struct PyModuleDef moduledef = {
     PyModuleDef_HEAD_INIT, kModuleName, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr};
 
@@ -71,6 +93,7 @@ extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() {
         static_assert(sizeof(&gil_acquire) == sizeof(void *),
                       "Function pointer must have the same size as void*");
         ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire)
+        ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release)
         ADD_FUNCTION("gil_acquire_inner_custom_funcaddr",
                      gil_acquire_inner<CustomAutoGIL, CustomAutoNoGIL>)
         ADD_FUNCTION("gil_acquire_nested_custom_funcaddr",

tests/test_gil_scoped.cpp

@@ -11,6 +11,9 @@
 
 #include "pybind11_tests.h"
 
+#include <string>
+#include <thread>
+
 #define CROSS_MODULE(Function)                                                                    \
     auto cm = py::module_::import("cross_module_gil_utils");                                      \
     auto target = reinterpret_cast<void (*)()>(PyLong_AsVoidPtr(cm.attr(Function).ptr()));
@@ -102,4 +105,30 @@ TEST_SUBMODULE(gil_scoped, m) {
         py::gil_scoped_acquire gil_acquired_inner;
         return py::str(obj);
     });
+    m.def("test_multi_acquire_release_cross_module", [](unsigned bits) {
+        py::set internals_ids;
+        internals_ids.add(PYBIND11_INTERNALS_ID);
+        {
+            py::gil_scoped_release gil_released;
+            auto thread_f = [bits, &internals_ids]() {
+                py::gil_scoped_acquire gil_acquired;
+                auto cm = py::module_::import("cross_module_gil_utils");
+                auto target = reinterpret_cast<std::string (*)(unsigned)>(
+                    PyLong_AsVoidPtr(cm.attr("gil_multi_acquire_release_funcaddr").ptr()));
+                std::string cm_internals_id = target(bits >> 3);
+                internals_ids.add(cm_internals_id);
+            };
+            if (bits & 0x1u) {
+                thread_f();
+            }
+            if (bits & 0x2u) {
+                std::thread non_python_thread(std::move(thread_f));
+                non_python_thread.join();
+            }
+            if (bits & 0x4u) {
+                thread_f();
+            }
+        }
+        return internals_ids;
+    });
 }

tests/test_gil_scoped.py

@@ -1,21 +1,9 @@
-import builtins
 import multiprocessing
 import threading
 
-import pytest
-
 from pybind11_tests import gil_scoped as m
 
 
-def get_pybind11_internals_keys():
-    keys = []
-    for key in dir(builtins):
-        if key.startswith("__pybind11_internals_"):
-            assert key.endswith("__")
-            keys.append(key)
-    return tuple(sorted(keys))
-
-
 def _run_in_process(target, *args, **kwargs):
     """Runs target in process and returns its exitcode after 10s (None if still alive)."""
     process = multiprocessing.Process(target=target, args=args, kwargs=kwargs)
@@ -45,6 +33,7 @@ def pure_virtual_func(self):
     m.test_callback_std_func(lambda: None)
     m.test_callback_virtual_func(extended)
     m.test_callback_pure_virtual_func(extended)
+
     m.test_cross_module_gil_released()
     m.test_cross_module_gil_acquired()
     m.test_cross_module_gil_inner_custom_released()
@@ -54,10 +43,15 @@ def pure_virtual_func(self):
     m.test_cross_module_gil_nested_custom_released()
     m.test_cross_module_gil_nested_custom_acquired()
     m.test_cross_module_gil_nested_pybind11_released()
-    # m.test_cross_module_gil_nested_pybind11_acquire() # this one dies in test_python_to_cpp_to_python_from_process
+    m.test_cross_module_gil_nested_pybind11_acquired()
+
     assert m.test_release_acquire(0xAB) == "171"
     assert m.test_nested_acquire(0xAB) == "171"
 
+    for bits in range(16 * 8):
+        internals_ids = m.test_multi_acquire_release_cross_module(bits)
+        assert len(internals_ids) == 2 if bits % 8 else 1
+
 
 def _python_to_cpp_to_python_from_threads(num_threads, parallel=False):
     """Calls different C++ functions that come back to Python, from Python threads."""
@@ -122,14 +116,6 @@ def test_cross_module_gil_acquired():
     m.test_cross_module_gil_acquired()  # Should not raise a SIGSEGV
 
 
-def test_report_builtins_internals_keys():
-    """For reporting, not an actual test."""
-    m.test_cross_module_gil_released()  # Any test that imports cross_module_gil_utils
-    keys = get_pybind11_internals_keys()
-    assert len(keys) != 0
-    pytest.skip("builtins internals keys: %s" % ", ".join(keys))
-
-
 def test_cross_module_gil_inner_custom_released():
     """Makes sure that the GIL can be acquired/released by another module
     from a GIL-released state using custom locking logic."""
@@ -184,3 +170,9 @@ def test_release_acquire():
 
 def test_nested_acquire():
     assert m.test_nested_acquire(0xAB) == "171"
+
+
+def test_multi_acquire_release_cross_module():
+    for bits in range(16 * 8):
+        internals_ids = m.test_multi_acquire_release_cross_module(bits)
+        assert len(internals_ids) == 2 if bits % 8 else 1