Add a test reproducing the #5827 crash

Signed-off-by: Rostan Tabet <rtabet@nvidia.com>

Author: rostan-t

tests/test_thread.cpp

@@ -15,6 +15,11 @@
 #include <chrono>
 #include <thread>
 
+#if defined(PYBIND11_CPP20) && defined(__has_include) && __has_include(<barrier>)
+#    define PYBIND11_HAS_BARRIER 1
+#    include <barrier>
+#endif
+
 namespace py = pybind11;
 
 namespace {
@@ -34,7 +39,6 @@ EmptyStruct SharedInstance;
 } // namespace
 
 TEST_SUBMODULE(thread, m) {
-
     py::class_<IntStruct>(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); }));
 
     // implicitly_convertible uses loader_life_support when an implicit
@@ -67,6 +71,39 @@ TEST_SUBMODULE(thread, m) {
     py::class_<EmptyStruct>(m, "EmptyStruct")
         .def_readonly_static("SharedInstance", &SharedInstance);
 
+#if defined(PYBIND11_HAS_BARRIER)
+    // In the free-threaded build, during PyThreadState_Clear, removing the thread from the biased
+    // reference counting table may call destructors. Make sure that it doesn't crash.
+    m.def("test_pythread_state_clear_destructor", [](py::type cls) {
+        py::handle obj;
+
+        std::barrier barrier{2};
+        std::thread thread1{[&]() {
+            py::gil_scoped_acquire gil;
+            obj = cls().release();
+            barrier.arrive_and_wait();
+        }};
+        std::thread thread2{[&]() {
+            py::gil_scoped_acquire gil;
+            barrier.arrive_and_wait();
+            // ob_ref_shared becomes negative; transition to the queued state
+            obj.dec_ref();
+        }};
+
+        // jthread is not supported by Apple Clang
+        thread1.join();
+        thread2.join();
+    });
+#endif
+
+    m.attr("has_barrier") =
+#ifdef PYBIND11_HAS_BARRIER
+        true;
+#else
+        false;
+#endif
+    m.def("acquire_gil", []() { py::gil_scoped_acquire gil_acquired; });
+
     // NOTE: std::string_view also uses loader_life_support to ensure that
     // the string contents remain alive, but that's a C++ 17 feature.
 }

tests/test_thread.py

@@ -5,6 +5,7 @@
 
 import pytest
 
+import env
 from pybind11_tests import thread as m
 
 
@@ -66,3 +67,14 @@ def access_shared_instance():
         thread.start()
     for thread in threads:
         thread.join()
+
+
+@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads")
+@pytest.mark.skipif(not m.has_barrier, reason="no <barrier>")
+@pytest.mark.skipif(env.sys_is_gil_enabled(), reason="Deadlock with the GIL")
+def test_pythread_state_clear_destructor():
+    class Foo:
+        def __del__(self):
+            m.acquire_gil()
+
+    m.test_pythread_state_clear_destructor(Foo)