feat: add semi-public API: pybind11::detail::is_holder_constructed (#5669)

* Add semi-public API: `pybind11::detail::is_holder_constructed`

* Update docs for `pybind11::custom_type_setup`

* Update tests in `test_custom_type_setup.cpp`

* Apply suggestions from code review

Author: XuehaiPan

docs/advanced/classes.rst

@@ -1384,13 +1384,21 @@ You can do that using ``py::custom_type_setup``:
            auto *type = &heap_type->ht_type;
            type->tp_flags |= Py_TPFLAGS_HAVE_GC;
            type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
-               auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
-               Py_VISIT(self.value.ptr());
+               // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse
+   #if PY_VERSION_HEX >= 0x03090000
+               Py_VISIT(Py_TYPE(self_base));
+   #endif
+               if (py::detail::is_holder_constructed(self_base)) {
+                   auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
+                   Py_VISIT(self.value.ptr());
+               }
                return 0;
            };
            type->tp_clear = [](PyObject *self_base) {
-               auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
-               self.value = py::none();
+               if (py::detail::is_holder_constructed(self_base)) {
+                   auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
+                   self.value = py::none();
+               }
                return 0;
            };
        }));

include/pybind11/detail/value_and_holder.h

@@ -74,5 +74,17 @@ struct value_and_holder {
     }
 };
 
+// This is a semi-public API to check if the corresponding instance has been constructed with a
+// holder. That is, if the instance has been constructed with a holder, the `__init__` method is
+// called and the C++ object is valid. Otherwise, the C++ object might only be allocated, but not
+// initialized. This will lead to **SEGMENTATION FAULTS** if the C++ object is used in any way.
+// Example usage: https://pybind11.readthedocs.io/en/stable/advanced/classes.html#custom-type-setup
+//                for `tp_traverse` and `tp_clear` implementations.
+// WARNING: The caller is responsible for ensuring that the `reinterpret_cast` is valid.
+inline bool is_holder_constructed(PyObject *obj) {
+    auto *const instance = reinterpret_cast<pybind11::detail::instance *>(obj);
+    return instance->get_value_and_holder().holder_constructed();
+}
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

tests/test_custom_type_setup.cpp

@@ -26,13 +26,21 @@ TEST_SUBMODULE(custom_type_setup, m) {
             auto *type = &heap_type->ht_type;
             type->tp_flags |= Py_TPFLAGS_HAVE_GC;
             type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
-                auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
-                Py_VISIT(self.value.ptr());
+        // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse
+#if PY_VERSION_HEX >= 0x03090000
+                Py_VISIT(Py_TYPE(self_base));
+#endif
+                if (py::detail::is_holder_constructed(self_base)) {
+                    auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
+                    Py_VISIT(self.value.ptr());
+                }
                 return 0;
             };
             type->tp_clear = [](PyObject *self_base) {
-                auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
-                self.value = py::none();
+                if (py::detail::is_holder_constructed(self_base)) {
+                    auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
+                    self.value = py::none();
+                }
                 return 0;
             };
         }));