Introduce PYBIND11_PROTOBUF_ASSUME_FULL_ABI_COMPATIBILITY (see proto_cast_util.h):

This macro can be defined by users certain about ABI compatibility between all Python extensions in their environment using protobufs. If defined, passing protos from Python to C++ may skip serialization/deserialization. — Note that this change makes the default behavior safer, but unless `PYBIND11_PROTOBUF_ASSUME_FULL_ABI_COMPATIBILITY` is defined, the runtime performance may be noticably slower.

PiperOrigin-RevId: 568608539

Author: rwgk

README.md

@@ -64,9 +64,12 @@ avoided, but not universally. Fundamentally, sharing C++ native protobuf
 objects between C++ and Python is unsafe because C++ assumes that it has
 exclusive ownership and may manipulate references in a way that undermines
 Python's much safer ownership semantics. Because of this, sharing mutable
-references or pointers between C++ and Python is not allowed. However, when
-passing a Python protobuf object to C++, the bindings may share the underlying
-C++ native protobuf object with C++ when passed by `const &` or `const *`.
+references or pointers between C++ and Python is not allowed.
+However, when passing a Python protobuf object to
+C++, and with `PYBIND11_PROTOBUF_ASSUME_FULL_ABI_COMPATIBILITY` defined
+(see proto_cast_util.h),
+the bindings will share the underlying C++ native protobuf object with C++ when
+passed by `const &` or `const *`.
 
 ### Protobuf Extensions
 

pybind11_protobuf/proto_cast_util.cc

@@ -575,6 +575,11 @@ void ImportProtoDescriptorModule(const Descriptor* descriptor) {
 }
 
 const Message* PyProtoGetCppMessagePointer(py::handle src) {
+#if !defined(PYBIND11_PROTOBUF_ASSUME_FULL_ABI_COMPATIBILITY)
+  return nullptr;
+#else
+  // Assume that C++ proto objects are compatible as long as there is a C++
+  // message pointer.
   assert(PyGILState_Check());
   if (!GlobalState::instance()->py_proto_api()) return nullptr;
   auto* ptr =
@@ -583,9 +588,9 @@ const Message* PyProtoGetCppMessagePointer(py::handle src) {
     // Clear the type_error set by GetMessagePointer sets a type_error when
     // src was not a wrapped C++ proto message.
     PyErr_Clear();
-    return nullptr;
   }
   return ptr;
+#endif
 }
 
 absl::optional<std::string> PyProtoDescriptorName(py::handle py_proto) {

pybind11_protobuf/proto_cast_util.h

@@ -14,6 +14,16 @@
 #include "google/protobuf/message.h"
 #include "absl/types/optional.h"
 
+// PYBIND11_PROTOBUF_ASSUME_FULL_ABI_COMPATIBILITY can be defined by users
+// certain about ABI compatibility between all Python extensions in their
+// environment using protobufs. If defined, passing protos from Python to C++
+// may skip serialization/deserialization.
+// Assuming full ABI compatibility means (roughly) that the following are
+// compatible between all involved Python extensions:
+// * Protobuf library versions.
+// * Compiler/linker & compiler/linker options.
+// #define PYBIND11_PROTOBUF_ASSUME_FULL_ABI_COMPATIBILITY
+
 namespace pybind11_protobuf {
 
 // Initialize internal proto cast dependencies, which includes importing

pybind11_protobuf/proto_caster_impl.h

@@ -99,10 +99,7 @@ struct proto_caster_load_impl<::google::protobuf::Message> {
     // Attempt to use the PyProto_API to get an underlying C++ message pointer
     // from the object.
     value = pybind11_protobuf::PyProtoGetCppMessagePointer(src);
-    if (value && value->GetDescriptor() && value->GetDescriptor()->file() &&
-        value->GetDescriptor()->file()->pool() ==
-            ::google::protobuf::DescriptorPool::generated_pool()) {
-      // Only messages in the same generated_pool() can be referenced directly.
+    if (value) {
       return true;
     }
 

pybind11_protobuf/tests/BUILD

@@ -1,9 +1,9 @@
 # Tests and examples for the pybind11_protobuf package.
 
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library")
+
 # Placeholder: load py_proto_library
 # Placeholder: load py_library
-
-load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library")
 load("@pybind11_bazel//:build_defs.bzl", "pybind_extension")
 # [internal] load cc_proto_library.bzl
 
@@ -235,6 +235,16 @@ py_test(
     ],
 )
 
+# Externally this is currently only built but not used.
+pybind_extension(
+    name = "very_large_proto_module",
+    srcs = ["very_large_proto_module.cc"],
+    deps = [
+        "//pybind11_protobuf:native_proto_caster",
+        "@com_google_protobuf//:protobuf",
+    ],
+)
+
 # tests for wrapped_proto_caster
 
 pybind_extension(

pybind11_protobuf/tests/very_large_proto_module.cc

@@ -0,0 +1,21 @@
+// Copyright (c) 2023 The Pybind Development Team. All rights reserved.
+//
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include <pybind11/pybind11.h>
+
+#include <cstddef>
+
+#include "google/protobuf/message.h"
+#include "pybind11_protobuf/native_proto_caster.h"
+
+PYBIND11_MODULE(very_large_proto_module, m) {
+  pybind11_protobuf::ImportNativeProtoCasters();
+
+  m.def("get_space_used_estimate",
+        [](const ::google::protobuf::Message& msg) -> std::size_t {
+          const ::google::protobuf::Reflection* refl = msg.GetReflection();
+          return refl != nullptr ? refl->SpaceUsedLong(msg) : 0;
+        });
+}

pybind11_protobuf/tests/very_large_proto_test.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2023 The Pybind Development Team. All rights reserved.
+#
+# All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+from absl.testing import absltest
+
+from pybind11_protobuf.tests import test_pb2
+from pybind11_protobuf.tests import very_large_proto_module as m
+
+
+class MessageTest(absltest.TestCase):
+
+  def test_greater_than_2gb_limit(self):
+    # This test is expected to fail if the Python-to-C++ conversion involves
+    # serialization/deserialization.
+    # Code exercised: IsCProtoSuitableForCopyFromCall()
+    kb = 1024
+    msg_size = 2 * kb**3 + kb  # A little over 2 GB.
+    msg = test_pb2.TestMessage(string_value='x' * msg_size)
+    space_used_estimate = m.get_space_used_estimate(msg)
+    self.assertGreater(space_used_estimate, msg_size)
+
+
+if __name__ == '__main__':
+  absltest.main()