feat: rework of arg/return type hints to support .noconvert() (#5486)

* Added rework of arg/return typing

* Changed `Path` to `pathlib.Path` for compatibility with pybind11-stubgen

* Removed old arg/return type hint implementation

* Added noconvert support for arg/return type hints

* Added commented failing tests for Literals with special characters

* Added return_descr/arg_descr for correct typing in typing::Callable

* Fixed clang-tidy issues

* Changed io_name to have explicit return type (for C++11 support)

* style: pre-commit fixes

* Added support for nested callables

* Fixed missing include

* Fixed is_return_value constructor call

* Fixed clang-tidy issue

* Uncommented test cases for special characters in literals

* Moved literal tests to correct test case

* Added escaping of special characters in typing::Literal

* Readded mistakenly deleted bracket

* Moved sanitize_string_literal to correct namespace

* Added test for Literal with `!` and changed StringLiteral template param name

* Added test for Literal with multiple and repeated special chars

* Simplified string literal sanitization function

* Added test for `->` in literal

* Added test for `->` with io_name

* Removed unused parameter name to prevent warning

* Added escaping of `-` in literal to prevent processing of `->`

* Fixed wrong computation of sanitized string literal length

* Added cast to prevent error with MSVC

* Simplified special character check

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: timohl

docs/advanced/cast/custom.rst

@@ -61,27 +61,25 @@ type is explicitly allowed.
 
     template <>
     struct type_caster<user_space::Point2D> {
-        // This macro inserts a lot of boilerplate code and sets the default type hint to `tuple`
-        PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple"));
-        // `arg_name` and `return_name` may optionally be used to specify type hints separately for
-        // arguments and return values.
+        // This macro inserts a lot of boilerplate code and sets the type hint.
+        // `io_name` is used to specify different type hints for arguments and return values.
         // The signature of our negate function would then look like:
         // `negate(Sequence[float]) -> tuple[float, float]`
-        static constexpr auto arg_name = const_name("Sequence[float]");
-        static constexpr auto return_name = const_name("tuple[float, float]");
+        PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]"));
 
         // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments
         // are used to indicate the return value policy and parent object (for
         // return_value_policy::reference_internal) and are often ignored by custom casters.
-        // The return value should reflect the type hint specified by `return_name`.
+        // The return value should reflect the type hint specified by the second argument of `io_name`.
         static handle
         cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) {
             return py::make_tuple(number.x, number.y).release();
         }
 
         // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The
         // second argument indicates whether implicit conversions should be allowed.
-        // The accepted types should reflect the type hint specified by `arg_name`.
+        // The accepted types should reflect the type hint specified by the first argument of
+        // `io_name`.
         bool load(handle src, bool /*convert*/) {
             // Check if handle is a Sequence
             if (!py::isinstance<py::sequence>(src)) {

include/pybind11/cast.h

@@ -34,39 +34,6 @@ PYBIND11_WARNING_DISABLE_MSVC(4127)
 
 PYBIND11_NAMESPACE_BEGIN(detail)
 
-// Type trait checker for `descr`
-template <typename>
-struct is_descr : std::false_type {};
-
-template <size_t N, typename... Ts>
-struct is_descr<descr<N, Ts...>> : std::true_type {};
-
-template <size_t N, typename... Ts>
-struct is_descr<const descr<N, Ts...>> : std::true_type {};
-
-// Use arg_name instead of name when available
-template <typename T, typename SFINAE = void>
-struct as_arg_type {
-    static constexpr auto name = T::name;
-};
-
-template <typename T>
-struct as_arg_type<T, typename std::enable_if<is_descr<decltype(T::arg_name)>::value>::type> {
-    static constexpr auto name = T::arg_name;
-};
-
-// Use return_name instead of name when available
-template <typename T, typename SFINAE = void>
-struct as_return_type {
-    static constexpr auto name = T::name;
-};
-
-template <typename T>
-struct as_return_type<T,
-                      typename std::enable_if<is_descr<decltype(T::return_name)>::value>::type> {
-    static constexpr auto name = T::return_name;
-};
-
 template <typename type, typename SFINAE = void>
 class type_caster : public type_caster_base<type> {};
 template <typename type>
@@ -1113,8 +1080,6 @@ struct pyobject_caster {
         return src.inc_ref();
     }
     PYBIND11_TYPE_CASTER(type, handle_type_name<type>::name);
-    static constexpr auto arg_name = as_arg_type<handle_type_name<type>>::name;
-    static constexpr auto return_name = as_return_type<handle_type_name<type>>::name;
 };
 
 template <typename T>
@@ -1668,7 +1633,7 @@ class argument_loader {
                   "py::args cannot be specified more than once");
 
     static constexpr auto arg_names
-        = ::pybind11::detail::concat(type_descr(as_arg_type<make_caster<Args>>::name)...);
+        = ::pybind11::detail::concat(type_descr(make_caster<Args>::name)...);
 
     bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); }
 

include/pybind11/detail/descr.h

@@ -99,6 +99,13 @@ constexpr descr<1, Type> const_name() {
     return {'%'};
 }
 
+// Use a different name based on whether the parameter is used as input or output
+template <size_t N1, size_t N2>
+constexpr descr<N1 + N2 + 1> io_name(char const (&text1)[N1], char const (&text2)[N2]) {
+    return const_name("@") + const_name(text1) + const_name("@") + const_name(text2)
+           + const_name("@");
+}
+
 // If "_" is defined as a macro, py::detail::_ cannot be provided.
 // It is therefore best to use py::detail::const_name universally.
 // This block is for backward compatibility only.
@@ -167,5 +174,15 @@ constexpr descr<N + 2, Ts...> type_descr(const descr<N, Ts...> &descr) {
     return const_name("{") + descr + const_name("}");
 }
 
+template <size_t N, typename... Ts>
+constexpr descr<N + 4, Ts...> arg_descr(const descr<N, Ts...> &descr) {
+    return const_name("@^") + descr + const_name("@!");
+}
+
+template <size_t N, typename... Ts>
+constexpr descr<N + 4, Ts...> return_descr(const descr<N, Ts...> &descr) {
+    return const_name("@$") + descr + const_name("@!");
+}
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/pybind11.h

@@ -22,6 +22,7 @@
 #include <cstring>
 #include <memory>
 #include <new>
+#include <stack>
 #include <string>
 #include <utility>
 #include <vector>
@@ -336,8 +337,8 @@ class cpp_function : public function {
 
         /* Generate a readable signature describing the function's arguments and return
            value types */
-        static constexpr auto signature = const_name("(") + cast_in::arg_names
-                                          + const_name(") -> ") + as_return_type<cast_out>::name;
+        static constexpr auto signature
+            = const_name("(") + cast_in::arg_names + const_name(") -> ") + cast_out::name;
         PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types();
 
         /* Register the function with Python from generic (non-templated) code */
@@ -440,6 +441,13 @@ class cpp_function : public function {
         std::string signature;
         size_t type_index = 0, arg_index = 0;
         bool is_starred = false;
+        // `is_return_value.top()` is true if we are currently inside the return type of the
+        // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops
+        // back to the previous state.
+        std::stack<bool> is_return_value({false});
+        // The following characters have special meaning in the signature parsing. Literals
+        // containing these are escaped with `!`.
+        std::string special_chars("!@%{}-");
         for (const auto *pc = text; *pc != '\0'; ++pc) {
             const auto c = *pc;
 
@@ -493,7 +501,57 @@ class cpp_function : public function {
                 } else {
                     signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name()));
                 }
+            } else if (c == '!' && special_chars.find(*(pc + 1)) != std::string::npos) {
+                // typing::Literal escapes special characters with !
+                signature += *++pc;
+            } else if (c == '@') {
+                // `@^ ... @!` and `@$ ... @!` are used to force arg/return value type (see
+                // typing::Callable/detail::arg_descr/detail::return_descr)
+                if (*(pc + 1) == '^') {
+                    is_return_value.emplace(false);
+                    ++pc;
+                    continue;
+                }
+                if (*(pc + 1) == '$') {
+                    is_return_value.emplace(true);
+                    ++pc;
+                    continue;
+                }
+                if (*(pc + 1) == '!') {
+                    is_return_value.pop();
+                    ++pc;
+                    continue;
+                }
+                // Handle types that differ depending on whether they appear
+                // in an argument or a return value position (see io_name<text1, text2>).
+                // For named arguments (py::arg()) with noconvert set, return value type is used.
+                ++pc;
+                if (!is_return_value.top()
+                    && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) {
+                    while (*pc != '\0' && *pc != '@') {
+                        signature += *pc++;
+                    }
+                    if (*pc == '@') {
+                        ++pc;
+                    }
+                    while (*pc != '\0' && *pc != '@') {
+                        ++pc;
+                    }
+                } else {
+                    while (*pc != '\0' && *pc != '@') {
+                        ++pc;
+                    }
+                    if (*pc == '@') {
+                        ++pc;
+                    }
+                    while (*pc != '\0' && *pc != '@') {
+                        signature += *pc++;
+                    }
+                }
             } else {
+                if (c == '-' && *(pc + 1) == '>') {
+                    is_return_value.emplace(true);
+                }
                 signature += c;
             }
         }

include/pybind11/stl/filesystem.h

@@ -106,9 +106,7 @@ struct path_caster {
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(T, const_name("os.PathLike"));
-    static constexpr auto arg_name = const_name("Union[os.PathLike, str, bytes]");
-    static constexpr auto return_name = const_name("Path");
+    PYBIND11_TYPE_CASTER(T, io_name("Union[os.PathLike, str, bytes]", "pathlib.Path"));
 };
 
 #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM)