Test different arg/kwarg arrangements for uninspectable normal args

Author: honno

array_api_tests/test_signatures.py

@@ -19,13 +19,14 @@ def squeeze(x, /, axis):
 
 """
 from collections import defaultdict
+from copy import copy
 from inspect import Parameter, Signature, signature
 from itertools import chain
 from types import FunctionType
-from typing import Callable, DefaultDict, Dict, List
+from typing import Any, Callable, DefaultDict, Dict, List, Literal, Sequence, get_args
 
 import pytest
-from hypothesis import given
+from hypothesis import given, note
 from hypothesis import strategies as st
 
 from . import dtype_helpers as dh
@@ -38,17 +39,23 @@ def squeeze(x, /, axis):
 
 pytestmark = pytest.mark.ci
 
-
-kind_to_str: Dict[Parameter, str] = {
+ParameterKind = Literal[
+    Parameter.POSITIONAL_ONLY,
+    Parameter.VAR_POSITIONAL,
+    Parameter.POSITIONAL_OR_KEYWORD,
+    Parameter.KEYWORD_ONLY,
+    Parameter.VAR_KEYWORD,
+]
+ALL_KINDS = get_args(ParameterKind)
+VAR_KINDS = (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD)
+kind_to_str: Dict[ParameterKind, str] = {
     Parameter.POSITIONAL_OR_KEYWORD: "normal argument",
     Parameter.POSITIONAL_ONLY: "pos-only argument",
     Parameter.KEYWORD_ONLY: "keyword-only argument",
     Parameter.VAR_POSITIONAL: "star-args (i.e. *args) argument",
     Parameter.VAR_KEYWORD: "star-kwargs (i.e. **kwargs) argument",
 }
 
-VAR_KINDS = (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD)
-
 
 def _test_inspectable_func(sig: Signature, stub_sig: Signature):
     params = list(sig.parameters.values())
@@ -89,11 +96,12 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature):
                 ], (
                     f"{param.name} is a {kind_to_str[param.kind]}, "
                     f"but should be a {f_stub_kind} "
-                    f"(or at least a {kind_to_str[Parameter.POSITIONAL_OR_KEYWORD]})"
+                    f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})"
                 )
             else:
                 pass
 
+
 def shapes(**kw) -> st.SearchStrategy[Shape]:
     if "min_side" not in kw.keys():
         kw["min_side"] = 1
@@ -111,7 +119,7 @@ def shapes(**kw) -> st.SearchStrategy[Shape]:
         **{k: st.just(()) for k in ["__bool__", "__int__", "__index__", "__float__"]},
         "sort": shapes(min_dims=1),  # for axis=-1,
         **{k: shapes(min_dims=2) for k in matrixy_funcs},
-        # Override for some matrixy functions
+        # Overwrite min_dims=2 shapes for some matrixy functions
         "cross": shapes(min_side=3, max_side=3, min_dims=3, max_dims=3),
         "outer": shapes(min_dims=1, max_dims=1),
     },
@@ -128,50 +136,98 @@ def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]:
         return xps.scalar_dtypes()
 
 
+func_to_example_values: Dict[str, Dict[ParameterKind, Dict[str, Any]]] = {
+    "broadcast_to": {
+        Parameter.POSITIONAL_ONLY: {"x": xp.asarray([0, 1])},
+        Parameter.POSITIONAL_OR_KEYWORD: {"shape": (1, 2)},
+    },
+    "cholesky": {
+        Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])}
+    },
+    "inv": {Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])}},
+}
+
+
+def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any]):
+    f_sig = f"{func_name}("
+    f_sig += ", ".join(str(a) for a in args)
+    if len(kwargs) != 0:
+        if len(args) != 0:
+            f_sig += ", "
+        f_sig += ", ".join(f"{k}={v}" for k, v in kwargs.items())
+    f_sig += ")"
+    return f_sig
+
+
 @given(data=st.data())
 def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature, data):
-    if func_name in ["cholesky", "inv"]:
-        func(xp.asarray([[1.0, 0.0], [0.0, 1.0]]))
-        return
-    elif func_name == "solve":
-        func(xp.asarray([[1.0, 2.0], [3.0, 5.0]]), xp.asarray([1.0, 2.0]))
-        return
-
-    pos_argname_to_example_value = {}
-    normal_argname_to_example_value = {}
-    kw_argname_to_example_value = {}
-    for stub_param in stub_sig.parameters.values():
-        if stub_param.name in ["x", "x1"]:
+    example_values: Dict[ParameterKind, Dict[str, Any]] = func_to_example_values.get(
+        func_name, {}
+    )
+    for kind in ALL_KINDS:
+        example_values.setdefault(kind, {})
+
+    for param in stub_sig.parameters.values():
+        for name_to_value in example_values.values():
+            if param.name in name_to_value.keys():
+                continue
+
+        if param.default != Parameter.empty:
+            example_value = param.default
+        elif param.name in ["x", "x1"]:
             dtypes = get_dtypes_strategy(func_name)
             shapes = func_to_shapes[func_name]
             example_value = data.draw(
-                xps.arrays(dtype=dtypes, shape=shapes), label=stub_param.name
+                xps.arrays(dtype=dtypes, shape=shapes), label=param.name
             )
-        elif stub_param.name == "x2":
-            assert "x1" in pos_argname_to_example_value.keys()  # sanity check
-            x1 = pos_argname_to_example_value["x1"]
+        elif param.name == "x2":
+            # sanity check
+            assert "x1" in example_values[Parameter.POSITIONAL_ONLY].keys()
+            x1 = example_values[Parameter.POSITIONAL_ONLY]["x1"]
             example_value = data.draw(
                 xps.arrays(dtype=x1.dtype, shape=x1.shape), label="x2"
             )
+        elif param.name == "axes":
+            example_value = ()
+        elif param.name == "shape":
+            example_value = ()
         else:
-            if stub_param.default != Parameter.empty:
-                example_value = stub_param.default
-            else:
-                pytest.skip(f"No example value for argument '{stub_param.name}'")
-
-        if stub_param.kind == Parameter.POSITIONAL_ONLY:
-            pos_argname_to_example_value[stub_param.name] = example_value
-        elif stub_param.kind == Parameter.POSITIONAL_OR_KEYWORD:
-            normal_argname_to_example_value[stub_param.name] = example_value
-        elif stub_param.kind == Parameter.KEYWORD_ONLY:
-            kw_argname_to_example_value[stub_param.name] = example_value
-        else:
-            pytest.skip()
+            pytest.skip(f"No example value for argument '{param.name}'")
 
-    if len(normal_argname_to_example_value) == 0:
-        func(*pos_argname_to_example_value.values(), **kw_argname_to_example_value)
+        if param.kind in VAR_KINDS:
+            pytest.skip("TODO")
+        example_values[param.kind][param.name] = example_value
+
+    if len(example_values[Parameter.POSITIONAL_OR_KEYWORD]) == 0:
+        f_func = make_pretty_func(
+            func_name,
+            example_values[Parameter.POSITIONAL_ONLY].values(),
+            example_values[Parameter.KEYWORD_ONLY],
+        )
+        note(f"trying {f_func}")
+        func(
+            *example_values[Parameter.POSITIONAL_ONLY].values(),
+            **example_values[Parameter.KEYWORD_ONLY],
+        )
     else:
-        pass  # TODO
+        either_argname_value_pairs = list(
+            example_values[Parameter.POSITIONAL_OR_KEYWORD].items()
+        )
+        n_either_args = len(either_argname_value_pairs)
+        for n_extra_args in reversed(range(n_either_args + 1)):
+            extra_args = [v for _, v in either_argname_value_pairs[:n_extra_args]]
+            if n_extra_args < n_either_args:
+                extra_kwargs = dict(either_argname_value_pairs[n_extra_args:])
+            else:
+                extra_kwargs = {}
+            args = list(example_values[Parameter.POSITIONAL_ONLY].values())
+            args += extra_args
+            kwargs = copy(example_values[Parameter.KEYWORD_ONLY])
+            if len(extra_kwargs) != 0:
+                kwargs.update(extra_kwargs)
+            f_func = make_pretty_func(func_name, args, kwargs)
+            note(f"trying {f_func}")
+            func(*args, **kwargs)
 
 
 def _test_func_signature(