Streamline uninspectable testing, just skipping awkward cases

Author: honno

array_api_tests/test_signatures.py

@@ -18,15 +18,13 @@ def squeeze(x, /, axis):
     ...
 
 """
-from collections import defaultdict
 from copy import copy
 from inspect import Parameter, Signature, signature
 from itertools import chain
 from types import FunctionType
-from typing import Any, Callable, DefaultDict, Dict, List, Literal, Sequence, get_args
+from typing import Any, Callable, Dict, List, Literal, Sequence, get_args
 
 import pytest
-from hypothesis import given, note
 from hypothesis import strategies as st
 
 from . import dtype_helpers as dh
@@ -35,7 +33,7 @@ def squeeze(x, /, axis):
 from ._array_module import _UndefinedStub
 from ._array_module import mod as xp
 from .stubs import array_methods, category_to_funcs, extension_to_funcs
-from .typing import DataType, Shape
+from .typing import DataType
 
 pytestmark = pytest.mark.ci
 
@@ -53,7 +51,7 @@ def squeeze(x, /, axis):
     Parameter.POSITIONAL_ONLY: "pos-only argument",
     Parameter.KEYWORD_ONLY: "keyword-only argument",
     Parameter.VAR_POSITIONAL: "star-args (i.e. *args) argument",
-    Parameter.VAR_KEYWORD: "star-kwargs (i.e. **kwargs) argument",
+    Parameter.VAR_KEYWORD: "star-kwonly (i.e. **kwonly) argument",
 }
 
 
@@ -63,14 +61,13 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature):
     # We're not interested if the array module has additional arguments, so we
     # only iterate through the arguments listed in the spec.
     for i, stub_param in enumerate(stub_params):
-        if sig is not None:
-            assert (
-                len(params) >= i + 1
-            ), f"Argument '{stub_param.name}' missing from signature"
-            param = params[i]
+        assert (
+            len(params) >= i + 1
+        ), f"Argument '{stub_param.name}' missing from signature"
+        param = params[i]
 
         # We're not interested in the name if it isn't actually used
-        if sig is not None and stub_param.kind not in [
+        if stub_param.kind not in [
             Parameter.POSITIONAL_ONLY,
             *VAR_KINDS,
         ]:
@@ -80,50 +77,17 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature):
 
         f_stub_kind = kind_to_str[stub_param.kind]
         if stub_param.kind in [Parameter.POSITIONAL_OR_KEYWORD, *VAR_KINDS]:
-            if sig is not None:
-                assert param.kind == stub_param.kind, (
-                    f"{param.name} is a {kind_to_str[param.kind]}, "
-                    f"but should be a {f_stub_kind}"
-                )
-            else:
-                pass
+            assert param.kind == stub_param.kind, (
+                f"{param.name} is a {kind_to_str[param.kind]}, "
+                f"but should be a {f_stub_kind}"
+            )
         else:
             # TODO: allow for kw-only args to be out-of-order
-            if sig is not None:
-                assert param.kind in [
-                    stub_param.kind,
-                    Parameter.POSITIONAL_OR_KEYWORD,
-                ], (
-                    f"{param.name} is a {kind_to_str[param.kind]}, "
-                    f"but should be a {f_stub_kind} "
-                    f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})"
-                )
-            else:
-                pass
-
-
-def shapes(**kw) -> st.SearchStrategy[Shape]:
-    if "min_side" not in kw.keys():
-        kw["min_side"] = 1
-    return hh.shapes(**kw)
-
-
-matrixy_funcs: List[str] = [
-    f.__name__
-    for f in chain(category_to_funcs["linear_algebra"], extension_to_funcs["linalg"])
-]
-matrixy_funcs += ["__matmul__", "triu", "tril"]
-func_to_shapes: DefaultDict[str, st.SearchStrategy[Shape]] = defaultdict(
-    shapes,
-    {
-        **{k: st.just(()) for k in ["__bool__", "__int__", "__index__", "__float__"]},
-        "sort": shapes(min_dims=1),  # for axis=-1,
-        **{k: shapes(min_dims=2) for k in matrixy_funcs},
-        # Overwrite min_dims=2 shapes for some matrixy functions
-        "cross": shapes(min_side=3, max_side=3, min_dims=3, max_dims=3),
-        "outer": shapes(min_dims=1, max_dims=1),
-    },
-)
+            assert param.kind in [stub_param.kind, Parameter.POSITIONAL_OR_KEYWORD,], (
+                f"{param.name} is a {kind_to_str[param.kind]}, "
+                f"but should be a {f_stub_kind} "
+                f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})"
+            )
 
 
 def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]:
@@ -136,97 +100,93 @@ def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]:
         return xps.scalar_dtypes()
 
 
-func_to_example_values: Dict[str, Dict[ParameterKind, Dict[str, Any]]] = {
-    "broadcast_to": {
-        Parameter.POSITIONAL_ONLY: {"x": xp.asarray([0, 1])},
-        Parameter.POSITIONAL_OR_KEYWORD: {"shape": (1, 2)},
-    },
-    "cholesky": {
-        Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])}
-    },
-    "inv": {Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])}},
-}
-
-
-def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any]):
+def make_pretty_func(func_name: str, args: Sequence[Any], kwonly: Dict[str, Any]):
     f_sig = f"{func_name}("
     f_sig += ", ".join(str(a) for a in args)
-    if len(kwargs) != 0:
+    if len(kwonly) != 0:
         if len(args) != 0:
             f_sig += ", "
-        f_sig += ", ".join(f"{k}={v}" for k, v in kwargs.items())
+        f_sig += ", ".join(f"{k}={v}" for k, v in kwonly.items())
     f_sig += ")"
     return f_sig
 
 
-@given(data=st.data())
-def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature, data):
-    example_values: Dict[ParameterKind, Dict[str, Any]] = func_to_example_values.get(
-        func_name, {}
-    )
-    for kind in ALL_KINDS:
-        example_values.setdefault(kind, {})
+matrixy_funcs: List[str] = [
+    f.__name__
+    for f in chain(category_to_funcs["linear_algebra"], extension_to_funcs["linalg"])
+]
+matrixy_funcs += ["__matmul__", "triu", "tril"]
 
-    for param in stub_sig.parameters.values():
-        for name_to_value in example_values.values():
-            if param.name in name_to_value.keys():
-                continue
 
-        if param.default != Parameter.empty:
-            example_value = param.default
+def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature):
+    skip_msg = (
+        f"Signature for {func_name}() is not inspectable "
+        "and is too troublesome to test for otherwise"
+    )
+    if func_name in [
+        "__bool__",
+        "__int__",
+        "__index__",
+        "__float__",
+        "pow",
+        "bitwise_left_shift",
+        "bitwise_right_shift",
+        "broadcast_to",
+        "permute_dims",
+        "sort",
+        *matrixy_funcs,
+    ]:
+        pytest.skip(skip_msg)
+
+    param_to_value: Dict[Parameter, Any] = {}
+    for param in stub_sig.parameters.values():
+        if param.kind in VAR_KINDS:
+            pytest.skip(skip_msg)
+        elif param.default != Parameter.empty:
+            value = param.default
         elif param.name in ["x", "x1"]:
             dtypes = get_dtypes_strategy(func_name)
-            shapes = func_to_shapes[func_name]
-            example_value = data.draw(
-                xps.arrays(dtype=dtypes, shape=shapes), label=param.name
-            )
+            value = xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)).example()
         elif param.name == "x2":
             # sanity check
-            assert "x1" in example_values[Parameter.POSITIONAL_ONLY].keys()
-            x1 = example_values[Parameter.POSITIONAL_ONLY]["x1"]
-            example_value = data.draw(
-                xps.arrays(dtype=x1.dtype, shape=x1.shape), label="x2"
-            )
-        elif param.name == "axes":
-            example_value = ()
-        elif param.name == "shape":
-            example_value = ()
+            assert "x1" in [p.name for p in param_to_value.keys()]
+            x1 = next(v for p, v in param_to_value.items() if p.name == "x1")
+            value = xps.arrays(dtype=x1.dtype, shape=x1.shape).example()
         else:
-            pytest.skip(f"No example value for argument '{param.name}'")
-
-        if param.kind in VAR_KINDS:
-            pytest.skip("TODO")
-        example_values[param.kind][param.name] = example_value
-
-    if len(example_values[Parameter.POSITIONAL_OR_KEYWORD]) == 0:
-        f_func = make_pretty_func(
-            func_name,
-            example_values[Parameter.POSITIONAL_ONLY].values(),
-            example_values[Parameter.KEYWORD_ONLY],
-        )
-        note(f"trying {f_func}")
-        func(
-            *example_values[Parameter.POSITIONAL_ONLY].values(),
-            **example_values[Parameter.KEYWORD_ONLY],
-        )
+            pytest.skip(skip_msg)
+        param_to_value[param] = value
+
+    posonly: List[Any] = [
+        v for p, v in param_to_value.items() if p.kind == Parameter.POSITIONAL_ONLY
+    ]
+    kwonly: Dict[str, Any] = {
+        p.name: v for p, v in param_to_value.items() if p.kind == Parameter.KEYWORD_ONLY
+    }
+    if (
+        sum(p.kind == Parameter.POSITIONAL_OR_KEYWORD for p in param_to_value.keys())
+        == 0
+    ):
+        f_func = make_pretty_func(func_name, posonly, kwonly)
+        print(f"trying {f_func}")
+        func(*posonly, **kwonly)
     else:
         either_argname_value_pairs = list(
-            example_values[Parameter.POSITIONAL_OR_KEYWORD].items()
+            (p.name, v)
+            for p, v in param_to_value.items()
+            if p.kind == Parameter.POSITIONAL_OR_KEYWORD
         )
         n_either_args = len(either_argname_value_pairs)
         for n_extra_args in reversed(range(n_either_args + 1)):
-            extra_args = [v for _, v in either_argname_value_pairs[:n_extra_args]]
+            extra_posargs = [v for _, v in either_argname_value_pairs[:n_extra_args]]
             if n_extra_args < n_either_args:
                 extra_kwargs = dict(either_argname_value_pairs[n_extra_args:])
             else:
                 extra_kwargs = {}
-            args = list(example_values[Parameter.POSITIONAL_ONLY].values())
-            args += extra_args
-            kwargs = copy(example_values[Parameter.KEYWORD_ONLY])
-            if len(extra_kwargs) != 0:
-                kwargs.update(extra_kwargs)
+            args = copy(posonly)
+            args += extra_posargs
+            kwargs = {**kwonly, **extra_kwargs}
             f_func = make_pretty_func(func_name, args, kwargs)
-            note(f"trying {f_func}")
+            print(f"trying {f_func}")
             func(*args, **kwargs)
 
 
@@ -279,11 +239,9 @@ def test_extension_func_signature(extension: str, stub: FunctionType):
 
 
 @pytest.mark.parametrize("stub", array_methods, ids=lambda f: f.__name__)
-@given(data=st.data())
-def test_array_method_signature(stub: FunctionType, data):
+def test_array_method_signature(stub: FunctionType):
     dtypes = get_dtypes_strategy(stub.__name__)
-    shapes = func_to_shapes[stub.__name__]
-    x = data.draw(xps.arrays(dtype=dtypes, shape=shapes), label="x")
+    x = xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)).example()
     assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}"
     method = getattr(x, stub.__name__)
     # Ignore 'self' arg in stub, which won't be present in instantiated objects.