Support PEP 646 syntax for Callable (#15951)

Fixes #15412

Two new things here as specified by PEP 646:
* Using star for an (explicit) type unpaking in callables, like
`Callable[[str, *tuple[int, ...]], None]`
* Allowing suffix items after a variadic item, like `Callable[[X,
Unpack[Ys], Z], bool]`

Implementation is straightforward. Btw while working in this I
accidentally fixed a nasty bug, tuple types were often not given any
line/column numbers, so if such type becomes a location of an error, it
is impossible to ignore.

Author: ilevkivskyi

mypy/exprtotype.py

@@ -17,6 +17,7 @@
     NameExpr,
     OpExpr,
     RefExpr,
+    StarExpr,
     StrExpr,
     TupleExpr,
     UnaryExpr,
@@ -35,6 +36,7 @@
     TypeOfAny,
     UnboundType,
     UnionType,
+    UnpackType,
 )
 
 
@@ -56,6 +58,7 @@ def expr_to_unanalyzed_type(
     options: Options | None = None,
     allow_new_syntax: bool = False,
     _parent: Expression | None = None,
+    allow_unpack: bool = False,
 ) -> ProperType:
     """Translate an expression to the corresponding type.
 
@@ -163,7 +166,10 @@ def expr_to_unanalyzed_type(
         return CallableArgument(typ, name, arg_const, expr.line, expr.column)
     elif isinstance(expr, ListExpr):
         return TypeList(
-            [expr_to_unanalyzed_type(t, options, allow_new_syntax, expr) for t in expr.items],
+            [
+                expr_to_unanalyzed_type(t, options, allow_new_syntax, expr, allow_unpack=True)
+                for t in expr.items
+            ],
             line=expr.line,
             column=expr.column,
         )
@@ -189,5 +195,7 @@ def expr_to_unanalyzed_type(
         return RawExpressionType(None, "builtins.complex", line=expr.line, column=expr.column)
     elif isinstance(expr, EllipsisExpr):
         return EllipsisType(expr.line)
+    elif allow_unpack and isinstance(expr, StarExpr):
+        return UnpackType(expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax))
     else:
         raise TypeTranslationError()

mypy/fastparse.py

@@ -115,6 +115,7 @@
     TypeOfAny,
     UnboundType,
     UnionType,
+    UnpackType,
 )
 from mypy.util import bytes_to_human_readable_repr, unnamed_function
 
@@ -1730,6 +1731,7 @@ def __init__(
         self.override_column = override_column
         self.node_stack: list[AST] = []
         self.is_evaluated = is_evaluated
+        self.allow_unpack = False
 
     def convert_column(self, column: int) -> int:
         """Apply column override if defined; otherwise return column.
@@ -2006,10 +2008,20 @@ def visit_Attribute(self, n: Attribute) -> Type:
         else:
             return self.invalid_type(n)
 
+    # Used for Callable[[X *Ys, Z], R]
+    def visit_Starred(self, n: ast3.Starred) -> Type:
+        return UnpackType(self.visit(n.value))
+
     # List(expr* elts, expr_context ctx)
     def visit_List(self, n: ast3.List) -> Type:
         assert isinstance(n.ctx, ast3.Load)
-        return self.translate_argument_list(n.elts)
+        old_allow_unpack = self.allow_unpack
+        # We specifically only allow starred expressions in a list to avoid
+        # confusing errors for top-level unpacks (e.g. in base classes).
+        self.allow_unpack = True
+        result = self.translate_argument_list(n.elts)
+        self.allow_unpack = old_allow_unpack
+        return result
 
 
 def stringify_name(n: AST) -> str | None:

mypy/typeanal.py

@@ -568,7 +568,9 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ
                 instance = self.named_type("builtins.tuple", [self.anal_type(t.args[0])])
                 instance.line = t.line
                 return instance
-            return self.tuple_type(self.anal_array(t.args, allow_unpack=True))
+            return self.tuple_type(
+                self.anal_array(t.args, allow_unpack=True), line=t.line, column=t.column
+            )
         elif fullname == "typing.Union":
             items = self.anal_array(t.args)
             return UnionType.make_union(items)
@@ -968,7 +970,10 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type:
         return t
 
     def visit_unpack_type(self, t: UnpackType) -> Type:
-        raise NotImplementedError
+        if not self.allow_unpack:
+            self.fail(message_registry.INVALID_UNPACK_POSITION, t.type, code=codes.VALID_TYPE)
+            return AnyType(TypeOfAny.from_error)
+        return UnpackType(self.anal_type(t.type))
 
     def visit_parameters(self, t: Parameters) -> Type:
         raise NotImplementedError("ParamSpec literals cannot have unbound TypeVars")
@@ -1364,12 +1369,22 @@ def analyze_callable_type(self, t: UnboundType) -> Type:
         assert isinstance(ret, CallableType)
         return ret.accept(self)
 
+    def refers_to_full_names(self, arg: UnboundType, names: Sequence[str]) -> bool:
+        sym = self.lookup_qualified(arg.name, arg)
+        if sym is not None:
+            if sym.fullname in names:
+                return True
+        return False
+
     def analyze_callable_args(
         self, arglist: TypeList
     ) -> tuple[list[Type], list[ArgKind], list[str | None]] | None:
         args: list[Type] = []
         kinds: list[ArgKind] = []
         names: list[str | None] = []
+        seen_unpack = False
+        unpack_types: list[Type] = []
+        invalid_unpacks = []
         for arg in arglist.items:
             if isinstance(arg, CallableArgument):
                 args.append(arg.typ)
@@ -1390,20 +1405,42 @@ def analyze_callable_args(
                     if arg.name is not None and kind.is_star():
                         self.fail(f"{arg.constructor} arguments should not have names", arg)
                         return None
-            elif isinstance(arg, UnboundType):
-                kind = ARG_POS
-                # Potentially a unpack.
-                sym = self.lookup_qualified(arg.name, arg)
-                if sym is not None:
-                    if sym.fullname in ("typing_extensions.Unpack", "typing.Unpack"):
-                        kind = ARG_STAR
-                args.append(arg)
-                kinds.append(kind)
-                names.append(None)
+            elif (
+                isinstance(arg, UnboundType)
+                and self.refers_to_full_names(arg, ("typing_extensions.Unpack", "typing.Unpack"))
+                or isinstance(arg, UnpackType)
+            ):
+                if seen_unpack:
+                    # Multiple unpacks, preserve them, so we can give an error later.
+                    invalid_unpacks.append(arg)
+                    continue
+                seen_unpack = True
+                unpack_types.append(arg)
+            else:
+                if seen_unpack:
+                    unpack_types.append(arg)
+                else:
+                    args.append(arg)
+                    kinds.append(ARG_POS)
+                    names.append(None)
+        if seen_unpack:
+            if len(unpack_types) == 1:
+                args.append(unpack_types[0])
             else:
-                args.append(arg)
-                kinds.append(ARG_POS)
-                names.append(None)
+                first = unpack_types[0]
+                if isinstance(first, UnpackType):
+                    # UnpackType doesn't have its own line/column numbers,
+                    # so use the unpacked type for error messages.
+                    first = first.type
+                args.append(
+                    UnpackType(self.tuple_type(unpack_types, line=first.line, column=first.column))
+                )
+            kinds.append(ARG_STAR)
+            names.append(None)
+        for arg in invalid_unpacks:
+            args.append(arg)
+            kinds.append(ARG_STAR)
+            names.append(None)
         # Note that arglist below is only used for error context.
         check_arg_names(names, [arglist] * len(args), self.fail, "Callable")
         check_arg_kinds(kinds, [arglist] * len(args), self.fail)
@@ -1713,9 +1750,11 @@ def check_unpacks_in_list(self, items: list[Type]) -> list[Type]:
             self.fail("More than one Unpack in a type is not allowed", final_unpack)
         return new_items
 
-    def tuple_type(self, items: list[Type]) -> TupleType:
+    def tuple_type(self, items: list[Type], line: int, column: int) -> TupleType:
         any_type = AnyType(TypeOfAny.special_form)
-        return TupleType(items, fallback=self.named_type("builtins.tuple", [any_type]))
+        return TupleType(
+            items, fallback=self.named_type("builtins.tuple", [any_type]), line=line, column=column
+        )
 
 
 TypeVarLikeList = List[Tuple[str, TypeVarLikeExpr]]

test-data/unit/check-typevar-tuple.test

@@ -509,6 +509,51 @@ call_prefix(target=func_prefix, args=(0, 'foo'))
 call_prefix(target=func2_prefix, args=(0, 'foo'))  # E: Argument "target" to "call_prefix" has incompatible type "Callable[[str, int, str], None]"; expected "Callable[[bytes, int, str], None]"
 [builtins fixtures/tuple.pyi]
 
+[case testTypeVarTuplePep646CallableSuffixSyntax]
+from typing import Callable, Tuple, TypeVar
+from typing_extensions import Unpack, TypeVarTuple
+
+x: Callable[[str, Unpack[Tuple[int, ...]], bool], None]
+reveal_type(x)  # N: Revealed type is "def (builtins.str, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.bool]])"
+
+T = TypeVar("T")
+S = TypeVar("S")
+Ts = TypeVarTuple("Ts")
+A = Callable[[T, Unpack[Ts], S], int]
+y: A[int, str, bool]
+reveal_type(y)  # N: Revealed type is "def (builtins.int, builtins.str, builtins.bool) -> builtins.int"
+z: A[Unpack[Tuple[int, ...]]]
+reveal_type(z)  # N: Revealed type is "def (builtins.int, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.int]]) -> builtins.int"
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarTuplePep646CallableInvalidSyntax]
+from typing import Callable, Tuple, TypeVar
+from typing_extensions import Unpack, TypeVarTuple
+
+Ts = TypeVarTuple("Ts")
+Us = TypeVarTuple("Us")
+a: Callable[[Unpack[Ts], Unpack[Us]], int]  # E: Var args may not appear after named or var args \
+                                            # E: More than one Unpack in a type is not allowed
+reveal_type(a)  # N: Revealed type is "def [Ts, Us] (*Unpack[Ts`-1]) -> builtins.int"
+b: Callable[[Unpack], int]  # E: Unpack[...] requires exactly one type argument
+reveal_type(b)  # N: Revealed type is "def (*Any) -> builtins.int"
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarTuplePep646CallableNewSyntax]
+from typing import Callable, Generic, Tuple
+from typing_extensions import ParamSpec
+
+x: Callable[[str, *Tuple[int, ...]], None]
+reveal_type(x)  # N: Revealed type is "def (builtins.str, *builtins.int)"
+y: Callable[[str, *Tuple[int, ...], bool], None]
+reveal_type(y)  # N: Revealed type is "def (builtins.str, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.bool]])"
+
+P = ParamSpec("P")
+class C(Generic[P]): ...
+bad: C[[int, *Tuple[int, ...], int]]  # E: Unpack is only valid in a variadic position
+reveal_type(bad)  # N: Revealed type is "__main__.C[[builtins.int, *Any]]"
+[builtins fixtures/tuple.pyi]
+
 [case testTypeVarTuplePep646UnspecifiedParameters]
 from typing import Tuple, Generic, TypeVar
 from typing_extensions import Unpack, TypeVarTuple
@@ -635,19 +680,6 @@ x: A[str, str]
 reveal_type(x)  # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str, builtins.str]"
 [builtins fixtures/tuple.pyi]
 
-[case testVariadicAliasWrongCallable]
-from typing import TypeVar, Callable
-from typing_extensions import Unpack, TypeVarTuple
-
-T = TypeVar("T")
-S = TypeVar("S")
-Ts = TypeVarTuple("Ts")
-
-A = Callable[[T, Unpack[Ts], S], int]  # E: Required positional args may not appear after default, named or var args
-x: A[int, str, int, str]
-reveal_type(x)  # N: Revealed type is "def (builtins.int, builtins.str, builtins.int, builtins.str) -> builtins.int"
-[builtins fixtures/tuple.pyi]
-
 [case testVariadicAliasMultipleUnpacks]
 from typing import Tuple, Generic, Callable
 from typing_extensions import Unpack, TypeVarTuple