Move ParamSpec validity checks to typeanal entirely

python · hauntsaninja · Jan 2, 2025 · Dec 11, 2024 · Dec 11, 2024 · Dec 11, 2024
commit 6ef6d89458cc0433b7c8b4ae34c13a3ea0c5a202
diff --git a/mypy/semanal.py b/mypy/semanal.py
@@ -71,7 +71,6 @@
 from mypy.nodes import (
     ARG_NAMED,
     ARG_POS,
-    ARG_STAR,
     ARG_STAR2,
     CONTRAVARIANT,
     COVARIANT,
@@ -980,7 +979,6 @@ def analyze_func_def(self, defn: FuncDef) -> None:
                 defn.type = result
                 self.add_type_alias_deps(analyzer.aliases_used)
                 self.check_function_signature(defn)
-                self.check_paramspec_definition(defn)
                 if isinstance(defn, FuncDef):
                     assert isinstance(defn.type, CallableType)
                     defn.type = set_callable_name(defn.type, defn)
@@ -1609,64 +1607,6 @@ def check_function_signature(self, fdef: FuncItem) -> None:
         elif len(sig.arg_types) > len(fdef.arguments):
             self.fail("Type signature has too many arguments", fdef, blocker=True)
 
-    def check_paramspec_definition(self, defn: FuncDef) -> None:
-        func = defn.type
-        assert isinstance(func, CallableType)
-
-        if not any(isinstance(var, ParamSpecType) for var in func.variables):
-            return  # Function does not have param spec variables
-
-        args = func.var_arg()
-        kwargs = func.kw_arg()
-        if args is None and kwargs is None:
-            return  # Looks like this function does not have starred args
-
-        args_defn_type = None
-        kwargs_defn_type = None
-        for arg_def, arg_kind in zip(defn.arguments, defn.arg_kinds):
-            if arg_kind == ARG_STAR:
-                args_defn_type = arg_def.type_annotation
-            elif arg_kind == ARG_STAR2:
-                kwargs_defn_type = arg_def.type_annotation
-
-        # This may happen on invalid `ParamSpec` args / kwargs definition,
-        # type analyzer sets types of arguments to `Any`, but keeps
-        # definition types as `UnboundType` for now.
-        if not (
-            (isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"))
-            or (
-                isinstance(kwargs_defn_type, UnboundType)
-                and kwargs_defn_type.name.endswith(".kwargs")
-            )
-        ):
-            # Looks like both `*args` and `**kwargs` are not `ParamSpec`
-            # It might be something else, skipping.
-            return
-
-        args_type = args.typ if args is not None else None
-        kwargs_type = kwargs.typ if kwargs is not None else None
-
-        if (
-            not isinstance(args_type, ParamSpecType)
-            or not isinstance(kwargs_type, ParamSpecType)
-            or args_type.name != kwargs_type.name
-        ):
-            if isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"):
-                param_name = args_defn_type.name.split(".")[0]
-            elif isinstance(kwargs_defn_type, UnboundType) and kwargs_defn_type.name.endswith(
-                ".kwargs"
-            ):
-                param_name = kwargs_defn_type.name.split(".")[0]
-            else:
-                # Fallback for cases that probably should not ever happen:
-                param_name = "P"
-
-            self.fail(
-                f'ParamSpec must have "*args" typed as "{param_name}.args" and "**kwargs" typed as "{param_name}.kwargs"',
-                func,
-                code=codes.VALID_TYPE,
-            )
-
     def visit_decorator(self, dec: Decorator) -> None:
         self.statement = dec
         # TODO: better don't modify them at all.

diff --git a/mypy/typeanal.py b/mypy/typeanal.py
@@ -1102,46 +1102,42 @@ def visit_callable_type(
                 variables, _ = self.bind_function_type_variables(t, t)
             type_guard = self.anal_type_guard(t.ret_type)
             type_is = self.anal_type_is(t.ret_type)
+
             arg_kinds = t.arg_kinds
-            if len(arg_kinds) >= 2 and arg_kinds[-2] == ARG_STAR and arg_kinds[-1] == ARG_STAR2:
-                arg_types = self.anal_array(t.arg_types[:-2], nested=nested) + [
-                    self.anal_star_arg_type(t.arg_types[-2], ARG_STAR, nested=nested),
-                    self.anal_star_arg_type(t.arg_types[-1], ARG_STAR2, nested=nested),
-                ]
-                # If nested is True, it means we are analyzing a Callable[...] type, rather
-                # than a function definition type. We need to "unpack" ** TypedDict annotation
-                # here (for function definitions it is done in semanal).
-                if nested and isinstance(arg_types[-1], UnpackType):
-                    # TODO: it would be better to avoid this get_proper_type() call.
-                    unpacked = get_proper_type(arg_types[-1].type)
-                    if isinstance(unpacked, TypedDictType):
-                        arg_types[-1] = unpacked
-                        unpacked_kwargs = True
-                    arg_types = self.check_unpacks_in_list(arg_types)
-            else:
-                star_index = None
-                if ARG_STAR in arg_kinds:
-                    star_index = arg_kinds.index(ARG_STAR)
-                star2_index = None
-                if ARG_STAR2 in arg_kinds:
-                    star2_index = arg_kinds.index(ARG_STAR2)
-                arg_types = []
-                for i, ut in enumerate(t.arg_types):
-                    at = self.anal_type(
-                        ut, nested=nested, allow_unpack=i in (star_index, star2_index)
-                    )
-                    if nested and isinstance(at, UnpackType) and i == star_index:
+            arg_types = []
+            has_pspec_args = has_pspec_kwargs = None
+            for kind, ut in zip(arg_kinds, t.arg_types):
+                if kind == ARG_STAR:
+                    has_pspec_args, at = self.anal_star_arg_type(ut, kind, nested=nested)
+                elif kind == ARG_STAR2:
+                    has_pspec_kwargs, at = self.anal_star_arg_type(ut, kind, nested=nested)
+                    if nested and isinstance(at, UnpackType):
                         # TODO: it would be better to avoid this get_proper_type() call.
                         p_at = get_proper_type(at.type)
                         if isinstance(p_at, TypedDictType) and not at.from_star_syntax:
                             # Automatically detect Unpack[Foo] in Callable as backwards
                             # compatible syntax for **Foo, if Foo is a TypedDict.
                             at = p_at
-                            arg_kinds[i] = ARG_STAR2
                             unpacked_kwargs = True
-                    arg_types.append(at)
-                if nested:
-                    arg_types = self.check_unpacks_in_list(arg_types)
+                else:
+                    if has_pspec_args:
+                        self.fail("Arguments not allowed after ParamSpec.args", t)
+                    at = self.anal_type(ut, nested=nested, allow_unpack=False)
+                arg_types.append(at)
+            if nested:
+                arg_types = self.check_unpacks_in_list(arg_types)
+
+            if has_pspec_args != has_pspec_kwargs:
+                name = has_pspec_args or has_pspec_kwargs
+                self.fail(
+                    f'ParamSpec must have "*args" typed as "{name}.args" and "**kwargs" typed as "{name}.kwargs"',
+                    t,
+                )
+                if ARG_STAR in arg_kinds:
+                    arg_types[arg_kinds.index(ARG_STAR)] = AnyType(TypeOfAny.from_error)
+                if ARG_STAR2 in arg_kinds:
+                    arg_types[arg_kinds.index(ARG_STAR2)] = AnyType(TypeOfAny.from_error)
+
             # If there were multiple (invalid) unpacks, the arg types list will become shorter,
             # we need to trim the kinds/names as well to avoid crashes.
             arg_kinds = t.arg_kinds[: len(arg_types)]
@@ -1196,7 +1192,7 @@ def anal_type_is_arg(self, t: UnboundType, fullname: str) -> Type | None:
             return self.anal_type(t.args[0])
         return None
 
-    def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
+    def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> tuple[str | None, Type]:
         """Analyze signature argument type for *args and **kwargs argument."""
         if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args:
             components = t.name.split(".")
@@ -1205,6 +1201,7 @@ def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
             if sym is not None and isinstance(sym.node, ParamSpecExpr):
                 tvar_def = self.tvar_scope.get_binding(sym)
                 if isinstance(tvar_def, ParamSpecType):
+                    # if t.line==25:breakpoint()
                     if kind == ARG_STAR:
                         make_paramspec = paramspec_args
                         if components[-1] != "args":
@@ -1223,15 +1220,15 @@ def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
                             )
                     else:
                         assert False, kind
-                    return make_paramspec(
+                    return tvar_name, make_paramspec(
                         tvar_def.name,
                         tvar_def.fullname,
                         tvar_def.id,
                         named_type_func=self.named_type,
                         line=t.line,
                         column=t.column,
                     )
-        return self.anal_type(t, nested=nested, allow_unpack=True)
+        return None, self.anal_type(t, nested=nested, allow_unpack=True)
 
     def visit_overloaded(self, t: Overloaded) -> Type:
         # Overloaded types are manually constructed in semanal.py by analyzing the

diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test
@@ -440,13 +440,13 @@ class C(Generic[P, P2]):
     def m1(self, *args: P.args, **kwargs: P.kwargs) -> None:
         self.m1(*args, **kwargs)
         self.m2(*args, **kwargs)  # E: Argument 1 to "m2" of "C" has incompatible type "*P.args"; expected "P2.args" \
-            # E: Argument 2 to "m2" of "C" has incompatible type "**P.kwargs"; expected "P2.kwargs"
+                                  # E: Argument 2 to "m2" of "C" has incompatible type "**P.kwargs"; expected "P2.kwargs"
         self.m1(*kwargs, **args)  # E: Argument 1 to "m1" of "C" has incompatible type "*P.kwargs"; expected "P.args" \
-            # E: Argument 2 to "m1" of "C" has incompatible type "**P.args"; expected "P.kwargs"
+                                  # E: Argument 2 to "m1" of "C" has incompatible type "**P.args"; expected "P.kwargs"
         self.m3(*args, **kwargs)  # E: Argument 1 to "m3" of "C" has incompatible type "*P.args"; expected "int" \
-            # E: Argument 2 to "m3" of "C" has incompatible type "**P.kwargs"; expected "int"
+                                  # E: Argument 2 to "m3" of "C" has incompatible type "**P.kwargs"; expected "int"
         self.m4(*args, **kwargs)  # E: Argument 1 to "m4" of "C" has incompatible type "*P.args"; expected "int" \
-            # E: Argument 2 to "m4" of "C" has incompatible type "**P.kwargs"; expected "int"
+                                  # E: Argument 2 to "m4" of "C" has incompatible type "**P.kwargs"; expected "int"
 
         self.m1(*args, **args)  # E: Argument 2 to "m1" of "C" has incompatible type "**P.args"; expected "P.kwargs"
         self.m1(*kwargs, **kwargs)  # E: Argument 1 to "m1" of "C" has incompatible type "*P.kwargs"; expected "P.args"
@@ -1264,7 +1264,7 @@ def f1(f: Callable[P, int], *args, **kwargs: P.kwargs) -> int: ...  # E: ParamSp
 def f2(f: Callable[P, int], *args: P.args, **kwargs) -> int: ...  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
 def f3(f: Callable[P, int], *args: P.args) -> int: ...  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
 def f4(f: Callable[P, int], **kwargs: P.kwargs) -> int: ...  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
-def f5(f: Callable[P, int], *args: P.args, extra_keyword_arg: int, **kwargs: P.kwargs) -> int: ...  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
+def f5(f: Callable[P, int], *args: P.args, extra_keyword_arg: int, **kwargs: P.kwargs) -> int: ...  # E: Arguments not allowed after ParamSpec.args
 
 # Error message test:
 P1 = ParamSpec('P1')
@@ -1294,7 +1294,10 @@ def f1(f: Callable[Concatenate[int, P], int], *args, **kwargs: P.kwargs) -> int:
 def f2(f: Callable[Concatenate[int, P], int], *args: P.args, **kwargs) -> int: ...  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
 def f3(f: Callable[Concatenate[int, P], int], *args: P.args) -> int: ...  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
 def f4(f: Callable[Concatenate[int, P], int], **kwargs: P.kwargs) -> int: ...  # E: ParamSpec must h
9E88
ave "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
-def f5(f: Callable[Concatenate[int, P], int], *args: P.args, extra_keyword_arg: int, **kwargs: P.kwargs) -> int: ...  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
+def f5(f: Callable[Concatenate[int, P], int], *args: P.args, extra_keyword_arg: int, **kwargs: P.kwargs) -> int: ...  # E: Arguments not allowed after ParamSpec.args
+
+
+
 [builtins fixtures/paramspec.pyi]
 
 
@@ -2474,12 +2477,6 @@ def run(func: Callable[Concatenate[int, str, P], T], *args: P.args, **kwargs: P.
 
 [builtins fixtures/paramspec.pyi]
 
-
-
-
-
-
-
 [case testParamSpecConformance1]
 #flags: --python-version 3.12
 from typing import Any, Callable, List, ParamSpec
@@ -2525,14 +2522,8 @@ def func5(*args: P, **kwargs: P) -> None:  # E  # E: Invalid location for ParamS
                                                 # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]"
     ...
 
-
 [builtins fixtures/paramspec.pyi]
 
-
-
-
-
-
 [case testParamSpecConformance2]
 #flags: --python-version 3.12
 from typing import Any, Callable, ParamSpec, assert_type
@@ -2555,7 +2546,7 @@ def puts_p_into_scope1(f: Callable[P, int]) -> None:
     def bad_kwargs1(*args: P.args, **kwargs: P.args) -> None:  # E  # E: Use "P.kwargs" for variadic "**" parameter
         pass
 
-    def bad_kwargs2(*args: P.args, **kwargs: Any) -> None:  # E
+    def bad_kwargs2(*args: P.args, **kwargs: Any) -> None:  # E  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
         pass
 
 
@@ -2567,10 +2558,10 @@ def puts_p_into_scope2(f: Callable[P, int]) -> None:
     stored_args: P.args  # E  # E: ParamSpec parts are not allowed here
     stored_kwargs: P.kwargs  # E  # E: ParamSpec parts are not allowed here
 
-    def just_args(*args: P.args) -> None:  # E  # E: ParamSpec parts are not allowed here
+    def just_args(*args: P.args) -> None:  # E  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
         pass
 
-    def just_kwargs(**kwargs: P.kwargs) -> None:  # E  # E: ParamSpec parts are not allowed here
+    def just_kwargs(**kwargs: P.kwargs) -> None:  # E  # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs"
         pass
 
 
@@ -2590,7 +2581,7 @@ def add(f: Callable[P, int]) -> Callable[Concatenate[str, P], None]:
     def foo(s: str, *args: P.args, **kwargs: P.kwargs) -> None:  # OK
         pass
 
-    def bar(*args: P.args, s: str, **kwargs: P.kwargs) -> None:  # E  # E: ParamSpec parts are not allowed here
+    def bar(*args: P.args, s: str, **kwargs: P.kwargs) -> None:  # E  # E: Arguments not allowed after ParamSpec.args
         pass
 
     return foo  # OK
@@ -2634,4 +2625,10 @@ twice(a_int_b_str, b="A", a=1)  # OK
 twice(a_int_b_str, "A", 1)  # E  # E: Argument 2 to "twice" has incompatible type "str"; expected "int" \
                                  # E: Argument 3 to "twice" has incompatible type "int"; expected "str"
 
+
+
+
+
+
+
 [builtins fixtures/paramspec.pyi]