Fix join/meet/combine of generic callables

python · ilevkivskyi · Jun 5, 2024 · Jun 1, 2024 · Jun 2, 2024 · Jun 2, 2024
commit ffc982e844b27e86d25bf70eebc49b59c9c76b12
diff --git a/mypy/join.py b/mypy/join.py
@@ -5,6 +5,7 @@
 from typing import Sequence, overload
 
 import mypy.typeops
+from mypy.expandtype import expand_type
 from mypy.maptype import map_instance_to_supertype
 from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT, VARIANCE_NOT_READY
 from mypy.state import state
@@ -36,6 +37,7 @@
     TypedDictType,
     TypeOfAny,
     TypeType,
+    TypeVarId,
     TypeVarLikeType,
     TypeVarTupleType,
     TypeVarType,
@@ -718,7 +720,35 @@ def is_similar_callables(t: CallableType, s: CallableType) -> bool:
     )
 
 
+def update_callable_ids(c: CallableType, ids: list[TypeVarId]) -> CallableType:
+    tv_map = {}
+    tvs = []
+    for (tv, new_id) in zip(c.variables, ids):
+        new_tv = tv.copy_modified(id=new_id)
+        tvs.append(new_tv)
+        tv_map[tv.id] = new_tv
+    return expand_type(c, tv_map).copy_modified(variables=tvs)
+
+
+def match_generic_callables(t: CallableType, s: CallableType) -> tuple[CallableType, CallableType]:
+    # The case where we combine/join/meet similar callables, situation where both are generic
+    # requires special care. A more principled solution may involve unify_generic_callable(),
+    # but it would have two problems:
+    #   * This adds risk of infinite recursion: e.g. join -> unification -> solver -> join
+    #   * Using unification is an incorrect thing for meets, as it "widens" the types
+    # Finally, this effectively falls back to an old behaviour before namespaces were added to
+    # type variables, and it worked relatively well.
+   
8000
 max_len = max(len(t.variables), len(s.variables))
+    min_len = min(len(t.variables), len(s.variables))
+    if min_len == 0:
+        return t, s
+    new_ids = [TypeVarId.new(meta_level=0) for _ in range(max_len)]
+    # Note: this relies on variables being in order they appear in function definition.
+    return update_callable_ids(t, new_ids), update_callable_ids(s, new_ids)
+
+
 def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
+    t, s = match_generic_callables(t, s)
     arg_types: list[Type] = []
     for i in range(len(t.arg_types)):
         arg_types.append(safe_meet(t.arg_types[i], s.arg_types[i]))
@@ -771,6 +801,7 @@ def safe_meet(t: Type, s: Type) -> Type:
 
 
 def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType:
+    t, s = match_generic_callables(t, s)
     arg_types: list[Type] = []
     for i in range(len(t.arg_types)):
         arg_types.append(safe_join(t.arg_types[i], s.arg_types[i]))

diff --git a/mypy/meet.py b/mypy/meet.py
@@ -1024,8 +1024,9 @@ def default(self, typ: Type) -> ProperType:
 
 
 def meet_similar_callables(t: CallableType, s: CallableType) -> CallableType:
-    from mypy.join import safe_join
+    from mypy.join import match_generic_callables, safe_join
 
+    t, s = match_generic_callables(t, s)
     arg_types: list[Type] = []
     for i in range(len(t.arg_types)):
         arg_types.append(safe_join(t.arg_types[i], s.arg_types[i]))

diff --git a/mypy/messages.py b/mypy/messages.py
@@ -2728,7 +2728,7 @@ def format_type(
     """
     Convert a type to a relatively short string suitable for error messages.
 
-    `verbosity` is a coarse grained control on the verbosity of the type
+    `verbosity` is a coarse-grained control on the verbosity of the type
 
     This function returns a string appropriate for unmodified use in error
     messages; this means that it will be quoted in most cases.  If
@@ -2744,7 +2744,7 @@ def format_type_bare(
     """
     Convert a type to a relatively short string suitable for error messages.
 
-    `verbosity` is a coarse grained control on the verbosity of the type
+    `verbosity` is a coarse-grained control on the verbosity of the type
     `fullnames` specifies a set of names that should be printed in full
 
     This function will return an unquoted string.  If a caller doesn't need to

diff --git a/mypy/subtypes.py b/mypy/subtypes.py
@@ -944,7 +944,7 @@ def visit_overloaded(self, left: Overloaded) -> bool:
                 # When it is the same overload, then the types are equal.
                 return True
 
-            # Ensure each overload in the right side (the supertype) is accounted for.
+            # Ensure each overload on the right side (the supertype) is accounted for.
             previous_match_left_index = -1
             matched_overloads = set()
 

diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test
@@ -3277,3 +3277,16 @@ def f(x: Callable[[A], A]) -> Callable[[B], B]:
 
 reveal_type(f(f))  # N: Revealed type is "def [B] (B`1) -> B`1"
 reveal_type(f(f)(f))  # N: Revealed type is "def [A] (x: def (A`-1) -> A`-1) -> def [B] (B`-2) -> B`-2"
+
+[case testGenericUnionFunctionJoin]
+from typing import TypeVar, Union
+
+T = TypeVar("T")
+S = TypeVar("S")
+
+def f(x: T, y: S) -> Union[T, S]: ...
+def g(x: T, y: S) -> Union[T, S]: ...
+
+x = [f, g]
+reveal_type(x)  # N: Revealed type is "builtins.list[def [T, S] (x: T`4, y: S`5) -> Union[T`4, S`5]]"
+[builtins fixtures/list.pyi]
diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test
@@ -2928,8 +2928,8 @@ def mix(fs: List[Callable[[S], T]]) -> Callable[[S], List[T]]:
 def id(__x: U) -> U:
     ...
 fs = [id, id, id]
-reveal_type(mix(fs))  # N: Revealed type is "def [S] (S`3) -> builtins.list[S`3]"
-reveal_type(mix([id, id, id]))  # N: Revealed type is "def [S] (S`5) -> builtins.list[S`5]"
+reveal_type(mix(fs))  # N: Revealed type is "def [S] (S`7) -> builtins.list[S`7]"
+reveal_type(mix([id, id, id]))  # N: Revealed type is "def [S] (S`9) -> builtins.list[S`9]"
 [builtins fixtures/list.pyi]
 
 [case testInferenceAgainstGenericCurry]