Merge branch 'master' into add-regex-plugin

python · Michael0x2a · Aug 24, 2019 · Oct 27, 2019 · Jan 5, 2022 · Jan 5, 2022
commit 03bd647f45707e115f82bad88fca563ddce80324
diff --git a/mypy/checker.py b/mypy/checker.py
@@ -53,9 +53,11 @@
 from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS
 from mypy.typeops import (
     map_type_from_supertype, bind_self, erase_to_bound, make_simplified_union,
-    erase_def_to_union_or_bound, erase_to_union_or_bound,
-    true_only, false_only, function_type, is_singleton_type,
-    try_expanding_enum_to_union, coerce_to_literal,
+    erase_def_to_union_or_bound, erase_to_union_or_bound, coerce_to_literal,
+    try_getting_str_literals_from_type, try_getting_int_literals_from_type,
+    tuple_fallback, is_singleton_type, try_expanding_sum_type_to_union,
+    true_only, false_only, function_type, get_type_vars, custom_special_method,
+    is_literal_type_like,
 )
 from mypy import message_registry
 from mypy.message_registry import ErrorMessage
@@ -68,8 +70,8 @@
 from mypy.maptype import map_instance_to_supertype
 from mypy.typevars import fill_typevars, has_no_typevars, fill_typevars_with_any
 from mypy.semanal import set_callable_name, refers_to_fullname
-from mypy.mro import calculate_mro
-from mypy.erasetype import erase_typevars, remove_instance_transient_info
+from mypy.mro import calculate_mro, MroError
+from mypy.erasetype import erase_typevars, remove_instance_transient_info, erase_type
 from mypy.expandtype import expand_type, expand_type_by_instance
 from mypy.visitor import NodeVisitor
 from mypy.join import join_types

diff --git a/mypy/erasetype.py b/mypy/erasetype.py
@@ -149,6 +149,18 @@ def remove_instance_transient_info(t: Type) -> Type:
 
 class TransientInstanceInfoEraser(TypeTranslator):
     def visit_instance(self, t: Instance) -> Type:
-        if t.last_known_value or t.metadata:
-            return t.copy_modified(last_known_value=None, metadata={})
+        if not t.last_known_value and not t.args and not t.metadata:
+            return t
+        new_t = t.copy_modified(
+            args=[a.accept(self) for a in t.args],
+            last_known_value=None,
+            metadata={},
+        )
+        new_t.can_be_true = t.can_be_true
+        new_t.can_be_false = t.can_be_false
+        return new_t
+
+    def visit_type_alias_type(self, t: TypeAliasType) -> Type:
+        # Type aliases can't contain literal values, because they are
+        # always constructed as explicit types.
         return t
diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py
@@ -23,8 +23,7 @@ class DefaultPlugin(Plugin):
 
     def get_function_hook(self, fullname: str
                           ) -> Optional[Callable[[FunctionContext], Type]]:
-        from mypy.plugins import ctypes
-        from mypy.plugins import regex
+        from mypy.plugins import ctypes, regex, singledispatch
 
         if fullname in ('contextlib.contextmanager', 'contextlib.asynccontextmanager'):
             return contextmanager_callback
@@ -36,6 +35,8 @@ def get_function_hook(self, fullname: str
             return regex.re_compile_callback
         elif fullname in regex.FUNCTIONS_PRODUCING_MATCH_OBJECT:
             return regex.re_direct_match_callback
+        elif fullname == 'functools.singledispatch':
+            return singledispatch.create_singledispatch_function_callback
         return None
 
     def get_method_signature_hook(self, fullname: str
@@ -58,8 +59,7 @@ def get_method_signature_hook(self, fullname: str
 
     def get_method_hook(self, fullname: str
                         ) -> Optional[Callable[[MethodContext], Type]]:
-        from mypy.plugins import ctypes
-        from mypy.plugins import regex
+        from mypy.plugins import ctypes, regex, singledispatch
 
         if fullname == 'typing.Mapping.get':
             return typed_dict_get_callback
@@ -87,6 +87,10 @@ def get_method_hook(self, fullname: str
             return regex.re_match_groups_callback
         elif fullname in regex.METHODS_PRODUCING_GROUP:
             return regex.re_match_group_callback
+        elif fullname == singledispatch.SINGLEDISPATCH_REGISTER_METHOD:
+            return singledispatch.singledispatch_register_callback
+        elif fullname == singledispatch.REGISTER_CALLABLE_CALL_METHOD:
+            return singledispatch.call_singledispatch_function_after_register_argument
         return None
 
     def get_attribute_hook(self, fullname: str

diff --git a/mypy/typeops.py b/mypy/typeops.py
@@ -19,8 +19,8 @@
     copy_type, TypeAliasType, TypeQuery, ParamSpecType
 )
 from mypy.nodes import (
-    FuncBase, FuncItem, OverloadedFuncDef, TypeInfo, TypeVar, ARG_STAR, ARG_STAR2, Expression,
-    StrExpr, Var
+    FuncBase, FuncItem, FuncDef, OverloadedFuncDef, TypeInfo, ARG_STAR, ARG_STAR2, ARG_POS,
+    Expression, StrExpr, Var, Decorator, SYMBOL_FUNCBASE_TYPES
 )
 from mypy.maptype import map_instance_to_supertype
 from mypy.expandtype import expand_type_by_instance, expand_type
@@ -631,7 +631,25 @@ def try_getting_literals_from_type(typ: Type,
                 return None
         else:
             return None
-    return strings
+    return literals
+
+
+def is_literal_type_like(t: Optional[Type]) -> bool:
+    """Returns 'true' if the given type context is potentially either a LiteralType,
+    a Union of LiteralType, or something similar.
+    """
+    t = get_proper_type(t)
+    if t is None:
+        return False
+    elif isinstance(t, LiteralType):
+        return True
+    elif isinstance(t, UnionType):
+        return any(is_literal_type_like(item) for item in t.items)
+    elif isinstance(t, TypeVarType):
+        return (is_literal_type_like(t.upper_bound)
+                or any(is_literal_type_like(item) for item in t.values))
+    else:
+        return False
 
 
 def get_enum_values(typ: Instance) -> List[str]:
@@ -656,15 +674,17 @@ def is_singleton_type(typ: Type) -> bool:
     constructing two distinct instances of 100001.
     """
     typ = get_proper_type(typ)
-    # TODO: Also make this return True if the type is a bool LiteralType.
+    # TODO:
     # Also make this return True if the type corresponds to ... (ellipsis) or NotImplemented?
     return (
-            isinstance(typ, NoneType) or (isinstance(typ, LiteralType) and typ.is_enum_literal())
+            isinstance(typ, NoneType)
+            or (isinstance(typ, LiteralType)
+                and (typ.is_enum_literal() or isinstance(typ.value, bool)))
             or (isinstance(typ, Instance) and typ.type.is_enum and len(get_enum_values(typ)) == 1)
     )
 
 
-def try_expanding_enum_to_union(typ: Type, target_fullname: str) -> ProperType:
+def try_expanding_sum_type_to_union(typ: Type, target_fullname: str) -> ProperType:
     """Attempts to recursively expand any enum Instances with the given target_fullname
     into a Union of all of its component LiteralTypes.
 
@@ -686,31 +706,75 @@ class Status(Enum):
     typ = get_proper_type(typ)
 
     if isinstance(typ, UnionType):
-        items = [try_expanding_enum_to_union(item, target_fullname) for item in typ.items]
-        return make_simplified_union(items)
-    elif isinstance(typ, Instance) and typ.type.is_enum and typ.type.fullname() == target_fullname:
-        new_items = []
-        for name, symbol in typ.type.names.items():
-            if not isinstance(symbol.node, Var):
-                continue
-            new_items.append(LiteralType(name, typ))
-        # SymbolTables are really just dicts, and dicts are guaranteed to preserve
-        # insertion order only starting with Python 3.7. So, we sort these for older
-        # versions of Python to help make tests deterministic.
-        #
-        # We could probably skip the sort for Python 3.6 since people probably run mypy
-        # only using CPython, but we might as well for the sake of full correctness.
-        if sys.version_info < (3, 7):
-            new_items.sort(key=lambda lit: lit.value)
-        return make_simplified_union(new_items)
-    else:
-        return typ
+        items = [try_expanding_sum_type_to_union(item, target_fullname) for item in typ.items]
+        return make_simplified_union(items, contract_literals=False)
+    elif isinstance(typ, Instance) and typ.type.fullname == target_fullname:
+        if typ.type.is_enum:
+            new_items = []
+            for name, symbol in typ.type.names.items():
+                if not isinstance(symbol.node, Var):
+                    continue
+                # Skip "_order_" and "__order__", since Enum will remove it
+                if name in ("_order_", "__order__"):
+                    continue
+                new_items.append(LiteralType(name, typ))
+            # SymbolTables are really just dicts, and dicts are guaranteed to preserve
+            # insertion order only starting with Python 3.7. So, we sort these for older
+            # versions of Python to help make tests deterministic.
+            #
+            # We could probably skip the sort for Python 3.6 since people probably run mypy
+            # only using CPython, but we might as well for the sake of full correctness.
+            if sys.version_info < (3, 7):
+                new_items.sort(key=lambda lit: lit.value)
+            return make_simplified_union(new_items, contract_literals=False)
+        elif typ.type.fullname == "builtins.bool":
+            return make_simplified_union(
+                [LiteralType(True, typ), LiteralType(False, typ)],
+                contract_literals=False
+            )
+
+    return typ
+
+
+def try_contracting_literals_in_union(types: Sequence[Type]) -> List[ProperType]:
+    """Contracts any literal types back into a sum type if possible.
+
+    Will replace the first instance of the literal with the sum type and
+    remove all others.
 
+    If we call `try_contracting_union(Literal[Color.RED, Color.BLUE, Color.YELLOW])`,
+    this function will return Color.
 
-def coerce_to_literal(typ: Type) -> ProperType:
+    We also treat `Literal[True, False]` as `bool`.
+    """
+    proper_types = [get_proper_type(typ) for typ in types]
+    sum_types: Dict[str, Tuple[Set[Any], List[int]]] = {}
+    marked_for_deletion = set()
+    for idx, typ in enumerate(proper_types):
+        if isinstance(typ, LiteralType):
+            fullname = typ.fallback.type.fullname
+            if typ.fallback.type.is_enum or isinstance(typ.value, bool):
+                if fullname not in sum_types:
+                    sum_types[fullname] = (set(get_enum_values(typ.fallback))
+                                           if typ.fallback.type.is_enum
+                                           else set((True, False)),
+                                           [])
+                literals, indexes = sum_types[fullname]
+                literals.discard(typ.value)
+                indexes.append(idx)
+                if not literals:
+                    first, *rest = indexes
+                    proper_types[first] = typ.fallback
+                    marked_for_deletion |= set(rest)
+    return list(itertools.compress(proper_types, [(i not in marked_for_deletion)
+                                                  for i in range(len(proper_types))]))
+
+
+def coerce_to_literal(typ: Type) -> Type:
     """Recursively converts any Instances that have a last_known_value or are
     instances of enum types with a single value into the corresponding LiteralType.
     """
+    original_type = typ
     typ = get_proper_type(typ)
     if isinstance(typ, UnionType):
         new_items = [coerce_to_literal(item) for item in typ.items]
@@ -722,4 +786,61 @@ def coerce_to_literal(typ: Type) -> ProperType:
             enum_values = get_enum_values(typ)
             if len(enum_values) == 1:
                 return LiteralType(value=enum_values[0], fallback=typ)
-    return typ
+    return original_type
+
+
+def get_type_vars(tp: Type) -> List[TypeVarType]:
+    return tp.accept(TypeVarExtractor())
+
+
+class TypeVarExtractor(TypeQuery[List[TypeVarType]]):
+    def __init__(self) -> None:
+        super().__init__(self._merge)
+
+    def _merge(self, iter: Iterable[List[TypeVarType]]) -> List[TypeVarType]:
+        out = []
+        for item in iter:
+            out.extend(item)
+        return out
+
+    def visit_type_var(self, t: TypeVarType) -> List[TypeVarType]:
+        return [t]
+
+
+def custom_special_method(typ: Type, name: str, check_all: bool = False) -> bool:
+    """Does this type have a custom special method such as __format__() or __eq__()?
+
+    If check_all is True ensure all items of a union have a custom method, not just some.
+    """
+    typ = get_proper_type(typ)
+    if isinstance(typ, Instance):
+        method = typ.type.get(name)
+        if method and isinstance(method.node, (SYMBOL_FUNCBASE_TYPES, Decorator, Var)):
+            if method.node.info:
+                return not method.node.info.fullname.startswith('builtins.')
+        return False
+    if isinstance(typ, UnionType):
+        if check_all:
+            return all(custom_special_method(t, name, check_all) for t in typ.items)
+        return any(custom_special_method(t, name) for t in typ.items)
+    if isinstance(typ, TupleType):
+        return custom_special_method(tuple_fallback(typ), name, check_all)
+    if isinstance(typ, CallableType) and typ.is_type_obj():
+        # Look up __method__ on the metaclass for class objects.
+        return custom_special_method(typ.fallback, name, check_all)
+    if isinstance(typ, AnyType):
+        # Avoid false positives in uncertain cases.
+        return True
+    # TODO: support other types (see ExpressionChecker.has_member())?
+    return False
+
+
+def is_redundant_literal_instance(general: ProperType, specific: ProperType) -> bool:
+    if not isinstance(general, Instance) or general.last_known_value is None:
+        return True
+    if isinstance(specific, Instance) and specific.last_known_value == general.last_known_value:
+        return True
+    if isinstance(specific, UninhabitedType):
+        return True
+
+    return False
diff --git a/mypy/types.py b/mypy/types.py
@@ -1017,10 +1017,11 @@ def serialize(self) -> Union[JsonDict, str]:
         type_ref = self.type.fullname
         if not self.args and not self.last_known_value:
             return type_ref
-        data = {'.class': 'Instance',
-                }  # type: JsonDict
-        data['type_ref'] = type_ref
-        data['args'] = [arg.serialize() for arg in self.args]
+        data: JsonDict = {
+            ".class": "Instance",
+        }
+        data["type_ref"] = type_ref
+        data["args"] = [arg.serialize() for arg in self.args]
         if self.metadata:
             data['metadata'] = self.metadata
         if self.last_known_value is not None:
@@ -1050,14 +1051,15 @@ def deserialize(cls, data: Union[JsonDict, str]) -> 'Instance':
     def copy_modified(self, *,
                       args: Bogus[List[Type]] = _dummy,
                       metadata: Bogus[Dict[str, JsonDict]] = _dummy,
+                      erased: Bogus[bool] = _dummy,
                       last_known_value: Bogus[Optional['LiteralType']] = _dummy) -> 'Instance':
         return Instance(
             self.type,
             args if args is not _dummy else self.args,
             self.line,
             self.column,
-            self.erased,
             metadata if metadata is not _dummy else self.metadata,
+            erased if erased is not _dummy else self.erased,
             last_known_value if last_known_value is not _dummy else self.last_known_value,
         )