From 784aa936661f4e4b805de103811c1e2ba60340b2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 10 Jun 2021 13:52:52 +0100 Subject: [PATCH 01/17] Support --install-types --non-interactive that doesn't prompt (#10616) It also doesn't show errors, making it useful in CI jobs. Example: ``` $ mypy --install-types --non-interactive -c 'import click' Installing missing stub packages: /Users/jukka/venv/mypy/bin/python3 -m pip install types-click Collecting types-click Using cached types_click-7.1.0-py2.py3-none-any.whl (13 kB) Installing collected packages: types-click Successfully installed types-click-7.1.0 ``` Work on #10600. --- mypy/main.py | 40 ++++++++++++++++++++++++++----------- mypy/options.py | 3 +++ test-data/unit/cmdline.test | 12 +++++++++++ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 201710f6cebc..693c7e8c6d13 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -72,13 +72,18 @@ def main(script_path: Optional[str], if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): # Since --install-types performs user input, we want regular stdout and stderr. - fail("--install-types not supported in this mode of running mypy", stderr, options) + fail("Error: --install-types not supported in this mode of running mypy", stderr, options) + + if options.non_interactive and not options.install_types: + fail("Error: --non-interactive is only supported with --install-types", stderr, options) if options.install_types and not sources: - install_types(options.cache_dir, formatter) + install_types(options.cache_dir, formatter, non_interactive=options.non_interactive) return def flush_errors(new_messages: List[str], serious: bool) -> None: + if options.non_interactive: + return if options.pretty: new_messages = formatter.fit_in_terminal(new_messages) messages.extend(new_messages) @@ -100,7 +105,10 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: blockers = True if not e.use_stdout: serious = True - if options.warn_unused_configs and options.unused_configs and not options.incremental: + if (options.warn_unused_configs + and options.unused_configs + and not options.incremental + and not options.non_interactive): print("Warning: unused section(s) in %s: %s" % (options.config_file, get_config_module_names(options.config_file, @@ -116,7 +124,7 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: code = 0 if messages: code = 2 if blockers else 1 - if options.error_summary: + if options.error_summary and not options.non_interactive: if messages: n_errors, n_files = util.count_stats(messages) if n_errors: @@ -130,7 +138,8 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: stdout.flush() if options.install_types: - install_types(options.cache_dir, formatter, after_run=True) + install_types(options.cache_dir, formatter, after_run=True, + non_interactive=options.non_interactive) return if options.fast_exit: @@ -751,6 +760,10 @@ def add_invertible_flag(flag: str, add_invertible_flag('--install-types', default=False, strict_flag=False, help="Install detected missing library stub packages using pip", group=other_group) + add_invertible_flag('--non-interactive', default=False, strict_flag=False, + help=("Install stubs without asking for confirmation and hide " + + "errors, with --install-types"), + group=other_group, inverse="--interactive") if server_options: # TODO: This flag is superfluous; remove after a short transition (2018-03-16) @@ -1072,7 +1085,9 @@ def fail(msg: str, stderr: TextIO, options: Options) -> None: def install_types(cache_dir: str, formatter: util.FancyFormatter, - after_run: bool = False) -> None: + *, + after_run: bool = False, + non_interactive: bool = False) -> None: """Install stub packages using pip if some missing stubs were detected.""" if not os.path.isdir(cache_dir): sys.stderr.write( @@ -1084,15 +1099,16 @@ def install_types(cache_dir: str, return with open(fnam) as f: packages = [line.strip() for line in f.readlines()] - if after_run: + if after_run and not non_interactive: print() print('Installing missing stub packages:') cmd = [sys.executable, '-m', 'pip', 'install'] + packages print(formatter.style(' '.join(cmd), 'none', bold=True)) print() - x = input('Install? [yN] ') - if not x.strip() or not x.lower().startswith('y'): - print(formatter.style('mypy: Skipping installation', 'red', bold=True)) - sys.exit(2) - print() + if not non_interactive: + x = input('Install? [yN] ') + if not x.strip() or not x.lower().startswith('y'): + print(formatter.style('mypy: Skipping installation', 'red', bold=True)) + sys.exit(2) + print() subprocess.run(cmd) diff --git a/mypy/options.py b/mypy/options.py index 82a4d34ea561..a163ae5ed014 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -297,6 +297,9 @@ def __init__(self) -> None: self.show_absolute_path = False # type: bool # Install missing stub packages if True self.install_types = False + # Install missing stub packages in non-interactive mode (don't prompt for + # confirmation, and don't show any errors) + self.non_interactive = False # When we encounter errors that may cause many additional errors, # skip most errors after this many messages have been reported. # -1 means unlimited. diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 2b5db57807b5..e469c48a48bf 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1276,3 +1276,15 @@ x = 0 # type: str y = 0 # type: str [out] pkg.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testCmdlineNonInteractiveWithoutInstallTypes] +# cmd: mypy --non-interactive -m pkg +[out] +Error: --non-interactive is only supported with --install-types +== Return code: 2 + +[case testCmdlineNonInteractiveInstallTypesNothingToDo] +# cmd: mypy --install-types --non-interactive -m pkg +[file pkg.py] +1() +[out] From e93a700c9de0e1b287590c03f427281aa400fda5 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 18 Jun 2021 02:56:48 -0700 Subject: [PATCH 02/17] Fix faulty casts in checker (#10560) Fixes #9615, fixes #9682 Both issue reports hit the faulty cast on L2803. This changes the logic so that that cast is actually always true. If not, we just end up doing whatever the fallback else clause does; this resulted in behaviour that matched what I expected. After that, the #9682 hit another crash in checker.py, where var.node was None instead of a Var. It seemed reasonable to just branch instead. Co-authored-by: hauntsaninja <> --- mypy/checker.py | 7 +++++-- test-data/unit/check-redefine.test | 9 +++++++++ test-data/unit/check-unreachable-code.test | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index dde8a3441c35..556f0ee13104 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2796,7 +2796,9 @@ def check_lvalue(self, lvalue: Lvalue) -> Tuple[Optional[Type], index_lvalue = None inferred = None - if self.is_definition(lvalue): + if self.is_definition(lvalue) and ( + not isinstance(lvalue, NameExpr) or isinstance(lvalue.node, Var) + ): if isinstance(lvalue, NameExpr): inferred = cast(Var, lvalue.node) assert isinstance(inferred, Var) @@ -3420,7 +3422,8 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: source = ('(exception variable "{}", which we do not ' 'accept outside except: blocks even in ' 'python 2)'.format(var.name)) - cast(Var, var.node).type = DeletedType(source=source) + if isinstance(var.node, Var): + var.node.type = DeletedType(source=source) self.binder.cleanse(var) if s.else_body: self.accept(s.else_body) diff --git a/test-data/unit/check-redefine.test b/test-data/unit/check-redefine.test index 5530777dd4db..4ac56edb24ef 100644 --- a/test-data/unit/check-redefine.test +++ b/test-data/unit/check-redefine.test @@ -474,3 +474,12 @@ with A() as x: reveal_type(x) # N: Revealed type is "builtins.int" with B() as x: x = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + +[case testRedefineModuleAsException] +import typing +try: + pass +except Exception as typing: + pass +[builtins fixtures/exception.pyi] diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index e4235e82fc98..21b0f1f0c60f 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -1305,3 +1305,19 @@ async def f_malformed_2() -> int: [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] + + +[case testConditionalTypeVarException] +# every part of this test case was necessary to trigger the crash +import sys +from typing import TypeVar + +T = TypeVar("T", int, str) + +def f(t: T) -> None: + if sys.platform == "lol": + try: + pass + except BaseException as e: + pass +[builtins fixtures/dict.pyi] From 5af694ee5d6fae7e8b73f75c4f1e7583003f0fce Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Thu, 10 Jun 2021 23:31:19 +0200 Subject: [PATCH 03/17] stubgen: Use PEP 604 instead of Optional (#10624) --- mypy/stubgen.py | 4 +--- test-data/unit/stubgen.test | 14 +++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index a12cd969e9ca..f731b95c8790 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -1159,10 +1159,8 @@ def get_str_type_of_node(self, rvalue: Expression, return 'bool' if can_infer_optional and \ isinstance(rvalue, NameExpr) and rvalue.name == 'None': - self.add_typing_import('Optional') self.add_typing_import('Any') - return '{}[{}]'.format(self.typing_name('Optional'), - self.typing_name('Any')) + return '{} | None'.format(self.typing_name('Any')) self.add_typing_import('Any') return self.typing_name('Any') diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index cfcd13eb45ad..46451391ec31 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -32,9 +32,9 @@ def g(b: int = ..., c: int = ...) -> None: ... [case testDefaultArgNone] def f(x=None): ... [out] -from typing import Any, Optional +from typing import Any -def f(x: Optional[Any] = ...) -> None: ... +def f(x: Any | None = ...) -> None: ... [case testDefaultArgBool] def f(x=True, y=False): ... @@ -772,12 +772,12 @@ class A: def method(self, a=None): self.x = [] [out] -from typing import Any, Optional +from typing import Any class A: x: Any = ... - def __init__(self, a: Optional[Any] = ...) -> None: ... - def method(self, a: Optional[Any] = ...) -> None: ... + def __init__(self, a: Any | None = ...) -> None: ... + def method(self, a: Any | None = ...) -> None: ... [case testAnnotationImportsFrom] import foo @@ -1700,10 +1700,10 @@ def Optional(): return 0 [out] -from typing import Any as _Any, Optional as _Optional +from typing import Any as _Any def f(x: _Any = ...): ... -def g(x: _Optional[_Any] = ...) -> None: ... +def g(x: _Any | None = ...) -> None: ... x: _Any From fee4422536803ea13f2eda8a42fbc5bdc24887ef Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Fri, 11 Jun 2021 00:37:07 +0200 Subject: [PATCH 04/17] stubgen: Use NamedTuple class syntax (#10625) ### Description Use `typing.NamedTuple` in stubgen, instead of `collections.namedtuple`. ## Test Plan I modified the existing tests and added another test for an empty namedtuple, to ensure that `Any` is not imported unnecessarily. --- mypy/stubgen.py | 18 ++++++++----- test-data/unit/stubgen.test | 52 +++++++++++++++++++++++++++---------- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index f731b95c8790..5efc25de4f41 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -536,7 +536,7 @@ def __init__(self, # Disable implicit exports of package-internal imports? self.export_less = export_less # Add imports that could be implicitly generated - self.import_tracker.add_import_from("collections", [("namedtuple", None)]) + self.import_tracker.add_import_from("typing", [("NamedTuple", None)]) # Names in __all__ are required for name in _all_ or (): if name not in IGNORED_DUNDERS: @@ -900,18 +900,24 @@ def is_namedtuple(self, expr: Expression) -> bool: def process_namedtuple(self, lvalue: NameExpr, rvalue: CallExpr) -> None: if self._state != EMPTY: self.add('\n') - name = repr(getattr(rvalue.args[0], 'value', ERROR_MARKER)) if isinstance(rvalue.args[1], StrExpr): - items = repr(rvalue.args[1].value) + items = rvalue.args[1].value.split(" ") elif isinstance(rvalue.args[1], (ListExpr, TupleExpr)): list_items = cast(List[StrExpr], rvalue.args[1].items) - items = '[%s]' % ', '.join(repr(item.value) for item in list_items) + items = [item.value for item in list_items] else: self.add('%s%s: Any' % (self._indent, lvalue.name)) self.import_tracker.require_name('Any') return - self.import_tracker.require_name('namedtuple') - self.add('%s%s = namedtuple(%s, %s)\n' % (self._indent, lvalue.name, name, items)) + self.import_tracker.require_name('NamedTuple') + self.add('{}class {}(NamedTuple):'.format(self._indent, lvalue.name)) + if len(items) == 0: + self.add(' ...\n') + else: + self.import_tracker.require_name('Any') + self.add('\n') + for item in items: + self.add('{} {}: Any\n'.format(self._indent, item)) self._state = CLASS def is_alias_expression(self, expr: Expression, top_level: bool = True) -> bool: diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 46451391ec31..66980cf110e5 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -591,18 +591,30 @@ class A: import collections, x X = collections.namedtuple('X', ['a', 'b']) [out] -from collections import namedtuple +from typing import Any, NamedTuple + +class X(NamedTuple): + a: Any + b: Any -X = namedtuple('X', ['a', 'b']) +[case testEmptyNamedtuple] +import collections +X = collections.namedtuple('X', []) +[out] +from typing import NamedTuple + +class X(NamedTuple): ... [case testNamedtupleAltSyntax] from collections import namedtuple, xx X = namedtuple('X', 'a b') xx [out] -from collections import namedtuple +from typing import Any, NamedTuple -X = namedtuple('X', 'a b') +class X(NamedTuple): + a: Any + b: Any [case testNamedtupleWithUnderscore] from collections import namedtuple as _namedtuple @@ -610,11 +622,13 @@ def f(): ... X = _namedtuple('X', 'a b') def g(): ... [out] -from collections import namedtuple +from typing import Any, NamedTuple def f() -> None: ... -X = namedtuple('X', 'a b') +class X(NamedTuple): + a: Any + b: Any def g() -> None: ... @@ -623,9 +637,11 @@ import collections, x _X = collections.namedtuple('_X', ['a', 'b']) class Y(_X): ... [out] -from collections import namedtuple +from typing import Any, NamedTuple -_X = namedtuple('_X', ['a', 'b']) +class _X(NamedTuple): + a: Any + b: Any class Y(_X): ... @@ -636,13 +652,19 @@ Y = namedtuple('Y', ('a',)) Z = namedtuple('Z', ('a', 'b', 'c', 'd', 'e')) xx [out] -from collections import namedtuple +from typing import Any, NamedTuple -X = namedtuple('X', []) +class X(NamedTuple): ... -Y = namedtuple('Y', ['a']) +class Y(NamedTuple): + a: Any -Z = namedtuple('Z', ['a', 'b', 'c', 'd', 'e']) +class Z(NamedTuple): + a: Any + b: Any + c: Any + d: Any + e: Any [case testDynamicNamedTuple] from collections import namedtuple @@ -2187,10 +2209,12 @@ from collections import namedtuple class C: N = namedtuple('N', ['x', 'y']) [out] -from collections import namedtuple +from typing import Any, NamedTuple class C: - N = namedtuple('N', ['x', 'y']) + class N(NamedTuple): + x: Any + y: Any [case testImports_directImportsWithAlias] import p.a as a From 2ce36778003a7eeac4278578226b973a9ec29a74 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Fri, 11 Jun 2021 01:24:58 +0200 Subject: [PATCH 05/17] stubgen: never generate variable initializer (#10623) Previously, the generation of a variable initialized (= ...) was inconsistent between top-level and class-level variables. Since the initializer has no function in stubs, stubgen will now never generate it. --- mypy/stubgen.py | 8 +++----- test-data/unit/stubgen.test | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 5efc25de4f41..f1835aa9acd9 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -72,7 +72,7 @@ Expression, IntExpr, UnaryExpr, StrExpr, BytesExpr, NameExpr, FloatExpr, MemberExpr, TupleExpr, ListExpr, ComparisonExpr, CallExpr, IndexExpr, EllipsisExpr, ClassDef, MypyFile, Decorator, AssignmentStmt, TypeInfo, - IfStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, TempNode, Block, + IfStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, Block, Statement, OverloadedFuncDef, ARG_POS, ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT ) from mypy.stubgenc import generate_stub_for_c_module @@ -678,7 +678,7 @@ def visit_decorator(self, o: Decorator) -> None: self.visit_func_def(o.func, is_abstract=is_abstract) def process_decorator(self, o: Decorator) -> Tuple[bool, bool]: - """Process a series of decorataors. + """Process a series of decorators. Only preserve certain special decorators such as @abstractmethod. @@ -1076,9 +1076,7 @@ def get_init(self, lvalue: str, rvalue: Expression, typename += '[{}]'.format(final_arg) else: typename = self.get_str_type_of_node(rvalue) - has_rhs = not (isinstance(rvalue, TempNode) and rvalue.no_rhs) - initializer = " = ..." if has_rhs and not self.is_top_level() else "" - return '%s%s: %s%s\n' % (self._indent, lvalue, typename, initializer) + return '%s%s: %s\n' % (self._indent, lvalue, typename) def add(self, string: str) -> None: """Add text to generated stub.""" diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 66980cf110e5..f2e102a5dcfe 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -161,7 +161,7 @@ class C: x = 1 [out] class C: - x: int = ... + x: int [case testInitTypeAnnotationPreserved] class C: @@ -178,7 +178,7 @@ class C: x.y = 2 [out] class C: - x: int = ... + x: int def __init__(self) -> None: ... [case testSelfAndClassBodyAssignment] @@ -192,7 +192,7 @@ class C: x: int class C: - x: int = ... + x: int def __init__(self) -> None: ... [case testEmptyClass] @@ -244,7 +244,7 @@ class A: _x: int class A: - _y: int = ... + _y: int [case testSpecialInternalVar] __all__ = [] @@ -438,7 +438,7 @@ class A: def f(self): ... [out] class A: - x: int = ... + x: int def f(self) -> None: ... [case testSkipMultiplePrivateDefs] @@ -742,7 +742,7 @@ class A: [out] class A: class B: - x: int = ... + x: int def f(self) -> None: ... def g(self) -> None: ... @@ -797,7 +797,7 @@ class A: from typing import Any class A: - x: Any = ... + x: Any def __init__(self, a: Any | None = ...) -> None: ... def method(self, a: Any | None = ...) -> None: ... @@ -933,7 +933,7 @@ class Foo: from typing import Any class Foo: - alias: Any = ... + alias: Any [case testAliasExceptions] noalias1 = None @@ -1583,14 +1583,14 @@ class B: from typing import Any class A: - y: str = ... + y: str @property def x(self): ... class B: @property def x(self): ... - y: str = ... + y: str @x.setter def x(self, value: Any) -> None: ... @@ -2196,7 +2196,7 @@ class C: from typing import Any class C: - x: Any = ... + x: Any def __init__(self, x: Any) -> None: ... def __lt__(self, other: Any) -> Any: ... def __le__(self, other: Any) -> Any: ... From ae3dcc76066771d3074332228407f6bcb2f5073b Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Fri, 11 Jun 2021 18:55:53 +0200 Subject: [PATCH 06/17] stubgen: Don't annotate unknown argument and return types (#10626) ### Description Don't annotate unknown argument and return types with explicit `Any`. Also, fix formatting of functions that have an explicit type and default argument. ## Test Plan I adapted the existing tests. I also added tests to ensure that explicit `Any`s remain in the stub. I also added a test for the formatting fix. --- mypy/stubgen.py | 48 +++++++----- test-data/unit/stubgen.test | 150 ++++++++++++++++++------------------ 2 files changed, 100 insertions(+), 98 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index f1835aa9acd9..7028b8da04b6 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -85,7 +85,7 @@ from mypy.options import Options as MypyOptions from mypy.types import ( Type, TypeStrVisitor, CallableType, UnboundType, NoneType, TupleType, TypeList, Instance, - AnyType + AnyType, get_proper_type ) from mypy.visitor import NodeVisitor from mypy.find_sources import create_source_list, InvalidSourceList @@ -624,26 +624,24 @@ def visit_func_def(self, o: FuncDef, is_abstract: bool = False, # name their 0th argument other than self/cls is_self_arg = i == 0 and name == 'self' is_cls_arg = i == 0 and name == 'cls' - if (annotated_type is None - and not arg_.initializer - and not is_self_arg - and not is_cls_arg): - self.add_typing_import("Any") - annotation = ": {}".format(self.typing_name("Any")) - elif annotated_type and not is_self_arg and not is_cls_arg: - annotation = ": {}".format(self.print_annotation(annotated_type)) - else: - annotation = "" + annotation = "" + if annotated_type and not is_self_arg and not is_cls_arg: + # Luckily, an argument explicitly annotated with "Any" has + # type "UnboundType" and will not match. + if not isinstance(get_proper_type(annotated_type), AnyType): + annotation = ": {}".format(self.print_annotation(annotated_type)) if arg_.initializer: - initializer = '...' if kind in (ARG_NAMED, ARG_NAMED_OPT) and not any(arg.startswith('*') for arg in args): args.append('*') if not annotation: - typename = self.get_str_type_of_node(arg_.initializer, True) - annotation = ': {} = ...'.format(typename) + typename = self.get_str_type_of_node(arg_.initializer, True, False) + if typename == '': + annotation = '=...' + else: + annotation = ': {} = ...'.format(typename) else: - annotation += '={}'.format(initializer) + annotation += ' = ...' arg = name + annotation elif kind == ARG_STAR: arg = '*%s%s' % (name, annotation) @@ -654,12 +652,16 @@ def visit_func_def(self, o: FuncDef, is_abstract: bool = False, args.append(arg) retname = None if o.name != '__init__' and isinstance(o.unanalyzed_type, CallableType): - retname = self.print_annotation(o.unanalyzed_type.ret_type) + if isinstance(get_proper_type(o.unanalyzed_type.ret_type), AnyType): + # Luckily, a return type explicitly annotated with "Any" has + # type "UnboundType" and will enter the else branch. + retname = None # implicit Any + else: + retname = self.print_annotation(o.unanalyzed_type.ret_type) elif isinstance(o, FuncDef) and (o.is_abstract or o.name in METHODS_WITH_RETURN_VALUE): # Always assume abstract methods return Any unless explicitly annotated. Also # some dunder methods should not have a None return type. - retname = self.typing_name('Any') - self.add_typing_import("Any") + retname = None # implicit Any elif not has_return_statement(o) and not is_abstract: retname = 'None' retfield = '' @@ -1148,7 +1150,8 @@ def is_private_member(self, fullname: str) -> bool: return False def get_str_type_of_node(self, rvalue: Expression, - can_infer_optional: bool = False) -> str: + can_infer_optional: bool = False, + can_be_any: bool = True) -> str: if isinstance(rvalue, IntExpr): return 'int' if isinstance(rvalue, StrExpr): @@ -1165,8 +1168,11 @@ def get_str_type_of_node(self, rvalue: Expression, isinstance(rvalue, NameExpr) and rvalue.name == 'None': self.add_typing_import('Any') return '{} | None'.format(self.typing_name('Any')) - self.add_typing_import('Any') - return self.typing_name('Any') + if can_be_any: + self.add_typing_import('Any') + return self.typing_name('Any') + else: + return '' def print_annotation(self, t: Type) -> str: printer = AnnotationPrinter(self) diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index f2e102a5dcfe..daee45a84082 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -15,18 +15,14 @@ def f(a, b): def g(arg): pass [out] -from typing import Any - -def f(a: Any, b: Any) -> None: ... -def g(arg: Any) -> None: ... +def f(a, b) -> None: ... +def g(arg) -> None: ... [case testDefaultArgInt] def f(a, b=2): ... def g(b=-1, c=0): ... [out] -from typing import Any - -def f(a: Any, b: int = ...) -> None: ... +def f(a, b: int = ...) -> None: ... def g(b: int = ..., c: int = ...) -> None: ... [case testDefaultArgNone] @@ -59,14 +55,14 @@ def f(x: float = ...) -> None: ... [case testDefaultArgOther] def f(x=ord): ... [out] -from typing import Any - -def f(x: Any = ...) -> None: ... +def f(x=...) -> None: ... [case testPreserveFunctionAnnotation] def f(x: Foo) -> Bar: ... +def g(x: Foo = Foo()) -> Bar: ... [out] def f(x: Foo) -> Bar: ... +def g(x: Foo = ...) -> Bar: ... [case testPreserveVarAnnotation] x: Foo @@ -81,16 +77,12 @@ x: Foo [case testVarArgs] def f(x, *y): ... [out] -from typing import Any - -def f(x: Any, *y: Any) -> None: ... +def f(x, *y) -> None: ... [case testKwVarArgs] def f(x, **y): ... [out] -from typing import Any - -def f(x: Any, **y: Any) -> None: ... +def f(x, **y) -> None: ... [case testVarArgsWithKwVarArgs] def f(a, *b, **c): ... @@ -99,13 +91,11 @@ def h(a, *b, c=1, **d): ... def i(a, *, b=1): ... def j(a, *, b=1, **c): ... [out] -from typing import Any - -def f(a: Any, *b: Any, **c: Any) -> None: ... -def g(a: Any, *b: Any, c: int = ...) -> None: ... -def h(a: Any, *b: Any, c: int = ..., **d: Any) -> None: ... -def i(a: Any, *, b: int = ...) -> None: ... -def j(a: Any, *, b: int = ..., **c: Any) -> None: ... +def f(a, *b, **c) -> None: ... +def g(a, *b, c: int = ...) -> None: ... +def h(a, *b, c: int = ..., **d) -> None: ... +def i(a, *, b: int = ...) -> None: ... +def j(a, *, b: int = ..., **c) -> None: ... [case testClass] class A: @@ -113,10 +103,8 @@ class A: x = 1 def g(): ... [out] -from typing import Any - class A: - def f(self, x: Any) -> None: ... + def f(self, x) -> None: ... def g() -> None: ... @@ -263,9 +251,7 @@ class B(A): ... @decorator def foo(x): ... [out] -from typing import Any - -def foo(x: Any) -> None: ... +def foo(x) -> None: ... [case testMultipleAssignment] x, y = 1, 2 @@ -293,10 +279,8 @@ y: Any def f(x, *, y=1): ... def g(x, *, y=1, z=2): ... [out] -from typing import Any - -def f(x: Any, *, y: int = ...) -> None: ... -def g(x: Any, *, y: int = ..., z: int = ...) -> None: ... +def f(x, *, y: int = ...) -> None: ... +def g(x, *, y: int = ..., z: int = ...) -> None: ... [case testProperty] class A: @@ -309,13 +293,11 @@ class A: def h(self): self.f = 1 [out] -from typing import Any - class A: @property def f(self): ... @f.setter - def f(self, x: Any) -> None: ... + def f(self, x) -> None: ... def h(self) -> None: ... [case testStaticMethod] @@ -323,11 +305,9 @@ class A: @staticmethod def f(x): ... [out] -from typing import Any - class A: @staticmethod - def f(x: Any) -> None: ... + def f(x) -> None: ... [case testClassMethod] class A: @@ -390,10 +370,8 @@ class A: def __getstate__(self): ... def __setstate__(self, state): ... [out] -from typing import Any - class A: - def __eq__(self) -> Any: ... + def __eq__(self): ... -- Tests that will perform runtime imports of modules. -- Don't use `_import` suffix if there are unquoted forward references. @@ -774,17 +752,13 @@ class A(X): ... def syslog(a): pass def syslog(a): pass [out] -from typing import Any - -def syslog(a: Any) -> None: ... +def syslog(a) -> None: ... [case testAsyncAwait_fast_parser] async def f(a): x = await y [out] -from typing import Any - -async def f(a: Any) -> None: ... +async def f(a) -> None: ... [case testInferOptionalOnlyFunc] class A: @@ -1527,11 +1501,10 @@ class Base(metaclass=ABCMeta): import abc from abc import abstractmethod from base import Base -from typing import Any class C(Base, metaclass=abc.ABCMeta): @abstractmethod - def other(self) -> Any: ... + def other(self): ... [case testInvalidNumberOfArgsInAnnotation] def f(x): @@ -1539,9 +1512,7 @@ def f(x): return '' [out] -from typing import Any - -def f(x: Any): ... +def f(x): ... [case testFunctionPartiallyAnnotated] def f(x) -> None: @@ -1555,13 +1526,45 @@ class A: pass [out] +def f(x) -> None: ... +def g(x, y: str): ... + +class A: + def f(self, x) -> None: ... + +[case testExplicitAnyArg] from typing import Any -def f(x: Any) -> None: ... -def g(x: Any, y: str) -> Any: ... +def f(x: Any): + pass +def g(x, y: Any) -> str: + pass +def h(x: Any) -> str: + pass -class A: - def f(self, x: Any) -> None: ... +[out] +from typing import Any + +def f(x: Any): ... +def g(x, y: Any) -> str: ... +def h(x: Any) -> str: ... + +[case testExplicitReturnedAny] +from typing import Any + +def f(x: str) -> Any: + pass +def g(x, y: str) -> Any: + pass +def h(x) -> Any: + pass + +[out] +from typing import Any + +def f(x: str) -> Any: ... +def g(x, y: str) -> Any: ... +def h(x) -> Any: ... [case testPlacementOfDecorators] class A: @@ -1580,8 +1583,6 @@ class B: self.y = 'y' [out] -from typing import Any - class A: y: str @property @@ -1592,7 +1593,7 @@ class B: def x(self): ... y: str @x.setter - def x(self, value: Any) -> None: ... + def x(self, value) -> None: ... [case testMisplacedTypeComment] def f(): @@ -1666,12 +1667,11 @@ class A: [out] import abc -from typing import Any class A(metaclass=abc.ABCMeta): @property @abc.abstractmethod - def x(self) -> Any: ... + def x(self): ... [case testAbstractProperty2_semanal] import other @@ -1683,12 +1683,11 @@ class A: [out] import abc -from typing import Any class A(metaclass=abc.ABCMeta): @property @abc.abstractmethod - def x(self) -> Any: ... + def x(self): ... [case testAbstractProperty3_semanal] import other @@ -1700,16 +1699,14 @@ class A: [out] import abc -from typing import Any class A(metaclass=abc.ABCMeta): @property @abc.abstractmethod - def x(self) -> Any: ... + def x(self): ... [case testClassWithNameAnyOrOptional] -def f(x=object()): - return 1 +Y = object() def g(x=None): pass @@ -1724,7 +1721,8 @@ def Optional(): [out] from typing import Any as _Any -def f(x: _Any = ...): ... +Y: _Any + def g(x: _Any | None = ...) -> None: ... x: _Any @@ -1858,9 +1856,7 @@ def g() -> None: ... def f(x, y): pass [out] -from typing import Any - -def f(x: Any, y: Any) -> None: ... +def f(x, y) -> None: ... [case testImportedModuleExits_import] # modules: a b c @@ -2197,11 +2193,11 @@ from typing import Any class C: x: Any - def __init__(self, x: Any) -> None: ... - def __lt__(self, other: Any) -> Any: ... - def __le__(self, other: Any) -> Any: ... - def __gt__(self, other: Any) -> Any: ... - def __ge__(self, other: Any) -> Any: ... + def __init__(self, x) -> None: ... + def __lt__(self, other): ... + def __le__(self, other): ... + def __gt__(self, other): ... + def __ge__(self, other): ... [case testNamedTupleInClass] from collections import namedtuple From f6e4c9ed65d184fd2cd802e7051ec715ad38e2c6 Mon Sep 17 00:00:00 2001 From: Smart <47581948+hellocoldworld@users.noreply.github.com> Date: Thu, 17 Jun 2021 03:44:26 -0300 Subject: [PATCH 07/17] Update docs for deferral of PEP 563 to 3.11 (#10655) Resolves #10648 --- docs/source/runtime_troubles.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/runtime_troubles.rst b/docs/source/runtime_troubles.rst index 39558464cf55..34d9dc795f0d 100644 --- a/docs/source/runtime_troubles.rst +++ b/docs/source/runtime_troubles.rst @@ -9,7 +9,7 @@ and explains how to get your code running again. Generally speaking, we have three tools at our disposal: * For Python 3.7 through 3.9, use of ``from __future__ import annotations`` - (:pep:`563`), made the default in Python 3.10 and later + (:pep:`563`), made the default in Python 3.11 and later * Use of string literal types or type comments * Use of ``typing.TYPE_CHECKING`` @@ -47,7 +47,7 @@ Future annotations import (PEP 563) ----------------------------------- Many of the issues described here are caused by Python trying to evaluate -annotations. From Python 3.10 on, Python will no longer attempt to evaluate +annotations. From Python 3.11 on, Python will no longer attempt to evaluate function and variable annotations. This behaviour is made available in Python 3.7 and later through the use of ``from __future__ import annotations``. From 4905aadc641b4916895131d6cf294ab7b00c2402 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 16 Jun 2021 16:32:37 +0200 Subject: [PATCH 08/17] Don't suggest to install stubs with py.typed files (#10652) The pallets libraries are now bundled with a `py.typed` file. Suggesting to install these stubs only leads to confusion. See python/typeshed#5423 and python/typeshed#5642 for context. --- mypy/stubinfo.py | 8 -------- mypy/test/teststubinfo.py | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py index 8cc72d0199a7..ad6f378a717f 100644 --- a/mypy/stubinfo.py +++ b/mypy/stubinfo.py @@ -32,12 +32,10 @@ def is_legacy_bundled_package(prefix: str, py_version: int) -> bool: 'certifi': StubInfo('types-certifi'), 'characteristic': StubInfo('types-characteristic'), 'chardet': StubInfo('types-chardet'), - 'click': StubInfo('types-click'), 'click_spinner': StubInfo('types-click-spinner'), 'concurrent': StubInfo('types-futures', py_version=2), 'contextvars': StubInfo('types-contextvars', py_version=3), 'croniter': StubInfo('types-croniter'), - 'cryptography': StubInfo('types-cryptography'), 'dataclasses': StubInfo('types-dataclasses', py_version=3), 'dateparser': StubInfo('types-dateparser'), 'datetimerange': StubInfo('types-DateTimeRange'), @@ -50,19 +48,14 @@ def is_legacy_bundled_package(prefix: str, py_version: int) -> bool: 'fb303': StubInfo('types-fb303', py_version=2), 'filelock': StubInfo('types-filelock', py_version=3), 'first': StubInfo('types-first'), - 'flask': StubInfo('types-Flask'), 'freezegun': StubInfo('types-freezegun', py_version=3), 'frozendict': StubInfo('types-frozendict', py_version=3), 'geoip2': StubInfo('types-geoip2'), 'gflags': StubInfo('types-python-gflags'), 'google.protobuf': StubInfo('types-protobuf'), 'ipaddress': StubInfo('types-ipaddress', py_version=2), - 'itsdangerous': StubInfo('types-itsdangerous'), - 'jinja2': StubInfo('types-Jinja2'), - 'jwt': StubInfo('types-jwt'), 'kazoo': StubInfo('types-kazoo', py_version=2), 'markdown': StubInfo('types-Markdown'), - 'markupsafe': StubInfo('types-MarkupSafe'), 'maxminddb': StubInfo('types-maxminddb'), 'mock': StubInfo('types-mock'), 'OpenSSL': StubInfo('types-openssl-python', py_version=2), @@ -95,6 +88,5 @@ def is_legacy_bundled_package(prefix: str, py_version: int) -> bool: 'tzlocal': StubInfo('types-tzlocal'), 'ujson': StubInfo('types-ujson'), 'waitress': StubInfo('types-waitress', py_version=3), - 'werkzeug': StubInfo('types-Werkzeug'), 'yaml': StubInfo('types-PyYAML'), } diff --git a/mypy/test/teststubinfo.py b/mypy/test/teststubinfo.py index e05ba879aea2..62346ce2bc9f 100644 --- a/mypy/test/teststubinfo.py +++ b/mypy/test/teststubinfo.py @@ -8,8 +8,8 @@ def test_is_legacy_bundled_packages(self) -> None: assert not is_legacy_bundled_package('foobar_asdf', 2) assert not is_legacy_bundled_package('foobar_asdf', 3) - assert is_legacy_bundled_package('click', 2) - assert is_legacy_bundled_package('click', 3) + assert is_legacy_bundled_package('certifi', 2) + assert is_legacy_bundled_package('certifi', 3) assert is_legacy_bundled_package('scribe', 2) assert not is_legacy_bundled_package('scribe', 3) From 0b6586624fdcf07c780cf9216c002623cbfb651c Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 16 Jun 2021 20:58:31 +0200 Subject: [PATCH 09/17] pyopenssl types package name was corrected (#10656) It's now also available for Python 3 --- mypy/stubinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py index ad6f378a717f..afa542b5c681 100644 --- a/mypy/stubinfo.py +++ b/mypy/stubinfo.py @@ -58,7 +58,7 @@ def is_legacy_bundled_package(prefix: str, py_version: int) -> bool: 'markdown': StubInfo('types-Markdown'), 'maxminddb': StubInfo('types-maxminddb'), 'mock': StubInfo('types-mock'), - 'OpenSSL': StubInfo('types-openssl-python', py_version=2), + 'OpenSSL': StubInfo('types-pyOpenSSL'), 'orjson': StubInfo('types-orjson', py_version=3), 'paramiko': StubInfo('types-paramiko'), 'pathlib2': StubInfo('types-pathlib2', py_version=2), From f5a340519e34b0c8781d7164613891ee658123b6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 13:00:58 +0100 Subject: [PATCH 10/17] Improve error reporting when --install-types has no cache (#10667) There are no tests since our test framework makes it tricky to write tests for these kinds of scenarios. Tested manually. Work on #10600. --- mypy/main.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 693c7e8c6d13..735d9c0778e8 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -77,6 +77,10 @@ def main(script_path: Optional[str], if options.non_interactive and not options.install_types: fail("Error: --non-interactive is only supported with --install-types", stderr, options) + if options.install_types and not options.incremental: + fail("Error: --install-types not supported with incremental mode disabled", + stderr, options) + if options.install_types and not sources: install_types(options.cache_dir, formatter, non_interactive=options.non_interactive) return @@ -1090,8 +1094,15 @@ def install_types(cache_dir: str, non_interactive: bool = False) -> None: """Install stub packages using pip if some missing stubs were detected.""" if not os.path.isdir(cache_dir): - sys.stderr.write( - "Error: no mypy cache directory (you must enable incremental mode)\n") + if not after_run: + sys.stderr.write( + "Error: Can't determine which types to install with no files to check " + + "(and no cache from previous mypy run)\n" + ) + else: + sys.stderr.write( + "Error: --install-types failed (no mypy cache directory)\n" + ) sys.exit(2) fnam = build.missing_stubs_file(cache_dir) if not os.path.isfile(fnam): From e8cf526913b81a9c3cc20a297f3d5915185c46f0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 16:58:07 +0100 Subject: [PATCH 11/17] Run build a second time when using --install-types --non-interactive (#10669) If the first build finds missing stub packages, run the build a second time after installing types. Only show errors from the final build. Example output: ``` $ mypy --install-types --non-interactive t.py Installing missing stub packages: /Users/jukka/venv/mypy/bin/python3 -m pip install types-redis Collecting types-redis Using cached types_redis-3.5.2-py2.py3-none-any.whl (11 kB) Installing collected packages: types-redis Successfully installed types-redis-3.5.2 t.py:2: error: Unsupported operand types for + ("int" and "str") Found 1 error in 1 file (checked 1 source file) ``` Work on #10600. --- mypy/main.py | 150 +++++++++++++++++++++++------------- test-data/unit/cmdline.test | 16 +++- 2 files changed, 111 insertions(+), 55 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 735d9c0778e8..da4eda6e04a0 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -67,59 +67,34 @@ def main(script_path: Optional[str], sources, options = process_options(args, stdout=stdout, stderr=stderr, fscache=fscache) - messages = [] formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): # Since --install-types performs user input, we want regular stdout and stderr. - fail("Error: --install-types not supported in this mode of running mypy", stderr, options) + fail("error: --install-types not supported in this mode of running mypy", stderr, options) if options.non_interactive and not options.install_types: - fail("Error: --non-interactive is only supported with --install-types", stderr, options) + fail("error: --non-interactive is only supported with --install-types", stderr, options) if options.install_types and not options.incremental: - fail("Error: --install-types not supported with incremental mode disabled", + fail("error: --install-types not supported with incremental mode disabled", stderr, options) if options.install_types and not sources: install_types(options.cache_dir, formatter, non_interactive=options.non_interactive) return - def flush_errors(new_messages: List[str], serious: bool) -> None: - if options.non_interactive: - return - if options.pretty: - new_messages = formatter.fit_in_terminal(new_messages) - messages.extend(new_messages) - f = stderr if serious else stdout - for msg in new_messages: - if options.color_output: - msg = formatter.colorize(msg) - f.write(msg + '\n') - f.flush() + res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) - serious = False - blockers = False - res = None - try: - # Keep a dummy reference (res) for memory profiling below, as otherwise - # the result could be freed. - res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) - except CompileError as e: - blockers = True - if not e.use_stdout: - serious = True - if (options.warn_unused_configs - and options.unused_configs - and not options.incremental - and not options.non_interactive): - print("Warning: unused section(s) in %s: %s" % - (options.config_file, - get_config_module_names(options.config_file, - [glob for glob in options.per_module_options.keys() - if glob in options.unused_configs])), - file=stderr) - maybe_write_junit_xml(time.time() - t0, serious, messages, options) + if options.non_interactive: + missing_pkgs = read_types_packages_to_install(options.cache_dir, after_run=True) + if missing_pkgs: + # Install missing type packages and rerun build. + install_types(options.cache_dir, formatter, after_run=True, non_interactive=True) + fscache.flush() + print() + res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) + show_messages(messages, stderr, formatter, options) if MEM_PROFILE: from mypy.memprofile import print_memory_profile @@ -128,7 +103,7 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: code = 0 if messages: code = 2 if blockers else 1 - if options.error_summary and not options.non_interactive: + if options.error_summary: if messages: n_errors, n_files = util.count_stats(messages) if n_errors: @@ -141,10 +116,13 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: stdout.write(formatter.format_success(len(sources), options.color_output) + '\n') stdout.flush() - if options.install_types: - install_types(options.cache_dir, formatter, after_run=True, - non_interactive=options.non_interactive) - return + if options.install_types and not options.non_interactive: + result = install_types(options.cache_dir, formatter, after_run=True, + non_interactive=False) + if result: + print() + print("note: Run mypy again for up-to-date results with installed types") + code = 2 if options.fast_exit: # Exit without freeing objects -- it's faster. @@ -158,6 +136,62 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: list([res]) +def run_build(sources: List[BuildSource], + options: Options, + fscache: FileSystemCache, + t0: float, + stdout: TextIO, + stderr: TextIO) -> Tuple[Optional[build.BuildResult], List[str], bool]: + formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) + + messages = [] + + def flush_errors(new_messages: List[str], serious: bool) -> None: + if options.pretty: + new_messages = formatter.fit_in_terminal(new_messages) + messages.extend(new_messages) + if options.non_interactive: + # Collect messages and possibly show them later. + return + f = stderr if serious else stdout + show_messages(new_messages, f, formatter, options) + + serious = False + blockers = False + res = None + try: + # Keep a dummy reference (res) for memory profiling afterwards, as otherwise + # the result could be freed. + res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) + except CompileError as e: + blockers = True + if not e.use_stdout: + serious = True + if (options.warn_unused_configs + and options.unused_configs + and not options.incremental + and not options.non_interactive): + print("Warning: unused section(s) in %s: %s" % + (options.config_file, + get_config_module_names(options.config_file, + [glob for glob in options.per_module_options.keys() + if glob in options.unused_configs])), + file=stderr) + maybe_write_junit_xml(time.time() - t0, serious, messages, options) + return res, messages, blockers + + +def show_messages(messages: List[str], + f: TextIO, + formatter: util.FancyFormatter, + options: Options) -> None: + for msg in messages: + if options.color_output: + msg = formatter.colorize(msg) + f.write(msg + '\n') + f.flush() + + # Make the help output a little less jarring. class AugmentedHelpFormatter(argparse.RawDescriptionHelpFormatter): def __init__(self, prog: str) -> None: @@ -1087,29 +1121,36 @@ def fail(msg: str, stderr: TextIO, options: Options) -> None: sys.exit(2) -def install_types(cache_dir: str, - formatter: util.FancyFormatter, - *, - after_run: bool = False, - non_interactive: bool = False) -> None: - """Install stub packages using pip if some missing stubs were detected.""" +def read_types_packages_to_install(cache_dir: str, after_run: bool) -> List[str]: if not os.path.isdir(cache_dir): if not after_run: sys.stderr.write( - "Error: Can't determine which types to install with no files to check " + + "error: Can't determine which types to install with no files to check " + "(and no cache from previous mypy run)\n" ) else: sys.stderr.write( - "Error: --install-types failed (no mypy cache directory)\n" + "error: --install-types failed (no mypy cache directory)\n" ) sys.exit(2) fnam = build.missing_stubs_file(cache_dir) if not os.path.isfile(fnam): - # If there are no missing stubs, generate no output. - return + # No missing stubs. + return [] with open(fnam) as f: - packages = [line.strip() for line in f.readlines()] + return [line.strip() for line in f.readlines()] + + +def install_types(cache_dir: str, + formatter: util.FancyFormatter, + *, + after_run: bool = False, + non_interactive: bool = False) -> bool: + """Install stub packages using pip if some missing stubs were detected.""" + packages = read_types_packages_to_install(cache_dir, after_run) + if not packages: + # If there are no missing stubs, generate no output. + return False if after_run and not non_interactive: print() print('Installing missing stub packages:') @@ -1123,3 +1164,4 @@ def install_types(cache_dir: str, sys.exit(2) print() subprocess.run(cmd) + return True diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index e469c48a48bf..92ef7e0690ed 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1280,7 +1280,7 @@ pkg.py:1: error: Incompatible types in assignment (expression has type "int", va [case testCmdlineNonInteractiveWithoutInstallTypes] # cmd: mypy --non-interactive -m pkg [out] -Error: --non-interactive is only supported with --install-types +error: --non-interactive is only supported with --install-types == Return code: 2 [case testCmdlineNonInteractiveInstallTypesNothingToDo] @@ -1288,3 +1288,17 @@ Error: --non-interactive is only supported with --install-types [file pkg.py] 1() [out] +pkg.py:1: error: "int" not callable + +[case testCmdlineNonInteractiveInstallTypesNothingToDoNoError] +# cmd: mypy --install-types --non-interactive -m pkg +[file pkg.py] +1 + 2 +[out] + +[case testCmdlineInteractiveInstallTypesNothingToDo] +# cmd: mypy --install-types -m pkg +[file pkg.py] +1() +[out] +pkg.py:1: error: "int" not callable From 64deb994f3671663f56d99c4dde1239cc8df03c3 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Mon, 21 Jun 2021 17:45:59 +0200 Subject: [PATCH 12/17] pkg_resources is now in types-setuptools (#10681) Depends on python/typeshed#5669 --- mypy/stubinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py index afa542b5c681..9945a12c121f 100644 --- a/mypy/stubinfo.py +++ b/mypy/stubinfo.py @@ -62,7 +62,7 @@ def is_legacy_bundled_package(prefix: str, py_version: int) -> bool: 'orjson': StubInfo('types-orjson', py_version=3), 'paramiko': StubInfo('types-paramiko'), 'pathlib2': StubInfo('types-pathlib2', py_version=2), - 'pkg_resources': StubInfo('types-pkg_resources', py_version=3), + 'pkg_resources': StubInfo('types-setuptools', py_version=3), 'polib': StubInfo('types-polib'), 'pycurl': StubInfo('types-pycurl'), 'pymssql': StubInfo('types-pymssql', py_version=2), From 680fded0a1c70686be28f6d17ee47f59367ff4f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 21 Jun 2021 18:49:23 +0100 Subject: [PATCH 13/17] Document --install-types --non-interactive (#10684) --- docs/source/command_line.rst | 14 ++++++++++++++ docs/source/running_mypy.rst | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 69d1a7b80fde..2d82d2b83517 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -903,6 +903,20 @@ Miscellaneous selection of third-party package stubs, instead of having them installed separately. +.. option:: --non-interactive + + When used together with :option:`--install-types `, this causes mypy to install all suggested stub + packages using pip without asking for confirmation, and then + continues to perform type checking using the installed stubs, if + some files or modules are provided to type check. + + This is implemented as up to two mypy runs internally. The first run + is used to find missing stub packages, and output is shown from + this run only if no missing stub packages were found. If missing + stub packages were found, they are installed and then another run + is performed. + .. option:: --junit-xml JUNIT_XML Causes mypy to generate a JUnit XML test result document with diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index 8a2fbc2df599..d4061fcdc103 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -182,10 +182,21 @@ mypy run. You can also use your normal mypy command line with the extra :option:`--install-types ` option to install missing stubs at the end of the run (if any were found). -You can also get this message if the stubs only support Python 3 and -your target Python version is Python 2, or vice versa. In this case -follow instructions in -:ref:`missing-type-hints-for-third-party-library`. +Use :option:`--install-types ` with +:option:`--non-interactive ` to install all suggested +stub packages without asking for confirmation, *and* type check your +code, in a single command: + +.. code-block:: text + + mypy --install-types --non-interactive src/ + +This can be useful in Continuous Integration jobs if you'd prefer not +to manage stub packages manually. This is somewhat slower than +explicitly installing stubs before running mypy, since it may type +check your code twice -- the first time to find the missing stubs, and +the second time to type check your code properly after mypy has +installed the stubs. .. _missing-type-hints-for-third-party-library: From 9637f992fbd9f0d8e5ae4a0fed91c6aa3fdb10f2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 21 Jun 2021 18:49:37 +0100 Subject: [PATCH 14/17] Fix crash with assignment to variable guarded with TypeGuard (#10683) This is a quick fix to unblock the 0.910 release. The type guard type can be unrelated to the original type, so we shouldn't join it. I'm not sure whether this is right from first principles, but it seems to address the issue. Fixes #10671. --- mypy/binder.py | 6 ++-- test-data/unit/check-typeguard.test | 49 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index c1b6862c9e6d..394c5ccf987c 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -5,7 +5,7 @@ from typing_extensions import DefaultDict from mypy.types import ( - Type, AnyType, PartialType, UnionType, TypeOfAny, NoneType, get_proper_type + Type, AnyType, PartialType, UnionType, TypeOfAny, NoneType, TypeGuardType, get_proper_type ) from mypy.subtypes import is_subtype from mypy.join import join_simple @@ -202,7 +202,9 @@ def update_from_options(self, frames: List[Frame]) -> bool: else: for other in resulting_values[1:]: assert other is not None - type = join_simple(self.declarations[key], type, other) + # Ignore the error about using get_proper_type(). + if not isinstance(other, TypeGuardType): # type: ignore[misc] + type = join_simple(self.declarations[key], type, other) if current_value is None or not is_same_type(type, current_value): self._put(key, type) changed = True diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index fa340cb04044..c19e3c812e29 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -82,6 +82,7 @@ def is_str_list(a: List[object]) -> TypeGuard[List[str]]: pass def main(a: List[object]): if is_str_list(a): reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(a) # N: Revealed type is "builtins.list[builtins.object]" [builtins fixtures/tuple.pyi] [case testTypeGuardUnionIn] @@ -91,6 +92,7 @@ def is_foo(a: Union[int, str]) -> TypeGuard[str]: pass def main(a: Union[str, int]) -> None: if is_foo(a): reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(a) # N: Revealed type is "Union[builtins.str, builtins.int]" [builtins fixtures/tuple.pyi] [case testTypeGuardUnionOut] @@ -315,3 +317,50 @@ def coverage(obj: Any) -> bool: return True return False [builtins fixtures/classmethod.pyi] + +[case testAssignToTypeGuardedVariable1] +from typing_extensions import TypeGuard + +class A: pass +class B(A): pass + +def guard(a: A) -> TypeGuard[B]: + pass + +a = A() +if not guard(a): + a = A() +[builtins fixtures/tuple.pyi] + +[case testAssignToTypeGuardedVariable2] +from typing_extensions import TypeGuard + +class A: pass +class B: pass + +def guard(a: A) -> TypeGuard[B]: + pass + +a = A() +if not guard(a): + a = A() +[builtins fixtures/tuple.pyi] + +[case testAssignToTypeGuardedVariable3] +from typing_extensions import TypeGuard + +class A: pass +class B: pass + +def guard(a: A) -> TypeGuard[B]: + pass + +a = A() +if guard(a): + reveal_type(a) # N: Revealed type is "__main__.B" + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(a) # N: Revealed type is "__main__.B" + a = A() + reveal_type(a) # N: Revealed type is "__main__.A" +reveal_type(a) # N: Revealed type is "__main__.A" +[builtins fixtures/tuple.pyi] From 96366d1d8417413019d688211e68bff3f9e92f2f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 22 Jun 2021 13:07:03 +0100 Subject: [PATCH 15/17] Don't ask to install a stub package if stubs are installed (#10670) If we encounter an import of a submodule of a package with installed stubs, and the submodule doesn't exist, don't ask to install stubs since that's not going to help. Also make it possible to ignore this error using `--ignore-missing-imports`. Work on #10645. --- mypy/build.py | 5 +++-- mypy/modulefinder.py | 10 +++++++--- test-data/unit/check-modules.test | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 59d46cdf9bb8..a39b78798d98 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2447,13 +2447,14 @@ def find_module_and_diagnose(manager: BuildManager, # Don't honor a global (not per-module) ignore_missing_imports # setting for modules that used to have bundled stubs, as # otherwise updating mypy can silently result in new false - # negatives. + # negatives. (Unless there are stubs but they are incomplete.) global_ignore_missing_imports = manager.options.ignore_missing_imports py_ver = options.python_version[0] if ((is_legacy_bundled_package(top_level, py_ver) or is_legacy_bundled_package(second_level, py_ver)) and global_ignore_missing_imports - and not options.ignore_missing_imports_per_module): + and not options.ignore_missing_imports_per_module + and result is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED): ignore_missing_imports = False if skip_diagnose: diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 4e608097289b..eb763652175e 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -230,10 +230,14 @@ def _find_module_non_stub_helper(self, components: List[str], elif not plausible_match and (self.fscache.isdir(dir_path) or self.fscache.isfile(dir_path + ".py")): plausible_match = True - if (is_legacy_bundled_package(components[0], self.python_major_ver) - or is_legacy_bundled_package('.'.join(components[:2]), self.python_major_ver)): + if is_legacy_bundled_package(components[0], self.python_major_ver): + if (len(components) == 1 + or (self.find_module(components[0]) is + ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED)): + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + if is_legacy_bundled_package('.'.join(components[:2]), self.python_major_ver): return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED - elif plausible_match: + if plausible_match: return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS else: return ModuleNotFoundReason.NOT_FOUND diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 9a68b00b6007..ad7322a2573f 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -3107,3 +3107,18 @@ from google.cloud import x main:1: error: Cannot find implementation or library stub for module named "google.cloud" main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports main:1: error: Cannot find implementation or library stub for module named "google" + +[case testMissingSubmoduleOfInstalledStubPackage] +import bleach.xyz +from bleach.abc import fgh +[file bleach/__init__.pyi] +[out] +main:1: error: Cannot find implementation or library stub for module named "bleach.xyz" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "bleach.abc" + +[case testMissingSubmoduleOfInstalledStubPackageIgnored] +# flags: --ignore-missing-imports +import bleach.xyz +from bleach.abc import fgh +[file bleach/__init__.pyi] From 46ce325c9ac613985ce96224e45f5c90e5cfd975 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 22 Jun 2021 13:38:34 +0100 Subject: [PATCH 16/17] Fix crash when inferring multiple assignment with overloaded function (#10689) When using lvalue context to re-infer call to an overloaded function, the inferred tuple type can switch to Any. Defensively accept this. It probably means that an Any component in argument types causes ambiguity. Fixes #10653. --- mypy/checker.py | 9 ++++-- test-data/unit/check-async-await.test | 41 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 556f0ee13104..cd489a2909d6 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2695,8 +2695,13 @@ def check_multi_assignment_from_tuple(self, lvalues: List[Lvalue], rvalue: Expre reinferred_rvalue_type, context, infer_lvalue_type) return - if isinstance(reinferred_rvalue_type, AnyType) and self.current_node_deferred: - # Doing more inference in deferred nodes can be hard, so give up for now. + if isinstance(reinferred_rvalue_type, AnyType): + # We can get Any if the current node is + # deferred. Doing more inference in deferred nodes + # is hard, so give up for now. We can also get + # here if reinferring types above changes the + # inferred return type for an overloaded function + # to be ambiguous. return assert isinstance(reinferred_rvalue_type, TupleType) rvalue_type = reinferred_rvalue_type diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index e6e5f4be8094..f4bf7c7cfa94 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -758,3 +758,44 @@ class Foo(Generic[T]): [builtins fixtures/async_await.pyi] [typing fixtures/typing-async.pyi] + +[case testAwaitOverloadSpecialCase] +from typing import Any, Awaitable, Iterable, overload, Tuple, List, TypeVar, Generic + +T = TypeVar("T") +FT = TypeVar("FT", bound='Future[Any]') + +class Future(Awaitable[T], Iterable[T]): + pass + +class Task(Future[T]): + pass + +@overload +def wait(fs: Iterable[FT]) -> Future[Tuple[List[FT], List[FT]]]: ... \ + # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def wait(fs: Iterable[Awaitable[T]]) -> Future[Tuple[List[Task[T]], List[Task[T]]]]: ... +def wait(fs: Any) -> Any: + pass + +async def imprecise1(futures: Iterable[Task[Any]]) -> None: + done: Any + pending: Any + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "Any" + +async def imprecise2(futures: Iterable[Awaitable[Any]]) -> None: + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "builtins.list[__main__.Task[Any]]" + +async def precise1(futures: Iterable[Future[int]]) -> None: + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "builtins.list[__main__.Future[builtins.int]]" + +async def precise2(futures: Iterable[Awaitable[int]]) -> None: + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "builtins.list[__main__.Task[builtins.int]]" + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] From f5fc579cf07f2078c9312044f6bcb132f891d746 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 22 Jun 2021 16:57:07 +0100 Subject: [PATCH 17/17] Bump version to 0.910 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 489dd0a5460a..16150dd9fb06 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -5,7 +5,7 @@ # - Release versions have the form "0.NNN". # - Dev versions have the form "0.NNN+dev" (PLUS sign to conform to PEP 440). # - For 1.0 we'll switch back to 1.2.3 form. -__version__ = '0.902' +__version__ = '0.910' base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))