From beb6bac89a1efd23e5e8e355197ed10168666a1d Mon Sep 17 00:00:00 2001
From: Michael Sullivan <sully@msully.net>
Date: Mon, 8 Jan 2018 14:20:37 -0800
Subject: [PATCH 1/3] Exit with code 2 on blocking errors

Exit with code 2 on blocking errors, while continuing to exit with 1
when there are nonblocking errors.  Add test infrastructure.

Fixes #2754.
---
 mypy/main.py                |  7 ++-
 mypy/test/testcmdline.py    |  2 +
 test-data/unit/cmdline.test | 92 +++++++++++++++++++++++++++++++++++++
 test-data/unit/reports.test |  5 ++
 4 files changed, 104 insertions(+), 2 deletions(-)

diff --git a/mypy/main.py b/mypy/main.py
index 35279c820253..8c072d48d7f9 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -72,12 +72,14 @@ def flush_errors(a: List[str], serious: bool) -> None:
                 f.write(m + '\n')
             f.flush()
         except BrokenPipeError:
-            sys.exit(1)
+            sys.exit(2)
 
     serious = False
+    blockers = False
     try:
         type_check_only(sources, bin_dir, options, flush_errors)
     except CompileError as e:
+        blockers = True
         if not e.use_stdout:
             serious = True
     if options.warn_unused_configs and options.unused_configs:
@@ -89,7 +91,8 @@ def flush_errors(a: List[str], serious: bool) -> None:
         t1 = time.time()
         util.write_junit_xml(t1 - t0, serious, messages, options.junit_xml)
     if messages:
-        sys.exit(1)
+        code = 2 if blockers else 1
+        sys.exit(code)
 
 
 def find_bin_directory(script_path: str) -> str:
diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py
index afdee5583968..d861a538f79c 100644
--- a/mypy/test/testcmdline.py
+++ b/mypy/test/testcmdline.py
@@ -58,6 +58,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
     outb = process.stdout.read()
     # Split output into lines.
     out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
+    result = process.wait()
     # Remove temp file.
     os.remove(program_path)
     # Compare actual output to expected.
@@ -78,6 +79,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
                                            path))
     else:
         out = normalize_error_messages(out)
+        out.append('== Return code: {}'.format(result))
         assert_string_arrays_equal(testcase.output, out,
                                    'Invalid output ({}, line {})'.format(
                                        testcase.file, testcase.line))
diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
index 27d57cd7f449..ebf2fafc5395 100644
--- a/test-data/unit/cmdline.test
+++ b/test-data/unit/cmdline.test
@@ -21,6 +21,7 @@ import pkg.subpkg.a
 [out]
 pkg/a.py:1: error: Name 'undef' is not defined
 pkg/subpkg/a.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testCmdlinePackageSlash]
 # cmd: mypy pkg/
@@ -34,6 +35,7 @@ import pkg.subpkg.a
 [out]
 pkg/a.py:1: error: Name 'undef' is not defined
 pkg/subpkg/a.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testCmdlineNonPackage]
 # cmd: mypy dir
@@ -43,6 +45,7 @@ undef
 undef
 [out]
 dir/a.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testCmdlineNonPackageSlash]
 # cmd: mypy dir/
@@ -52,6 +55,7 @@ undef
 undef
 [out]
 dir/a.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testCmdlinePackageContainingSubdir]
 # cmd: mypy pkg
@@ -62,6 +66,7 @@ undef
 undef
 [out]
 pkg/a.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testCmdlineNonPackageContainingPackage]
 # cmd: mypy dir
@@ -74,6 +79,7 @@ undef
 [out]
 dir/subpkg/a.py:1: error: Name 'undef' is not defined
 dir/a.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testCmdlineInvalidPackageName]
 # cmd: mypy dir/sub.pkg/a.py
@@ -82,6 +88,7 @@ dir/a.py:1: error: Name 'undef' is not defined
 undef
 [out]
 sub.pkg is not a valid Python package name
+== Return code: 1
 
 [case testBadFileEncoding]
 # cmd: mypy a.py
@@ -89,6 +96,7 @@ sub.pkg is not a valid Python package name
 # coding: uft-8
 [out]
 mypy: can't decode file 'a.py': unknown encoding: uft-8
+== Return code: 2
 
 [case testCannotIgnoreDuplicateModule]
 # cmd: mypy one/mod/__init__.py two/mod/__init__.py
@@ -98,6 +106,7 @@ mypy: can't decode file 'a.py': unknown encoding: uft-8
 # type: ignore
 [out]
 two/mod/__init__.py: error: Duplicate module named 'mod'
+== Return code: 2
 
 [case testFlagsFile]
 # cmd: mypy @flagsfile
@@ -110,6 +119,8 @@ def f():
         1/0
     except ZeroDivisionError, err:
         print err
+[out]
+== Return code: 0
 
 [case testConfigFile]
 # cmd: mypy main.py
@@ -122,6 +133,8 @@ def f():
         1/0
     except ZeroDivisionError, err:
         print err
+[out]
+== Return code: 0
 
 [case testErrorContextConfig]
 # cmd: mypy main.py
@@ -134,6 +147,7 @@ def f() -> None:
 [out]
 main.py: note: In function "f":
 main.py:2: error: Unsupported operand types for + ("int" and "str")
+== Return code: 1
 
 [case testAltConfigFile]
 # cmd: mypy --config-file config.ini main.py
@@ -146,6 +160,8 @@ def f():
         1/0
     except ZeroDivisionError, err:
         print err
+[out]
+== Return code: 0
 
 [case testPerFileConfigSection]
 # cmd: mypy x.py y.py z.py
@@ -175,6 +191,7 @@ def g(a: int) -> int:
 z.py:1: error: Function is missing a type annotation
 z.py:4: error: Call to untyped function "f" in typed context
 x.py:1: error: Function is missing a type annotation
+== Return code: 1
 
 [case testPerFileConfigSectionMultipleMatches]
 # cmd: mypy xx.py xy.py yx.py yy.py
@@ -203,6 +220,7 @@ yx.py:2: error: Call to untyped function "f" in typed context
 xy.py:1: error: Function is missing a type annotation
 xy.py:2: error: Call to untyped function "f" in typed context
 xx.py:1: error: Function is missing a type annotation
+== Return code: 1
 
 [case testMultipleGlobConfigSection]
 # cmd: mypy x.py y.py z.py
@@ -219,12 +237,14 @@ def f(a): pass
 [out]
 z.py:1: error: Function is missing a type annotation
 x.py:1: error: Function is missing a type annotation
+== Return code: 1
 
 [case testConfigErrorNoSection]
 # cmd: mypy -c pass
 [file mypy.ini]
 [out]
 mypy.ini: No [mypy] section in config file
+== Return code: 0
 
 [case testConfigErrorUnknownFlag]
 # cmd: mypy -c pass
@@ -233,6 +253,7 @@ mypy.ini: No [mypy] section in config file
 bad = 0
 [out]
 mypy.ini: [mypy]: Unrecognized option: bad = 0
+== Return code: 0
 
 [case testConfigErrorBadBoolean]
 # cmd: mypy -c pass
@@ -241,6 +262,7 @@ mypy.ini: [mypy]: Unrecognized option: bad = 0
 ignore_missing_imports = nah
 [out]
 mypy.ini: [mypy]: ignore_missing_imports: Not a boolean: nah
+== Return code: 0
 
 [case testConfigErrorNotPerFile]
 # cmd: mypy -c pass
@@ -250,6 +272,7 @@ mypy.ini: [mypy]: ignore_missing_imports: Not a boolean: nah
 python_version = 3.4
 [out]
 mypy.ini: [mypy-*]: Per-module sections should only specify per-module flags (python_version)
+== Return code: 0
 
 [case testConfigMypyPath]
 # cmd: mypy file.py
@@ -275,6 +298,7 @@ baz(bar(foo('oof')))
 file.py:1: error: Cannot find module named 'no_stubs'
 file.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 file.py:6: error: Argument 1 to "foo" has incompatible type "str"; expected "int"
+== Return code: 1
 
 [case testIgnoreErrorsConfig]
 # cmd: mypy x.py y.py
@@ -288,6 +312,7 @@ ignore_errors = True
 "" + 0
 [out]
 y.py:1: error: Unsupported operand types for + ("str" and "int")
+== Return code: 1
 
 [case testConfigFollowImportsNormal]
 # cmd: mypy main.py
@@ -312,6 +337,7 @@ main.py:3: error: Unsupported operand types for + ("int" and "str")
 main.py:6: error: Unsupported operand types for + ("int" and "str")
 main.py:7: error: Module has no attribute "y"
 main.py:8: error: Unsupported operand types for + (Module and "int")
+== Return code: 1
 
 [case testConfigFollowImportsSilent]
 # cmd: mypy main.py
@@ -333,6 +359,7 @@ main.py:2: error: Unsupported operand types for + ("int" and "str")
 main.py:4: error: Unsupported operand types for + ("int" and "str")
 main.py:5: error: Module has no attribute "y"
 main.py:6: error: Unsupported operand types for + (Module and "int")
+== Return code: 1
 
 [case testConfigFollowImportsSkip]
 # cmd: mypy main.py
@@ -349,6 +376,7 @@ follow_imports = skip
 [out]
 main.py:2: error: Revealed type is 'Any'
 main.py:4: error: Revealed type is 'Any'
+== Return code: 1
 
 [case testConfigFollowImportsError]
 # cmd: mypy main.py
@@ -367,6 +395,7 @@ main.py:1: note: Import of 'a' ignored
 main.py:1: note: (Using --follow-imports=error, module not passed on command line)
 main.py:2: error: Revealed type is 'Any'
 main.py:4: error: Revealed type is 'Any'
+== Return code: 1
 
 [case testConfigFollowImportsSelective]
 # cmd: mypy main.py
@@ -407,6 +436,7 @@ main.py:5: error: Revealed type is 'builtins.int'
 main.py:6: error: Revealed type is 'builtins.int'
 main.py:7: error: Revealed type is 'Any'
 main.py:8: error: Revealed type is 'Any'
+== Return code: 1
 
 [case testConfigSilentMissingImportsOff]
 # cmd: mypy main.py
@@ -420,6 +450,7 @@ ignore_missing_imports = False
 main.py:1: error: Cannot find module named 'missing'
 main.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 main.py:2: error: Revealed type is 'Any'
+== Return code: 1
 
 [case testConfigSilentMissingImportsOn]
 # cmd: mypy main.py
@@ -431,6 +462,7 @@ reveal_type(missing.x)  # Expect Any
 ignore_missing_imports = True
 [out]
 main.py:2: error: Revealed type is 'Any'
+== Return code: 1
 
 [case testDotInFilenameOKScript]
 # cmd: mypy a.b.py c.d.pyi
@@ -441,6 +473,7 @@ whatever
 [out]
 c.d.pyi:1: error: Name 'whatever' is not defined
 a.b.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testDotInFilenameOKFolder]
 # cmd: mypy my.folder
@@ -448,6 +481,7 @@ a.b.py:1: error: Name 'undef' is not defined
 undef
 [out]
 my.folder/tst.py:1: error: Name 'undef' is not defined
+== Return code: 1
 
 [case testDotInFilenameNoImport]
 # cmd: mypy main.py
@@ -459,6 +493,7 @@ whatever
 main.py:1: error: Cannot find module named 'a'
 main.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 main.py:1: error: Cannot find module named 'a.b'
+== Return code: 1
 
 [case testPythonVersionTooOld10]
 # cmd: mypy -c pass
@@ -467,6 +502,7 @@ main.py:1: error: Cannot find module named 'a.b'
 python_version = 1.0
 [out]
 mypy.ini: [mypy]: python_version: Python major version '1' out of range (must be 2 or 3)
+== Return code: 0
 
 [case testPythonVersionTooOld26]
 # cmd: mypy -c pass
@@ -475,6 +511,7 @@ mypy.ini: [mypy]: python_version: Python major version '1' out of range (must be
 python_version = 2.6
 [out]
 mypy.ini: [mypy]: python_version: Python 2.6 is not supported (must be 2.7)
+== Return code: 0
 
 [case testPythonVersionTooOld32]
 # cmd: mypy -c pass
@@ -483,6 +520,7 @@ mypy.ini: [mypy]: python_version: Python 2.6 is not supported (must be 2.7)
 python_version = 3.2
 [out]
 mypy.ini: [mypy]: python_version: Python 3.2 is not supported (must be 3.3 or higher)
+== Return code: 0
 
 [case testPythonVersionTooNew28]
 # cmd: mypy -c pass
@@ -491,6 +529,7 @@ mypy.ini: [mypy]: python_version: Python 3.2 is not supported (must be 3.3 or hi
 python_version = 2.8
 [out]
 mypy.ini: [mypy]: python_version: Python 2.8 is not supported (must be 2.7)
+== Return code: 0
 
 [case testPythonVersionTooNew40]
 # cmd: mypy -c pass
@@ -499,6 +538,7 @@ mypy.ini: [mypy]: python_version: Python 2.8 is not supported (must be 2.7)
 python_version = 4.0
 [out]
 mypy.ini: [mypy]: python_version: Python major version '4' out of range (must be 2 or 3)
+== Return code: 0
 
 [case testPythonVersionAccepted27]
 # cmd: mypy -c pass
@@ -506,6 +546,7 @@ mypy.ini: [mypy]: python_version: Python major version '4' out of range (must be
 [[mypy]
 python_version = 2.7
 [out]
+== Return code: 0
 
 [case testPythonVersionAccepted33]
 # cmd: mypy -c pass
@@ -513,6 +554,7 @@ python_version = 2.7
 [[mypy]
 python_version = 3.3
 [out]
+== Return code: 0
 
 [case testPythonVersionAccepted36]
 # cmd: mypy -c pass
@@ -520,6 +562,7 @@ python_version = 3.3
 [[mypy]
 python_version = 3.6
 [out]
+== Return code: 0
 
 [case testDisallowAnyUnimported]
 # cmd: mypy main.py
@@ -533,6 +576,7 @@ from unreal import F
 def f(x: F) -> None: pass
 [out]
 main.py:3: error: Argument 1 to "f" becomes "Any" due to an unfollowed import
+== Return code: 1
 
 [case testDisallowAnyExplicitDefSignature]
 # cmd: mypy m.py
@@ -557,6 +601,7 @@ def h() -> List[Any]:
 m.py:3: error: Explicit "Any" is not allowed
 m.py:6: error: Explicit "Any" is not allowed
 m.py:9: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitVarDeclaration]
 # cmd: mypy m.py
@@ -577,6 +622,7 @@ class X:
 m.py:2: error: Explicit "Any" is not allowed
 m.py:3: error: Explicit "Any" is not allowed
 m.py:5: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitGenericVarDeclaration]
 # cmd: mypy m.py
@@ -591,6 +637,7 @@ from typing import Any, List
 v: List[Any] = []
 [out]
 m.py:2: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitInheritance]
 # cmd: mypy m.py
@@ -611,6 +658,7 @@ class D(List[Any]):
 [out]
 m.py:3: error: Explicit "Any" is not allowed
 m.py:6: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitAlias]
 # cmd: mypy m.py
@@ -633,6 +681,7 @@ def foo(x: X) -> Y:  # no error
 [out]
 m.py:3: error: Explicit "Any" is not allowed
 m.py:4: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitGenericAlias]
 # cmd: mypy m.py
@@ -658,6 +707,7 @@ def goo(x: TupleAny[Any]) -> None:  # error
 [out]
 m.py:5: error: Explicit "Any" is not allowed
 m.py:10: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitCast]
 # cmd: mypy m.py
@@ -676,6 +726,7 @@ z = cast(List[Any], x)
 [out]
 m.py:4: error: Explicit "Any" is not allowed
 m.py:5: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitNamedTuple]
 # cmd: mypy m.py
@@ -693,6 +744,7 @@ Point = NamedTuple('Point', [('x', List[Any]),
 
 [out]
 m.py:3: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitTypeVarConstraint]
 # cmd: mypy m.py
@@ -708,6 +760,7 @@ from typing import Any, List, TypeVar
 T = TypeVar('T', Any, List[Any])
 [out]
 m.py:3: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitNewType]
 # cmd: mypy m.py
@@ -726,6 +779,7 @@ Bar = NewType('Bar', List[Any])
 [out]
 m.py:3: error: Argument 2 to NewType(...) must be subclassable (got "Any")
 m.py:4: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitTypedDictSimple]
 # cmd: mypy m.py
@@ -744,6 +798,7 @@ M(x='x', y=2)  # no error
 def f(m: M) -> None: pass  # no error
 [out]
 m.py:4: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyExplicitTypedDictGeneric]
 # cmd: mypy m.py
@@ -761,6 +816,7 @@ M = TypedDict('M', {'x': str, 'y': List[Any]})  # error
 N = TypedDict('N', {'x': str, 'y': List})  # no error
 [out]
 m.py:4: error: Explicit "Any" is not allowed
+== Return code: 1
 
 [case testDisallowAnyGenericsTupleNoTypeParams]
 # cmd: mypy m.py
@@ -782,6 +838,7 @@ x: Tuple = ()  # error
 m.py:3: error: Missing type parameters for generic type
 m.py:4: error: Missing type parameters for generic type
 m.py:8: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowAnyGenericsTupleWithNoTypeParamsGeneric]
 # cmd: mypy m.py
@@ -797,6 +854,7 @@ def f(s: List[Tuple]) -> None: pass  # error
 def g(s: List[Tuple[str, str]]) -> None: pass  # no error
 [out]
 m.py:3: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowAnyGenericsTypeType]
 # cmd: mypy m.py
@@ -817,6 +875,7 @@ x: Type = g(0)  # error
 [out]
 m.py:4: error: Missing type parameters for generic type
 m.py:8: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowAnyGenericsAliasGenericType]
 # cmd: mypy m.py
@@ -834,6 +893,7 @@ def f(l: L) -> None: pass  # error
 def g(l: L[str]) -> None: pass  # no error
 [out]
 m.py:5: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowAnyGenericsGenericAlias]
 # cmd: mypy m.py
@@ -858,6 +918,7 @@ x: A = ('a', 'b', 1)  # error
 m.py:6: error: Missing type parameters for generic type
 m.py:7: error: Missing type parameters for generic type
 m.py:11: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowAnyGenericsPlainList]
 # cmd: mypy m.py
@@ -882,6 +943,7 @@ m.py:5: error: Missing type parameters for generic type
 m.py:6: error: Missing type parameters for generic type
 m.py:8: error: Need type annotation for variable
 m.py:9: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowAnyGenericsCustomGenericClass]
 # cmd: mypy m.py
@@ -905,6 +967,7 @@ y: G = x  # error
 [out]
 m.py:6: error: Missing type parameters for generic type
 m.py:10: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowAnyGenericsBuiltinCollections]
 # cmd: mypy m.py
@@ -927,6 +990,7 @@ m.py:4: error: Implicit generic "Any". Use 'typing.List' and specify generic par
 m.py:5: error: Implicit generic "Any". Use 'typing.Dict' and specify generic parameters
 m.py:6: error: Implicit generic "Any". Use 'typing.Set' and specify generic parameters
 m.py:7: error: Implicit generic "Any". Use 'typing.FrozenSet' and specify generic parameters
+== Return code: 1
 
 [case testDisallowAnyGenericsTypingCollections]
 # cmd: mypy m.py
@@ -949,6 +1013,7 @@ m.py:4: error: Missing type parameters for generic type
 m.py:5: error: Missing type parameters for generic type
 m.py:6: error: Missing type parameters for generic type
 m.py:7: error: Missing type parameters for generic type
+== Return code: 1
 
 [case testDisallowSubclassingAny]
 # cmd: mypy m.py y.py
@@ -973,6 +1038,7 @@ x = None  # type: Any
 class ShouldNotBeFine(x): ...
 [out]
 y.py:5: error: Class cannot subclass 'x' (has type 'Any')
+== Return code: 1
 
 [case testDeterministicSectionOrdering]
 # cmd: mypy a
@@ -996,6 +1062,7 @@ ignore_errors = True
 ignore_errors = False
 [out]
 a/b/c/d/e/__init__.py:1: error: "int" not callable
+== Return code: 1
 
 [case testDisallowUntypedDefsAndGenerics]
 # cmd: mypy a.py
@@ -1008,3 +1075,28 @@ def get_tasks(self):
     return 'whatever'
 [out]
 a.py:1: error: Function is missing a type annotation
+== Return code: 1
+
+[case testMissingFile]
+# cmd: mypy nope.py
+[out]
+mypy: can't read file 'nope.py': No such file or directory
+== Return code: 2
+
+[case testParseError]
+# cmd: mypy a.py
+[file a.py]
+def foo(
+[out]
+a.py:1: error: unexpected EOF while parsing
+== Return code: 2
+
+[case testParseErrorAnnots]
+# cmd: mypy a.py
+[file a.py]
+def foo(x):
+    # type: (str, int) -> None
+    return
+[out]
+a.py:1: error: Type signature has too many arguments
+== Return code: 2
diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test
index 27ad06d66755..54a79d9f8eae 100644
--- a/test-data/unit/reports.test
+++ b/test-data/unit/reports.test
@@ -12,6 +12,7 @@
 bad_report = .
 [out]
 mypy.ini: [mypy]: Unrecognized report type: bad_report
+== Return code: 0
 
 [case testCoberturaParser]
 # cmd: mypy --cobertura-xml-report build pkg
@@ -71,6 +72,8 @@ def untyped_function():
 
 [case testAnyExprReportDivisionByZero]
 # cmd: mypy --any-exprs-report=out -c 'pass'
+[out]
+== Return code: 0
 
 [case testClassDefIsTreatedAsEmpty]
 # cmd: mypy --html-report report n.py
@@ -336,3 +339,5 @@ def foo():
     n             0          0            0                  0       0              0
 ---------------------------------------------------------------------------------------
 Total             0          0            0                  0       0              0
+[out]
+== Return code: 0

From 744a96b4c3149c5899430646bd6987aaa5a583a0 Mon Sep 17 00:00:00 2001
From: Michael Sullivan <sully@msully.net>
Date: Mon, 8 Jan 2018 15:52:49 -0800
Subject: [PATCH 2/3] Reduce the amount of noise in the tests

---
 mypy/test/testcmdline.py    |  4 ++-
 test-data/unit/cmdline.test | 65 +++----------------------------------
 test-data/unit/reports.test |  4 ---
 3 files changed, 7 insertions(+), 66 deletions(-)

diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py
index d861a538f79c..05ea436569f2 100644
--- a/mypy/test/testcmdline.py
+++ b/mypy/test/testcmdline.py
@@ -79,7 +79,9 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
                                            path))
     else:
         out = normalize_error_messages(out)
-        out.append('== Return code: {}'.format(result))
+        obvious_result = 1 if out else 0
+        if obvious_result != result:
+            out.append('== Return code: {}'.format(result))
         assert_string_arrays_equal(testcase.output, out,
                                    'Invalid output ({}, line {})'.format(
                                        testcase.file, testcase.line))
diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
index ebf2fafc5395..c6d82add8079 100644
--- a/test-data/unit/cmdline.test
+++ b/test-data/unit/cmdline.test
@@ -4,7 +4,10 @@
 -- The initial line specifies the command line, in the format
 --
 --   # cmd: mypy <options>
-
+--
+-- '== Return code: <value>' is added to the output when the process return code
+-- is "nonobvious" -- that is, when it is something other than 0 if there are no
+-- messages and 1 if there are.
 
 -- Directories/packages on the command line
 -- ----------------------------------------
@@ -21,7 +24,6 @@ import pkg.subpkg.a
 [out]
 pkg/a.py:1: error: Name 'undef' is not defined
 pkg/subpkg/a.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testCmdlinePackageSlash]
 # cmd: mypy pkg/
@@ -35,7 +37,6 @@ import pkg.subpkg.a
 [out]
 pkg/a.py:1: error: Name 'undef' is not defined
 pkg/subpkg/a.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testCmdlineNonPackage]
 # cmd: mypy dir
@@ -45,7 +46,6 @@ undef
 undef
 [out]
 dir/a.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testCmdlineNonPackageSlash]
 # cmd: mypy dir/
@@ -55,7 +55,6 @@ undef
 undef
 [out]
 dir/a.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testCmdlinePackageContainingSubdir]
 # cmd: mypy pkg
@@ -66,7 +65,6 @@ undef
 undef
 [out]
 pkg/a.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testCmdlineNonPackageContainingPackage]
 # cmd: mypy dir
@@ -79,7 +77,6 @@ undef
 [out]
 dir/subpkg/a.py:1: error: Name 'undef' is not defined
 dir/a.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testCmdlineInvalidPackageName]
 # cmd: mypy dir/sub.pkg/a.py
@@ -88,7 +85,6 @@ dir/a.py:1: error: Name 'undef' is not defined
 undef
 [out]
 sub.pkg is not a valid Python package name
-== Return code: 1
 
 [case testBadFileEncoding]
 # cmd: mypy a.py
@@ -119,8 +115,6 @@ def f():
         1/0
     except ZeroDivisionError, err:
         print err
-[out]
-== Return code: 0
 
 [case testConfigFile]
 # cmd: mypy main.py
@@ -133,8 +127,6 @@ def f():
         1/0
     except ZeroDivisionError, err:
         print err
-[out]
-== Return code: 0
 
 [case testErrorContextConfig]
 # cmd: mypy main.py
@@ -147,7 +139,6 @@ def f() -> None:
 [out]
 main.py: note: In function "f":
 main.py:2: error: Unsupported operand types for + ("int" and "str")
-== Return code: 1
 
 [case testAltConfigFile]
 # cmd: mypy --config-file config.ini main.py
@@ -160,8 +151,6 @@ def f():
         1/0
     except ZeroDivisionError, err:
         print err
-[out]
-== Return code: 0
 
 [case testPerFileConfigSection]
 # cmd: mypy x.py y.py z.py
@@ -191,7 +180,6 @@ def g(a: int) -> int:
 z.py:1: error: Function is missing a type annotation
 z.py:4: error: Call to untyped function "f" in typed context
 x.py:1: error: Function is missing a type annotation
-== Return code: 1
 
 [case testPerFileConfigSectionMultipleMatches]
 # cmd: mypy xx.py xy.py yx.py yy.py
@@ -220,7 +208,6 @@ yx.py:2: error: Call to untyped function "f" in typed context
 xy.py:1: error: Function is missing a type annotation
 xy.py:2: error: Call to untyped function "f" in typed context
 xx.py:1: error: Function is missing a type annotation
-== Return code: 1
 
 [case testMultipleGlobConfigSection]
 # cmd: mypy x.py y.py z.py
@@ -237,7 +224,6 @@ def f(a): pass
 [out]
 z.py:1: error: Function is missing a type annotation
 x.py:1: error: Function is missing a type annotation
-== Return code: 1
 
 [case testConfigErrorNoSection]
 # cmd: mypy -c pass
@@ -298,7 +284,6 @@ baz(bar(foo('oof')))
 file.py:1: error: Cannot find module named 'no_stubs'
 file.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 file.py:6: error: Argument 1 to "foo" has incompatible type "str"; expected "int"
-== Return code: 1
 
 [case testIgnoreErrorsConfig]
 # cmd: mypy x.py y.py
@@ -312,7 +297,6 @@ ignore_errors = True
 "" + 0
 [out]
 y.py:1: error: Unsupported operand types for + ("str" and "int")
-== Return code: 1
 
 [case testConfigFollowImportsNormal]
 # cmd: mypy main.py
@@ -337,7 +321,6 @@ main.py:3: error: Unsupported operand types for + ("int" and "str")
 main.py:6: error: Unsupported operand types for + ("int" and "str")
 main.py:7: error: Module has no attribute "y"
 main.py:8: error: Unsupported operand types for + (Module and "int")
-== Return code: 1
 
 [case testConfigFollowImportsSilent]
 # cmd: mypy main.py
@@ -359,7 +342,6 @@ main.py:2: error: Unsupported operand types for + ("int" and "str")
 main.py:4: error: Unsupported operand types for + ("int" and "str")
 main.py:5: error: Module has no attribute "y"
 main.py:6: error: Unsupported operand types for + (Module and "int")
-== Return code: 1
 
 [case testConfigFollowImportsSkip]
 # cmd: mypy main.py
@@ -376,7 +358,6 @@ follow_imports = skip
 [out]
 main.py:2: error: Revealed type is 'Any'
 main.py:4: error: Revealed type is 'Any'
-== Return code: 1
 
 [case testConfigFollowImportsError]
 # cmd: mypy main.py
@@ -395,7 +376,6 @@ main.py:1: note: Import of 'a' ignored
 main.py:1: note: (Using --follow-imports=error, module not passed on command line)
 main.py:2: error: Revealed type is 'Any'
 main.py:4: error: Revealed type is 'Any'
-== Return code: 1
 
 [case testConfigFollowImportsSelective]
 # cmd: mypy main.py
@@ -436,7 +416,6 @@ main.py:5: error: Revealed type is 'builtins.int'
 main.py:6: error: Revealed type is 'builtins.int'
 main.py:7: error: Revealed type is 'Any'
 main.py:8: error: Revealed type is 'Any'
-== Return code: 1
 
 [case testConfigSilentMissingImportsOff]
 # cmd: mypy main.py
@@ -450,7 +429,6 @@ ignore_missing_imports = False
 main.py:1: error: Cannot find module named 'missing'
 main.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 main.py:2: error: Revealed type is 'Any'
-== Return code: 1
 
 [case testConfigSilentMissingImportsOn]
 # cmd: mypy main.py
@@ -462,7 +440,6 @@ reveal_type(missing.x)  # Expect Any
 ignore_missing_imports = True
 [out]
 main.py:2: error: Revealed type is 'Any'
-== Return code: 1
 
 [case testDotInFilenameOKScript]
 # cmd: mypy a.b.py c.d.pyi
@@ -473,7 +450,6 @@ whatever
 [out]
 c.d.pyi:1: error: Name 'whatever' is not defined
 a.b.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testDotInFilenameOKFolder]
 # cmd: mypy my.folder
@@ -481,7 +457,6 @@ a.b.py:1: error: Name 'undef' is not defined
 undef
 [out]
 my.folder/tst.py:1: error: Name 'undef' is not defined
-== Return code: 1
 
 [case testDotInFilenameNoImport]
 # cmd: mypy main.py
@@ -493,7 +468,6 @@ whatever
 main.py:1: error: Cannot find module named 'a'
 main.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 main.py:1: error: Cannot find module named 'a.b'
-== Return code: 1
 
 [case testPythonVersionTooOld10]
 # cmd: mypy -c pass
@@ -545,24 +519,18 @@ mypy.ini: [mypy]: python_version: Python major version '4' out of range (must be
 [file mypy.ini]
 [[mypy]
 python_version = 2.7
-[out]
-== Return code: 0
 
 [case testPythonVersionAccepted33]
 # cmd: mypy -c pass
 [file mypy.ini]
 [[mypy]
 python_version = 3.3
-[out]
-== Return code: 0
 
 [case testPythonVersionAccepted36]
 # cmd: mypy -c pass
 [file mypy.ini]
 [[mypy]
 python_version = 3.6
-[out]
-== Return code: 0
 
 [case testDisallowAnyUnimported]
 # cmd: mypy main.py
@@ -576,7 +544,6 @@ from unreal import F
 def f(x: F) -> None: pass
 [out]
 main.py:3: error: Argument 1 to "f" becomes "Any" due to an unfollowed import
-== Return code: 1
 
 [case testDisallowAnyExplicitDefSignature]
 # cmd: mypy m.py
@@ -601,7 +568,6 @@ def h() -> List[Any]:
 m.py:3: error: Explicit "Any" is not allowed
 m.py:6: error: Explicit "Any" is not allowed
 m.py:9: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitVarDeclaration]
 # cmd: mypy m.py
@@ -622,7 +588,6 @@ class X:
 m.py:2: error: Explicit "Any" is not allowed
 m.py:3: error: Explicit "Any" is not allowed
 m.py:5: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitGenericVarDeclaration]
 # cmd: mypy m.py
@@ -637,7 +602,6 @@ from typing import Any, List
 v: List[Any] = []
 [out]
 m.py:2: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitInheritance]
 # cmd: mypy m.py
@@ -658,7 +622,6 @@ class D(List[Any]):
 [out]
 m.py:3: error: Explicit "Any" is not allowed
 m.py:6: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitAlias]
 # cmd: mypy m.py
@@ -681,7 +644,6 @@ def foo(x: X) -> Y:  # no error
 [out]
 m.py:3: error: Explicit "Any" is not allowed
 m.py:4: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitGenericAlias]
 # cmd: mypy m.py
@@ -707,7 +669,6 @@ def goo(x: TupleAny[Any]) -> None:  # error
 [out]
 m.py:5: error: Explicit "Any" is not allowed
 m.py:10: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitCast]
 # cmd: mypy m.py
@@ -726,7 +687,6 @@ z = cast(List[Any], x)
 [out]
 m.py:4: error: Explicit "Any" is not allowed
 m.py:5: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitNamedTuple]
 # cmd: mypy m.py
@@ -744,7 +704,6 @@ Point = NamedTuple('Point', [('x', List[Any]),
 
 [out]
 m.py:3: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitTypeVarConstraint]
 # cmd: mypy m.py
@@ -760,7 +719,6 @@ from typing import Any, List, TypeVar
 T = TypeVar('T', Any, List[Any])
 [out]
 m.py:3: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitNewType]
 # cmd: mypy m.py
@@ -779,7 +737,6 @@ Bar = NewType('Bar', List[Any])
 [out]
 m.py:3: error: Argument 2 to NewType(...) must be subclassable (got "Any")
 m.py:4: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitTypedDictSimple]
 # cmd: mypy m.py
@@ -798,7 +755,6 @@ M(x='x', y=2)  # no error
 def f(m: M) -> None: pass  # no error
 [out]
 m.py:4: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyExplicitTypedDictGeneric]
 # cmd: mypy m.py
@@ -816,7 +772,6 @@ M = TypedDict('M', {'x': str, 'y': List[Any]})  # error
 N = TypedDict('N', {'x': str, 'y': List})  # no error
 [out]
 m.py:4: error: Explicit "Any" is not allowed
-== Return code: 1
 
 [case testDisallowAnyGenericsTupleNoTypeParams]
 # cmd: mypy m.py
@@ -838,7 +793,6 @@ x: Tuple = ()  # error
 m.py:3: error: Missing type parameters for generic type
 m.py:4: error: Missing type parameters for generic type
 m.py:8: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowAnyGenericsTupleWithNoTypeParamsGeneric]
 # cmd: mypy m.py
@@ -854,7 +808,6 @@ def f(s: List[Tuple]) -> None: pass  # error
 def g(s: List[Tuple[str, str]]) -> None: pass  # no error
 [out]
 m.py:3: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowAnyGenericsTypeType]
 # cmd: mypy m.py
@@ -875,7 +828,6 @@ x: Type = g(0)  # error
 [out]
 m.py:4: error: Missing type parameters for generic type
 m.py:8: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowAnyGenericsAliasGenericType]
 # cmd: mypy m.py
@@ -893,7 +845,6 @@ def f(l: L) -> None: pass  # error
 def g(l: L[str]) -> None: pass  # no error
 [out]
 m.py:5: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowAnyGenericsGenericAlias]
 # cmd: mypy m.py
@@ -918,7 +869,6 @@ x: A = ('a', 'b', 1)  # error
 m.py:6: error: Missing type parameters for generic type
 m.py:7: error: Missing type parameters for generic type
 m.py:11: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowAnyGenericsPlainList]
 # cmd: mypy m.py
@@ -943,7 +893,6 @@ m.py:5: error: Missing type parameters for generic type
 m.py:6: error: Missing type parameters for generic type
 m.py:8: error: Need type annotation for variable
 m.py:9: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowAnyGenericsCustomGenericClass]
 # cmd: mypy m.py
@@ -967,7 +916,6 @@ y: G = x  # error
 [out]
 m.py:6: error: Missing type parameters for generic type
 m.py:10: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowAnyGenericsBuiltinCollections]
 # cmd: mypy m.py
@@ -990,7 +938,6 @@ m.py:4: error: Implicit generic "Any". Use 'typing.List' and specify generic par
 m.py:5: error: Implicit generic "Any". Use 'typing.Dict' and specify generic parameters
 m.py:6: error: Implicit generic "Any". Use 'typing.Set' and specify generic parameters
 m.py:7: error: Implicit generic "Any". Use 'typing.FrozenSet' and specify generic parameters
-== Return code: 1
 
 [case testDisallowAnyGenericsTypingCollections]
 # cmd: mypy m.py
@@ -1013,7 +960,6 @@ m.py:4: error: Missing type parameters for generic type
 m.py:5: error: Missing type parameters for generic type
 m.py:6: error: Missing type parameters for generic type
 m.py:7: error: Missing type parameters for generic type
-== Return code: 1
 
 [case testDisallowSubclassingAny]
 # cmd: mypy m.py y.py
@@ -1038,7 +984,6 @@ x = None  # type: Any
 class ShouldNotBeFine(x): ...
 [out]
 y.py:5: error: Class cannot subclass 'x' (has type 'Any')
-== Return code: 1
 
 [case testDeterministicSectionOrdering]
 # cmd: mypy a
@@ -1062,7 +1007,6 @@ ignore_errors = True
 ignore_errors = False
 [out]
 a/b/c/d/e/__init__.py:1: error: "int" not callable
-== Return code: 1
 
 [case testDisallowUntypedDefsAndGenerics]
 # cmd: mypy a.py
@@ -1075,7 +1019,6 @@ def get_tasks(self):
     return 'whatever'
 [out]
 a.py:1: error: Function is missing a type annotation
-== Return code: 1
 
 [case testMissingFile]
 # cmd: mypy nope.py
diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test
index 54a79d9f8eae..8343212fc8df 100644
--- a/test-data/unit/reports.test
+++ b/test-data/unit/reports.test
@@ -72,8 +72,6 @@ def untyped_function():
 
 [case testAnyExprReportDivisionByZero]
 # cmd: mypy --any-exprs-report=out -c 'pass'
-[out]
-== Return code: 0
 
 [case testClassDefIsTreatedAsEmpty]
 # cmd: mypy --html-report report n.py
@@ -339,5 +337,3 @@ def foo():
     n             0          0            0                  0       0              0
 ---------------------------------------------------------------------------------------
 Total             0          0            0                  0       0              0
-[out]
-== Return code: 0

From 36a338c0b8761bac4198da0d8a81bae16fe83104 Mon Sep 17 00:00:00 2001
From: Michael Sullivan <sully@msully.net>
Date: Mon, 8 Jan 2018 15:54:34 -0800
Subject: [PATCH 3/3] More test noise reduction

---
 test-data/unit/cmdline.test | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
index c6d82add8079..b892757123a3 100644
--- a/test-data/unit/cmdline.test
+++ b/test-data/unit/cmdline.test
@@ -519,18 +519,21 @@ mypy.ini: [mypy]: python_version: Python major version '4' out of range (must be
 [file mypy.ini]
 [[mypy]
 python_version = 2.7
+[out]
 
 [case testPythonVersionAccepted33]
 # cmd: mypy -c pass
 [file mypy.ini]
 [[mypy]
 python_version = 3.3
+[out]
 
 [case testPythonVersionAccepted36]
 # cmd: mypy -c pass
 [file mypy.ini]
 [[mypy]
 python_version = 3.6
+[out]
 
 [case testDisallowAnyUnimported]
 # cmd: mypy main.py