From 0a2007dad63bd30ac66925f802feb9ee6418b40e Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 19 Feb 2023 11:41:15 -0400 Subject: [PATCH 01/11] Refactor test_program to allow for multiple test suites --- Lib/test/test_unittest/test_program.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 3645bcf4b43562..2fb048102e6d1b 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -71,15 +71,19 @@ def testExpectedFailure(self): def testUnexpectedSuccess(self): pass - class FooBarLoader(unittest.TestLoader): - """Test loader that returns a suite containing FooBar.""" + class TestLoader(unittest.TestLoader): + """Test loader that returns a suite containing testsuite.""" + + def __init__(self, testcase): + self.testcase = testcase + def loadTestsFromModule(self, module): return self.suiteClass( - [self.loadTestsFromTestCase(Test_TestProgram.FooBar)]) + [self.loadTestsFromTestCase(self.testcase)]) def loadTestsFromNames(self, names, module): return self.suiteClass( - [self.loadTestsFromTestCase(Test_TestProgram.FooBar)]) + [self.loadTestsFromTestCase(self.testcase)]) def test_defaultTest_with_string(self): class FakeRunner(object): @@ -92,7 +96,7 @@ def run(self, test): runner = FakeRunner() program = unittest.TestProgram(testRunner=runner, exit=False, defaultTest='test.test_unittest', - testLoader=self.FooBarLoader()) + testLoader=self.TestLoader(self.FooBar)) sys.argv = old_argv self.assertEqual(('test.test_unittest',), program.testNames) @@ -108,7 +112,7 @@ def run(self, test): program = unittest.TestProgram( testRunner=runner, exit=False, defaultTest=['test.test_unittest', 'test.test_unittest2'], - testLoader=self.FooBarLoader()) + testLoader=self.TestLoader(self.FooBar)) sys.argv = old_argv self.assertEqual(['test.test_unittest', 'test.test_unittest2'], program.testNames) @@ -118,7 +122,7 @@ def test_NonExit(self): program = unittest.main(exit=False, argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), - testLoader=self.FooBarLoader()) + testLoader=self.TestLoader(self.FooBar)) self.assertTrue(hasattr(program, 'result')) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) @@ -136,7 +140,7 @@ def test_Exit(self): argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), exit=True, - testLoader=self.FooBarLoader()) + testLoader=self.TestLoader(self.FooBar)) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) @@ -152,7 +156,7 @@ def test_ExitAsDefault(self): unittest.main, argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), - testLoader=self.FooBarLoader()) + testLoader=self.TestLoader(self.FooBar)) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) From 653220060dd7ec586e7e07c444c08faa45a2cdd9 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 19 Feb 2023 11:42:24 -0400 Subject: [PATCH 02/11] Refactor test_program to use assertRaises as a context manager --- Lib/test/test_unittest/test_program.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 2fb048102e6d1b..a8700689a1b0b8 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -134,13 +134,12 @@ def test_NonExit(self): def test_Exit(self): stream = BufferedWriter() - self.assertRaises( - SystemExit, - unittest.main, - argv=["foobar"], - testRunner=unittest.TextTestRunner(stream=stream), - exit=True, - testLoader=self.TestLoader(self.FooBar)) + with self.assertRaises(SystemExit): + unittest.main( + argv=["foobar"], + testRunner=unittest.TextTestRunner(stream=stream), + exit=True, + testLoader=self.TestLoader(self.FooBar)) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) @@ -151,12 +150,11 @@ def test_Exit(self): def test_ExitAsDefault(self): stream = BufferedWriter() - self.assertRaises( - SystemExit, - unittest.main, - argv=["foobar"], - testRunner=unittest.TextTestRunner(stream=stream), - testLoader=self.TestLoader(self.FooBar)) + with self.assertRaises(SystemExit): + unittest.main( + argv=["foobar"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.FooBar)) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) From ebba337b01eaa4bc7cf9db0fa6690e022bc1ff18 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 19 Feb 2023 11:53:31 -0400 Subject: [PATCH 03/11] unittest: Print an error message if no tests ran --- Lib/test/test_unittest/test_runner.py | 10 ++++++++++ Lib/unittest/runner.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index df584b7620d092..9152047488517c 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -574,6 +574,16 @@ def test(self): 'inner setup', 'inner test', 'inner cleanup', 'end outer test', 'outer cleanup']) + def test_run_empty_suite_error_message(self): + class EmptyTest(unittest.TestCase): + pass + + suite = unittest.defaultTestLoader.loadTestsFromTestCase(EmptyTest) + runner = getRunner() + runner.run(suite) + + self.assertIn("\nNO TESTS RUN\n", runner.stream.getvalue()) + class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 6678adb6a7d813..7c0c3da176a0a2 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -238,6 +238,8 @@ def run(self, test): infos.append("failures=%d" % failed) if errored: infos.append("errors=%d" % errored) + elif run == 0: + self.stream.write("NO TESTS RUN") else: self.stream.write("OK") if skipped: From 5e1a88de871f75701e1f2e93a001a6355b4d5b83 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 19 Feb 2023 11:54:18 -0400 Subject: [PATCH 04/11] unittest runner: Exit code 5 if no tests were run As discussed in https://discuss.python.org/t/unittest-fail-if-zero-tests-were-discovered/21498/7 It is common for test runner misconfiguration to fail to find any tests, this should be an error. Fixes: #62432 --- Doc/library/unittest.rst | 3 ++- Lib/test/test_unittest/test_program.py | 17 ++++++++++++++++- Lib/unittest/main.py | 7 ++++++- ...023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst | 3 +++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 1577149e976474..00a4acb7039442 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -2255,7 +2255,8 @@ Loading and running tests The *testRunner* argument can either be a test runner class or an already created instance of it. By default ``main`` calls :func:`sys.exit` with - an exit code indicating success or failure of the tests run. + an exit code indicating success (0) or failure (1) of the tests run. + An exit code of 5 indicates that no tests were run. The *testLoader* argument has to be a :class:`TestLoader` instance, and defaults to :data:`defaultTestLoader`. diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index a8700689a1b0b8..2b5fbfadccb8b1 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -71,6 +71,9 @@ def testExpectedFailure(self): def testUnexpectedSuccess(self): pass + class Empty(unittest.TestCase): + pass + class TestLoader(unittest.TestLoader): """Test loader that returns a suite containing testsuite.""" @@ -134,12 +137,13 @@ def test_NonExit(self): def test_Exit(self): stream = BufferedWriter() - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit) as cm: unittest.main( argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), exit=True, testLoader=self.TestLoader(self.FooBar)) + self.assertEqual(cm.exception.code, 1) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) @@ -163,6 +167,17 @@ def test_ExitAsDefault(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + def test_ExitEmptySuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["empty"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.Empty)) + self.assertEqual(cm.exception.code, 5) + out = stream.getvalue() + self.assertIn('\nNO TESTS RUN\n', out) + class InitialisableProgram(unittest.TestProgram): exit = False diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 046fbd3a45dcf8..7aff0cae5d24eb 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -273,6 +273,11 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - sys.exit(not self.result.wasSuccessful()) + if self.result.testsRun == 0: + sys.exit(5) + elif not self.result.wasSuccessful(): + sys.exit(1) + else: + sys.exit(0) main = TestProgram diff --git a/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst b/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst new file mode 100644 index 00000000000000..430c88551507ce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst @@ -0,0 +1,3 @@ +The :mod:`unittest` runner will now exit with status code 5, if no tests +were run. It is common for test runner misconfiguration to fail to find any +tests, this should be an error. From fb6771468ffe1d499ff3866f905cfdb849e6f327 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 19 Feb 2023 11:50:57 -0400 Subject: [PATCH 05/11] Count the test run in our stub test so that the suite succeeds --- Lib/test/test_unittest/test_result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index efd9c902350506..2f1c147fed71f3 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -465,6 +465,7 @@ def testFailFastSetByRunner(self): stream = BufferedWriter() runner = unittest.TextTestRunner(stream=stream, failfast=True) def test(result): + result.testsRun += 1 self.assertTrue(result.failfast) result = runner.run(test) stream.flush() From 0687ca8080d1844438ccc1617e43ae4c623bc8e7 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 19 Feb 2023 12:40:48 -0400 Subject: [PATCH 06/11] Update ACKs --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index ca92608868f23f..e465bd085f8ccb 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1501,6 +1501,7 @@ Vlad Riscutia Wes Rishel Daniel Riti Juan M. Bello Rivas +Stefano Rivera Llandy Riveron Del Risco Mohd Sanad Zaki Rizvi Davide Rizzo From 3ac2e71e766fb43db587f459f4bf1fdabf01f0e2 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 20 Feb 2023 07:45:34 -0400 Subject: [PATCH 07/11] Put success first in the if ladder --- Lib/unittest/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 7aff0cae5d24eb..8adc6e067bfea3 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -275,9 +275,10 @@ def runTests(self): if self.exit: if self.result.testsRun == 0: sys.exit(5) - elif not self.result.wasSuccessful(): - sys.exit(1) - else: + elif self.result.wasSuccessful(): sys.exit(0) + else: + sys.exit(1) + main = TestProgram From 747077946d45a1e830c239f9a9b3729ec728b396 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 20 Feb 2023 07:46:57 -0400 Subject: [PATCH 08/11] NO TESTS RAN, in past tense --- Lib/test/test_unittest/test_program.py | 2 +- Lib/test/test_unittest/test_runner.py | 2 +- Lib/unittest/runner.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 2b5fbfadccb8b1..390a85d911a247 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -176,7 +176,7 @@ def test_ExitEmptySuite(self): testLoader=self.TestLoader(self.Empty)) self.assertEqual(cm.exception.code, 5) out = stream.getvalue() - self.assertIn('\nNO TESTS RUN\n', out) + self.assertIn('\nNO TESTS RAN\n', out) class InitialisableProgram(unittest.TestProgram): diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 9152047488517c..0a94c555ef0c05 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -582,7 +582,7 @@ class EmptyTest(unittest.TestCase): runner = getRunner() runner.run(suite) - self.assertIn("\nNO TESTS RUN\n", runner.stream.getvalue()) + self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue()) class TestModuleCleanUp(unittest.TestCase): diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 7c0c3da176a0a2..6686e15672b59d 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -239,7 +239,7 @@ def run(self, test): if errored: infos.append("errors=%d" % errored) elif run == 0: - self.stream.write("NO TESTS RUN") + self.stream.write("NO TESTS RAN") else: self.stream.write("OK") if skipped: From ecb363cebc2050357de58bf58e53c4691ba96986 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 20 Feb 2023 07:48:04 -0400 Subject: [PATCH 09/11] The parameter name was testcase --- Lib/test/test_unittest/test_program.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 390a85d911a247..bb37ae81da7e33 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -75,7 +75,7 @@ class Empty(unittest.TestCase): pass class TestLoader(unittest.TestLoader): - """Test loader that returns a suite containing testsuite.""" + """Test loader that returns a suite containing the supplied testcase.""" def __init__(self, testcase): self.testcase = testcase From 7d6c3227c08e769fe45c8af599cdd9cae0d8f737 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 20 Feb 2023 07:48:45 -0400 Subject: [PATCH 10/11] Drop a comma --- .../Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst b/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst index 430c88551507ce..a8d66ea48c3278 100644 --- a/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst +++ b/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst @@ -1,3 +1,3 @@ -The :mod:`unittest` runner will now exit with status code 5, if no tests -were run. It is common for test runner misconfiguration to fail to find any -tests, this should be an error. +The :mod:`unittest` runner will now exit with status code 5 if no tests +were run. It is common for test runner misconfiguration to fail to find +any tests, this should be an error. From 9fbab3619093bebe8217c0b231df4b8a4e08677c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 26 Apr 2023 18:01:39 -0700 Subject: [PATCH 11/11] move 5 to a _NO_TESTS_EXITCODE constant. --- Lib/unittest/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index cf31eb8753a183..51b81a6c3728bb 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -9,6 +9,7 @@ from .signals import installHandler __unittest = True +_NO_TESTS_EXITCODE = 5 MAIN_EXAMPLES = """\ Examples: @@ -280,7 +281,7 @@ def runTests(self): self.result = testRunner.run(self.test) if self.exit: if self.result.testsRun == 0: - sys.exit(5) + sys.exit(_NO_TESTS_EXITCODE) elif self.result.wasSuccessful(): sys.exit(0) else: