8000 bpo-46523: fix tests rerun when `setUp[Class|Module]` fails (#30895) · python/cpython@9953860 · GitHub
[go: up one dir, main page]
Skip to content

Navigation Menu

Sign in

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 9953860

Browse files
sobolevnJelleZijlstraambv
authored
bpo-46523: fix tests rerun when setUp[Class|Module] fails (#30895)
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com> Co-authored-by: Łukasz Langa <lukasz@langa.pl>
1 parent 059bb04 commit 9953860

File tree

3 files changed

+184
-3
lines changed

3 files changed

+184
-3
lines changed

‎Lib/test/libregrtest/main.py

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@
2929
# Must be smaller than buildbot "1200 seconds without output" limit.
3030
EXIT_TIMEOUT = 120.0
3131

32+
# gh-90681: When rerunning tests, we might need to rerun the whole
33+
# class or module suite if some its life-cycle hooks fail.
34+
# Test level hooks are not affected.
35+
_TEST_LIFECYCLE_HOOKS = frozenset((
36+
'setUpClass', 'tearDownClass',
37+
'setUpModule', 'tearDownModule',
38+
))
39+
3240
EXITCODE_BAD_TEST = 2
3341
EXITCODE_INTERRUPTED = 130
3442
EXITCODE_ENV_CHANGED = 3
@@ -337,8 +345,12 @@ def rerun_failed_tests(self):
337345

338346
errors = result.errors or []
339347
failures = result.failures or []
340-
error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
341-
failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
348+
error_names = [
349+
self.normalize_test_name(test_full_name, is_error=True)
350+
for (test_full_name, *_) in errors]
351+
failure_names = [
352+
self.normalize_test_name(test_full_name)
353+
for (test_full_name, *_) in failures]
342354
self.ns.verbose = True
343355
orig_match_tests = self.ns.match_tests
344356
if errors or failures:
@@ -364,6 +376,21 @@ def rerun_failed_tests(self):
364376

365377
self.display_result()
366378

379+
def normalize_test_name(self, test_full_name, *, is_error=False):
380+
short_name = test_full_name.split(" ")[0]
381+
if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
382+
# This means that we have a failure in a life-cycle hook,
383+
# we need to rerun the whole module or class suite.
384+
# Basically the error looks like this:
385+
# ERROR: setUpClass (test.test_reg_ex.RegTest)
386+
# or
387+
# ERROR: setUpModule (test.test_reg_ex)
388+
# So, we need to parse the class / module name.
389+
lpar = test_full_name.index('(')
390+
rpar = test_full_name.index(')')
391+
return test_full_name[lpar + 1: rpar].split('.')[-1]
392+
return short_name
393+
367394
def display_result(self):
368395
# If running the test suite for PGO then no one cares about results.
369396
if self.ns.pgo:

‎Lib/test/support/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1108,7 +1108,7 @@ def _run_suite(suite):
11081108
if junit_xml_list is not None:
11091109
junit_xml_list.append(result.get_xml_element())
11101110

1111-
if not result.testsRun and not result.skipped:
1111+
if not result.testsRun and not result.skipped and not result.errors:
11121112
raise TestDidNotRun
11131113
if not result.wasSuccessful():
11141114
if len(result.errors) == 1 and not result.failures:

‎Lib/test/test_regrtest.py

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1120,6 +1120,160 @@ def test_fail_once(self):
11201120
self.check_executed_tests(output, [testname],
11211121
rerun={testname: "test_fail_once"})
11221122

1123+
def test_rerun_setup_class_hook_failure(self):
1124+
# FAILURE then FAILURE
1125+
code = textwrap.dedent("""
1126+
import unittest
1127+
1128+
class ExampleTests(unittest.TestCase):
1129+
@classmethod
1130+
def setUpClass(self):
1131+
raise RuntimeError('Fail')
1132+
1133+
def test_success(self):
1134+
return
1135+
""")
1136+
testname = self.create_test(code=code)
1137+
1138+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1139+
self.check_executed_tests(output, testname,
1140+
failed=[testname],
1141+
rerun={testname: "ExampleTests"})
1142+
1143+
def test_rerun_teardown_class_hook_failure(self):
1144+
# FAILURE then FAILURE
1145+
code = textwrap.dedent("""
1146+
import unittest
1147+
1148+
class ExampleTests(unittest.TestCase):
1149+
@classmethod
1150+
def tearDownClass(self):
1151+
raise RuntimeError('Fail')
1152+
1153+
def test_success(self):
1154+
return
1155+
""")
1156+
testname = self.create_test(code=code)
1157+
1158+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1159+
self.check_executed_tests(output, testname,
1160+
failed=[testname],
1161+
rerun={testname: "ExampleTests"})
1162+
1163+
def test_rerun_setup_module_hook_failure(self):
1164+
# FAILURE then FAILURE
1165+
code = textwrap.dedent("""
1166+
import unittest
1167+
1168+
def setUpModule():
1169+
raise RuntimeError('Fail')
1170+
1171+
class ExampleTests(unittest.TestCase):
1172+
def test_success(self):
1173+
return
1174+
""")
1175+
testname = self.create_test(code=code)
1176+
1177+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1178+
self.check_executed_tests(output, testname,
1179+
failed=[testname],
1180+
rerun={testname: testname})
1181+
1182+
def test_rerun_teardown_module_hook_failure(self):
1183+
# FAILURE then FAILURE
1184+
code = textwrap.dedent("""
1185+
import unittest
1186+
1187+
def tearDownModule():
1188+
raise RuntimeError('Fail')
1189+
1190+
class ExampleTests(unittest.TestCase):
1191+
def test_success(self):
1192+
return
1193+
""")
1194+
testname = self.create_test(code=code)
1195+
1196+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1197+
self.check_executed_tests(output, testname,
1198+
failed=[testname],
1199+
rerun={testname: testname})
1200+
1201+
def test_rerun_setup_hook_failure(self):
1202+
# FAILURE then FAILURE
1203+
code = textwrap.dedent("""
1204+
import unittest
1205+
1206+
class ExampleTests(unittest.TestCase):
1207+
def setUp(self):
1208+
raise RuntimeError('Fail')
1209+
1210+
def test_success(self):
1211+
return
1212+
""")
1213+
testname = self.create_test(code=code)
1214+
1215+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1216+
self.check_executed_tests(output, testname,
1217+
failed=[testname],
1218+
rerun={testname: "test_success"})
1219+
1220+
def test_rerun_teardown_hook_failure(self):
1221+
# FAILURE then FAILURE
1222+
code = textwrap.dedent("""
1223+
import unittest
1224+
1225+
< F987 span class=pl-s> class ExampleTests(unittest.TestCase):
1226+
def tearDown(self):
1227+
raise RuntimeError('Fail')
1228+
1229+
def test_success(self):
1230+
return
1231+
""")
1232+
testname = self.create_test(code=code)
1233+
1234+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1235+
self.check_executed_tests(output, testname,
1236+
failed=[testname],
1237+
rerun={testname: "test_success"})
1238+
1239+
def test_rerun_async_setup_hook_failure(self):
1240+
# FAILURE then FAILURE
1241+
code = textwrap.dedent("""
1242+
import unittest
1243+
1244+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1245+
async def asyncSetUp(self):
1246+
raise RuntimeError('Fail')
1247+
1248+
async def test_success(self):
1249+
return
1250+
""")
1251+
testname = self.create_test(code=code)
1252+
1253+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1254+
self.check_executed_tests(output, testname,
1255+
failed=[testname],
1256+
rerun={testname: "test_success"})
1257+
1258+
def test_rerun_async_teardown_hook_failure(self):
1259+
# FAILURE then FAILURE
1260+
code = textwrap.dedent("""
1261+
import unittest
1262+
1263+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1264+
async def asyncTearDown(self):
1265+
raise RuntimeError('Fail')
1266+
1267+
async def test_success(self):
1268+
return
1269+
""")
1270+
testname = self.create_test(code=code)
1271+
1272+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1273+
self.check_executed_tests(output, testname,
1274+
failed=[testname],
1275+
rerun={testname: "test_success"})
1276+
11231277
def test_no_tests_ran(self):
11241278
code = textwrap.dedent("""
11251279
import unittest

0 commit comments

Comments
 (0)
0