8000 regrtest computes statistics · python/cpython@8b407a2 · GitHub
[go: up one dir, main page]
Skip to content

Commit 8b407a2

Browse files
vstinnerAlexWaygood
vstinner
and
AlexWaygood
committed
regrtest computes statistics
* Rewrite TestResult as a dataclass with a new State class. * Add test.support.TestStats class and Regrtest.stats_dict attribute. * libregrtest.runtest functions now modify a TestResult instance in-place. * libregrtest summary lists the number of run tests and skipped tests, and denied resources. * test_main() of test_netrc, test_pep646_syntax and test_xml_etree now return results. * Add TestResult.has_meaningful_duration() method * Use match/case syntadx to have different code depending on the test state. * Rename CHILD_ERROR to MULTIPROCESSING_ERROR * Compute TestResult duration in the upper function Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
1 parent 594b000 commit 8b407a2

File tree

10 files changed

+479
-294
lines changed

10 files changed

+479
-294
lines changed

‎Lib/test/libregrtest/main.py

Lines changed: 68 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,14 @@
1111
import unittest
1212
from test.libregrtest.cmdline import _parse_args
1313
from test.libregrtest.runtest import (
14-
findtests, split_test_packages, runtest, get_abs_module, is_failed,
15-
PROGRESS_MIN_TIME,
16-
Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted,
17-
ChildError, DidNotRun)
14+
findtests, split_test_packages, runtest, get_abs_module,
15+
PROGRESS_MIN_TIME, State)
1816
from test.libregrtest.setup import setup_tests
1917
from test.libregrtest.pgo import setup_pgo_tests
2018
from test.libregrtest.utils import (removepy, count, format_duration,
2119
printlist, get_build_info)
2220
from test import support
21+
from test.support import TestStats
2322
from test.support import os_helper
2423
from test.support import threading_helper
2524

@@ -85,6 +84,7 @@ def __init__(self):
8584
self.rerun = []
8685
self.first_result = None
8786
self.interrupted = False
87+
self.stats_dict: dict[str, TestStats] = {}
8888

8989
# used by --slow
9090
self.test_times = []
@@ -115,32 +115,37 @@ def get_executed(self):
115115
| set(self.run_no_tests))
116116

117117
def accumulate_result(self, result, rerun=False):
118-
test_name = result.name
119-
120-
if not isinstance(result, (ChildError, Interrupted)) and not rerun:
121-
self.test_times.append((result.duration_sec, test_name))
122-
123-
if isinstance(result, Passed):
124-
self.good.append(test_name)
125-
elif isinstance(result, ResourceDenied):
126-
self.skipped.append(test_name)
127-
self.resource_denieds.append(test_name)
128-
elif isinstance(result, Skipped):
129-
self.skipped.append(test_name)
130-
elif isinstance(result, EnvChanged):
131-
self.environment_changed.append(test_name)
132-
elif isinstance(result, Failed):
133-
if not rerun:
134-
self.bad.append(test_name)
135-
self.need_rerun.append(result)
136-
elif isinstance(result, DidNotRun):
137-
self.run_no_tests.append(test_name)
138-
elif isinstance(result, Interrupted):
139-
self.interrupted = True
140-
else:
141-
raise ValueError("invalid test result: %r" % result)
118+
test_name = result.test_name
119+
120+
if result.has_meaningful_duration() and not rerun:
121+
self.test_times.append((result.duration, test_name))
142122

143-
if rerun and not isinstance(result, (Failed, Interrupted)):
123+
match result.state:
124+
case State.PASSED:
125+
self.good.append(test_name)
126+
case State.ENV_CHANGED:
127+
self.environment_changed.append(test_name)
128+
case State.SKIPPED:
129+
self.skipped.append(test_name)
130+
case State.RESOURCE_DENIED:
131+
self.skipped.append(test_name)
132+
self.resource_denieds.append(test_name)
133+
case State.INTERRUPTED:
134+
self.interrupted = True
135+
case State.DID_NOT_RUN:
136+
self.run_no_tests.append(test_name)
137+
case _:
138+
if result.is_failed(self.ns.fail_env_changed):
139+
if not rerun:
140+
self.bad.append(test_name)
141+
self.need_rerun.append(result)
142+
else:
143+
raise ValueError(f"invalid test state: {state!r}")
144+
145+
if result.stats is not None:
146+
self.stats_dict[result.test_name] = result.stats
147+
148+
if rerun and not(result.is_failed(False) or result.state == State.INTERRUPTED):
144149
self.bad.remove(test_name)
145150

146151
xml_data = result.xml_data
@@ -337,7 +342,7 @@ def rerun_failed_tests(self):
337342
rerun_list = list(self.need_rerun)
338343
self.need_rerun.clear()
339344
for result in rerun_list:
340-
test_name = result.name
345+
test_name = result.test_name
341346
self.rerun.append(test_name)
342347

343348
errors = result.errors or []
@@ -364,7 +369,7 @@ def rerun_failed_tests(self):
364369

365370
self.accumulate_result(result, rerun=True)
366371

367-
if isinstance(result, Interrupted):
372+
if result.state == State.INTERRUPTED:
368373
break
369374

370375
if self.bad:
@@ -480,14 +485,14 @@ def run_tests_sequential(self):
480485
result = runtest(self.ns, test_name)
481486
self.accumulate_result(result)
482487

483-
if isinstance(result, Interrupted):
488+
if result.state == State.INTERRUPTED:
484489
break
485490

486491
previous_test = str(result)
487492
test_time = time.monotonic() - start_time
488493
if test_time >= PROGRESS_MIN_TIME:
489494
previous_test = "%s in %s" % (previous_test, format_duration(test_time))
490-
elif isinstance(result, Passed):
495+
elif result.state == State.PASSED:
491496
# be quiet: say nothing if the test passed shortly
492497
previous_test = None
493498

@@ -496,7 +501,7 @@ def run_tests_sequential(self):
496501
if module not in save_modules and module.startswith("test."):
497502
support.unload(module)
498503

499-
if self.ns.failfast and is_failed 10000 (result, self.ns):
504+
if self.ns.failfast and result.is_failed(self.ns.fail_env_changed):
500505
break
501506

502507
if previous_test:
@@ -638,13 +643,39 @@ def finalize(self):
638643
coverdir=self.ns.coverdir)
639644

640645
print()
641-
duration = time.monotonic() - self.start_time
642-
print("Total duration: %s" % format_duration(duration))
643-
print("Tests result: %s" % self.get_tests_result())
646+
self.display_summary()
644647

645648
if self.ns.runleaks:
646649
os.system("leaks %d" % os.getpid())
647650

651+
def display_summary(self):
652+
duration = time.monotonic() - self.start_time
653+
total = TestStats()
654+
for stats in self.stats_dict.values():
655+
total.accumulate(stats)
656+
stats = [f'run={total.tests_run:,}']
657+
if total.failures:
658+
stats.append(f'failures={total.failures:,}')
659+
if total.skipped:
660+
stats.append(f'skipped={total.skipped:,}')
661+
print(f"Total tests: {' '.join(stats)}")
662+
print("Total duration: %s" % format_duration(duration))
663+
664+
result = self.get_tests_result()
665+
skipped = len(self.skipped)
666+
if skipped:
667+
skipped = f"{skipped} skipped"
668+
denied = len(self.resource_denieds)
669+
if denied:
670+
denied = f"{denied} resource denied"
671+
if skipped:
672+
skipped = f"{skipped}; {denied}"
673+
else:
674+
skipped = denied
675+
if skipped:
676+
result = f"{result} ({skipped})"
677+
print(f"Tests result: {result}")
678+
648679
def save_xml_result(self):
649680
if not self.ns.xmlpath and not self.testsuite_xml:
650681
return

‎Lib/test/libregrtest/refleak.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,12 @@ def get_pooled_int(value):
8383
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
8484
flush=True)
8585

86+
results = None
8687
dash_R_cleanup(fs, ps, pic, zdc, abcs)
8788
support.gc_collect()
8889

8990
for i in rep_range:
90-
test_func()
91+
results = test_func()
9192

9293
dash_R_cleanup(fs, ps, pic, zdc, abcs)
9394
support.gc_collect()
@@ -151,7 +152,7 @@ def check_fd_deltas(deltas):
151152
print(msg, file=refrep)
152153
refrep.flush()
153154
failed = True
154-
return failed
155+
return (failed, results)
155156

156157

157158
def dash_R_cleanup(fs, ps, pic, zdc, abcs):

0 commit comments

Comments
 (0)
0