8000 [3.7] bpo-36719: sync regrtest with master branch (GH-12967) · python/cpython@1069d38 · GitHub
[go: up one dir, main page]
Skip to content

Commit 1069d38

Browse files
vstinnervstinner
authored
[3.7] bpo-36719: sync regrtest with master branch (GH-12967)
* Clean up code which checked presence of os.{stat,lstat,chmod} (GH-11643) (cherry picked from commit 8377cd4) * bpo-36725: regrtest: add TestResult type (GH-12960) * Add TestResult and MultiprocessResult types to ensure that results always have the same fields. * runtest() now handles KeyboardInterrupt * accumulate_result() and format_test_result() now takes a TestResult * cleanup_test_droppings() is now called by runtest() and mark the test as ENV_CHANGED if the test leaks support.TESTFN file. * runtest() now includes code "around" the test in the test timing * Add print_warning() in test.libregrtest.utils to standardize how libregrtest logs warnings to ease parsing the test output. * support.unload() is now called with abstest rather than test_name * Rename 'test' variable/parameter to 'test_name' * dash_R(): remove unused the_module parameter * Remove unused imports (cherry picked from commit 4d29983) * bpo-36725: Refactor regrtest multiprocessing code (GH-12961) Rewrite run_tests_multiprocess() function as a new MultiprocessRunner class with multiple methods to better report errors and stop immediately when needed. Changes: * Worker processes are now killed immediately if tests are interrupted or if a test does crash (CHILD_ERROR): worker processes are killed. * Rewrite how errors in a worker thread are reported to the main thread. No longer ignore BaseException or parsing errors silently. * Remove 'finished' variable: use worker.is_alive() instead * Always compute omitted tests. Add Regrtest.get_executed() method. (cherry picked from commit 3cde440) * bpo-36719: regrtest always detect uncollectable objects (GH-12951) regrtest now always detects uncollectable objects. Previously, the check was only enabled by --findleaks. The check now also works with -jN/--multiprocess N. --findleaks becomes a deprecated alias to --fail-env-changed. (cherry picked from commit 75120d2) * bpo-34060: Report system load when running test suite for Windows (GH-8357) While Windows exposes the system processor queue length, the raw value used for load calculations on Unix systems, it does not provide an API to access the averaged value. Hence to calculate the load we must track and average it ourselves. We can't use multiprocessing or a thread to read it in the background while the tests run since using those would conflict with test_multiprocessing and test_xxsubprocess. Thus, we use Window's asynchronous IO API to run the tracker in the background with it sampling at the correct rate. When we wish to access the load we check to see if there's new data on the stream, if there is, we update our load values. (cherry picked from commit e16467a) * bpo-36719: Fix regrtest re-run (GH-12964) Properly handle a test which fail but then pass. Add test_rerun_success() unit test. (cherry picked from commit 837acc1) * bpo-36719: regrtest closes explicitly WindowsLoadTracker (GH-12965) Regrtest.finalize() now closes explicitly the WindowsLoadTracker instance. (cherry picked from commit 00db7c7)
1 parent 3076a3e commit 1069d38

13 files changed

+667
-383
lines changed

‎Lib/test/libregrtest/cmdline.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,9 @@ def _create_parser():
226226
'(instead of the Python stdlib test suite)')
227227

228228
group = parser.add_argument_group('Special runs')
229-
group.add_argument('-l', '--findleaks', action='store_true',
230-
help='if GC is available detect tests that leak memory')
229+
group.add_argument('-l', '--findleaks', action='store_const', const=2,
230+
default=1,
231+
help='deprecated alias to --fail-env-changed')
231232
group.add_argument('-L', '--runleaks', action='store_true',
232233
help='run the leaks(1) command just before exit.' +
233234
more_details)
@@ -309,7 +310,7 @@ def _parse_args(args, **kwargs):
309310
# Defaults
310311
ns = argparse.Namespace(testdir=None, verbose=0, quiet=False,
311312
exclude=False, single=False, randomize=False, fromfile=None,
312-
findleaks=False, use_resources=None, trace=False, coverdir='coverage',
313+
findleaks=1, use_resources=None, trace=False, coverdir='coverage',
313314
runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
314315
random_seed=None, use_mp=None, verbose3=False, forever=False,
315316
header=False, failfast=False, match_tests=None, pgo=False)
@@ -330,12 +331,13 @@ def _parse_args(args, **kwargs):
330331
parser.error("unrecognized arguments: %s" % arg)
331332
sys.exit(1)
332333

334+
if ns.findleaks > 1:
335+
# --findleaks implies --fail-env-changed
336+
ns.fail_env_changed = True
333337
if ns.single and ns.fromfile:
334338
parser.error("-s and -f don't go together!")
335339
if ns.use_mp is not None and ns.trace:
336340
parser.error("-T and -j don't go together!")
337-
if ns.use_mp is not None and ns.findleaks:
338-
parser.error("-l and -j don't go together!")
339341
if ns.failfast and not (ns.verbose or ns.verbose3):
340342
parser.error("-G/--failfast needs either -v or -W")
341343
if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):

‎Lib/test/libregrtest/main.py

Lines changed: 94 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,6 @@
2020
from test.libregrtest.setup import setup_tests
2121
from test.libregrtest.utils import removepy, count, format_duration, printlist
2222
from test import support
23-
try:
24-
import gc
25-
except ImportError:
26-
gc = None
2723

2824

2925
# When tests are run from the Python build directory, it is best practice
@@ -79,8 +75,8 @@ def __init__(self):
7975
self.skipped = []
8076
self.resource_denieds = []
8177
self.environment_changed = []
82-
self.rerun = []
8378
self.run_no_tests = []
79+
self.rerun = []
8480
self.first_result = None
8581
self.interrupted = False
8682

@@ -90,9 +86,6 @@ def __init__(self):
9086
# used by --coverage, trace.Trace instance
9187
self.tracer = None
9288

93-
# used by --findleaks, store for gc.garbage
94-
self.found_garbage = []
95-
9689
# used to display the progress bar "[ 3/100]"
9790
self.start_time = time.monotonic()
9891
self.test_count = ''
@@ -105,26 +98,43 @@ def __init__(self):
10598
# used by --junit-xml
10699
self.testsuite_xml = None
107100

108-
def accumulate_result(self, test, result):
109-
ok, test_time, xml_data = result
110-
if ok not in (CHILD_ERROR, INTERRUPTED):
111-
self.test_times.append((test_time, test))
101+
self.win_load_tracker = None
102+
103+
def get_executed(self):
104+
return (set(self.good) | set(self.bad) | set(self.skipped)
105+
| set(self.resource_denieds) | set(self.environment_changed)
106+
| set(self.run_no_tests))
107+
108+
def accumulate_result(self, result, rerun=False):
109+
test_name = result.test_name
110+
ok = result.result
111+
112+
if ok not in (CHILD_ERROR, INTERRUPTED) and not rerun:
113+
self.test_times.append((result.test_time, test_name))
114+
112115
if ok == PASSED:
113-
self.good.append(test)
116+
self.good.append(test_name)
114117
elif ok in (FAILED, CHILD_ERROR):
115-
self.bad.append(test)
118+
if not rerun:
119+
self.bad.append(test_name)
116120
elif ok == ENV_CHANGED:
117-
self.environment_changed.append(test)
121+
self.environment_changed.append(test_name)
118122
elif ok == SKIPPED:
119-
self.skipped.append(test)
123+
self.skipped.append(test_name)
120124
elif ok == RESOURCE_DENIED:
121-
self.skipped.append(test)
122-
self.resource_denieds.append(test)
125+
self.skipped.append(test_name)
126+
self.resource_denieds.append(test_name)
123127
elif ok == TEST_DID_NOT_RUN:
124-
self.run_no_tests.append(test)
125-
elif ok != INTERRUPTED:
128+
self.run_no_tests.append(test_name)
129+
elif ok == INTERRUPTED:
130+
self.interrupted = True
131+
else:
126132
raise ValueError("invalid test result: %r" % ok)
127133

134+
if rerun and ok not in {FAILED, CHILD_ERROR, INTERRUPTED}:
135+
self.bad.remove(test_name)
136+
137+
xml_data = result.xml_data
128138
if xml_data:
129139
import xml.etree.ElementTree as ET
130140
for e in xml_data:
@@ -134,7 +144,7 @@ def accumulate_result(self, test, result):
134144
print(xml_data, file=sys.__stderr__)
135145
raise
136146

137-
def display_progress(self, test_index, test):
147+
def display_progress(self, test_index, text):
138148
if self.ns.quiet:
139149
return
140150

@@ -143,12 +153,12 @@ def display_progress(self, test_index, test):
143153
fails = len(self.bad) + len(self.environment_changed)
144154
if fails and not self.ns.pgo:
145155
line = f"{line}/{fails}"
146-
line = f"[{line}] {test}"
156+
line = f"[{line}] {text}"
147157

148158
# add the system load prefix: "load avg: 1.80 "
149-
if hasattr(os, 'getloadavg'):
150-
load_avg_1min = os.getloadavg()[0]
151-
line = f"load avg: {load_avg_1min:.2f} {line}"
159+
load_avg = self.getloadavg()
160+
if load_avg is not None:
161+
line = f"load avg: {load_avg:.2f} {line}"
152162

153163
# add the timestamp prefix: "0:01:05 "
154164
test_time = time.monotonic() - self.start_time
@@ -164,22 +174,6 @@ def parse_args(self, kwargs):
164174
"faulthandler.dump_traceback_later", file=sys.stderr)
165175
ns.timeout = None
166176

167-
if ns.threshold is not None and gc is None:
168-
print('No GC available, ignore --threshold.', file=sys.stderr)
169-
ns.threshold = None
170-
171-
if ns.findleaks:
172-
if gc is not None:
173-
# Uncomment the line below to report garbage that is not
174-
# freeable by reference counting alone. By default only
175-
# garbage that is not collectable by the GC is reported.
176-
pass
177-
#gc.set_debug(gc.DEBUG_SAVEALL)
178-
else:
179-
print('No GC available, disabling --findleaks',
180-
file=sys.stderr)
181-< 1241 /span>
ns.findleaks = False
182-
183177
if ns.xmlpath:
184178
support.junit_xml_list = self.testsuite_xml = []
185179

@@ -275,13 +269,13 @@ def list_cases(self):
275269
support.verbose = False
276270
support.set_match_tests(self.ns.match_tests)
277271

278-
for test in self.selected:
279-
abstest = get_abs_module(self.ns, test)
272+
for test_name in self.selected:
273+
abstest = get_abs_module(self.ns, test_name)
280274
try:
281275
suite = unittest.defaultTestLoader.loadTestsFromName(abstest)
282276
self._list_cases(suite)
283277
except unittest.SkipTest:
284-
self.skipped.append(test)
278+
self.skipped.append(test_name)
285279

286280
if self.skipped:
287281
print(file=sys.stderr)
@@ -298,23 +292,19 @@ def rerun_failed_tests(self):
298292
print()
299293
print("Re-running failed tests in verbose mode")
300294
self.rerun = self.bad[:]
301-
for test in self.rerun:
302-
print("Re-running test %r in verbose mode" % test, flush=True)
303-
try:
304-
self.ns.verbose = True
305-
ok = runtest(self.ns, test)
306-
except KeyboardInterrupt:
307-
self.interrupted = True
308-
# print a newline separate from the ^C
309-
print()
295+
for test_name in self.rerun:
296+
print(f"Re-running {test_name} in verbose mode", flush=True)
297+
self.ns.verbose = True
298+
result = runtest(self.ns, test_name)
299+
300+
self.accumulate_result(result, rerun=True)
301+
302+
if result.result == INTERRUPTED:
310303
break
311-
else:
312-
if ok[ 10000 0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}:
313-
self.bad.remove(test)
314-
else:
315-
if self.bad:
316-
print(count(len(self.bad), 'test'), "failed again:")
317-
printlist(self.bad)
304+
305+
if self.bad:
306+
print(count(len(self.bad), 'test'), "failed again:")
307+
printlist(self.bad)
318308

319309
self.display_result()
320310

@@ -327,11 +317,11 @@ def display_result(self):
327317
print("== Tests result: %s ==" % self.get_tests_result())
328318

329319
if self.interrupted:
330-
print()
331-
# print a newline after ^C
332320
print("Test suite interrupted by signal SIGINT.")
333-
executed = set(self.good) | set(self.bad) | set(self.skipped)
334-
omitted = set(self.selected) - executed
321+
322+
omitted = set(self.selected) - self.get_executed()
323+
if omitted:
324+
print()
335325
print(count(len(omitted), "test"), "omitted:")
336326
printlist(omitted)
337327

@@ -348,8 +338,8 @@ def display_result(self):
348338
self.test_times.sort(reverse=True)
349339
print()
350340
print("10 slowest tests:")
351-
for time, test in self.test_times[:10]:
352-
print("- %s: %s" % (test, format_duration(time)))
341+
for test_time, test in self.test_times[:10]:
342+
print("- %s: %s" % (test, format_duration(test_time)))
353343

354344
if self.bad:
355345
print()
@@ -387,50 +377,37 @@ def run_tests_sequential(self):
387377
print("Run tests sequentially")
388378

389379
previous_test = None
390-
for test_index, test in enumerate(self.tests, 1):
380+
for test_index, test_name in enumerate(self.tests, 1):
391381
start_time = time.monotonic()
392382

393-
text = test
383+
text = test_name
394384
if previous_test:
395385
text = '%s -- %s' % (text, previous_test)
396386
self.display_progress(test_index, text)
397387

398388
if self.tracer:
399389
# If we're tracing code coverage, then we don't exit with status
400390
# if on a false return value from main.
401-
cmd = ('result = runtest(self.ns, test); '
402-
'self.accumulate_result(test, result)')
391+
cmd = ('result = runtest(self.ns, test_name); '
392+
'self.accumulate_result(result)')
403393
ns = dict(locals())
404394
self.tracer.runctx(cmd, globals=globals(), locals=ns)
405395
result = ns['result']
406396
else:
407-
try:
408-
result = runtest(self.ns, test)
409-
except KeyboardInterrupt:
410-
self.interrupted = True
411-
self.accumulate_result(test, (INTERRUPTED, None, None))
412-
break
413-
else:
414-
self.accumulate_result(test, result)
415-
416-
previous_test = format_test_result(test, result[0])
397+
result = runtest(self.ns, test_name)
398+
self.accumulate_result(result)
399+
400+
if result.result == INTERRUPTED:
401+
break
402+
403+
previous_test = format_test_result(result)
417404
test_time = time.monotonic() - start_time
418405
if test_time >= PROGRESS_MIN_TIME:
419406
previous_test = "%s in %s" % (previous_test, format_duration(test_time))
420407
elif result[0] == PASSED:
421408
# be quiet: say nothing if the test passed shortly
422409
previous_test = None
423410

424-
if self.ns.findleaks:
425-
gc.collect()
426-
if gc.garbage:
427-
print("Warning: test created", len(gc.garbage), end=' ')
428-
print("uncollectable object(s).")
429-
# move the uncollectable objects somewhere so we don't see
430-
# them again
431-
self.found_garbage.extend(gc.garbage)
432-
del gc.garbage[:]
433-
434411
# Unload the newly imported modules (best effort finalization)
435412
for module in sys.modules.keys():
436413
if module not in save_modules and module.startswith("test."):
@@ -441,8 +418,8 @@ def run_tests_sequential(self):
441418

442419
def _test_forever(self, tests):
443420
while True:
444-
for test in tests:
445-
yield test
421+
for test_name in tests:
422+
yield test_name
446423
if self.bad:
447424
return
448425
if self.ns.fail_env_changed and self.environment_changed:
@@ -515,6 +492,10 @@ def run_tests(self):
515492
self.run_tests_sequential()
516493

517494
def finalize(self):
495+
if self.win_load_tracker is not None:
496+
self.win_load_tracker.close()
497+
self.win_load_tracker = None
498+
518499
if self.next_single_filename:
519500
if self.next_single_test:
520501
with open(self.next_single_filename, 'w') as fp:
@@ -585,6 +566,15 @@ def main(self, tests=None, **kwargs):
585566
with support.temp_cwd(test_cwd, quiet=True):
586567
self._main(tests, kwargs)
587568

569+
def getloadavg(self):
570+
if self.win_load_tracker is not None:
571+
return self.win_load_tracker.getloadavg()
572+
573+
if hasattr(os, 'getloadavg'):
574+
return os.getloadavg()[0]
575+
576+
return None
577+
588578
def _main(self, tests, kwargs):
589579
if self.ns.huntrleaks:
590580
warmup, repetitions, _ = self.ns.huntrleaks
@@ -616,6 +606,18 @@ def _main(self, tests, kwargs):
616606
self.list_cases()
617607
sys.exit(0)
618608

609+
# If we're on windows and this is the parent runner (not a worker),
610+
# track the load average.
611+
if sys.platform == 'win32' and (self.ns.worker_args is None):
612+
from test.libregrtest.win_utils import WindowsLoadTracker
613+
614+
try:
615+
self.win_load_tracker = WindowsLoadTracker()
616+
except FileNotFoundError as error:
617+
# Windows IoT Core and Windows Nano Server do not provide
618+
# typeperf.exe for x64, x86 or ARM
619+
print(f'Failed to create WindowsLoadTracker: {error}')
620+
619621
self.run_tests()
620622
self.display_result()
621623

‎Lib/test/libregrtest/refleak.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import errno
21
import os
32
import re
43
import sys
@@ -18,7 +17,7 @@ def _get_dump(cls):
1817
cls._abc_negative_cache, cls._abc_negative_cache_version)
1918

2019

21-
def dash_R(ns, the_module, test_name, test_func):
20+
def dash_R(ns, test_name, test_func):
2221
"""Run a test multiple times, looking for reference leaks.
2322
2423
Returns:

0 commit comments

Comments
0 (0)
0