8000 [3.11] Add test.support.busy_retry() (#93770) (#110341) · python/cpython@f9ac377 · GitHub
[go: up one dir, main page]
Skip to content

Commit f9ac377

Browse files
vstinnervstinner
authored
[3.11] Add test.support.busy_retry() (#93770) (#110341)
Add test.support.busy_retry() (#93770) Add busy_retry() and sleeping_retry() functions to test.support. (cherry picked from commit 7e9eaad)
1 parent 6c98c73 commit f9ac377

12 files changed

+185
-99
lines changed

‎Doc/library/test.rst

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,51 @@ The :mod:`test.support` module defines the following constants:
404404

405405
The :mod:`test.support` module defines the following functions:
406406

407+
.. function:: busy_retry(timeout, err_msg=None, /, *, error=True)
408+
409+
Run the loop body until ``break`` stops the loop.
410+
411+
After *timeout* seconds, raise an :exc:`AssertionError` if *error* is true,
412+
or just stop the loop if *error* is false.
413+
414+
Example::
415+
416+
for _ in support.busy_retry(support.SHORT_TIMEOUT):
417+
if check():
418+
break
419+
420+
Example of error=False usage::
421+
422+
for _ in support.busy_retry(support.SHORT_TIMEOUT, error=False):
423+
if check():
424+
break
425+
else:
426+
raise RuntimeError('my custom error')
427+
428+
.. function:: sleeping_retry(timeout, err_msg=None, /, *, init_delay=0.010, max_delay=1.0, error=True)
429+
430+
Wait strategy that applies exponential backoff.
431+
432+
Run the loop body until ``break`` stops the loop. Sleep at each loop
433+
iteration, but not at the first iteration. The sleep delay is doubled at
434+
each iteration (up to *max_delay* seconds).
435+
436+
See :func:`busy_retry` documentation for the par B41A ameters usage.
437+
438+
Example raising an exception after SHORT_TIMEOUT seconds::
439+
440+
for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
441+
if check():
442+
break
443+
444+
Example of error=False usage::
445+
446+
for _ in support.sleeping_retry(support.SHORT_TIMEOUT, error=False):
447+
if check():
448+
break
449+
else:
450+
raise RuntimeError('my custom error')
451+
407452
.. function:: is_resource_enabled(resource)
408453

409454
Return ``True`` if *resource* is enabled and available. The list of

‎Lib/test/_test_multiprocessing.py

Lines changed: 25 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -4376,18 +4376,13 @@ def test_shared_memory_cleaned_after_process_termination(self):
43764376
p.terminate()
43774377
p.wait()
43784378

4379-
deadline = time.monotonic() + support.LONG_TIMEOUT
4380-
t = 0.1
4381-
while time.monotonic() < deadline:
4382-
time.sleep(t)
4383-
t = min(t*2, 5)
4379+
err_msg = ("A SharedMemory segment was leaked after "
4380+
"a process was abruptly terminated")
4381+
for _ in support.sleeping_retry(support.LONG_TIMEOUT, err_msg):
43844382
try:
43854383
smm = shared_memory.SharedMemory(name, create=False)
43864384
except FileNotFoundError:
43874385
break
4388-
else:
4389-
raise AssertionError("A SharedMemory segment was leaked after"
4390-
" a process was abruptly terminated.")
43914386

43924387
if os.name == 'posix':
43934388
# Without this line it was raising warnings like:
@@ -5458,20 +5453,18 @@ def create_and_register_resource(rtype):
54585453
p.terminate()
54595454
p.wait()
54605455

5461-
deadline = time.monotonic() + support.LONG_TIMEOUT
5462-
while time.monotonic() < deadline:
5463-
time.sleep(.5)
5456+
err_msg = (f"A {rtype} resource was leaked after a process was "
5457+
f"abruptly terminated")
5458+
for _ in support.sleeping_retry(support.SHORT_TIMEOUT,
5459+
err_msg):
54645460
try:
54655461
_resource_unlink(name2, rtype)
54665462
except OSError as e:
54675463
# docs say it should be ENOENT, but OSX seems to give
54685464
# EINVAL
54695465
self.assertIn(e.errno, (errno.ENOENT, errno.EINVAL))
54705466
break
5471-
else:
5472-
raise AssertionError(
5473-
f"A {rtype} resource was leaked after a process was "
5474-
f"abruptly terminated.")
5467+
54755468
err = p.stderr.read().decode('utf-8')
54765469
p.stderr.close()
54775470
expected = ('resource_tracker: There appear to be 2 leaked {} '
@@ -5707,18 +5700,17 @@ def wait_proc_exit(self):
57075700
# but this can take a bit on slow machines, so wait a few seconds
57085701
# if there are other children too (see #17395).
57095702
join_process(self.proc)
5703+
57105704
start_time = time.monotonic()
5711-
t = 0.01
5712-
while len(multiprocessing.active_children()) > 1:
5713-
time.sleep(t)
5714-
t *= 2
5715-
dt = time.monotonic() - start_time
5716-
if dt >= 5.0:
5717-
test.support.environment_altered = True
5718-
support.print_warning(f"multiprocessing.Manager still has "
5719-
f"{multiprocessing.active_children()} "
5720-
f"active children after {dt} seconds")
5705+
for _ in support.sleeping_retry(5.0, error=False):
5706+
if len(multiprocessing.active_children()) <= 1:
57215707
break
5708+
else:
5709+
dt = time.monotonic() - start_time
5710+
support.environment_altered = True
5711+
support.print_warning(f"multiprocessing.Manager still has "
5712+
f"{multiprocessing.active_children()} "
5713+
f"active children after {dt:.1f} seconds")
57225714

57235715
def run_worker(self, worker, obj):
57245716
self.proc = multiprocessing.Process(target=worker, args=(obj, ))
@@ -6031,17 +6023,15 @@ def tearDownClass(cls):
60316023
# but this can take a bit on slow machines, so wait a few seconds
60326024
# if there are other children too (see #17395)
60336025
start_time = time.monotonic()
6034-
t = 0.01
6035-
while len(multiprocessing.active_children()) > 1:
6036-
time.sleep(t)
6037-
t *= 2
6038-
dt = time.monotonic() - start_time
6039-
if dt >= 5.0:
6040-
test.support.environment_altered = True
6041-
support.print_warning(f"multiprocessing.Manager still has "
6042-
f"{multiprocessing.active_children()} "
6043-
f"active children after {dt} seconds")
6026+
for _ in support.sleeping_retry(5.0, error=False):
6027+
if len(multiprocessing.active_children()) <= 1:
60446028
break
6029+
else:
6030+
dt = time.monotonic() - start_time
6031+
support.environment_altered = True
6032+
support.print_warning(f"multiprocessing.Manager still has "
6033+
f"{multiprocessing.active_children()} "
6034+
f"active children after {dt:.1f} seconds")
60456035

60466036
gc.collect() # do garbage collection
60476037
if cls.manager._number_of_objects() != 0:

‎Lib/test/fork_wait.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,8 @@ def test_wait(self):
5454
self.threads.append(thread)
5555

5656
# busy-loop to wait for threads
57-
deadline = time.monotonic() + support.SHORT_TIMEOUT
58-
while len(self.alive) < NUM_THREADS:
59-
time.sleep(0.1)
60-
if deadline < time.monotonic():
57+
for _ in support.sleeping_retry(support.SHORT_TIMEOUT, error=False):
58+
if len(self.alive) >= NUM_THREADS:
6159
break
6260

6361
a = sorted(self.alive.keys())

‎Lib/test/support/__init__.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2324,6 +2324,82 @@ def requires_venv_with_pip():
23242324
return unittest.skipUnless(ctypes, 'venv: pip requires ctypes')
23252325

23262326

2327+
def busy_retry(timeout, err_msg=None, /, *, error=True):
2328+
"""
2329+
Run the loop body until "break" stops the loop.
2330+
2331+
After *timeout* seconds, raise an AssertionError if *error* is true,
2332+
or just stop if *error is false.
2333+
2334+
Example:
2335+
2336+
for _ in support.busy_retry(support.SHORT_TIMEOUT):
2337+
if check():
2338+
break
2339+
2340+
Example of error=False usage:
2341+
2342+
for _ in support.busy_retry(support.SHORT_TIMEOUT, error=False):
2343+
if check():
2344+
break
2345+
else:
2346+
raise RuntimeError('my custom error')
2347+
2348+
"""
2349+
if timeout <= 0:
2350+
raise ValueError("timeout must be greater than zero")
2351+
2352+
start_time = time.monotonic()
2353+
deadline = start_time + timeout
2354+
2355+
while True:
2356+
yield
2357+
2358+
if time.monotonic() >= deadline:
2359+
break
2360+
2361+
if error:
2362+
dt = time.monotonic() - start_time
2363+
msg = f"timeout ({dt:.1f} seconds)"
2364+
if err_msg:
2365+
msg = f"{msg}: {err_msg}"
2366+
raise AssertionError(msg)
2367+
2368+
2369+
def sleeping_retry(timeout, err_msg=None, /,
2370+
*, init_delay=0.010, max_delay=1.0, error=True):
2371+
"""
2372+
Wait strategy that applies exponential backoff.
2373+
2374+
Run the loop body until "break" stops the loop. Sleep at each loop
2375+
iteration, but not at the first iteration. The sleep delay is doubled at
2376+
each iteration (up to *max_delay* seconds).
2377+
2378+
See busy_retry() documentation for the parameters usage.
2379+
2380+
Example raising an exception after SHORT_TIMEOUT seconds:
2381+
2382+
for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
2383+
if check():
2384+
break
2385+
2386+
Example of error=False usage:
2387+
2388+
for _ in support.sleeping_retry(support.SHORT_TIMEOUT, error=False):
2389+
if check():
2390+
break
2391+
else:
2392+
raise RuntimeError('my custom error')
2393+
"""
2394+
2395+
delay = init_delay
2396+
for _ in busy_retry(timeout, err_msg, error=error):
2397+
yield
2398+
2399+
time.sleep(delay)
2400+
delay = min(delay * 2, max_delay)
2401+
2402+
23272403
@contextlib.contextmanager
23282404
def adjust_int_max_str_digits(max_digits):
23292405
"""Temporarily change the integer string conversion length limit."""

‎Lib/test/test__xxsubinterpreters.py

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,11 @@ def _wait_for_interp_to_run(interp, timeout=None):
4545
# run subinterpreter eariler than the main thread in multiprocess.
4646
if timeout is None:
4747
timeout = support.SHORT_TIMEOUT
48-
start_time = time.monotonic()
49-
deadline = start_time + timeout
50-
while not interpreters.is_running(interp):
51-
if time.monotonic() > deadline:
52-
raise RuntimeError('interp is not running')
53-
time.sleep(0.010)
48+
for _ in support.sleeping_retry(timeout, error=False):
49+
if interpreters.is_running(interp):
50+
break
51+
else:
52+
raise RuntimeError('interp is not running')
5453

5554

5655
@contextlib.contextmanager

‎Lib/test/test_concurrent_futures/test_init.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,10 @@ def test_initializer(self):
7878
future.result()
7979

8080
# At some point, the executor should break
81-
t1 = time.monotonic()
82-
while not self.executor._broken:
83-
if time.monotonic() - t1 > 5:
84-
self.fail("executor not broken after 5 s.")
85-
time.sleep(0.01)
81+
for _ in support.sleeping_retry(5, "executor not broken"):
82+
if self.executor._broken:
83+
break
84+
8685
# ... and from this point submit() is guaranteed to fail
8786
with self.assertRaises(BrokenExecutor):
8887
self.executor.submit(get_init_status)

‎Lib/test/test_multiprocessing_main_handling.py

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import sys
4141
import time
4242
from multiprocessing import Pool, set_start_method
43+
from test import support
4344
4445
# We use this __main D7AE __ defined function in the map call below in order to
4546
# check that multiprocessing in correctly running the unguarded
@@ -59,13 +60,11 @@ def f(x):
5960
results = []
6061
with Pool(5) as pool:
6162
pool.map_async(f, [1, 2, 3], callback=results.extend)
62-
start_time = time.monotonic()
63-
while not results:
64-
time.sleep(0.05)
65-
# up to 1 min to report the results
66-
dt = time.monotonic() - start_time
67-
if dt > 60.0:
68-
raise RuntimeError("Timed out waiting for results (%.1f sec)" % dt)
63+
64+
# up to 1 min to report the results
65+
for _ in support.sleeping_retry(60, "Timed out waiting for results"):
66+
if results:
67+
break
6968
7069
results.sort()
7170
print(start_method, "->", results)
@@ -86,19 +85,17 @@ def f(x):
8685
import sys
8786
import time
8887
from multiprocessing import Pool, set_start_method
88+
from test import support
8989
9090
start_method = sys.argv[1]
9191
set_start_method(start_method)
9292
results = []
9393
with Pool(5) as pool:
9494
pool.map_async(int, [1, 4, 9], callback=results.extend)
95-
start_time = time.monotonic()
96-
while not results:
97-
time.sleep(0.05)
98-
# up to 1 min to report the results
99-
dt = time.monotonic() - start_time
100-
if dt > 60.0:
101-
raise RuntimeError("Timed out waiting for results (%.1f sec)" % dt)
95+
# up to 1 min to report the results
96+
for _ in support.sleeping_retry(60, "Timed out waiting for results"):
97+
if results:
98+
break
10299
103100
results.sort()
104101
print(start_method, "->", results)

‎Lib/test/test_signal.py

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -813,13 +813,14 @@ def test_itimer_virtual(self):
813813
signal.signal(signal.SIGVTALRM, self.sig_vtalrm)
814814
signal.setitimer(self.itimer, 0.3, 0.2)
815815

816-
start_time = time.monotonic()
817-
while time.monotonic() - start_time < 60.0:
816+
for _ in support.busy_retry(60.0, error=False):
818817
# use up some virtual time by doing real work
819818
_ = pow(12345, 67890, 10000019)
820819
if signal.getitimer(self.itimer) == (0.0, 0.0):
821-
break # sig_vtalrm handler stopped this itimer
822-
else: # Issue 8424
820+
# sig_vtalrm handler stopped this itimer
821+
break
822+
else:
823+
# bpo-8424
823824
self.skipTest("timeout: likely cause: machine too slow or load too "
824825
"high")
825826

@@ -833,13 +834,14 @@ def test_itimer_prof(self):
833834
signal.signal(signal.SIGPROF, self.sig_prof)
834835
signal.setitimer(self.itimer, 0.2, 0.2)
835836

836-
start_time = time.monotonic()
837-
while time.monotonic() - start_time < 60.0:
837+
for _ in support.busy_retry(60.0, error=False):
838838
# do some work
839839
_ = pow(12345, 67890, 10000019)
840840
if signal.getitimer(self.itimer) == (0.0, 0.0):
841-
break # sig_prof handler stopped this itimer
842-
else: # Issue 8424
841+
# sig_prof handler stopped this itimer
842+
break
843+
else:
844+
# bpo-8424
843845
self.skipTest("timeout: likely cause: machine too slow or load too "
844846
"high")
845847

@@ -1308,8 +1310,6 @@ def handler(signum, frame):
13081310
self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL
13091311

13101312
expected_sigs = 0
1311-
deadline = time.monotonic() + support.SHORT_TIMEOUT
1312-
13131313
while expected_sigs < N:
13141314
# Hopefully the SIGALRM will be received somewhere during
13151315
# initial processing of SIGUSR1.
@@ -1318,8 +1318,9 @@ def handler(signum, frame):
13181318

13191319
expected_sigs += 2
13201320
# Wait for handlers to run to avoid signal coalescing
1321-
while len(sigs) < expected_sigs and time.monotonic() < deadline:
1322-
time.sleep(1e-5)
1321+
for _ in support.sleeping_retry(support.SHORT_TIMEOUT, error=False):
1322+
if len(sigs) >= expected_sigs:
1323+
break
13231324

13241325
# All ITIMER_REAL signals should have been delivered to the
13251326
# Python handler

0 commit comments

Comments
 (0)
0