8000 bpo-36560: Fix reference leak hunting in regrtest (GH-12744) · python/cpython@5aaac94 · GitHub
[go: up one dir, main page]
Skip to content

Commit 5aaac94

Browse files
vstinnervstinner
authored
bpo-36560: Fix reference leak hunting in regrtest (GH-12744)
Fix reference leak hunting in regrtest: compute also deltas (of reference count, allocated memory blocks, file descriptor count) during warmup, to ensure that everything is initialized before starting to hunt reference leaks. Other changes: * Replace gc.collect() with support.gc_collect() * Move calls to read memory statistics from dash_R_cleanup() to dash_R() * Pass regrtest 'ns' to dash_R() * dash_R() is now more quiet with --quiet option (don't display progress). * Precompute the full range for "for it in range(repcount):" to ensure that the iteration doesn't allocate anything new. * dash_R() now is responsible to call warm_caches().
1 parent e16467a commit 5aaac94

File tree

4 files changed

+46
-33
lines changed

4 files changed

+46
-33
lines changed

‎Lib/test/libregrtest/refleak.py

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def _get_dump(cls):
1818
cls._abc_negative_cache, cls._abc_negative_cache_version)
1919

2020

21-
def dash_R(the_module, test, indirect_test, huntrleaks):
21+
def dash_R(ns, the_module, test_name, test_func):
2222
"""Run a test multiple times, looking for reference leaks.
2323
2424
Returns:
@@ -32,6 +32,10 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
3232
raise Exception("Tracking reference leaks requires a debug build "
3333
"of Python")
3434

35+
# Avoid false positives due to various caches
36+
# filling slowly with random data:
37+
warm_caches()
38+
3539
# Save current values for dash_R_cleanup() to restore.
3640
fs = warnings.filters[:]
3741
ps = copyreg.dispatch_table.copy()
@@ -57,31 +61,50 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
5761
def get_pooled_int(value):
5862
return int_pool.setdefault(value, value)
5963

60-
nwarmup, ntracked, fname = huntrleaks
64+
nwarmup, ntracked, fname = ns.huntrleaks
6165
fname = os.path.join(support.SAVEDCWD, fname)
6266
repcount = nwarmup + ntracked
67+
68+
# Pre-allocate to ensure that the loop doesn't allocate anything new
69+
rep_range = list(range(repcount))
6370
rc_deltas = [0] * repcount
6471
alloc_deltas = [0] * repcount
6572
fd_deltas = [0] * repcount
73+
getallocatedblocks = sys.getallocatedblocks
74+
gettotalrefcount = sys.gettotalrefcount
75+
fd_count = support.fd_count
6676

67-
print("beginning", repcount, "repetitions", file=sys.stderr)
68-
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
69-
flush=True)
7077
# initialize variables to make pyflakes quiet
7178
rc_before = alloc_before = fd_before = 0
72-
for i in range(repcount):
73-
indirect_test()
74-
alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc,
75-
abcs)
76-
print('.', end='', file=sys.stderr, flush=True)
77-
if i >= nwarmup:
78-
rc_deltas[i] = get_pooled_int(rc_after - rc_before)
79-
alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
80-
fd_deltas[i] = get_pooled_int(fd_after - fd_before)
79+
80+
if not ns.quiet:
81+
print("beginning", repcount, "repetitions", file=sys.stderr)
82+
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
83+
flush=True)
84+
85+
for i in rep_range:
86+
test_func()
87+
dash_R_cleanup(fs, ps, pic, zdc, abcs)
88+
89+
# Collect cyclic trash and read memory statistics immediately after.
90+
support.gc_collect()
91+
alloc_after = getallocatedblocks()
92+
rc_after = gettotalrefcount()
93+
fd_after = fd_count()
94+
95+
if not ns.quiet:
96+
print('.', end='', file=sys.stderr, flush=True)
97+
98+
rc_deltas[i] = get_pooled_int(rc_after - rc_before)
99+
alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
100+
fd_deltas[i] = get_pooled_int(fd_after - fd_before)
101+
81102
alloc_before = alloc_after
82103
rc_before = rc_after
83104
fd_before = fd_after
84-
print(file=sys.stderr)
105+
106+
if not ns.quiet:
107+
print(file=sys.stderr)
85108

86109
# These checkers return False on success, True on failure
87110
def check_rc_deltas(deltas):
@@ -112,7 +135,7 @@ def check_fd_deltas(deltas):
112135
deltas = deltas[nwarmup:]
113136
if checker(deltas):
114137
msg = '%s leaked %s %s, sum=%s' % (
115-
test, deltas, item_name, sum(deltas))
138+
test_name, deltas, item_name, sum(deltas))
116139
print(msg, file=sys.stderr, flush=True)
117140
with open(fname, "a") as refrep:
118141
print(msg, file=refrep)
@@ -122,7 +145,7 @@ def check_fd_deltas(deltas):
122145

123146

124147
def dash_R_cleanup(fs, ps, pic, zdc, abcs):
125-
import gc, copyreg
148+
import copyreg
126149
import collections.abc
127150

128151
# Restore some original values.
@@ -154,16 +177,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
154177

155178
clear_caches()
156179

157-
# Collect cyclic trash and read memory statistics immediately after.
158-
func1 = sys.getallocatedblocks
159-
func2 = sys.gettotalrefcount
160-
gc.collect()
161-
return func1(), func2(), support.fd_count()
162-
163180

164181
def clear_caches():
165-
import gc
166-
167182
# Clear the warnings registry, so they can be displayed again
168183
for mod in sys.modules.values():
169184
if hasattr(mod, '__warningregistry__'):
@@ -256,7 +271,7 @@ def clear_caches():
256271
for f in typing._cleanups:
257272
f()
258273

259-
gc.collect()
274+
support.gc_collect()
260275

261276

262277
def warm_caches():

‎Lib/test/libregrtest/runtest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ def test_runner():
177177
raise Exception("errors while loading tests")
178178
support.run_unittest(tests)
179179
if ns.huntrleaks:
180-
refleak = dash_R(the_module, test, test_runner, ns.huntrleaks)
180+
refleak = dash_R(ns, the_module, test, test_runner)
181181
else:
182182
test_runner()
183183
test_time = time.perf_counter() - start_time

‎Lib/test/libregrtest/setup.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@
1010
except ImportError:
1111
gc = None
1212

13-
from test.libregrtest.refleak import warm_caches
14-
1513

1614
def setup_tests(ns):
1715
try:
@@ -79,10 +77,6 @@ def setup_tests(ns):
7977
if ns.huntrleaks:
8078
unittest.BaseTestSuite._cleanup = False
8179

82-
# Avoid false positives due to various caches
83-
# filling slowly with random data:
84-
warm_caches()
85-
8680
if ns.memlimit is not None:
8781
support.set_memlimit(ns.memlimit)
8882

‎Misc/NEWS.d/next/Tests/2019-04-09-14-08-02.bpo-36560._ejeOr.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix reference leak hunting in regrtest: compute also deltas (of reference
2+
count, allocated memory blocks, file descriptor count) during warmup, to
3+
ensure that everything is initialized before starting to hunt reference
4+
leaks.

0 commit comments

Comments
 (0)
0