8000 GH-117108: Change the size of the GC increment to about 1% of the tot… · python/cpython@e28477f · GitHub
[go: up one dir, main page]
Skip to content

Commit e28477f

Browse files
markshannonmarkshannon
authored
GH-117108: Change the size of the GC increment to about 1% of the total heap size. (GH-117120)
1 parent e2e0b4b commit e28477f

File tree

6 files changed

+47
-28
lines changed

6 files changed

+47
-28
lines changed

‎Include/internal/pycore_gc.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,7 @@ struct _gc_runtime_state {
282282
/* a list of callbacks to be invoked when collection is performed */
283283
PyObject *callbacks;
284284

285+
Py_ssize_t heap_size;
285286
Py_ssize_t work_to_do;
286287
/* Which of the old spaces is the visited space */
287288
int visited_space;
@@ -321,7 +322,7 @@ extern void _PyGC_Unfreeze(PyInterpreterState *interp);
321322
/* Number of frozen objects */
322323
extern Py_ssize_t _PyGC_GetFreezeCount(PyInterpreterState *interp);
323324

324-
extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation);
325+
extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, int generation);
325326
extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs);
326327

327328
// Functions to clear types free lists

‎Lib/test/test_gc.py

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,7 +1058,19 @@ class Z:
10581058
callback.assert_not_called()
10591059
gc.enable()
10601060

1061+
1062+
class IncrementalGCTests(unittest.TestCase):
1063+
1064+
def setUp(self):
1065+
# Reenable GC as it is disabled module-wide
1066+
gc.enable()
1067+
1068+
def tearDown(self):
1069+
gc.disable()
1070+
10611071
@unittest.skipIf(Py_GIL_DISABLED, "Free threading does not support incremental GC")
1072+
# Use small increments to emulate longer running process in a shorter time
1073+
@gc_threshold(200, 10)
10621074
def test_incremental_gc_handles_fast_cycle_creation(self):
10631075

10641076
class LinkedList:
@@ -1080,28 +1092,31 @@ def make_ll(depth):
10801092
head = LinkedList(head, head.prev)
10811093
return head
10821094

1083-
head = make_ll(10000)
1084-
count = 10000
1095+
head = make_ll(1000)
1096+
count = 1000
10851097

1086-
# We expect the counts to go negative eventually
1087-
# as there will some objects we aren't counting,
1088-
# e.g. the gc stats dicts. The test merely checks
1089-
# that the counts don't grow.
1098+
# There will be some objects we aren't counting,
1099+
# e.g. the gc stats dicts. This test checks
1100+
# that the counts don't grow, so we try to
1101+
# correct for the uncounted objects
1102+
# This is just an estimate.
1103+
CORRECTION = 20
10901104

10911105
enabled = gc.isenabled()
10921106
gc.enable()
10931107
olds = []
1094-
for i in range(1000):
1095-
newhead = make_ll(200)
1096-
count += 200
1108+
for i in range(20_000):
1109+
newhead = make_ll(20)
1110+
count += 20
10971111
newhead.surprise = head
10981112
olds.append(newhead)
1099-
if len(olds) == 50:
1113+
if len(olds) == 20:
11001114
stats = gc.get_stats()
11011115
young = stats[0]
11021116
incremental = stats[1]
11031117
old = stats[2]
11041118
collected = young['collected'] + incremental['collected'] + old['collected']
1119+
count += CORRECTION
11051120
live = count - collected
11061121
self.assertLess(live, 25000)
11071122
del olds[:]

‎Misc/NEWS.d/next/Core and Builtins/2024-03-21-12-10-11.gh-issue-117108._6jIrB.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The cycle GC now chooses the size of increments based on the total heap
2+
size, instead of the rate of object creation. This ensures that it can keep
3+
up with growing heaps.

‎Modules/gcmodule.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ gc_get_objects_impl(PyObject *module, Py_ssize_t generation)
326326
}
327327

328328
PyInterpreterState *interp = _PyInterpreterState_GET();
329-
return _PyGC_GetObjects(interp, generation);
329+
return _PyGC_GetObjects(interp, (int)generation);
330330
}
331331

332332
/*[clinic input]

‎Python/gc.c

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ _PyGC_Init(PyInterpreterState *interp)
182182
if (gcstate->callbacks == NULL) {
183183
return _PyStatus_NO_MEMORY();
184184
}
185+
gcstate->heap_size = 0;
185186

186187
return _PyStatus_OK();
187188
}
@@ -1232,7 +1233,7 @@ gc_collect_region(PyThreadState *tstate,
12321233
struct gc_collection_stats *stats);
12331234

12341235
static inline Py_ssize_t
1235-
gc_list_set_space(PyGC_Head *list, uintptr_t space)
1236+
gc_list_set_space(PyGC_Head *list, int space)
12361237
{
12371238
Py_ssize_t size = 0;
12381239
PyGC_Head *gc;
@@ -1258,9 +1259,9 @@ gc_list_set_space(PyGC_Head *list, uintptr_t space)
12581259
* N == 1.4 (1 + 4/threshold)
12591260
*/
12601261

1261-
/* Multiply by 4 so that the default incremental threshold of 10
1262-
* scans objects at 20% the rate of object creation */
1263-
#define SCAN_RATE_MULTIPLIER 2
1262+
/* Divide by 10, so that the default incremental threshold of 10
1263+
* scans objects at 1% of the heap size */
1264+
#define SCAN_RATE_DIVISOR 10
12641265

12651266
static void
12661267
add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats)
@@ -1313,7 +1314,7 @@ gc_collect_young(PyThreadState *tstate,
13131314
if (scale_factor < 1) {
13141315
scale_factor = 1;
13151316
}
1316-
gcstate->work_to_do += survivor_count + survivor_count * SCAN_RATE_MULTIPLIER / scale_factor;
1317+
gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor;
13171318
add_stats(gcstate, 0, stats);
13181319
}
13191320

@@ -1384,12 +1385,12 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat
13841385
static void
13851386
completed_cycle(GCState *gcstate)
13861387
{
1388+
#ifdef Py_DEBUG
13871389
PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head;
13881390
assert(gc_list_is_empty(not_visited));
1391+
#endif
13891392
gcstate->visited_space = flip_old_space(gcstate->visited_space);
1390-
if (gcstate->work_to_do > 0) {
1391-
gcstate->work_to_do = 0;
1392-
}
1393+
gcstate->work_to_do = 0;
13931394
}
13941395

13951396
static void
@@ -1404,13 +1405,13 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
14041405
if (scale_factor < 1) {
14051406
scale_factor = 1;
14061407
}
1407-
Py_ssize_t increment_size = 0;
14081408
gc_list_merge(&gcstate->young.head, &increment);
14091409
gcstate->young.count = 0;
14101410
if (gcstate->visited_space) {
14111411
/* objects in visited space have bit set, so we set it here */
14121412
gc_list_set_space(&increment, 1);
14131413
}
1414+
Py_ssize_t increment_size = 0;
14141415
while (increment_size < gcstate->work_to_do) {
14151416
if (gc_list_is_empty(not_visited)) {
14161417
break;
@@ -1425,14 +1426,11 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
14251426
PyGC_Head survivors;
14261427
gc_list_init(&survivors);
14271428
gc_collect_region(tstate, &increment, &survivors, UNTRACK_TUPLES, stats);
1428-
Py_ssize_t survivor_count = gc_list_size(&survivors);
14291429
gc_list_merge(&survivors, visited);
14301430
assert(gc_list_is_empty(&increment));
1431-
gcstate->work_to_do += survivor_count + survivor_count * SCAN_RATE_MULTIPLIER / scale_factor;
1431+
gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor;
14321432
gcstate->work_to_do -= increment_size;
1433-
if (gcstate->work_to_do < 0) {
1434-
gcstate->work_to_do = 0;
1435-
}
1433+
14361434
validate_old(gcstate);
14371435
add_stats(gcstate, 1, stats);
14381436
if (gc_list_is_empty(not_visited)) {
@@ -1678,7 +1676,7 @@ _PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs)
16781676
}
16791677

16801678
PyObject *
1681-
_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation)
1679+
_PyGC_GetObjects(PyInterpreterState *interp, int generation)
16821680
{
16831681
assert(generation >= -1 && generation < NUM_GENERATIONS);
16841682
GCState *gcstate = &interp->gc;
@@ -1974,6 +1972,7 @@ _PyObject_GC_Link(PyObject *op)
19741972
gc->_gc_next = 0;
19751973
gc->_gc_prev = 0;
19761974
gcstate->young.count++; /* number of allocated GC objects */
1975+
gcstate->heap_size++;
19771976
if (gcstate->young.count > gcs D8A8 tate->young.threshold &&
19781977
gcstate->enabled &&
19791978
gcstate->young.threshold &&
@@ -2095,6 +2094,7 @@ PyObject_GC_Del(void *op)
20952094
if (gcstate->young.count > 0) {
20962095
gcstate->young.count--;
20972096
}
2097+
gcstate->heap_size--;
20982098
PyObject_Free(((char *)op)-presize);
20992099
}
21002100

‎Python/gc_free_threading.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1305,7 +1305,7 @@ visit_get_objects(const mi_heap_t *heap, const mi_heap_area_t *area,
13051305
}
13061306

13071307
PyObject *
1308-
_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation)
1308+
_PyGC_GetObjects(PyInterpreterState *interp, int generation)
13091309
{
13101310
PyObject *result = PyList_New(0);
13111311
if (!result) {

0 commit comments

Comments
 (0)
0