8000 gh-115103: Implement delayed free mechanism for free-threaded builds … · python/cpython@e3ad6ca · GitHub
[go: up one dir, main page]
Skip to content

Commit e3ad6ca

Browse files
colesburycolesbury
authored
gh-115103: Implement delayed free mechanism for free-threaded builds (#115367)
This adds `_PyMem_FreeDelayed()` and supporting functions. The `_PyMem_FreeDelayed()` function frees memory with the same allocator as `PyMem_Free()`, but after some delay to ensure that concurrent lock-free readers have finished.
1 parent d207c7c commit e3ad6ca

File tree

8 files changed

+226
-0
lines changed

8 files changed

+226
-0
lines changed

‎Include/internal/pycore_interp.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ struct _is {
231231

232232
struct _Py_dict_state dict_state;
233233
struct _Py_exc_state exc_state;
234+
struct _Py_mem_interp_free_queue mem_free_queue;
234235

235236
struct ast_state ast;
236237
struct types_state types;

‎Include/internal/pycore_pymem.h

Lines changed: 19 additions & 0 deletions
+
PyMutex mutex; // protects the queue
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#ifndef Py_INTERNAL_PYMEM_H
22
#define Py_INTERNAL_PYMEM_H
33

4+
#include "pycore_llist.h" // struct llist_node
45
#include "pycore_lock.h" // PyMutex
56

67
#ifdef __cplusplus
@@ -48,6 +49,11 @@ struct _pymem_allocators {
4849
PyObjectArenaAllocator obj_arena;
4950
};
5051

52+
struct _Py_mem_interp_free_queue {
53+
int has_work; // true if the queue is not empty
54
55+
struct llist_node head; // queue of _mem_work_chunk items
56+
};
5157

5258
/* Set the memory allocator of the specified domain to the default.
5359
Save the old allocator into *old_alloc if it's non-NULL.
@@ -110,6 +116,19 @@ extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator);
110116
/* Is the debug allocator enabled? */
111117
extern int _PyMem_DebugEnabled(void);
112118

119+
// Enqueue a pointer to be freed possibly after some delay.
120+
extern void _PyMem_FreeDelayed(void *ptr);
121+
122+
// Periodically process delayed free requests.
123+
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
124+
125+
// Abandon all thread-local delayed free requests and push them to the
126+
// interpreter's queue.
127+
extern void _PyMem_AbandonDelayed(PyThreadState *tstate);
128+
129+
// On interpreter shutdown, frees all delayed free requests.
130+
extern void _PyMem_FiniDelayed(PyInterpreterState *interp);
131+
113132
#ifdef __cplusplus
114133
}
115134
#endif

‎Include/internal/pycore_pymem_init.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,11 @@ extern void _PyMem_ArenaFree(void *, void *, size_t);
9292
{ NULL, _PyMem_ArenaAlloc, _PyMem_ArenaFree }
9393

9494

95+
#define _Py_mem_free_queue_INIT(queue) \
96+
{ \
97+
.head = LLIST_INIT(queue.head), \
98+
}
99+
95100
#ifdef __cplusplus
96101
}
97102
#endif

‎Include/internal/pycore_runtime_init.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ extern PyTypeObject _PyExc_MemoryError;
176176
}, \
177177
.dtoa = _dtoa_state_INIT(&(INTERP)), \
178178
.dict_state = _dict_state_INIT, \
179+
.mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \
179180
.func_state = { \
180181
.next_version = 1, \
181182
}, \

‎Include/internal/pycore_tstate.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ typedef struct _PyThreadStateImpl {
2929
PyThreadState base;
3030

3131
struct _qsbr_thread_state *qsbr; // only used by free-threaded build
32+
struct llist_node mem_free_queue; // delayed free queue
3233

3334
#ifdef Py_GIL_DISABLED
3435
struct _gc_thread_state gc;

‎Objects/obmalloc.c

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -948,6 +948,196 @@ _PyMem_Strdup(const char *str)
948948
return copy;
949949
}
950950

951+
/***********************************************/
952+
/* Delayed freeing support for Py_GIL_DISABLED */
953+
/***********************************************/
954+
955+
// So that sizeof(struct _mem_work_chunk) is 4096 bytes on 64-bit platforms.
956+
#define WORK_ITEMS_PER_CHUNK 254
957+
958+
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
959+
struct _mem_work_item {
960+
void *ptr;
961+
uint64_t qsbr_goal;
962+
};
963+
964+
// A fixed-size buffer of pointers to be freed
965+
struct _mem_work_chunk {
966+
// Linked list node of chunks in queue
967+
struct llist_node node;
968+
969+
Py_ssize_t rd_idx; // index of next item to read
970+
Py_ssize_t wr_idx; // index of next item to write
971+
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
972+
};
973+
974+
void
975+
_PyMem_FreeDelayed(void *ptr)
976+
{
977+
#ifndef Py_GIL_DISABLED
978+
PyMem_Free(ptr);
979+
#else
980+
if (_PyRuntime.stoptheworld.world_stopped) {
981+
// Free immediately if the world is stopped, including during
982+
// interpreter shutdown.
983+
PyMem_Free(ptr);
984+
return;
985+
}
986+
987+
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
988+
struct llist_node *head = &tstate->mem_free_queue;
989+
990+
struct _mem_work_chunk *buf = NULL;
991+
if (!llist_empty(head)) {
992+
// Try to re-use the last buffer
993+
buf = llist_data(head->prev, struct _mem_work_chunk, node);
994+
if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {
995+
// already full
996+
buf = NULL;
997+
}
998+
}
999+
1000+
if (buf == NULL) {
1001+
buf = PyMem_Calloc(1, sizeof(*buf));
1002+
if (buf != NULL) {
1003+
llist_insert_tail(head, &buf->node);
1004+
}
1005+
}
1006+
1007+
if (buf == NULL) {
1008+
// failed to allocate a buffer, free immediately
1009+
_PyEval_StopTheWorld(tstate->base.interp);
1010+
PyMem_Free(ptr);
1011+
_PyEval_StartTheWorld(tstate->base.interp);
1012+
return;
1013+
}
1014+
1015+
assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK);
1016+
uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr);
1017+
buf->array[buf->wr_idx].ptr = ptr;
1018+
buf->array[buf->wr_idx].qsbr_goal = seq;
1019+
buf->wr_idx++;
1020+
1021+
if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {
1022+
_PyMem_ProcessDelayed((PyThreadState *)tstate);
1023+
}
1024+
#endif
1025+
}
1026+
1027+
static struct _mem_work_chunk *
1028+
work_queue_first(struct llist_node *head)
1029+
{
1030+
return llist_data(head->next, struct _mem_work_chunk, node);
1031+
}
1032+
1033+
static void
1034+
process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
1035+
bool keep_empty)
1036+
{
1037+
while (!llist_empty(head)) {
1038+
struct _mem_work_chunk *buf = work_queue_first(head);
1039+
1040+
while (buf->rd_idx < buf->wr_idx) {
1041+
struct _mem_work_item *item = &buf->array[buf->rd_idx];
1042+
if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) {
1043+
return;
1044+
}
1045+
1046+
PyMem_Free(item->ptr);
1047+
buf->rd_idx++;
1048+
}
1049+
1050+
assert(buf->rd_idx == buf->wr_idx);
1051+
if (keep_empty && buf->node.next == head) {
1052+
// Keep the last buffer in the queue to reduce re-allocations
1053+
buf->rd_idx = buf->wr_idx = 0;
1054+
return;
1055+
}
1056+
1057+
llist_remove(&buf->node);
1058+
PyMem_Free(buf);
1059+
}
1060+
}
1061+
1062+
static void
1063+
process_interp_queue(struct _Py_mem_interp_free_queue *queue,
1064+
struct _qsbr_thread_state *qsbr)
1065+
{
1066+
if (!_Py_atomic_load_int_relaxed(&queue->has_work)) {
1067+
return;
1068+
}
1069+
1070+
// Try to acquire the lock, but don't block if it's already held.
1071+
if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) {
1072+
process_queue(&queue->head, qsbr, false);
1073+
1074+
int more_work = !llist_empty(&queue->head);
1075+
_Py_atomic_store_int_relaxed(&queue->has_work, more_work);
1076+
1077+
PyMutex_Unlock(&queue->mutex);
1078+
}
1079+
}
1080+
1081+
void
1082+
_PyMem_ProcessDelayed(PyThreadState *tstate)
1083+
{
1084+
PyInterpreterState *interp = tstate->interp;
1085+
_PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
1086+
1087+
// Process thread-local work
1088+
process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true);
1089+
1090+
// Process shared interpreter work
1091+
process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr);
1092+
}
1093+
1094+
void
1095+
_PyMem_AbandonDelayed(PyThreadState *tstate)
1096+
{
1097+
PyInterpreterState *interp = tstate->interp;
1098+
struct llist_node *queue = &((_PyThreadStateImpl *)tstate)->mem_free_queue;
1099+
1100+
if (llist_empty(queue)) {
1101+
return;
1102+
}
1103+
1104+
// Check if the queue contains one empty buffer
1105+
struct _mem_work_chunk *buf = work_queue_first(queue);
1106+
if (buf->rd_idx == buf->wr_idx) {
1107+
llist_remove(&buf->node);
1108+
PyMem_Free(buf);
1109+
assert(llist_empty(queue));
1110+
return;
1111+
}
1112+
1113+
// Merge the thread's work queue into the interpreter's work queue.
1114+
PyMutex_Lock(&interp->mem_free_queue.mutex);
1115+
llist_concat(&interp->mem_free_queue.head, queue);
1116+
_Py_atomic_store_int_relaxed(&interp->mem_free_queue.has_work, 1);
1117+
PyMutex_Unlock(&interp->mem_free_queue.mutex);
1118+
1119+
assert(llist_empty(queue)); // the thread's queue is now empty
1120+
}
1121+
1122+
void
1123+
_PyMem_FiniDelayed(PyInterpreterState *interp)
1124+
{
1125+
struct llist_node *head = &interp->mem_free_queue.head;
1126+
while (!llist_empty(head)) {
1127+
struct _mem_work_chunk *buf = work_queue_first(head);
1128+
1129+
while (buf->rd_idx < buf->wr_idx) {
1130+
// Free the remaining items immediately. There should be no other
1131+
// threads accessing the memory at this point during shutdown.
1132+
struct _mem_work_item *item = &buf->array[buf->rd_idx];
1133+
PyMem_Free(item->ptr);
1134+
buf->rd_idx++;
1135+
}
1136+
1137+
llist_remove(&buf->node);
1138+
PyMem_Free(buf);
1139+
}
1140+
}
9511141

9521142
/**************************/
9531143
/* the "object" allocator */

‎Python/pylifecycle.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1837,6 +1837,9 @@ finalize_interp_clear(PyThreadState *tstate)
18371837

18381838
finalize_interp_types(tstate->interp);
18391839

1840+
/* Free any delayed free requests immediately */
1841+
_PyMem_FiniDelayed(tstate->interp);
1842+
18401843
/* finalize_interp_types may allocate Python objects so we may need to
18411844
abandon mimalloc segments again */
18421845
_PyThreadState_ClearMimallocHeaps(tstate);

‎Python/pystate.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,7 @@ init_interpreter(PyInterpreterState *interp,
617617
#ifdef Py_GIL_DISABLED
618618
_Py_brc_init_state(interp);
619619
#endif
620+
llist_init(&interp->mem_free_queue.head);
620621
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
621622
interp->monitors.tools[i] = 0;
622623
}
@@ -1353,6 +1354,7 @@ init_threadstate(_PyThreadStateImpl *_tstate,
13531354
// Initialize biased reference counting inter-thread queue
13541355
_Py_brc_init_thread(tstate);
13551356
#endif
1357+
llist_init(&_tstate->mem_free_queue);
13561358

13571359
if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) {
13581360
// Start in the suspended state if there is an ongoing stop-the-world.
@@ -1574,6 +1576,7 @@ PyThreadState_Clear(PyThreadState *tstate)
15741576
// don't call _PyInterpreterState_SetNotRunningMain() yet.
15751577
tstate->on_delete(tstate->on_delete_data);
15761578
}
1579+
15771580
#ifdef Py_GIL_DISABLED
15781581
// Each thread should clear own freelists in free-threading builds.
15791582
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
@@ -1583,6 +1586,9 @@ PyThreadState_Clear(PyThreadState *tstate)
15831586
_Py_brc_remove_thread(tstate);
15841587
#endif
15851588

1589+
// Merge our queue of pointers to be freed into the interpreter queue.
1590+
_PyMem_AbandonDelayed(tstate);
1591+
15861592
_PyThreadState_ClearMimallocHeaps(tstate);
15871593

15881594
tstate->_status.cleared = 1;

0 commit comments

Comments
 (0)
0