8000 gh-112075: Support freeing object memory via QSBR (#116344) · python/cpython@7db871e · GitHub
[go: up one dir, main page]
Skip to content

Commit 7db871e

Browse files
DinoVDinoV
authored
gh-112075: Support freeing object memory via QSBR (#116344)
Free objects with qsbr if shared
1 parent 6183158 commit 7db871e

File tree

4 files changed

+68
-13
lines changed

4 files changed

+68
-13
lines changed

‎Include/internal/pycore_gc.h

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
4444
# define _PyGC_BITS_UNREACHABLE (4)
4545
# define _PyGC_BITS_FROZEN (8)
4646
# define _PyGC_BITS_SHARED (16)
47+
# define _PyGC_BITS_SHARED_INLINE (32)
4748
#endif
4849

4950
/* True if the object is currently tracked by the GC. */
@@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) {
7172

7273
#ifdef Py_GIL_DISABLED
7374

74-
/* True if an object is shared between multiple threads and
75-
* needs special purpose when freeing to do the possibility
76-
* of in-flight lock-free reads occurring */
75+
/* True if memory the object references is shared between
76+
* multiple threads and needs special purpose when freeing
77+
* those references due to the possibility of in-flight
78+
* lock-free reads occurring. The object is responsible
79+
* for calling _PyMem_FreeDelayed on the referenced
80+
* memory. */
7781
static inline int _PyObject_GC_IS_SHARED(PyObject *op) {
7882
return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0;
7983
}
@@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
8488
}
8589
#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op))
8690

91+
/* True if the memory of the object is shared between multiple
92+
* threads and needs special purpose when freeing due to
93+
* the possibility of in-flight lock-free reads occurring.
94+
* Objects with this bit that are GC objects will automatically
95+
* delay-freed by PyObject_GC_Del. */
96+
static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
97+
return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
98+
}
99+
#define _PyObject_GC_IS_SHARED_INLINE(op) \
100+
_PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op))
101+
102+
static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) {
103+
op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE;
104+
}
105+
#define _PyObject_GC_SET_SHARED_INLINE(op) \
106+
_PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op))
107+
87108
#endif
88109

89110
/* Bit flags for _gc_prev */

‎Include/internal/pycore_pymem.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void);
119119
// Enqueue a pointer to be freed possibly after some delay.
120120
extern void _PyMem_FreeDelayed(void *ptr);
121121

122+
// Enqueue an object to be freed possibly after some delay
123+
extern void _PyObject_FreeDelayed(void *ptr);
124+
122125
// Periodically process delayed free requests.
123126
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
124127

‎Objects/obmalloc.c

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1070,7 +1070,7 @@ _PyMem_Strdup(const char *str)
10701070

10711071
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
10721072
struct _mem_work_item {
1073-
void *ptr;
1073+
uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
10741074
uint64_t qsbr_goal;
10751075
};
10761076

@@ -1084,16 +1084,27 @@ struct _mem_work_chunk {
10841084
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
10851085
};
10861086

1087-
void
1088-
_PyMem_FreeDelayed(void *ptr)
1087+
static void
1088+
free_work_item(uintptr_t ptr)
1089+
{
1090+
if (ptr & 0x01) {
1091+
PyObject_Free((char *)(ptr - 1));
1092+
}
1093+
else {
1094+
PyMem_Free((void *)ptr);
1095+
}
1096+
}
1097+
1098+
static void
1099+
free_delayed(uintptr_t ptr)
10891100
{
10901101
#ifndef Py_GIL_DISABLED
1091-
PyMem_Free(ptr);
1102+
free_work_item(ptr);
10921103
#else
10931104
if (_PyRuntime.stoptheworld.world_stopped) {
10941105
// Free immediately if the world is stopped, including during
10951106
// interpreter shutdown.
1096-
PyMem_Free(ptr);
1107+
free_work_item(ptr);
10971108
return;
10981109
}
10991110

@@ -1120,7 +1131,7 @@ _PyMem_FreeDelayed(void *ptr)
11201131
if (buf == NULL) {
11211132
// failed to allocate a buffer, free immediately
11221133
_PyEval_StopTheWorld(tstate->base.interp);
1123-
PyMem_Free(ptr);
1134+
free_work_item(ptr);
11241135
_PyEval_StartTheWorld(tstate->base.interp);
11251136
return;
11261137
}
@@ -1137,6 +1148,20 @@ _PyMem_FreeDelayed(void *ptr)
11371148
#endif
11381149
}
11391150

1151+
void
1152+
_PyMem_FreeDelayed(void *ptr)
1153+
{
1154+
assert(!((uintptr_t)ptr & 0x01));
1155+
free_delayed((uintptr_t)ptr);
1156+
}
1157+
1158+
void
1159+
_PyObject_FreeDelayed(void *ptr)
1160+
{
1161+
assert(!((uintptr_t)ptr & 0x01));
1162+
free_delayed(((uintptr_t)ptr)|0x01);
1163+
}
1164+
11401165
static struct _mem_work_chunk *
11411166
work_queue_first(struct llist_node *head)
11421167
{
@@ -1156,7 +1181,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
11561181
return;
11571182
}
11581183

1159-
PyMem_Free(item->ptr);
1184+
free_work_item(item->ptr);
11601185
buf->rd_idx++;
11611186
}
11621187

@@ -1243,7 +1268,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
12431268
// Free the remaining items immediately. There should be no other
12441269
// threads accessing the memory at this point during shutdown.
12451270
struct _mem_work_item *item = &buf->array[buf->rd_idx];
1246-
PyMem_Free(item->ptr);
1271+
free_work_item(item->ptr);
12471272
buf->rd_idx++;
12481273
}
12491274

‎Python/gc_free_threading.c

Lines changed: 8 additions & 2 deletions
D922
Original file line numberDiff line numberDiff line change
@@ -1695,6 +1695,7 @@ PyObject_GC_Del(void *op)
16951695
{
16961696
size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
16971697
if (_PyObject_GC_IS_TRACKED(op)) {
1698+
_PyObject_GC_UNTRACK(op);
16981699
#ifdef Py_DEBUG
16991700
PyObject *exc = PyErr_GetRaisedException();
17001701
if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,
@@ -1707,8 +1708,13 @@ PyObject_GC_Del(void *op)
17071708
}
17081709

17091710
record_deallocation(_PyThreadState_GET());
1710-
1711-
PyObject_Free(((char *)op)-presize);
1711+
PyObject *self = (PyObject *)op;
1712+
if (_PyObject_GC_IS_SHARED_INLINE(self)) {
1713+
_PyObject_FreeDelayed(((char *)op)-presize);
1714+
}
1715+
else {
1716+
PyObject_Free(((char *)op)-presize);
1717+
}
17121718
}
17131719

17141720
int

0 commit comments

Comments
 (0)
0