8000 Free objects with qsbr if shared · python/cpython@faa4484 · GitHub
[go: up one dir, main page]
Skip to content

Commit faa4484

Browse files
DinoVDinoV
committed
Free objects with qsbr if shared
1 parent d2f1b0e commit faa4484

File tree

4 files changed

+68
-13
lines changed

4 files changed

+68
-13
lines changed

‎Include/internal/pycore_gc.h

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
4444
# define _PyGC_BITS_UNREACHABLE (4)
4545
# define _PyGC_BITS_FROZEN (8)
4646
# define _PyGC_BITS_SHARED (16)
47+
# define _PyGC_BITS_SHARED_INLINE (32)
4748
#endif
4849

4950
/* True if the object is currently tracked by the GC. */
@@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) {
7172

7273
#ifdef Py_GIL_DISABLED
7374

74-
/* True if an object is shared between multiple threads and
75-
* needs special purpose when freeing to do the possibility
76-
* of in-flight lock-free reads occurring */
75+
/* True if memory the object references is shared between
76+
* multiple threads and needs special purpose when freeing
77+
* those references due to the possibility of in-flight
78+
* lock-free reads occurring. The object is responsible
79+
* for calling _PyMem_FreeDelayed on the referenced
80+
* memory. */
7781
static inline int _PyObject_GC_IS_SHARED(PyObject *op) {
7882
return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0;
7983
}
@@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
8488
}
8589
#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op))
8690

91+
/* True if the memory of the object is shared between multiple
92+
* threads and needs special purpose when freeing due to
93+
* the possibility of in-flight lock-free reads occurring.
94+
* Objects with this bit that are GC objects will automatically
95+
* delay-freed by PyObject_GC_Del. */
96+
static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
97+
return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
98+
}
99+
#define _PyObject_GC_IS_SHARED_INLINE(op) \
100+
_PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op))
101+
102+
static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) {
103+
op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE;
104+
}
105+
#define _PyObject_GC_SET_SHARED_INLINE(op) \
106+
_PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op))
107+
87108
#endif
88109

89110
/* Bit flags for _gc_prev */

‎Include/internal/pycore_pymem.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void);
119119
// Enqueue a pointer to be freed possibly after some delay.
120120
extern void _PyMem_FreeDelayed(void *ptr);
121121

122+
// Enqueue an object to be freed possibly after some delay
123+
extern void _PyObject_FreeDelayed(void *ptr);
124+
122125
// Periodically process delayed free requests.
123126
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
124127

‎Objects/obmalloc.c

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -957,7 +957,7 @@ _PyMem_Strdup(const char *str)
957957

958958
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
959959
struct _mem_work_item {
960-
void *ptr;
960+
uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
961961
uint64_t qsbr_goal;
962962
};
963963

@@ -971,16 +971,27 @@ struct _mem_work_chunk {
971971
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
972972
};
973973

974-
void
975-
_PyMem_FreeDelayed(void *ptr)
974+
static void
975+
free_work_item(uintptr_t ptr)
976+
{
977+
if (ptr & 0x01) {
978+
PyObject_Free((char *)(ptr - 1));
979+
}
980+
else {
981+
PyMem_Free((void *)ptr);
982+
}
983+
}
984+
985+
static void
986+
free_delayed(uintptr_t ptr)
976987
{
977988
#ifndef Py_GIL_DISABLED
978-
PyMem_Free(ptr);
989+
free_work_item(ptr);
979990
#else
980991
if (_PyRuntime.stoptheworld.world_stopped) {
981992
// Free immediately if the world is stopped, including during
982993
// interpreter shutdown.
983-
PyMem_Free(ptr);
994+
free_work_item(ptr);
984995
return;
985996
}
986997

@@ -1007,7 +1018,7 @@ _PyMem_FreeDelayed(void *ptr)
10071018
if (buf == NULL) {
10081019
// failed to allocate a buffer, free immediately
10091020
_PyEval_StopTheWorld(tstate->base.interp);
1010-
PyMem_Free(ptr);
1021+
free_work_item(ptr);
10111022
_PyEval_StartTheWorld(tstate->base.interp);
10121023
return;
10131024
}
@@ -1024,6 +1035,20 @@ _PyMem_FreeDelayed(void *ptr)
10241035
#endif
10251036
}
10261037

1038+
void
1039+
_PyMem_FreeDelayed(void *ptr)
1040+
{
1041+
assert(!((uintptr_t)ptr & 0x01));
1042+
free_delayed((uintptr_t)ptr);
1043+
}
1044+
1045+
void
1046+
_PyObject_FreeDelayed(void *ptr)
1047+
{
1048+
assert(!((uintptr_t)ptr & 0x01));
1049+
free_delayed(((uintptr_t)ptr)|0x01);
1050+
}
1051+
10271052
static struct _mem_work_chunk *
10281053
work_queue_first(struct llist_node *head)
10291054
{
@@ -1043,7 +1068,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
10431068
return;
10441069
}
10451070

1046-
PyMem_Free(item->ptr);
1071+
free_work_item(item->ptr);
10471072
buf->rd_idx++;
10481073
}
10491074

@@ -1130,7 +1155,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
11301155
// Free the remaining items immediately. There should be no other
11311156
// threads accessing the memory at this point during shutdown.
11321157
struct _mem_work_item *item = &buf->array[buf->rd_idx];
1133-
PyMem_Free(item->ptr);
1158+
free_work_item(item->ptr);
11341159
buf->rd_idx++;
11351160
}
11361161

‎Python/gc_free_threading.c

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1692,6 +1692,7 @@ PyObject_GC_Del(void *op)
16921692
{
16931693
size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
16941694
if (_PyObject_GC_IS_TRACKED(op)) {
1695+
_PyObject_GC_UNTRACK(op);
16951696
#ifdef Py_DEBUG
16961697
PyObject *exc = PyErr_GetRaisedException();
16971698
if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,
@@ -1704,8 +1705,13 @@ PyObject_GC_Del(void *op)
17041705
}
17051706

17061707
record_deallocation(_PyThreadState_GET());
1707-
1708-
PyObject_Free(((char *)op)-presize);
1708+
PyObject *self = (PyObject *)op;
1709+
if (_PyObject_GC_IS_SHARED_INLINE(self)) {
1710+
_PyObject_FreeDelayed(((char *)op)-presize);
1711+
}
1712+
else {
1713+
PyObject_Free(((char *)op)-presize);
1714+
}
17091715
}
17101716

17111717
int

0 commit comments

Comments
 (0)
0