8000 gh-112075: Add try-incref functions from nogil branch for use in dict… · python/cpython@4850410 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4850410

Browse files
DinoVcolesbury
DinoV
and
colesbury
authored
gh-112075: Add try-incref functions from nogil branch for use in dict thread safety (#114512)
* Bring in a subset of biased reference counting: colesbury/nogil@b6b12a9a94e The NoGIL branch has functions for attempting to do an incref on an object which may or may not be in flight. This just brings those functions over so that they will be usable from in the dict implementation to get items w/o holding a lock. There's a handful of small simple modifications: Adding inline to the force inline functions to avoid a warning, and switching from _Py_ALWAYS_INLINE to Py_ALWAYS_INLINE as that's available Remove _Py_REF_LOCAL_SHIFT as it doesn't exist yet (and is currently 0 in the 3.12 nogil branch anyway) ob_ref_shared is currently Py_ssize_t and not uint32_t, so use that _PY_LIKELY doesn't exist, so drop it _Py_ThreadLocal becomes _Py_IsOwnedByCurrentThread Add '_PyInterpreterState_GET()' to _Py_IncRefTotal calls. Co-Authored-By: Sam Gross <colesbury@gmail.com>
1 parent 8278fa2 commit 4850410

File tree

1 file changed

+136
-0
lines changed

1 file changed

+136
-0
lines changed

‎Include/internal/pycore_object.h

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,142 @@ static inline void _PyObject_GC_UNTRACK(
376376
_PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op))
377377
#endif
378378

379+
#ifdef Py_GIL_DISABLED
380+
381+
/* Tries to increment an object's reference count
382+
*
383+
* This is a specialized version of _Py_TryIncref that only succeeds if the
384+
* object is immortal or local to this thread. It does not handle the case
385+
* where the reference count modification requires an atomic operation. This
386+
* allows call sites to specialize for the immortal/local case.
387+
*/
388+
static inline int
389+
_Py_TryIncrefFast(PyObject *op) {
390+
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
391+
local += 1;
392+
if (local == 0) {
393+
// immortal
394+
return 1;
395+
}
396+
if (_Py_IsOwnedByCurrentThread(op)) {
397+
_Py_INCREF_STAT_INC();
398+
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
399+
#ifdef Py_REF_DEBUG
400+
_Py_IncRefTotal(_PyInterpreterState_GET());
401+
#endif
402+
return 1;
403+
}
404+
return 0;
405+
}
406+
407+
static inline int
408+
_Py_TryIncRefShared(PyObject *op)
409+
{
410+
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
411+
for (;;) {
412+
// If the shared refcount is zero and the object is either merged
413+
// or may not have weak references, then we cannot incref it.
414+
if (shared == 0 || shared == _Py_REF_MERGED) {
415+
return 0;
416+
}
417+
418+
if (_Py_atomic_compare_exchange_ssize(
419+
&op->ob_ref_shared,
420+
&shared,
421+
shared + (1 << _Py_REF_SHARED_SHIFT))) {
422+
#ifdef Py_REF_DEBUG
423+
_Py_IncRefTotal(_PyInterpreterState_GET());
424+
#endif
425+
_Py_INCREF_STAT_INC();
426+
return 1;
427+
}
428+
}
429+
}
430+
431+
/* Tries to incref the object op and ensures that *src still points to it. */
432+
static inline int
433+
_Py_TryIncref(PyObject **src, PyObject *op)
434+
{
435+
if (_Py_TryIncrefFast(op)) {
436+
return 1;
437+
}
438+
if (!_Py_TryIncRefShared(op)) {
439+
return 0;
440+
}
441+
if (op != _Py_atomic_load_ptr(src)) {
442+
Py_DECREF(op);
443+
return 0;
444+
}
445+
return 1;
446+
}
447+
448+
/* Loads and increfs an object from ptr, which may contain a NULL value.
449+
Safe with concurrent (atomic) updates to ptr.
450+
NOTE: The writer must set maybe-weakref on the stored object! */
451+
static inline PyObject *
452+
_Py_XGetRef(PyObject **ptr)
453+
{
454+
for (;;) {
455+
PyObject *value = _Py_atomic_load_ptr(ptr);
456+
if (value == NULL) {
457+
return value;
458+
}
459+
if (_Py_TryIncref(ptr, value)) {
460+
return value;
461+
}
462+
}
463+
}
464+
465+
/* Attempts to loads and increfs an object from ptr. Returns NULL
466+
on failure, which may be due to a NULL value or a concurrent update. */
467+
static inline PyObject *
468+
_Py_TryXGetRef(PyObject **ptr)
469+
{
470+
PyObject *value = _Py_atomic_load_ptr(ptr);
471+
if (value == NULL) {
472+
return value;
473+
}
474+
if (_Py_TryIncref(ptr, value)) {
475+
return value;
476+
}
477+
return NULL;
478+
}
479+
480+
/* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF
481+
on objects owned by a different thread. */
482+
static inline PyObject *
483+
_Py_NewRefWithLock(PyObject *op)
484+
{
485+
if (_Py_TryIncrefFast(op)) {
486+
return op;
487+
}
488+
_Py_INCREF_STAT_INC();
489+
for (;;) {
490+
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
491+
Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT);
492+
if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) {
493+
new_shared |= _Py_REF_MAYBE_WEAKREF;
494+
}
495+
if (_Py_atomic_compare_exchange_ssize(
496+
&op->ob_ref_shared,
497+
&shared,
498+
new_shared)) {
499+
return op;
500+
}
501+
}
502+
}
503+
504+
static inline PyObject *
505+
_Py_XNewRefWithLock(PyObject *obj)
506+
{
507+
if (obj == NULL) {
508+
return NULL;
509+
}
510+
return _Py_NewRefWithLock(obj);
511+
}
512+
513+
#endif
514+
379515
#ifdef Py_REF_DEBUG
380516
extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *);
381517
extern void _Py_FinalizeRefTotal(_PyRuntimeState *);

0 commit comments

Comments
 (0)
0