8000 gh-128942: make arraymodule.c free-thread safe by tom-pytel · Pull Request #128943 · python/cpython · GitHub
[go: up one dir, main page]

Skip to content

gh-128942: make arraymodule.c free-thread safe #128943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 40 commits into from
Feb 27, 2025
Merged
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
7c8edcd
gh-128942: make arraymodule.c free-thread safe
tom-pytel Jan 17, 2025
b4c38d6
📜🤖 Added by blurb_it.
blurb-it[bot] Jan 17, 2025
523c049
Update 2025-01-17-13-53-32.gh-issue-128942.DxzaIg.rst
tom-pytel Jan 17, 2025
714da01
Update 2025-01-17-13-53-32.gh-issue-128942.DxzaIg.rst
tom-pytel Jan 17, 2025
6893468
refactor array methods to established patterns
tom-pytel Jan 17, 2025
65b4a63
made arrayiter freethread safe
tom-pytel Jan 17, 2025
761d27a
cleanup to match other segfault fix PR
tom-pytel Jan 17, 2025
0bcd3c7
style nit from other PR
tom-pytel Jan 17, 2025
5676c92
add _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED()
tom-pytel Jan 17, 2025
c18290c
misc style
tom-pytel Jan 17, 2025
6ee2303
misc cleanups
tom-pytel Jan 18, 2025
7d5de1f
misc
tom-pytel Jan 18, 2025
02fc79c
Merge branch 'main' into fix-issue-128942
tom-pytel Jan 18, 2025
0ad9800
3 og 4 requested changes
tom-pytel Jan 18, 2025
325c241
not locking bytes
tom-pytel Jan 18, 2025
7958c55
array_array_frombytes get buffer from clinic
tom-pytel Jan 18, 2025
0532d93
more safe
tom-pytel Jan 19, 2025
6afec34
protect ob_exports exclusively with critical section
tom-pytel Jan 20, 2025
f0d5824
Merge branch 'main' into fix-issue-128942
tom-pytel Jan 20, 2025
f4058c5
Merge branch 'main' into fix-issue-128942
tom-pytel Jan 20, 2025
1bf45fc
fix clinic stuff that seems to have changed
tom-pytel Jan 20, 2025
6e7b65a
Merge branch 'main' into fix-issue-128942
tom-pytel Jan 20, 2025
bb4bf90
iterator safe from multi-crit deadlocks
tom-pytel Jan 20, 2025
4d5bb3a
remove unnecessary crit sect in setstate
tom-pytel Jan 21, 2025
e50e653
protect ob_exports atomically
tom-pytel Jan 22, 2025
841a870
2 missed misc atomic writes
tom-pytel Jan 23, 2025
a0e36db
Merge branch 'main' into fix-issue-128942
tom-pytel Feb 12, 2025
d80c1a5
misc tweaks
tom-pytel Feb 12, 2025
9aa4540
Merge branch 'main' into fix-issue-128942
tom-pytel Feb 12, 2025
dffc9cd
add test
tom-pytel Feb 12, 2025
b3665fd
misc fix, b'\xdd' -> 0xdd
tom-pytel Feb 12, 2025
3814e53
use support.Py_GIL_DISABLED
tom-pytel Feb 14, 2025
cb0345f
Merge branch 'main' into fix-issue-128942
tom-pytel Feb 15, 2025
4a5c568
add critical section held assertions
tom-pytel Feb 15, 2025
cc8d715
Py_CLEAR(it->ao)
tom-pytel Feb 15, 2025
5f352a3
make ob_exports non-atomic everywhere
tom-pytel Feb 15, 2025
98f1433
check type before lock in array_ass_subscr()
tom-pytel Feb 16, 2025
2a45a44
Merge branch 'main' into fix-issue-128942
tom-pytel Feb 16, 2025
c0799be
Merge branch 'main' into fix-issue-128942
kumaraditya303 Feb 27, 2025
0bcde5c
simplify array_arrayiterator___reduce___impl
kumaraditya303 Feb 27, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'main' into fix-issue-128942
  • Loading branch information
tom-pytel committed Feb 12, 2025
commit a0e36db98145bb2799b0b1c796f58e0fea5fa3ca
151 changes: 80 additions & 71 deletions Modules/arraymodule.c
F438
Original file line number Diff line number Diff line change
Expand Up @@ -742,16 +742,17 @@ array_dealloc(PyObject *op)
PyTypeObject *tp = Py_TYPE(op);
PyObject_GC_UnTrack(op);

if (FT_ATOMIC_LOAD_SSIZE_RELAXED(op->ob_exports) > 0) {
arrayobject *self = arrayobject_CAST(op);
if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_exports) > 0) {
PyErr_SetString(PyExc_SystemError,
"deallocated array object has exported buffers");
PyErr_Print();
}
if (op->weakreflist != NULL) {
PyObject_ClearWeakRefs((PyObject *) op);
if (self->weakreflist != NULL) {
PyObject_ClearWeakRefs(op);
}
if (op->ob_item != NULL) {
PyMem_Free(op->ob_item);
if (self->ob_item != NULL) {
PyMem_Free(self->ob_item);
}
tp->tp_free(op);
Py_DECREF(tp);
Expand Down Expand Up @@ -882,26 +883,27 @@ array_richcompare(PyObject *v, PyObject *w, int op)
static Py_ssize_t
array_length(PyObject *op)
{
return Pyarrayobject_GET_SIZE(a);
arrayobject *self = arrayobject_CAST(op);
return Pyarrayobject_GET_SIZE(self);
}

static PyObject *
array_item_lock_held(arrayobject *a, Py_ssize_t i)
array_item_lock_held(PyObject *op, Py_ssize_t i)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a);
if (i < 0 || i >= Py_SIZE(a)) {
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
if (i < 0 || i >= Py_SIZE(op)) {
PyErr_SetString(PyExc_IndexError, "array index out of range");
return NULL;
}
return getarrayitem(op, i);
}

static PyObject *
array_item(arrayobject *a, Py_ssize_t i)
array_item(PyObject *op, Py_ssize_t i)
{
PyObject *ret;
Py_BEGIN_CRITICAL_SECTION(a);
ret = array_item_lock_held(a, i);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_item_lock_held(op, i);
Py_END_CRITICAL_SECTION();
return ret;
}
Expand Down Expand Up @@ -981,10 +983,11 @@ array_array___deepcopy___impl(arrayobject *self, PyObject *unused)
}

static PyObject *
array_concat_lock_held(arrayobject *a, PyObject *bb)
array_concat_lock_held(PyObject *op, PyObject *bb)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb);
arrayobject *a = arrayobject_CAST(op);
array_state *state = find_array_state_by_type(Py_TYPE(a));
Py_ssize_t size;
arrayobject *np;
Expand Down Expand Up @@ -1019,19 +1022,20 @@ array_concat_lock_held(arrayobject *a, PyObject *bb)
}

static PyObject *
array_concat(arrayobject *a, PyObject *bb)
array_concat(PyObject *op, PyObject *bb)
{
PyObject *ret;
Py_BEGIN_CRITICAL_SECTION2(a, bb);
ret = array_concat_lock_held(a, bb);
Py_BEGIN_CRITICAL_SECTION2(op, bb);
ret = array_concat_lock_held(op, bb);
Py_END_CRITICAL_SECTION2();
return ret;
}

static PyObject *
array_repeat_lock_held(arrayobject *a, Py_ssize_t n)
array_repeat_lock_held(PyObject *op, Py_ssize_t n)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
arrayobject *a = arrayobject_CAST(op);
array_state *state = find_array_state_by_type(Py_TYPE(a));

if (n < 0)
Expand All @@ -1055,11 +1059,11 @@ array_repeat_lock_held(arrayobject *a, Py_ssize_t n)
}

static PyObject *
array_repeat(arrayobject *a, Py_ssize_t n)
array_repeat(PyObject *op, Py_ssize_t n)
{
PyObject *ret;
Py_BEGIN_CRITICAL_SECTION(a);
ret = array_repeat_lock_held(a, n);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_repeat_lock_held(op, n);
Py_END_CRITICAL_SECTION();
return ret;
}
Expand Down Expand Up @@ -1100,9 +1104,10 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh)
}

static int
array_ass_item_lock_held(arrayobject *a, Py_ssize_t i, PyObject *v)
array_ass_item_lock_held(PyObject *op, Py_ssize_t i, PyObject *v)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
arrayobject *a = arrayobject_CAST(op);
if (i < 0 || i >= Py_SIZE(a)) {
PyErr_SetString(PyExc_IndexError,
"array assignment index out of range");
Expand All @@ -1114,11 +1119,11 @@ array_ass_item_lock_held(arrayobject *a, Py_ssize_t i, PyObject *v)
}

static int
array_ass_item(arrayobject *a, Py_ssize_t i, PyObject *v)
array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v)
{
int ret;
Py_BEGIN_CRITICAL_SECTION(a);
ret = array_ass_item_lock_held(a, i, v);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_ass_item_lock_held(op, i, v);
Py_END_CRITICAL_SECTION();
return ret;
}
Expand Down Expand Up @@ -1219,9 +1224,10 @@ array_inplace_concat(PyObject *op, PyObject *bb)
}

static PyObject *
array_inplace_repeat_lock_held(arrayobject *self, Py_ssize_t n)
array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
arrayobject *self = arrayobject_CAST(op);
const Py_ssize_t array_size = Py_SIZE(self);

if (array_size > 0 && n != 1 ) {
Expand All @@ -1244,11 +1250,11 @@ array_inplace_repeat_lock_held(arrayobject *self, Py_ssize_t n)
}

static PyObject *
array_inplace_repeat(arrayobject *self, Py_ssize_t n)
array_inplace_repeat(PyObject *op, Py_ssize_t n)
{
PyObject *ret;
Py_BEGIN_CRITICAL_SECTION(self);
ret = array_inplace_repeat_lock_held(self, n);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_inplace_repeat_lock_held(op, n);
Py_END_CRITICAL_SECTION();
return ret;
}
Expand Down Expand Up @@ -1339,28 +1345,28 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start,
}

static int
array_contains_lock_held(arrayobject *self, PyObject *v)
array_contains_lock_held(PyObject *op, PyObject *v)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
Py_ssize_t i;
int cmp;

for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(self); i++) {
PyObject *selfi = getarrayitem(self, i);
if (selfi == NULL)
for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) {
PyObject *opi = getarrayitem(op, i);
if (opi == NULL)
return -1;
cmp = PyObject_RichCompareBool(selfi, v, Py_EQ);
Py_DECREF(selfi);
cmp = PyObject_RichCompareBool(opi, v, Py_EQ);
Py_DECREF(opi);
}
return cmp;
}

static int
array_contains(arrayobject *self, PyObject *v)
array_contains(PyObject *op, PyObject *v)
{
int ret;
Py_BEGIN_CRITICAL_SECTION(self);
ret = array_contains_lock_held(self, v);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_contains_lock_held(op, v);
Py_END_CRITICAL_SECTION();
return ret;
}
Expand Down Expand Up @@ -2524,9 +2530,9 @@ static PyMethodDef array_methods[] = {
};

static PyObject *
array_repr_lock_held(arrayobject *a)
array_repr_lock_held(PyObject *op)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
char typecode;
PyObject *s, *v = NULL;
Py_ssize_t len;
Expand All @@ -2553,18 +2559,19 @@ array_repr_lock_held(arrayobject *a)
}

static PyObject *
array_repr(arrayobject *a)
array_repr(PyObject *op)
{
PyObject *ret;
Py_BEGIN_CRITICAL_SECTION(a);
ret = array_repr_lock_held(a);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_repr_lock_held(op);
Py_END_CRITICAL_SECTION();
return ret;
}

static PyObject*
array_subscr_lock_held(arrayobject* self, PyObject* item)
array_subscr_lock_held(PyObject *op, PyObject *item)
{
arrayobject *self = arrayobject_CAST(op);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
array_state *state = find_array_state_by_type(Py_TYPE(self));

Expand Down Expand Up @@ -2626,27 +2633,27 @@ array_subscr_lock_held(arrayobject* self, PyObject* item)
}
}

static PyObject*
array_subscr(arrayobject* self, PyObject* item)
static PyObject *
array_subscr(PyObject *op, PyObject *item)
{
PyObject *ret;
Py_BEGIN_CRITICAL_SECTION(self);
ret = array_subscr_lock_held(self, item);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_subscr_lock_held(op, item);
Py_END_CRITICAL_SECTION();
return ret;
}

static int
array_ass_subscr_lock_held(arrayobject* self, PyObject* item, PyObject* value)
array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
arrayobject *self = arrayobject_CAST(op);
#ifdef Py_DEBUG
if (value != NULL) {
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(value);
}
#endif
Py_ssize_t start, stop, step, slicelength, needed;
arrayobject *self = arrayobject_CAST(op);
array_state* state = find_array_state_by_type(Py_TYPE(self));
arrayobject* other;
int itemsize;
Expand Down Expand Up @@ -2698,7 +2705,7 @@ array_ass_subscr_lock_held(arrayobject* self, PyObject* item, PyObject* value)
value = array_slice(other, 0, needed);
if (value == NULL)
return -1;
ret = array_ass_subscr_lock_held(self, item, value);
ret = array_ass_subscr_lock_held(op, item, value);
Py_DECREF(value);
return ret;
}
Expand Down Expand Up @@ -2803,17 +2810,17 @@ array_ass_subscr_lock_held(arrayobject* self, PyObject* item, PyObject* value)
}

static int
array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value)
array_ass_subscr(PyObject *op, PyObject* item, PyObject* value)
{
int ret;
if (value == NULL) {
Py_BEGIN_CRITICAL_SECTION(self);
ret = array_ass_subscr_lock_held(self, item, value);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_ass_subscr_lock_held(op, item, value);
Py_END_CRITICAL_SECTION();
}
else {
Py_BEGIN_CRITICAL_SECTION2(self, value);
ret = array_ass_subscr_lock_held(self, item, value);
Py_BEGIN_CRITICAL_SECTION2(op, value);
ret = array_ass_subscr_lock_held(op, item, value);
Py_END_CRITICAL_SECTION2();
}
return ret;
Expand All @@ -2823,16 +2830,16 @@ static const void *emptybuf = "";


static int
array_buffer_getbuf_lock_held(arrayobject *self, Py_buffer *view, int flags)
array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
arrayobject *self = arrayobject_CAST(op);
if (view == NULL) {
PyErr_SetString(PyExc_BufferError,
"array_buffer_getbuf: view==NULL argument is obsolete");
return -1;
}

arrayobject *self = arrayobject_CAST(op);
view->buf = (void *)self->ob_item;
view->obj = Py_NewRef(self);
if (view->buf == NULL)
Expand Down Expand Up @@ -2869,18 +2876,19 @@ array_buffer_getbuf_lock_held(arrayobject *self, Py_buffer *view, int flags)
}

static int
array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags)
array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags)
{
int ret;
Py_BEGIN_CRITICAL_SECTION(self);
ret = array_buffer_getbuf_lock_held(self, view, flags);
Py_BEGIN_CRITICAL_SECTION(op);
ret = array_buffer_getbuf_lock_held(op, view, flags);
Py_END_CRITICAL_SECTION();
return ret;
}

static void
array_buffer_relbuf(PyObject *op, Py_buffer *Py_UNUSED(view))
{
arrayobject *self = arrayobject_CAST(op);
#ifdef Py_GIL_DISABLED
assert(_Py_atomic_add_ssize(&self->ob_exports, -1) >= 1);
#else
Expand Down Expand Up @@ -3233,6 +3241,7 @@ array_iter(PyObject *op)
static PyObject *
arrayiter_next(PyObject *op)
{
arrayiterobject *it = arrayiterobject_CAST(op);
assert(it != NULL);
Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->index);
if (index < 0) {
Expand All @@ -3245,9 +3254,10 @@ arrayiter_next(PyObject *op)
assert(array_Check(it->ao, state));
#endif

Py_BEGIN_CRITICAL_SECTION(it->ao);
if (index < Py_SIZE(it->ao)) {
ret = (*it->getitem)(it->ao, index);
arrayobject *ao = it->ao;
Py_BEGIN_CRITICAL_SECTION(ao);
if (index < Py_SIZE(ao)) {
ret = (*it->getitem)(ao, index);
}
else {
ret = NULL;
Expand All @@ -3260,7 +3270,6 @@ arrayiter_next(PyObject *op)
else {
FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, -1);
#ifndef Py_GIL_DISABLED
arrayobject *ao = it->ao;
it->ao = NULL;
Py_DECREF(ao);
#endif
Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.
0