8000 BUG: reference cycle in np.vectorize (#11977) · numpy/numpy@f5b6850 · GitHub
[go: up one dir, main page]
Skip to content

Commit f5b6850

Browse files
mattipseberg
mattip
authored and
seberg
committed
BUG: reference cycle in np.vectorize (#11977)
This implements cyclic support by adding `tp_traverse` to ufuns which may contain a user provided object function (`np.frompyfunc`). Ufuncs that do not add this are not added to the circular reference count tracking. The ufunc does not need to implement `tp_clear` because it is an immutable object.
1 parent ad0e902 commit f5b6850

File tree

4 files changed

+94
-12
lines changed

4 files changed

+94
-12
lines changed

‎numpy/core/include/numpy/ufuncobject.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,11 @@ typedef struct _tagPyUFuncObject {
120120
*/
121121
int nin, nout, nargs;
122122

123-
/* Identity for reduction, either PyUFunc_One or PyUFunc_Zero */
123+
/*
124+
* Identity for reduction, any of PyUFunc_One, PyUFunc_Zero
125+
* PyUFunc_MinusOne, PyUFunc_None, PyUFunc_ReorderableNone,
126+
* PyUFunc_IdentityValue.
127+
*/
124128
int identity;
125129

126130
/* Array of one-dimensional core loops */
@@ -301,7 +305,7 @@ typedef struct _tagPyUFuncObject {
301305
*/
302306
#define PyUFunc_ReorderableNone -2
303307
/*
304-
* UFunc unit is in identity_value, and the order of operations can be reordered
308+
* UFunc unit is an identity_value, and the order of operations can be reordered
305309
* This case allows reduction with multiple axes at once.
306310
*/
307311
#define PyUFunc_IdentityValue -3

‎numpy/core/src/umath/ufunc_object.c

Lines changed: 40 additions & 8 deletions
0, /* tp_setattro */
Original file line numberDiff line numberDiff line change
@@ -4928,26 +4928,46 @@ PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, voi
49284928
return NULL;
49294929
}
49304930

4931-
ufunc = PyArray_malloc(sizeof(PyUFuncObject));
4931+
ufunc = PyObject_GC_New(PyUFuncObject, &PyUFunc_Type);
4932+
/*
4933+
* We use GC_New here for ufunc->obj, but do not use GC_Track since
4934+
* ufunc->obj is still NULL at the end of this function.
4935+
* See ufunc_frompyfunc where ufunc->obj is set and GC_Track is called.
4936+
*/
49324937
if (ufunc == NULL) {
49334938
return NULL;
49344939
}
4935-
memset(ufunc, 0, sizeof(PyUFuncObject));
4936-
PyObject_Init((PyObject *)ufunc, &PyUFunc_Type);
49374940

49384941
ufunc->nin = nin;
49394942
ufunc->nout = nout;
49404943
ufunc->nargs = nin+nout;
49414944
ufunc->identity = identity;
49424945
if (ufunc->identity == PyUFunc_IdentityValue) {
49434946
Py_INCREF(identity_value);
4947+
ufunc->identity_value = identity_value;
4948+
}
4949+
else {
4950+
ufunc->identity_value = NULL;
49444951
}
4945-
ufunc->identity_value = identity_value;
49464952

49474953
ufunc->functions = func;
49484954
ufunc->data = data;
49494955
ufunc->types = types;
49504956
ufunc->ntypes = ntypes;
4957+
ufunc->core_signature = NULL;
4958+
ufunc->core_enabled = 0;
4959+
ufunc->obj = NULL;
4960+
ufunc->core_num_dims = NULL;
4961+
ufunc->core_num_dim_ix = 0;
4962+
ufunc->core_offsets = NULL;
4963+
ufunc->core_dim_ixs = NULL;
4964+
ufunc->core_dim_sizes = NULL;
4965+
ufunc->core_dim_flags = NULL;
4966+
ufunc->userloops = NULL;
4967+
ufunc->ptr = NULL;
4968+
ufunc->reserved2 = NULL;
4969+
ufunc->reserved1 = 0< 6D4E /span>;
4970+
ufunc->iter_flags = 0;
49514971

49524972
/* Type resolution and inner loop selection functions */
49534973
ufunc->type_resolver = &PyUFunc_DefaultTypeResolver;
@@ -5313,6 +5333,7 @@ PyUFunc_RegisterLoopForType(PyUFuncObject *ufunc,
53135333
static void
53145334
ufunc_dealloc(PyUFuncObject *ufunc)
53155335
{
5336+
PyObject_GC_UnTrack((PyObject *)ufunc);
53165337
PyArray_free(ufunc->core_num_dims);
53175338
PyArray_free(ufunc->core_dim_ixs);
53185339
PyArray_free(ufunc->core_dim_sizes);
@@ -5322,11 +5343,13 @@ ufunc_dealloc(PyUFuncObject *ufunc)
53225343
PyArray_free(ufunc->ptr);
53235344
PyArray_free(ufunc->op_flags);
53245345
Py_XDECREF(ufunc->userloops);
5325-
Py_XDECREF(ufunc->obj);
53265346
if (ufunc->identity == PyUFunc_IdentityValue) {
53275347
Py_DECREF(ufunc->identity_value);
53285348
}
5329-
PyArray_free(ufunc);
5349+
if (ufunc->obj != NULL) {
5350+
Py_DECREF(ufunc->obj);
5351+
}
5352+
PyObject_GC_Del(ufunc);
53305353
}
53315354

53325355
static PyObject *
@@ -5335,6 +5358,15 @@ ufunc_repr(PyUFuncObject *ufunc)
53355358
return PyUString_FromFormat("<ufunc '%s'>", ufunc->name);
53365359
}
53375360

5361+
static int
5362+
ufunc_traverse(PyUFuncObject *self, visitproc visit, void *arg)
5363+
{
5364+
Py_VISIT(self->obj);
5365+
if (self->identity == PyUFunc_IdentityValue) {
5366+
Py_VISIT(self->identity_value);
5367+
}
5368+
return 0;
5369+
}
53385370

53395371
/******************************************************************************
53405372
*** UFUNC METHODS ***
@@ -6051,9 +6083,9 @@ NPY_NO_EXPORT PyTypeObject PyUFunc_Type = {
60516083
0, /* tp_getattro */
60526084
60536085
0, /* tp_as_buffer */
6054-
Py_TPFLAGS_DEFAULT, /* tp_flags */
6086+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
60556087
0, /* tp_doc */
6056-
0, /* tp_traverse */
6088+
(traverseproc)ufunc_traverse, /* tp_traverse */
60576089
0, /* tp_clear */
60586090
0, /* tp_richcompare */
60596091
0, /* tp_weaklistoffset */

‎numpy/core/src/umath/umathmodule.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ ufunc_frompyfunc(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *NPY_UNUS
161161

162162
self->type_resolver = &object_ufunc_type_resolver;
163163
self->legacy_inner_loop_selector = &object_ufunc_loop_selector;
164+
PyObject_GC_Track(self);
164165

165166
return (PyObject *)self;
166167
}

‎numpy/lib/tests/test_function_base.py

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import warnings
55
import sys
66
import decimal
7+
import types
78
import pytest
89

910
import numpy as np
@@ -24,6 +25,7 @@
2425

2526
from numpy.compat import long
2627

28+
PY2 = sys.version_info[0] == 2
2729

2830
def get_mat(n):
2931
data = np.arange(n)
@@ -353,9 +355,9 @@ class subclass(np.ndarray):
353355
assert_equal(type(np.average(a, weights=w)), subclass)
354356

355357
def test_upcasting(self):
356-
types = [('i4', 'i4', 'f8'), ('i4', 'f4', 'f8'), ('f4', 'i4', 'f8'),
358+
typs = [('i4', 'i4', 'f8'), ('i4', 'f4', 'f8'), ('f4', 'i4', 'f8'),
357359
('f4', 'f4', 'f4'), ('f4', 'f8', 'f8')]
358-
for at, wt, rt in types:
360+
for at, wt, rt in typs:
359361
a = np.array([[1,2],[3,4]], dtype=at)
360362
w = np.array([[1,2],[3,4]], dtype=wt)
361363
assert_equal(np.average(a, weights=w).dtype, np.dtype(rt))
@@ -1498,6 +1500,49 @@ def test_size_zero_output(self):
14981500
f(x)
14991501

15001502

1503+
class TestLeaks(object):
1504+
class A(object):
1505+
iters = 20
1506+
1507+
def bound(self, *args):
1508+
return 0
1509+
1510+
@staticmethod
1511+
def unbound(*args):
1512+
return 0
1513+
1514+
@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
1515+
@pytest.mark.parametrize('name, incr', [
1516+
('bound', A.iters),
1517+
('unbound', 0),
1518+
])
1519+
def test_frompyfunc_leaks(self, name, incr):
1520+
# exposed in gh-11867 as np.vectorized, but the problem stems from
1521+
# frompyfunc.
1522+
# class.attribute = np.frompyfunc(<method>) creates a
1523+
# reference cycle if <method> is a bound class method. It requires a
1524+
# gc collection cycle to break the cycle (on CPython 3)
1525+
import gc
1526+
A_func = getattr(self.A, name)
1527+
gc.disable()
1528+
try:
1529+
refcount = sys.getrefcount(A_func)
1530+
for i in ra A19E nge(self.A.iters):
1531+
a = self.A()
1532+
a.f = np.frompyfunc(getattr(a, name), 1, 1)
1533+
out = a.f(np.arange(10))
1534+
a = None
1535+
if PY2:
1536+
assert_equal(sys.getrefcount(A_func), refcount)
1537+
else:
1538+
# A.func is part of a reference cycle if incr is non-zero
1539+
assert_equal(sys.getrefcount(A_func), refcount + incr)
1540+
for i in range(5):
1541+
gc.collect()
1542+
assert_equal(sys.getrefcount(A_func), refcount)
1543+
finally:
1544+
gc.enable()
1545+
15011546
class TestDigitize(object):
15021547

15031548
def test_forward(self):

0 commit comments

Comments
 (0)
0