10000 BUG: use cyclic garbage collection semantics in np.frompyfunc · numpy/numpy@0267264 · GitHub
[go: up one dir, main page]
Skip to content

Commit 0267264

Browse files
mattipmattip
committed
BUG: use cyclic garbage collection semantics in np.frompyfunc
1 parent 81fe95c commit 0267264

File tree

3 files changed

+92
-9
lines changed

3 files changed

+92
-9
lines changed

‎numpy/core/src/umath/ufunc_object.c

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4892,12 +4892,15 @@ PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, voi
48924892
return NULL;
48934893
}
48944894

4895-
ufunc = PyArray_malloc(sizeof(PyUFuncObject));
4895+
ufunc = PyObject_GC_New(PyUFuncObject, &PyUFunc_Type);
4896+
/*
4897+
* We use GC_New here for ufunc->obj, but do not use GC_Track since
4898+
* ufunc->obj is still NULL at the end of this function.
4899+
* See ufunc_frompyfunc where ufunc->obj is set and GC_Track is called.
4900+
*/
48964901
if (ufunc == NULL) {
48974902
return NULL;
48984903
}
4899-
memset(ufunc, 0, sizeof(PyUFuncObject));
4900-
PyObject_Init((PyObject *)ufunc, &PyUFunc_Type);
49014904

49024905
ufunc->nin = nin;
49034906
ufunc->nout = nout;
@@ -4912,6 +4915,20 @@ PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, voi
49124915
ufunc->data = data;
49134916
ufunc->types = types;
49144917
ufunc->ntypes = ntypes;
4918+
ufunc->core_signature = NULL;
4919+
ufunc->core_enabled = 0;
4920+
ufunc->obj = NULL;
4921+
ufunc->core_num_dims = NULL;
4922+
ufunc->core_num_dim_ix = 0;
4923+
ufunc->core_offsets = NULL;
4924+
ufunc->core_dim_ixs = NULL;
4925+
ufunc->core_dim_sizes = NULL;
4926+
ufunc->core_dim_flags = NULL;
4927+
ufunc->userloops = NULL;
4928+
ufunc->ptr = NULL;
4929+
ufunc->reserved2 = NULL;
4930+
ufunc->reserved1 = 0;
4931+
ufunc->iter_flags = 0;
49154932

49164933
/* Type resolution and inner loop selection functions */
49174934
ufunc->type_resolver = &PyUFunc_DefaultTypeResolver;
@@ -5284,11 +5301,14 @@ ufunc_dealloc(PyUFuncObject *ufunc)
52845301
PyArray_free(ufunc->ptr);
52855302
PyArray_free(ufunc->op_flags);
52865303
Py_XDECREF(ufunc->userloops);
5287-
Py_XDECREF(ufunc->obj);
52885304
if (ufunc->identity == PyUFunc_IdentityValue) {
52895305
Py_DECREF(ufunc->identity_value);
52905306
}
5291-
PyArray_free(ufunc);
5307+
if (ufunc->obj != NULL) {
5308+
PyObject_GC_UnTrack((PyObject *)ufunc);
5309+
Py_DECREF(ufunc->obj);
5310+
}
5311+
PyObject_GC_Del(ufunc);
52925312
}
52935313

52945314
static PyObject *
@@ -5297,6 +5317,16 @@ ufunc_repr(PyUFuncObject *ufunc)
52975317
return PyUString_FromFormat("<ufunc '%s'>", ufunc->name);
52985318
}
52995319

5320+
static int
5321+
ufunc_traverse(PyObject *self, visitproc visit, void *arg)
5322+
{
5323+
PyUFuncObject *ufunc = (PyUFuncObject*)self;
5324+
Py_VISIT(ufunc->obj);
5325+
if (ufunc->identity == PyUFunc_IdentityValue) {
5326+
Py_VISIT(ufunc->identity_value);
5327+
}
5328+
return 0;
5329+
}
53005330

53015331
/******************************************************************************
53025332
*** UFUNC METHODS ***
@@ -6013,9 +6043,9 @@ NPY_NO_EXPORT PyTypeObject PyUFunc_Type = {
60136043
0, /* tp_getattro */
60146044
0, /* tp_setattro */
60156045
0, /* tp_as_buffer */
6016-
Py_TPFLAGS_DEFAULT, /* tp_flags */
6046+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
60176047
0, /* tp_doc */
6018-
0, /* tp_traverse */
6048+
(traverseproc)ufunc_traverse, /* tp_traverse */
60196049
0, /* tp_clear */
60206050
0, /* tp_richcompare */
60216051
0, /* tp_weaklistoffset */

‎numpy/core/src/umath/umathmodule.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ ufunc_frompyfunc(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *NPY_UNUS
161161

162162
self->type_resolver = &object_ufunc_type_resolver;
163163
self->legacy_inner_loop_selector = &object_ufunc_loop_selector;
164+
PyObject_GC_Track(self);
164165

165166
return (PyObject *)self;
166167
}

‎numpy/lib/tests/test_function_base.py

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import warnings
55
import sys
66
import decimal
7+
import types
78
import pytest
89

910
import numpy as np
@@ -24,6 +25,7 @@
2425

2526
from numpy.compat import long
2627

28+
PY2 = sys.version_info[0] == 2
2729

2830
def get_mat(n):
2931
data = np.arange(n)
@@ -353,9 +355,9 @@ class subclass(np.ndarray):
353355
assert_equal(type(np.average(a, weights=w)), subclass)
354356

355357
def test_upcasting(self):
356-
types = [('i4', 'i4', 'f8'), ('i4', 'f4', 'f8'), ('f4', 'i4', 'f8'),
358+
typs = [('i4', 'i4', 'f8'), ('i4', 'f4', 'f8'), ('f4', 'i4', 'f8'),
357359
('f4', 'f4', 'f4'), ('f4', 'f8', 'f8')]
358-
for at, wt, rt in types:
360+
for at, wt, rt in typs:
359361
a = np.array([[1,2],[3,4]], dtype=at)
360362
w = np.array([[1,2],[3,4]], dtype=wt)
361363
assert_equal(np.average(a, weights=w).dtype, np.dtype(rt))
@@ -1498,6 +1500,56 @@ def test_size_zero_output(self):
14981500
f(x)
14991501

15001502

1503+
class TestLeaks(object):
1504+
class A(object):
1505+
iters = 20
1506+
1507+
def bound(self, *args):
1508+
return np.int(0)
1509+
1510+
@staticmethod
1511+
def unbound(*args):
1512+
return np.int(0)
1513+
1514+
def npy_bound(self, a):
1515+
return types.MethodType(np.frompyfunc(self.bound, 1, 1), a)
1516+
1517+
def npy_unbound(self, a):
1518+
return np.frompyfunc(self.unbound, 1, 1)
1519+
1520+
@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
1521+
@pytest.mark.parametrize('func, npy_func, incr', [
1522+
# Test with both an unbound method and a bound one
1523+
(A.bound, A.npy_bound, A.iters),
1524+
(A.unbound, A.npy_unbound, 0),
1525+
])
1526+
def test_frompyfunc_leaks(self, func, npy_func, incr):
1527+
# exposed in gh-11867 as np.vectorized, but the problem stems from
1528+
# frompyfunc.
1529+
# class.attribute = np.frompyfunc(<method>) creates a
1530+
# reference cycle that requires a gc collection cycle to break
1531+
# (on CPython 3)
1532+
import gc
1533+
1534+
gc.disable()
1535+
try:
1536+
refcount = sys.getrefcount(func)
1537+
for i in range(self.A.iters):
1538+
a = self.A()
1539+
a.f = npy_func(a, a)
1540+
out = a.f(np.arange(10))
1541+
a = None
1542+
if PY2:
1543+
assert_equal(sys.getrefcount(func), refcount)
1544+
else:
1545+
# func is part of a reference cycle
1546+
assert_equal(sys.getrefcount(func), refcount + incr)
1547+
for i in range(5):
1548+
gc.collect()
1549+
assert_equal(sys.getrefcount(func), refcount)
1550+
finally:
1551+
gc.enable()
1552+
15011553
class TestDigitize(object):
15021554

15031555
def test_forward(self):

0 commit comments

Comments
 (0)
0