8000 BUG, MAINT: Ufunc reduce reference leak by mhvk · Pull Request #10194 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

BUG, MAINT: Ufunc reduce reference leak #10194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 11, 2017
Merged

BUG, MAINT: Ufunc reduce reference leak #10194

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1cb7ffd
BUG: Failure to DECREF in PyUFunc_GenericReduction.
mhvk Dec 10, 2017
2fadede
MAINT: Use single failure path in Py_UFunc_GenericReduction.
mhvk Dec 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
MAINT: Use single failure path in Py_UFunc_GenericReduction. 
This should help avoid reference leaks in future additions.
  • Loading branch information
@mhvk
mhvk committed Dec 11, 2017
commit 2fadedee0799df7c4ce8f81caf5a1283df767e57
67 changes: 25 additions & 42 deletions numpy/core/src/umath/ufunc_object.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3656,7 +3656,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
int i, naxes=0, ndim;
int axes[NPY_MAXDIMS];
PyObject *axes_in = NULL;
PyArrayObject *mp, *ret = NULL;
PyArrayObject *mp = NULL, *ret = NULL;
PyObject *op, *res = NULL;
PyObject *obj_ind, *context;
PyArrayObject *indices = NULL;
Expand Down Expand Up @@ -3712,19 +3712,17 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
PyArray_Descr *indtype;
indtype = PyArray_DescrFromType(NPY_INTP);
if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO&O&:reduceat", reduceat_kwlist,
&op,
&obj_ind,
&axes_in,
PyArray_DescrConverter2, &otype,
PyArray_OutputConverter, &out)) {
Py_XDECREF(otype);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

otype cannot be defined if this fails.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's not true - this was correct before - it could fail on the OutputConverter.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In fact, this should maybe xdecref out too.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I looked for other usages, and they always DECREF only stuff before OutputConverter - so, I guess one does not have to worry about the one that does fail, just what comes before. Anyway, I'll change it back.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or, rather, change it to goto fail as it is for the other options.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess one does not have to worry about the one that does fail, just what comes before.

Looking at the implementation of PyArg_ParseTupleAndKeywords in 3.6, I think I can break this by providing an argument by keyword and position, which happens after the other arguments are parsed. I certainly don't think that cpython guarantees the last object will never need freeing

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You may well be right, but since this is separate from the fix here (in that it is done everywhere OutputConverter is used), I think is should be a separate issue: #10197

Copy link
Member
@eric-wieser eric-wieser Dec 11, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wrong, PyArray_OutputConverter borrows a reference. Would be good to distinguish the Converters that lend/create references in the name somehow, but we're way too late to make that decision.

return NULL;
&op,
&obj_ind,
&axes_in,
PyArray_DescrConverter2, &otype,
PyArray_OutputConverter, &out)) {
goto fail;
}
indices = (PyArrayObject *)PyArray_FromAny(obj_ind, indtype,
1, 1, NPY_ARRAY_CARRAY, NULL);
if (indices == NULL) {
Py_XDECREF(otype);
return NULL;
goto fail;
}
}
else if (operation == UFUNC_ACCUMULATE) {
Expand All @@ -3734,8 +3732,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
&axes_in,
PyArray_DescrConverter2, &otype,
PyArray_OutputConverter, &out)) {
Py_XDECREF(otype);
return NULL;
goto fail;
}
}
else {
Expand All @@ -3746,8 +3743,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
PyArray_DescrConverter2, &otype,
PyArray_OutputConverter, &out,
&keepdims)) {
Py_XDECREF(otype);
return NULL;
goto fail;
}
}
/* Ensure input is an array */
Expand All @@ -3760,7 +3756,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
mp = (PyArrayObject *)PyArray_FromAny(op, NULL, 0, 0, 0, context);
Py_XDECREF(context);
if (mp == NULL) {
return NULL;
goto fail;
}

ndim = PyArray_NDIM(mp);
Expand All @@ -3771,9 +3767,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
PyErr_Format(PyExc_TypeError,
"cannot perform %s with flexible type",
_reduce_type[operation]);
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
}

/* Convert the 'axis' parameter into a list of axes */
Expand All @@ -3793,22 +3787,16 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
if (naxes < 0 || naxes > NPY_MAXDIMS) {
PyErr_SetString(PyExc_ValueError,
"too many values for 'axis'");
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
}
for (i = 0; i < naxes; ++i) {
PyObject *tmp = PyTuple_GET_ITEM(axes_in, i);
int axis = PyArray_PyIntAsInt(tmp);
if (error_converting(axis)) {
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
}
if (check_and_adjust_axis(&axis, ndim) < 0) {
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this would be clearer as two separate ifs, like it was before - it makes it look more like the other uses of check_and_adjust_axis.

}
axes[i] = (int)axis;
}
Expand All @@ -3818,18 +3806,14 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
int axis = PyArray_PyIntAsInt(axes_in);
/* TODO: PyNumber_Index would be good to use here */
if (error_converting(axis)) {
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
}
/* Special case letting axis={0 or -1} slip through for scalars */
if (ndim == 0 && (axis == 0 || axis == -1)) {
axis = 0;
}
else if (check_and_adjust_axis(&axis, ndim) < 0) {
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is where the DECREF was missing.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Careless of me, good catch

}
axes[0] = (int)axis;
naxes = 1;
Expand All @@ -3849,9 +3833,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
(naxes == 0 || (naxes == 1 && axes[0] == 0)))) {
PyErr_Format(PyExc_TypeError, "cannot %s on a scalar",
_reduce_type[operation]);
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
}
}

Expand Down Expand Up @@ -3897,9 +3879,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
if (naxes != 1) {
PyErr_SetString(PyExc_ValueError,
"accumulate does not allow multiple axes");
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
}
ret = (PyArrayObject *)PyUFunc_Accumulate(ufunc, mp, out, axes[0],
otype->type_num);
Expand All @@ -3908,9 +3888,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
if (naxes != 1) {
PyErr_SetString(PyExc_ValueError,
"reduceat does not allow multiple axes");
Py_XDECREF(otype);
Py_DECREF(mp);
return NULL;
goto fail;
}
ret = (PyArrayObject *)PyUFunc_Reduceat(ufunc, mp, indices, out,
axes[0], otype->type_num);
Expand Down Expand Up @@ -3943,6 +3921,11 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args,
}
}
return PyArray_Return(ret);

fail:
Py_XDECREF(otype);
Py_XDECREF(mp);
return NULL;
}

/*
Expand Down
0