diff --git a/doc/release/1.15.0-notes.rst b/doc/release/1.15.0-notes.rst index 1f3bf9bf3653..8961de300a13 100644 --- a/doc/release/1.15.0-notes.rst +++ b/doc/release/1.15.0-notes.rst @@ -74,8 +74,7 @@ Deprecations * Users of ``nditer`` should use the nditer object as a context manager anytime one of the iterator operands is writeable, so that numpy can manage writeback semantics, or should call ``it.close()``. A - `RuntimeWarning` will be emitted otherwise in these cases. Users of the C-API - should call ``NpyIter_Close`` before ``NpyIter_Deallocate``. + `RuntimeWarning` may be emitted otherwise in these cases. * The ``normed`` argument of ``np.histogram``, deprecated long ago in 1.6.0, now emits a ``DeprecationWarning``. @@ -170,9 +169,6 @@ in the user guide. C API changes ============= -* ``NpyIter_Close`` has been added and should be called before - ``NpyIter_Deallocate`` to resolve possible writeback-enabled arrays. - * Functions ``npy_get_floatstatus_barrier`` and ``npy_clear_floatstatus_barrier`` have been added and should be used in place of the ``npy_get_floatstatus``and ``npy_clear_status`` functions. Optimizing compilers like GCC 8.1 and Clang diff --git a/doc/source/reference/c-api.iterator.rst b/doc/source/reference/c-api.iterator.rst index 392dcb730a07..940452d3cf0d 100644 --- a/doc/source/reference/c-api.iterator.rst +++ b/doc/source/reference/c-api.iterator.rst @@ -110,7 +110,6 @@ number of non-zero elements in an array. /* Increment the iterator to the next inner loop */ } while(iternext(iter)); - NpyIter_Close(iter) /* best practice, not strictly required in this case */ NpyIter_Deallocate(iter); return nonzero_count; @@ -195,7 +194,6 @@ is used to control the memory layout of the allocated result, typically ret = NpyIter_GetOperandArray(iter)[1]; Py_INCREF(ret); - NpyIter_Close(iter); if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { Py_DECREF(ret); return NULL; @@ -495,7 +493,7 @@ Construction and Destruction per operand. Using ``NPY_ITER_READWRITE`` or ``NPY_ITER_WRITEONLY`` for a user-provided operand may trigger `WRITEBACKIFCOPY`` semantics. The data will be written back to the original array - when ``NpyIter_Close`` is called. + when ``NpyIter_Deallocate`` is called. .. c:var:: NPY_ITER_COPY @@ -507,13 +505,13 @@ Construction and Destruction Triggers :c:data:`NPY_ITER_COPY`, and when an array operand is flagged for writing and is copied, causes the data - in a copy to be copied back to ``op[i]`` when ``NpyIter_Close`` is - called. + in a copy to be copied back to ``op[i]`` when + ``NpyIter_Deallocate`` is called. If the operand is flagged as write-only and a copy is needed, an uninitialized temporary array will be created and then copied - to back to ``op[i]`` on calling ``NpyIter_Close``, instead of doing - the unnecessary copy operation. + to back to ``op[i]`` on calling ``NpyIter_Deallocate``, instead of + doing the unnecessary copy operation. .. c:var:: NPY_ITER_NBO .. c:var:: NPY_ITER_ALIGNED @@ -709,9 +707,7 @@ Construction and Destruction the functions will pass back errors through it instead of setting a Python exception. - :c:func:`NpyIter_Deallocate` must be called for each copy. One call to - :c:func:`NpyIter_Close` is sufficient to trigger writeback resolution for - all copies since they share buffers. + :c:func:`NpyIter_Deallocate` must be called for each copy. .. c:function:: int NpyIter_RemoveAxis(NpyIter* iter, int axis)`` @@ -763,23 +759,9 @@ Construction and Destruction Returns ``NPY_SUCCEED`` or ``NPY_FAIL``. -.. c:function:: int NpyIter_Close(NpyIter* iter) - - Resolves any needed writeback resolution. Should be called before - :c:func::`NpyIter_Deallocate`. After this call it is not safe to use the operands. - When using :c:func:`NpyIter_Copy`, only one call to :c:func:`NpyIter_Close` - is sufficient to resolve any writebacks, since the copies share buffers. - - Returns ``0`` or ``-1`` if unsuccessful. - .. c:function:: int NpyIter_Deallocate(NpyIter* iter) - Deallocates the iterator object. - - :c:func:`NpyIter_Close` should be called before this. If not, and if - writeback is needed, it will be performed at this point in order to maintain - backward-compatibility with older code, and a deprecation warning will be - emitted. Old code should be updated to call `NpyIter_Close` beforehand. + Deallocates the iterator object and resolves any needed writebacks. Returns ``NPY_SUCCEED`` or ``NPY_FAIL``. diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index cc6c3a5fbf6d..43c32eac6edd 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -39,8 +39,7 @@ 0x0000000b = edb1ba83730c650fd9bc5772a919cda7 # Version 12 (NumPy 1.14) Added PyArray_ResolveWritebackIfCopy, +# Version 12 (NumPy 1.15) No change. # PyArray_SetWritebackIfCopyBase and deprecated PyArray_SetUpdateIfCopyBase. 0x0000000c = a1bc756c5782853ec2e3616cf66869d8 -# Version 13 (NumPy 1.15) Added NpyIter_Close -0x0000000d = 4386e829d65aafce6bd09a85b142d585 diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index 6cfbbbcc7b25..d8a9ee6b4903 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -350,8 +350,6 @@ 'PyArray_ResolveWritebackIfCopy': (302,), 'PyArray_SetWritebackIfCopyBase': (303,), # End 1.14 API - 'NpyIter_Close': (304,), - # End 1.15 API } ufunc_types_api = { diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py index 70a43046c3e4..356482b07248 100644 --- a/numpy/core/setup_common.py +++ b/numpy/core/setup_common.py @@ -40,8 +40,8 @@ # 0x0000000a - 1.12.x # 0x0000000b - 1.13.x # 0x0000000c - 1.14.x -# 0x0000000d - 1.15.x -C_API_VERSION = 0x0000000d +# 0x0000000c - 1.15.x +C_API_VERSION = 0x0000000c class MismatchCAPIWarning(Warning): pass diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index cba96a4c2b5e..67c9a333cc4d 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -1041,76 +1041,6 @@ test_nditer_too_large(PyObject *NPY_UNUSED(self), PyObject *args) { return NULL; } -static PyObject * -test_nditer_writeback(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) -{ - /* like npyiter_init */ - PyObject *op_in = NULL, *op_dtypes_in = NULL, *value = NULL; - PyArrayObject * opview; - int iop, nop = 0; - PyArrayObject *op[NPY_MAXARGS]; - npy_uint32 flags = 0; - NPY_ORDER order = NPY_KEEPORDER; - NPY_CASTING casting = NPY_EQUIV_CASTING; - npy_uint32 op_flags[NPY_MAXARGS]; - PyArray_Descr *op_request_dtypes[NPY_MAXARGS]; - int retval; - unsigned char do_close; - int buffersize = 0; - NpyIter *iter = NULL; - static char *kwlist[] = {"value", "do_close", "input", "op_dtypes", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "ObO|O:test_nditer_writeback", kwlist, - &value, - &do_close, - &op_in, - &op_dtypes_in)) { - return NULL; - } - /* op and op_flags */ - if (! PyArray_Check(op_in)) { - return NULL; - } - nop = 1; - op[0] = (PyArrayObject*)op_in; - op_flags[0] = NPY_ITER_READWRITE|NPY_ITER_UPDATEIFCOPY; - - /* Set the dtypes */ - for (iop=0; iopiter = NULL; self->nested_child = NULL; - self->is_closed = 0; } return (PyObject *)self; @@ -1173,10 +1174,29 @@ NpyIter_NestedIters(PyObject *NPY_UNUSED(self), return NULL; } + static void npyiter_dealloc(NewNpyArrayIterObject *self) { if (self->iter) { + if (npyiter_has_writeback(self->iter)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "Temporary data has not been written back to one of the " + "operands. Typically nditer is used as a context manager " + "otherwise 'close' must be called before reading iteration " + "results.", 1) < 0) { + PyObject *s; + + s = PyUString_FromString("npyiter_dealloc"); + if (s) { + PyErr_WriteUnraisable(s); + Py_DECREF(s); + } + else { + PyErr_WriteUnraisable(Py_None); + } + } + } NpyIter_Deallocate(self->iter); self->iter = NULL; Py_XDECREF(self->nested_child); @@ -1418,12 +1438,6 @@ static PyObject *npyiter_value_get(NewNpyArrayIterObject *self) ret = npyiter_seq_item(self, 0); } else { - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return NULL; - } - ret = PyTuple_New(nop); if (ret == NULL) { return NULL; @@ -1453,12 +1467,6 @@ static PyObject *npyiter_operands_get(NewNpyArrayIterObject *self) "Iterator is invalid"); return NULL; } - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return NULL; - } - nop = NpyIter_GetNOp(self->iter); operands = self->operands; @@ -1487,13 +1495,6 @@ static PyObject *npyiter_itviews_get(NewNpyArrayIterObject *self) "Iterator is invalid"); return NULL; } - - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return NULL; - } - nop = NpyIter_GetNOp(self->iter); ret = PyTuple_New(nop); @@ -1517,7 +1518,7 @@ static PyObject * npyiter_next(NewNpyArrayIterObject *self) { if (self->iter == NULL || self->iternext == NULL || - self->finished || self->is_closed) { + self->finished) { return NULL; } @@ -1911,13 +1912,6 @@ static PyObject *npyiter_dtypes_get(NewNpyArrayIterObject *self) "Iterator is invalid"); return NULL; } - - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return NULL; - } - nop = NpyIter_GetNOp(self->iter); ret = PyTuple_New(nop); @@ -2011,13 +2005,6 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) "and no reset has been done yet"); return NULL; } - - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return NULL; - } - nop = NpyIter_GetNOp(self->iter); /* Negative indexing */ @@ -2090,13 +2077,6 @@ npyiter_seq_slice(NewNpyArrayIterObject *self, "and no reset has been done yet"); return NULL; } - - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return NULL; - } - nop = NpyIter_GetNOp(self->iter); if (ilow < 0) { ilow = 0; @@ -2156,13 +2136,6 @@ npyiter_seq_ass_item(NewNpyArrayIterObject *self, Py_ssize_t i, PyObject *v) "and no reset has been done yet"); return -1; } - - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return -1; - } - nop = NpyIter_GetNOp(self->iter); /* Negative indexing */ @@ -2234,13 +2207,6 @@ npyiter_seq_ass_slice(NewNpyArrayIterObject *self, Py_ssize_t ilow, "and no reset has been done yet"); return -1; } - - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return -1; - } - nop = NpyIter_GetNOp(self->iter); if (ilow < 0) { ilow = 0; @@ -2292,12 +2258,6 @@ npyiter_subscript(NewNpyArrayIterObject *self, PyObject *op) return NULL; } - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return NULL; - } - if (PyInt_Check(op) || PyLong_Check(op) || (PyIndex_Check(op) && !PySequence_Check(op))) { npy_intp i = PyArray_PyIntAsIntp(op); @@ -2347,12 +2307,6 @@ npyiter_ass_subscript(NewNpyArrayIterObject *self, PyObject *op, return -1; } - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, - "Iterator is closed"); - return -1; - } - if (PyInt_Check(op) || PyLong_Check(op) || (PyIndex_Check(op) && !PySequence_Check(op))) { npy_intp i = PyArray_PyIntAsIntp(op); @@ -2387,10 +2341,6 @@ npyiter_enter(NewNpyArrayIterObject *self) PyErr_SetString(PyExc_RuntimeError, "operation on non-initialized iterator"); return NULL; } - if (self->is_closed) { - PyErr_SetString(PyExc_ValueError, "cannot reuse closed iterator"); - return NULL; - } Py_INCREF(self); return (PyObject *)self; } @@ -2403,8 +2353,8 @@ npyiter_close(NewNpyArrayIterObject *self) if (self->iter == NULL) { Py_RETURN_NONE; } - ret = NpyIter_Close(iter); - self->is_closed = 1; + ret = NpyIter_Deallocate(iter); + self->iter = NULL; if (ret < 0) { return NULL; } diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 5e92bc9919f6..59fc5aa20a42 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -1281,7 +1281,6 @@ iterator_loop(PyUFuncObject *ufunc, PyArrayObject **op_it; npy_uint32 iter_flags; - int retval; NPY_BEGIN_THREADS_DEF; @@ -1355,7 +1354,6 @@ iterator_loop(PyUFuncObject *ufunc, /* Call the __array_prepare__ functions for the new array */ if (prepare_ufunc_output(ufunc, &op[nin+i], arr_prep[i], full_args, i) < 0) { - NpyIter_Close(iter); NpyIter_Deallocate(iter); return -1; } @@ -1384,7 +1382,6 @@ iterator_loop(PyUFuncObject *ufunc, baseptrs[i] = PyArray_BYTES(op_it[i]); } if (NpyIter_ResetBasePointers(iter, baseptrs, NULL) != NPY_SUCCEED) { - NpyIter_Close(iter); NpyIter_Deallocate(iter); return -1; } @@ -1392,7 +1389,6 @@ iterator_loop(PyUFuncObject *ufunc, /* Get the variables needed for the loop */ iternext = NpyIter_GetIterNext(iter, NULL); if (iternext == NULL) { - NpyIter_Close(iter); NpyIter_Deallocate(iter); return -1; } @@ -1410,9 +1406,7 @@ iterator_loop(PyUFuncObject *ufunc, NPY_END_THREADS; } - retval = NpyIter_Close(iter); - NpyIter_Deallocate(iter); - return retval; + return NpyIter_Deallocate(iter); } /* @@ -1597,7 +1591,7 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, PyObject **arr_prep, ufunc_full_args full_args) { - int retval, i, nin = ufunc->nin, nout = ufunc->nout; + int i, nin = ufunc->nin, nout = ufunc->nout; int nop = nin + nout; npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; @@ -1709,7 +1703,6 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, if (prepare_ufunc_output(ufunc, &op_tmp, arr_prep[i], full_args, i) < 0) { - NpyIter_Close(iter); NpyIter_Deallocate(iter); return -1; } @@ -1720,7 +1713,6 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, "The __array_prepare__ functions modified the data " "pointer addresses in an invalid fashion"); Py_DECREF(op_tmp); - NpyIter_Close(iter); NpyIter_Deallocate(iter); return -1; } @@ -1755,7 +1747,6 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, wheremask != NULL ? fixed_strides[nop] : fixed_strides[nop + nin], &innerloop, &innerloopdata, &needs_api) < 0) { - NpyIter_Close(iter); NpyIter_Deallocate(iter); return -1; } @@ -1763,7 +1754,6 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, /* Get the variables needed for the loop */ iternext = NpyIter_GetIterNext(iter, NULL); if (iternext == NULL) { - NpyIter_Close(iter); NpyIter_Deallocate(iter); return -1; } @@ -1787,9 +1777,7 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, NPY_AUXDATA_FREE(innerloopdata); } - retval = NpyIter_Close(iter); - NpyIter_Deallocate(iter); - return retval; + return NpyIter_Deallocate(iter); } static npy_bool @@ -2300,7 +2288,7 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, int nin, nout; int i, j, idim, nop; const char *ufunc_name; - int retval = 0, subok = 1; + int retval, subok = 1; int needs_api = 0; PyArray_Descr *dtypes[NPY_MAXARGS]; @@ -2809,16 +2797,11 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, goto fail; } - /* Write back any temporary data from PyArray_SetWritebackIfCopyBase */ - if (NpyIter_Close(iter) < 0) { - goto fail; - } - PyArray_free(inner_strides); - if (NpyIter_Close(iter) < 0) { - goto fail; + if (NpyIter_Deallocate(iter) < 0) { + retval = -1; } - NpyIter_Deallocate(iter); + /* The caller takes ownership of all the references in op */ for (i = 0; i < nop; ++i) { Py_XDECREF(dtypes[i]); @@ -2831,14 +2814,13 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, Py_XDECREF(full_args.in); Py_XDECREF(full_args.out); - NPY_UF_DBG_PRINT("Returning Success\n"); + NPY_UF_DBG_PRINT1("Returning code %d\n", reval); - return 0; + return retval; fail: NPY_UF_DBG_PRINT1("Returning failure code %d\n", retval); PyArray_free(inner_strides); - NpyIter_Close(iter); NpyIter_Deallocate(iter); for (i = 0; i < nop; ++i) { Py_XDECREF(op[i]); @@ -3031,7 +3013,7 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, Py_XDECREF(full_args.out); Py_XDECREF(wheremask); - NPY_UF_DBG_PRINT("Returning Success\n"); + NPY_UF_DBG_PRINT("Returning success code 0\n"); return 0; @@ -3722,12 +3704,6 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, } finish: - if (NpyIter_Close(iter) < 0) { - goto fail; - } - if (NpyIter_Close(iter_inner) < 0) { - goto fail; - } Py_XDECREF(op_dtypes[0]); NpyIter_Deallocate(iter); NpyIter_Deallocate(iter_inner); @@ -4110,9 +4086,6 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, } finish: - if (NpyIter_Close(iter) < 0) { - goto fail; - } Py_XDECREF(op_dtypes[0]); NpyIter_Deallocate(iter); @@ -5544,7 +5517,6 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) iternext = NpyIter_GetIterNext(iter_buffer, NULL); if (iternext == NULL) { - NpyIter_Close(iter_buffer); NpyIter_Deallocate(iter_buffer); goto fail; } @@ -5614,7 +5586,6 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) PyErr_SetString(PyExc_ValueError, err_msg); } - NpyIter_Close(iter_buffer); NpyIter_Deallocate(iter_buffer); Py_XDECREF(op2_array); @@ -5632,7 +5603,7 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) } fail: - /* iter_buffer has already been deallocated, don't use NpyIter_Close */ + /* iter_buffer has already been deallocated, don't use NpyIter_Dealloc */ if (op1_array != (PyArrayObject*)op1) { PyArray_DiscardWritebackIfCopy(op1_array); } diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index a0096efdbc4b..13bc6b34a45b 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -2830,10 +2830,6 @@ def test_writebacks(): x[:] = 123 # x.data still valid assert_equal(au, 6) # but not connected to au - do_close = 1 - # test like above, only in C, and with an option to skip the NpyIter_Close - _multiarray_tests.test_nditer_writeback(3, do_close, au, op_dtypes=[np.dtype('f4')]) - assert_equal(au, 3) it = nditer(au, [], [['readwrite', 'updateifcopy']], casting='equiv', op_dtypes=[np.dtype('f4')]) @@ -2862,7 +2858,7 @@ def test_writebacks(): x[...] = 123 # make sure we cannot reenter the closed iterator enter = it.__enter__ - assert_raises(ValueError, enter) + assert_raises(RuntimeError, enter) def test_close_equivalent(): ''' using a context amanger and using nditer.close are equivalent @@ -2897,12 +2893,13 @@ def test_close_raises(): assert_raises(StopIteration, next, it) assert_raises(ValueError, getattr, it, 'operands') +@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") def test_warn_noclose(): a = np.arange(6, dtype='f4') au = a.byteswap().newbyteorder() - do_close = 0 with suppress_warnings() as sup: sup.record(RuntimeWarning) - # test like above, only in C, and with an option to skip the NpyIter_Close - _multiarray_tests.test_nditer_writeback(3, do_close, au, op_dtypes=[np.dtype('f4')]) + it = np.nditer(au, [], [['readwrite', 'updateifcopy']], + casting='equiv', op_dtypes=[np.dtype('f4')]) + del it assert len(sup.log) == 1