From 847aefc8a5dc239f2937cf369de0895e94e78533 Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Wed, 28 Aug 2019 12:52:18 -0400 Subject: [PATCH 1/4] BUG: view with fieldless dtype should raise if itemsize != 0 Fixes #14344 --- numpy/core/include/numpy/ndarraytypes.h | 3 ++- numpy/core/tests/test_dtype.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 1221aeecebe2..bda000da944e 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1695,7 +1695,8 @@ PyArray_CLEARFLAGS(PyArrayObject *arr, int flags) #define PyDataType_ISOBJECT(obj) PyTypeNum_ISOBJECT(((PyArray_Descr*)(obj))->type_num) #define PyDataType_HASFIELDS(obj) (((PyArray_Descr *)(obj))->names != NULL) #define PyDataType_HASSUBARRAY(dtype) ((dtype)->subarray != NULL) -#define PyDataType_ISUNSIZED(dtype) ((dtype)->elsize == 0) +#define PyDataType_ISUNSIZED(dtype) ((dtype)->elsize == 0 && \ + !PyDataType_HASFIELDS(dtype)) #define PyDataType_MAKEUNSIZED(dtype) ((dtype)->elsize = 0) #define PyArray_ISBOOL(obj) PyTypeNum_ISBOOL(PyArray_TYPE(obj)) diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index f60eab6965fd..caaeac92d3fe 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -419,6 +419,11 @@ def test_partial_dict(self): assert_raises(ValueError, np.dtype, {'formats': ['i4', 'i4'], 'f0': ('i4', 0), 'f1':('i4', 4)}) + def test_fieldless_views(self): + a = np.zeros(2, dtype={'names':[], 'formats':[], 'offsets':[], + 'itemsize':8}) + assert_raises(ValueError, a.view, np.dtype([])) + class TestSubarray(object): def test_single_subarray(self): From 9b740ef1f7ccb8f5e0b5d018eb5fa4c26c7f0712 Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Sat, 31 Aug 2019 14:50:59 -0400 Subject: [PATCH 2/4] BUG: PyDataType_ISUNSIZED should be False for structured types --- numpy/core/src/multiarray/arrayobject.c | 3 ++- numpy/core/src/multiarray/ctors.c | 10 ++++++++-- numpy/core/src/multiarray/methods.c | 2 +- numpy/core/tests/test_dtype.py | 14 ++++++++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index eb939f47c0e9..4676c40d590f 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -1200,7 +1200,8 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op) } } if (res == NULL && !PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, "No fields found."); + /* these dtypes had no fields */ + return cmp_op == Py_EQ ? Py_True : Py_False; } return res; } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index f0a15505df9c..6b4b9aa1ff31 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -3843,7 +3843,13 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type, s = (npy_intp)ts - offset; n = (npy_intp)count; itemsize = type->elsize; - if (n < 0 ) { + if (n < 0) { + if (itemsize == 0) { + PyErr_SetString(PyExc_ValueError, + "cannot determine count if itemsize is 0"); + Py_DECREF(type); + return NULL; + } if (s % itemsize != 0) { PyErr_SetString(PyExc_ValueError, "buffer size must be a multiple"\ @@ -4036,7 +4042,7 @@ PyArray_FromIter(PyObject *obj, PyArray_Descr *dtype, npy_intp count) } for (i = 0; (i < count || count == -1) && (value = PyIter_Next(iter)); i++) { - if (i >= elcount) { + if (i >= elcount && elsize != 0) { npy_intp nbytes; /* Grow PyArray_DATA(ret): diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 79c60aa2e04d..8073071dde33 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1861,7 +1861,7 @@ array_reduce_ex(PyArrayObject *self, PyObject *args) PyDataType_FLAGCHK(descr, NPY_ITEM_HASOBJECT) || (PyType_IsSubtype(((PyObject*)self)->ob_type, &PyArray_Type) && ((PyObject*)self)->ob_type != &PyArray_Type) || - PyDataType_ISUNSIZED(descr)) { + descr->elsize == 0) { /* The PickleBuffer class from version 5 of the pickle protocol * can only be used for arrays backed by a contiguous data buffer. * For all other cases we fallback to the generic array_reduce diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index caaeac92d3fe..14c1857b412d 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -424,6 +424,20 @@ def test_fieldless_views(self): 'itemsize':8}) assert_raises(ValueError, a.view, np.dtype([])) + d = np.dtype((np.dtype([]), 10)) + assert_equal(d.shape, (10,)) + assert_equal(d.itemsize, 0) + assert_equal(d.base, np.dtype([])) + + arr = np.fromiter((() for i in range(10)), []) + assert_equal(arr.dtype, np.dtype([])) + assert_raises(ValueError, np.frombuffer, b'', dtype=[]) + assert_equal(np.frombuffer(b'', dtype=[], count=2), + np.empty(2, dtype=[])) + + assert_raises(ValueError, np.dtype, ([], 'f8')) + assert_raises(ValueError, np.zeros(1, dtype='i4').view, []) + class TestSubarray(object): def test_single_subarray(self): From 7c3adfb7436a37d995721f1c793e5e3ced6d9b7e Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Sat, 31 Aug 2019 15:30:32 -0400 Subject: [PATCH 3/4] BUG: Fix comparison of fieldless structured arrays --- doc/source/reference/c-api/array.rst | 3 +++ numpy/core/src/multiarray/arrayobject.c | 12 +++++++----- numpy/core/tests/test_dtype.py | 6 ++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index a2b56cee7a0b..215584b62d06 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -997,6 +997,9 @@ argument must be a :c:type:`PyObject *` that can be directly interpret called on flexible dtypes. Types that are attached to an array will always be sized, hence the array form of this macro not existing. + ..versionchanged:: 1.18.0 + For structured datatypes with no fields this function now return False. + .. c:function:: PyTypeNum_ISUSERDEF(num) .. c:function:: PyDataType_ISUSERDEF(descr) diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 4676c40d590f..05e1d301185d 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -1201,15 +1201,17 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op) } if (res == NULL && !PyErr_Occurred()) { /* these dtypes had no fields */ - return cmp_op == Py_EQ ? Py_True : Py_False; + res = PyArray_NewLikeArray(self, NPY_ANYORDER, + PyArray_DescrFromType(NPY_BOOL), 1); + if (res) { + PyArray_FILLWBYTE((PyArrayObject*)res, + cmp_op == Py_EQ ? 1 : 0); + } } return res; } else { - /* - * compare as a string. Assumes self and - * other have same descr->type - */ + /* compare as a string. Assumes self and other have same descr->type */ return _strings_richcompare(self, other, cmp_op, 0); } } diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 14c1857b412d..6230b27b7679 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -438,6 +438,12 @@ def test_fieldless_views(self): assert_raises(ValueError, np.dtype, ([], 'f8')) assert_raises(ValueError, np.zeros(1, dtype='i4').view, []) + assert_equal(np.zeros(2, dtype=[]) == np.zeros(2, dtype=[]), + np.ones(2, dtype=bool)) + + assert_equal(np.zeros(2, dtype=[]) == a, + np.ones(2, dtype=bool)) + class TestSubarray(object): def test_single_subarray(self): From d6f7524defc808f32219528350429d5569696183 Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Sat, 31 Aug 2019 20:43:13 -0400 Subject: [PATCH 4/4] BUG: Fix fieldless comparison broadcasting Fixes #13438 --- doc/release/upcoming_changes/14393.c_api.rst | 5 +++++ doc/source/reference/c-api/array.rst | 5 +++-- numpy/core/src/multiarray/arrayobject.c | 18 ++++++++++++++---- numpy/core/tests/test_dtype.py | 4 ++-- 4 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 doc/release/upcoming_changes/14393.c_api.rst diff --git a/doc/release/upcoming_changes/14393.c_api.rst b/doc/release/upcoming_changes/14393.c_api.rst new file mode 100644 index 000000000000..0afd275849b9 --- /dev/null +++ b/doc/release/upcoming_changes/14393.c_api.rst @@ -0,0 +1,5 @@ +PyDataType_ISUNSIZED(descr) now returns False for structured datatypes +---------------------------------------------------------------------- +Previously this returned True for any datatype of itemsize 0, but now this +returns false for the non-flexible datatype with itemsize 0, ``np.dtype([])``. + diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 215584b62d06..8936a4752ca8 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -997,8 +997,9 @@ argument must be a :c:type:`PyObject *` that can be directly interpret called on flexible dtypes. Types that are attached to an array will always be sized, hence the array form of this macro not existing. - ..versionchanged:: 1.18.0 - For structured datatypes with no fields this function now return False. + .. versionchanged:: 1.18 + + For structured datatypes with no fields this function now returns False. .. c:function:: PyTypeNum_ISUSERDEF(num) diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 05e1d301185d..074496deea9a 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -1200,11 +1200,21 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op) } } if (res == NULL && !PyErr_Occurred()) { - /* these dtypes had no fields */ - res = PyArray_NewLikeArray(self, NPY_ANYORDER, - PyArray_DescrFromType(NPY_BOOL), 1); + /* these dtypes had no fields. Use a MultiIter to broadcast them + * to an output array, and fill with True (for EQ)*/ + PyArrayMultiIterObject *mit = (PyArrayMultiIterObject *) + PyArray_MultiIterNew(2, self, other); + if (mit == NULL) { + return NULL; + } + + res = PyArray_NewFromDescr(&PyArray_Type, + PyArray_DescrFromType(NPY_BOOL), + mit->nd, mit->dimensions, + NULL, NULL, 0, NULL); + Py_DECREF(mit); if (res) { - PyArray_FILLWBYTE((PyArrayObject*)res, + PyArray_FILLWBYTE((PyArrayObject *)res, cmp_op == Py_EQ ? 1 : 0); } } diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 6230b27b7679..f5ca775ef5ac 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -441,8 +441,8 @@ def test_fieldless_views(self): assert_equal(np.zeros(2, dtype=[]) == np.zeros(2, dtype=[]), np.ones(2, dtype=bool)) - assert_equal(np.zeros(2, dtype=[]) == a, - np.ones(2, dtype=bool)) + assert_equal(np.zeros((1, 2), dtype=[]) == a, + np.ones((1, 2), dtype=bool)) class TestSubarray(object):