From 4e86e89be1afc9b6e72e959bdc21567a31f5e1e4 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Sat, 26 Mar 2016 16:08:34 +0100 Subject: [PATCH 1/2] BUG: fix array too big error for wide dtypes. The error was not being raised when arr.size * arr.dtype.itemsize was too large, but only when arr.size was too large alone. Closes gh-7451 --- numpy/core/src/multiarray/ctors.c | 10 ++++++---- numpy/core/tests/test_multiarray.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 79c2b16b168f..5671f2251fc7 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -929,8 +929,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, return NULL; } - /* Check dimensions */ - size = 1; + /* Check datatype element size */ sd = (size_t) descr->elsize; if (sd == 0) { if (!PyDataType_ISSTRING(descr)) { @@ -950,6 +949,8 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, } } + /* Check dimensions */ + size = sd; for (i = 0; i < nd; i++) { npy_intp dim = dims[i]; @@ -976,7 +977,8 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, */ if (npy_mul_with_overflow_intp(&size, size, dim)) { PyErr_SetString(PyExc_ValueError, - "array is too big."); + "array is too big; `arr.size * arr.dtype.itemsize` " + "is larger than the maximum possible size."); Py_DECREF(descr); return NULL; } @@ -1025,7 +1027,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, * the memory, but be careful with this... */ memcpy(fa->strides, strides, sizeof(npy_intp)*nd); - sd *= size; + sd = size; } } else { diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 14a902b9c74a..94731f20c795 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -730,6 +730,21 @@ def __len__(self): d = A([1,2,3]) assert_equal(len(np.array(d)), 3) + def test_array_too_big(self): + # Test that array creation succeeds for arrays addressable by intp + # on the byte level and fails for too large arrays. + buf = np.zeros(100) + + max_bytes = np.iinfo(np.intp).max + for dtype in ["intp", "S20", "b"]: + dtype = np.dtype(dtype) + itemsize = dtype.itemsize + + np.ndarray(buffer=buf, strides=(0,), + shape=(max_bytes//itemsize,), dtype=dtype) + assert_raises(ValueError, np.ndarray, buffer=buf, strides=(0,), + shape=(max_bytes//itemsize + 1,), dtype=dtype) + class TestStructured(TestCase): def test_subarray_field_access(self): From 3e8c2b00ee7d0c438226057bebeb973c4ab3458f Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Sat, 26 Mar 2016 19:19:50 +0100 Subject: [PATCH 2/2] MAINT: Rename size and sd to nbytes in NewFromDescr_int --- numpy/core/src/multiarray/ctors.c | 38 +++++++++++++++---------------- numpy/core/src/multiarray/ctors.h | 2 +- numpy/core/src/multiarray/shape.c | 6 ++--- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 5671f2251fc7..0017de0ad6e7 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -901,8 +901,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, { PyArrayObject_fields *fa; int i; - size_t sd; - npy_intp size; + npy_intp nbytes; if (descr->subarray) { PyObject *ret; @@ -930,8 +929,8 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, } /* Check datatype element size */ - sd = (size_t) descr->elsize; - if (sd == 0) { + nbytes = descr->elsize; + if (nbytes == 0) { if (!PyDataType_ISSTRING(descr)) { PyErr_SetString(PyExc_TypeError, "Empty data-type"); Py_DECREF(descr); @@ -942,15 +941,14 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, return NULL; } if (descr->type_num == NPY_STRING) { - sd = descr->elsize = 1; + nbytes = descr->elsize = 1; } else { - sd = descr->elsize = sizeof(npy_ucs4); + nbytes = descr->elsize = sizeof(npy_ucs4); } } - /* Check dimensions */ - size = sd; + /* Check dimensions and multiply them to nbytes */ for (i = 0; i < nd; i++) { npy_intp dim = dims[i]; @@ -975,7 +973,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, * multiplying the dimensions together to get the total size of the * array. */ - if (npy_mul_with_overflow_intp(&size, size, dim)) { + if (npy_mul_with_overflow_intp(&nbytes, nbytes, dim)) { PyErr_SetString(PyExc_ValueError, "array is too big; `arr.size * arr.dtype.itemsize` " "is larger than the maximum possible size."); @@ -1017,9 +1015,9 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, } fa->strides = fa->dimensions + nd; memcpy(fa->dimensions, dims, sizeof(npy_intp)*nd); - if (strides == NULL) { /* fill it in */ - sd = _array_fill_strides(fa->strides, dims, nd, sd, - flags, &(fa->flags)); + if (strides == NULL) { /* fill it in */ + _array_fill_strides(fa->strides, dims, nd, descr->elsize, + flags, &(fa->flags)); } else { /* @@ -1027,7 +1025,6 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, * the memory, but be careful with this... */ memcpy(fa->strides, strides, sizeof(npy_intp)*nd); - sd = size; } } else { @@ -1041,19 +1038,18 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, * e.g. shape=(0,) -- otherwise buffer exposure * (a.data) doesn't work as it should. */ - - if (sd == 0) { - sd = descr->elsize; + if (nbytes == 0) { + nbytes = descr->elsize; } /* * It is bad to have uninitialized OBJECT pointers * which could also be sub-fields of a VOID array */ if (zeroed || PyDataType_FLAGCHK(descr, NPY_NEEDS_INIT)) { - data = npy_alloc_cache_zero(sd); + data = npy_alloc_cache_zero(nbytes); } else { - data = npy_alloc_cache(sd); + data = npy_alloc_cache(nbytes); } if (data == NULL) { PyErr_NoMemory(); @@ -3774,9 +3770,11 @@ PyArray_FromIter(PyObject *obj, PyArray_Descr *dtype, npy_intp count) * If data is not given but created here, then flags will be NPY_ARRAY_DEFAULT * and a non-zero flags argument can be used to indicate a FORTRAN style * array is desired. + * + * Dimensions and itemsize must have been checked for validity. */ -NPY_NO_EXPORT size_t +NPY_NO_EXPORT void _array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize, int inflag, int *objflags) { @@ -3855,7 +3853,7 @@ _array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize, *objflags |= (NPY_ARRAY_C_CONTIGUOUS|NPY_ARRAY_F_CONTIGUOUS); } } - return itemsize; + return; } /* diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h index 757362fb8455..783818deff28 100644 --- a/numpy/core/src/multiarray/ctors.h +++ b/numpy/core/src/multiarray/ctors.h @@ -51,7 +51,7 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order); /* FIXME: remove those from here */ -NPY_NO_EXPORT size_t +NPY_NO_EXPORT void _array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize, int inflag, int *objflags); diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index a307bed9cfec..942475f84091 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -149,9 +149,9 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck, } /* make new_strides variable */ - sd = (size_t) PyArray_DESCR(self)->elsize; - sd = (size_t) _array_fill_strides(new_strides, new_dimensions, new_nd, sd, - PyArray_FLAGS(self), &(((PyArrayObject_fields *)self)->flags)); + _array_fill_strides( + new_strides, new_dimensions, new_nd, PyArray_DESCR(self)->elsize, + PyArray_FLAGS(self), &(((PyArrayObject_fields *)self)->flags)); memmove(PyArray_DIMS(self), new_dimensions, new_nd*sizeof(npy_intp)); memmove(PyArray_STRIDES(self), new_strides, new_nd*sizeof(npy_intp)); Py_RETURN_NONE;