8000 BUG: fix array too big error for wide dtypes. by seberg · Pull Request #7463 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

BUG: fix array too big error for wide dtypes. #7463

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 26, 2016
Merged

BUG: fix array too big error for wide dtypes. #7463

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4e86e89
BUG: fix array too big error for wide dtypes.
seberg Mar 26, 2016
3e8c2b0
MAINT: Rename size and sd to nbytes in NewFromDescr_int
seberg Mar 26, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
8000 Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 21 additions & 21 deletions numpy/core/src/multiarray/ctors.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -901,8 +901,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
{
PyArrayObject_fields *fa;
int i;
size_t sd;
npy_intp size;
npy_intp nbytes;

if (descr->subarray) {
PyObject *ret;
Expand All @@ -929,10 +928,9 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
return NULL;
}

/* Check dimensions */
size = 1;
sd = (size_t) descr->els 8000 ize;
if (sd == 0) {
/* Check datatype element size */
nbytes = descr->elsize;
if (nbytes == 0) {
if (!PyDataType_ISSTRING(descr)) {
PyErr_SetString(PyExc_TypeError, "Empty data-type");
Py_DECREF(descr);
Expand All @@ -943,13 +941,14 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
return NULL;
}
if (descr->type_num == NPY_STRING) {
sd = descr->elsize = 1;
nbytes = descr->elsize = 1;
}
else {
sd = descr->elsize = sizeof(npy_ucs4);
nbytes = descr->elsize = sizeof(npy_ucs4);
}
}

/* Check dimensions and multiply them to nbytes */
for (i = 0; i < nd; i++) {
npy_intp dim = dims[i];

Expand All @@ -974,9 +973,10 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
* multiplying the dimensions together to get the total size of the
* array.
*/
if (npy_mul_with_overflow_intp(&size, size, dim)) {
if (npy_mul_with_overflow_intp(&nbytes, nbytes, dim)) {
PyErr_SetString(PyExc_ValueError,
"array is too big.");
"array is too big; `arr.size * arr.dtype.itemsize` "
"is larger than the maximum possible size.");
Py_DECREF(descr);
return NULL;
}
Expand Down Expand Up @@ -1015,17 +1015,16 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
}
fa->strides = fa->dimensions + nd;
memcpy(fa->dimensions, dims, sizeof(npy_intp)*nd);
if (strides == NULL) { /* fill it in */
sd = _array_fill_strides(fa->strides, dims, nd, sd,
flags, &(fa->flags));
if (strides == NULL) { /* fill it in */
_array_fill_strides(fa->strides, dims, nd, descr->elsize,
flags, &(fa->flags));
}
else {
/*
* we allow strides even when we create
* the memory, but be careful with this...
*/
memcpy(fa->strides, strides, sizeof(npy_intp)*nd);
sd *= size;
}
}
else {
Expand All @@ -1039,19 +1038,18 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
* e.g. shape=(0,) -- otherwise buffer exposure
* (a.data) doesn't work as it should.
*/

if (sd == 0) {
sd = descr->elsize;
if (nbytes == 0) {
nbytes = descr->elsize;
}
/*
* It is bad to have uninitialized OBJECT pointers
* which could also be sub-fields of a VOID array
*/
if (zeroed || PyDataType_FLAGCHK(descr, NPY_NEEDS_INIT)) {
data = npy_alloc_cache_zero(sd);
data = npy_alloc_cache_zero(nbytes);
}
else {
data = npy_alloc_cache(sd);
data = npy_alloc_cache(nbytes);
}
if (data == NULL) {
PyErr_NoMemory();
Expand Down Expand Up @@ -3772,9 +3770,11 @@ PyArray_FromIter(PyObject *obj, PyArray_Descr *dtype, npy_intp count)
* If data is not given but created here, then flags will be NPY_ARRAY_DEFAULT
* and a non-zero flags argument can be used to indicate a FORTRAN style
* array is desired.
*
* Dimensions and itemsize must have been checked for validity.
*/

NPY_NO_EXPORT size_t
NPY_NO_EXPORT void
_array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize,
int inflag, int *objflags)
{
Expand Down Expand Up @@ -3853,7 +3853,7 @@ _array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize,
*objflags |= (NPY_ARRAY_C_CONTIGUOUS|NPY_ARRAY_F_CONTIGUOUS);
}
}
return itemsize;
return;
}

/*
Expand Down
2 changes: 1 addition & 1 deletion numpy/core/src/multiarray/ctors.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src,
NPY_ORDER order);

/* FIXME: remove those from here */
NPY_NO_EXPORT size_t
NPY_NO_EXPORT void
_array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize,
int inflag, int *objflags);

Expand Down
6 changes: 3 additions & 3 deletions numpy/core/src/multiarray/shape.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,9 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck,
}

/* make new_strides variable */
sd = (size_t) PyArray_DESCR(self)->elsize;
sd = (size_t) _array_fill_strides(new_strides, new_dimensions, new_nd, sd,
PyArray_FLAGS(self), &(((PyArrayObject_fields *)self)->flags));
_array_fill_strides(
new_strides, new_dimensions, new_nd, PyArray_DESCR(self)->elsize,
PyArray_FLAGS(self), &(((PyArrayObject_fields *)self)->flags));
memmove(PyArray_DIMS(self), new_dimensions, new_nd*sizeof(npy_intp));
memmove(PyArray_STRIDES(self), new_strides, new_nd*sizeof(npy_intp));
Py_RETURN_NONE;
Expand Down
15 changes: 15 additions & 0 deletions numpy/core/tests/test_multiarray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -730,6 +730,21 @@ def __len__(self):
d = A([1,2,3])
assert_equal(len(np.array(d)), 3)

def test_array_too_big(self):
# Test that array creation succeeds for arrays addressable by intp
# on the byte level and fails for too large arrays.
buf = np.zeros(100)

max_bytes = np.iinfo(np.intp).max
for dtype in ["intp", "S20", "b"]:
dtype = np.dtype(dtype)
itemsize = dtype.itemsize

np.ndarray(buffer=buf, strides=(0,),
shape=(max_bytes//itemsize,), dtype=dtype)
assert_raises(ValueError, np.ndarray, buffer=buf, strides=(0,),
shape=(max_bytes//itemsize + 1,), dtype=dtype)


class TestStructured(TestCase):
def test_subarray_field_access(self):
Expand Down
0