diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index ba5121306476..5174bd889ed7 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -3580,6 +3580,7 @@ array_fromfile_binary(FILE *fp, PyArray_Descr *dtype, npy_intp num, size_t *nrea { PyArrayObject *r; npy_off_t start, numbytes; + int elsize; if (num < 0) { int fail = 0; @@ -3606,16 +3607,21 @@ array_fromfile_binary(FILE *fp, PyArray_Descr *dtype, npy_intp num, size_t *nrea } num = numbytes / dtype->elsize; } + + /* + * Array creation may move sub-array dimensions from the dtype to array + * dimensions, so we need to use the original element size when reading. + */ + elsize = dtype->elsize; + r = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &num, NULL, NULL, 0, NULL); if (r == NULL) { return NULL; } - /* In some cases NewFromDescr can replace the dtype, so fetch new one */ - dtype = PyArray_DESCR(r); NPY_BEGIN_ALLOW_THREADS; - *nread = fread(PyArray_DATA(r), dtype->elsize, num, fp); + *nread = fread(PyArray_DATA(r), elsize, num, fp); NPY_END_ALLOW_THREADS; return r; } @@ -3642,14 +3648,19 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, size = (num >= 0) ? num : FROM_BUFFER_SIZE; + /* + * Array creation may move sub-array dimensions from the dtype to array + * dimensions, so we need to use the original dtype when reading. + */ + Py_INCREF(dtype); + r = (PyArrayObject *) PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &size, NULL, NULL, 0, NULL); if (r == NULL) { + Py_DECREF(dtype); return NULL; } - /* In some cases NewFromDescr can replace the dtype, so fetch new one */ - dtype = PyArray_DESCR(r); clean_sep = swab_separator(sep); if (clean_sep == NULL) { @@ -3710,6 +3721,7 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, if (PyErr_Occurred()) { /* If an error is already set (unlikely), do not create new one */ Py_DECREF(r); + Py_DECREF(dtype); return NULL; } /* 2019-09-12, NumPy 1.18 */ @@ -3721,6 +3733,7 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, } fail: + Py_DECREF(dtype); if (err == 1) { PyErr_NoMemory(); } @@ -3986,6 +3999,11 @@ PyArray_FromString(char *data, npy_intp slen, PyArray_Descr *dtype, return NULL; } } + /* + * NewFromDescr may replace dtype to absorb subarray shape + * into the array, so get size beforehand. + */ + npy_intp size_to_copy = num*dtype->elsize; ret = (PyArrayObject *) PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &num, NULL, NULL, @@ -3993,7 +4011,7 @@ PyArray_FromString(char *data, npy_intp slen, PyArray_Descr *dtype, if (ret == NULL) { return NULL; } - memcpy(PyArray_DATA(ret), data, num*dtype->elsize); + memcpy(PyArray_DATA(ret), data, size_to_copy); } else { /* read from character-based string */ diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 88db357b2af2..9b124f603b2a 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -5023,6 +5023,19 @@ def test_locale(self): self.test_tofile_sep() self.test_tofile_format() + def test_fromfile_subarray_binary(self): + # Test subarray dtypes which are absorbed into the shape + x = np.arange(24, dtype="i4").reshape(2, 3, 4) + x.tofile(self.filename) + res = np.fromfile(self.filename, dtype="(3,4)i4") + assert_array_equal(x, res) + + x_str = x.tobytes() + with assert_warns(DeprecationWarning): + # binary fromstring is deprecated + res = np.fromstring(x_str, dtype="(3,4)i4") + assert_array_equal(x, res) + class TestFromBuffer(object): @pytest.mark.parametrize('byteorder', ['<', '>'])