8000 Merge pull request #14657 from pv/fix-fromfile · numpy/numpy@d235f37 · GitHub
[go: up one dir, main page]

Skip to content

Commit d235f37

Browse files
authored
Merge pull request #14657 from pv/fix-fromfile
BUG: fix fromfile behavior when reading sub-array dtypes
2 parents 9451d6f + 0927f7b commit d235f37

File tree

2 files changed

+37
-6
lines changed

2 files changed

+37
-6
lines changed

numpy/core/src/multiarray/ctors.c

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3580,6 +3580,7 @@ array_fromfile_binary(FILE *fp, PyArray_Descr *dtype, npy_intp num, size_t *nrea
35803580
{
35813581
PyArrayObject *r;
35823582
npy_off_t start, numbytes;
3583+
int elsize;
35833584

35843585
if (num < 0) {
35853586
int fail = 0;
@@ -3606,16 +3607,21 @@ array_fromfile_binary(FILE *fp, PyArray_Descr *dtype, npy_intp num, size_t *nrea
36063607
}
36073608
num = numbytes / dtype->elsize;
36083609
}
3610+
3611+
/*
3612+
* Array creation may move sub-array dimensions from the dtype to array
3613+
* dimensions, so we need to use the original element size when reading.
3614+
*/
3615+
elsize = dtype->elsize;
3616+
36093617
r = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &num,
36103618
NULL, NULL, 0, NULL);
36113619
if (r == NULL) {
36123620
return NULL;
36133621
}
3614-
/* In some cases NewFromDescr can replace the dtype, so fetch new one */
3615-
dtype = PyArray_DESCR(r);
36163622

36173623
NPY_BEGIN_ALLOW_THREADS;
3618-
*nread = fread(PyArray_DATA(r), dtype->elsize, num, fp);
3624+
*nread = fread(PyArray_DATA(r), elsize, num, fp);
36193625
NPY_END_ALLOW_THREADS;
36203626
return r;
36213627
}
@@ -3642,14 +3648,19 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread,
36423648

36433649
size = (num >= 0) ? num : FROM_BUFFER_SIZE;
36443650

3651+
/*
3652+
* Array creation may move sub-array dimensions from the dtype to array
3653+
* dimensions, so we need to use the original dtype when reading.
3654+
*/
3655+
Py_INCREF(dtype);
3656+
36453657
r = (PyArrayObject *)
36463658
PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &size,
36473659
NULL, NULL, 0, NULL);
36483660
if (r == NULL) {
3661+
Py_DECREF(dtype);
36493662
return NULL;
36503663
}
3651-
/* In some cases NewFromDescr can replace the dtype, so fetch new one */
3652-
dtype = PyArray_DESCR(r);
36533664

36543665
clean_sep = swab_separator(sep);
36553666
if (clean_sep == NULL) {
@@ -3710,6 +3721,7 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread,
37103721
if (PyErr_Occurred()) {
37113722
/* If an error is already set (unlikely), do not create new one */
37123723
Py_DECREF(r);
3724+
Py_DECREF(dtype);
37133725
return NULL;
37143726
}
37153727
/* 2019-09-12, NumPy 1.18 */
@@ -3721,6 +3733,7 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread,
37213733
}
37223734

37233735
fail:
3736+
Py_DECREF(dtype);
37243737
if (err == 1) {
37253738
PyErr_NoMemory();
37263739
}
@@ -3986,14 +3999,19 @@ PyArray_FromString(char *data, npy_intp slen, PyArray_Descr *dtype,
39863999
return NULL;
39874000
}
39884001
}
4002+
/*
4003+
* NewFromDescr may replace dtype to absorb subarray shape
4004+
* into the array, so get size beforehand.
4005+
*/
4006+
npy_intp size_to_copy = num*dtype->elsize;
39894007
ret = (PyArrayObject *)
39904008
PyArray_NewFromDescr(&PyArray_Type, dtype,
39914009
1, &num, NULL, NULL,
39924010
0, NULL);
39934011
if (ret == NULL) {
39944012
return NULL;
39954013
}
3996-
memcpy(PyArray_DATA(ret), data, num*dtype->elsize);
4014+
memcpy(PyArray_DATA(ret), data, size_to_copy);
39974015
}
39984016
else {
39994017
/* read from character-based string */

numpy/core/tests/test_multiarray.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5023,6 +5023,19 @@ def test_locale(self):
50235023
self.test_tofile_sep()
50245024
self.test_tofile_format()
50255025

5026+
def test_fromfile_subarray_binary(self):
5027+
# Test subarray dtypes which are absorbed into the shape
5028+
x = np.arange(24, dtype="i4").reshape(2, 3, 4)
5029+
x.tofile(self.filename)
5030+
res = np.fromfile(self.filename, dtype="(3,4)i4")
5031+
assert_array_equal(x, res)
5032+
5033+
x_str = x.tobytes()
5034+
with assert_warns(DeprecationWarning):
5035+
# binary fromstring is deprecated
5036+
res = np.fromstring(x_str, dtype="(3,4)i4")
5037+
assert_array_equal(x, res)
5038+
50265039

50275040
class TestFromBuffer(object):
50285041
@pytest.mark.parametrize('byteorder', ['<', '>'])

0 commit comments

Comments
 (0)
0