8000 BUG: core: ensure file handle positions are in sync · pv/numpy@dd05022 · GitHub
[go: up one dir, main page]
Skip to content

Commit dd05022

Browse files
pvpv
committed
BUG: core: ensure file handle positions are in sync
The Python3 file handles keep records of current file positions, so that the raw handle needs to be restored to the original position in order to not confuse the mechanism. Fixes numpygh-4118
1 parent 927cfcf commit dd05022

File tree

4 files changed

+83
-8
lines changed

4 files changed

+83
-8
lines changed

‎numpy/core/include/numpy/npy_3kcompat.h

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,13 @@ PyUnicode_Concat2(PyObject **left, PyObject *right)
146146
* Get a FILE* handle to the file represented by the Python object
147147
*/
148148
static NPY_INLINE FILE*
149-
npy_PyFile_Dup(PyObject *file, char *mode)
149+
npy_PyFile_Dup(PyObject *file, char *mode, Py_ssize_t *orig_pos)
150150
{
151151
int fd, fd2;
152152
PyObject *ret, *os;
153153
Py_ssize_t pos;
154154
FILE *handle;
155+
155156
/* Flush first to ensure things end up in the file in the correct order */
156157
ret = PyObject_CallMethod(file, "flush", "");
157158
if (ret == NULL) {
@@ -162,6 +163,9 @@ npy_PyFile_Dup(PyObject *file, char *mode)
162163
if (fd == -1) {
163164
return NULL;
164165
}
166+
167+
/* The handle needs to be dup'd because we have to call fclose
168+
at the end */
165169
os = PyImport_ImportModule("os");
166170
if (os == NULL) {
167171
return NULL;
@@ -173,6 +177,8 @@ npy_PyFile_Dup(PyObject *file, char *mode)
173177
}
174178
fd2 = PyNumber_AsSsize_t(ret, NULL);
175179
Py_DECREF(ret);
180+
181+
/* Convert to FILE* handle */
176182
#ifdef _WIN32
177183
handle = _fdopen(fd2, mode);
178184
#else
@@ -182,6 +188,11 @@ npy_PyFile_Dup(PyObject *file, char *mode)
182188
PyErr_SetString(PyExc_IOError,
183189
"Getting a FILE* from a Python file object failed");
184190
}
191+
192+
/* Record the original raw file handle position */
193+
*orig_pos = npy_ftell(handle);
194+
195+
/* Seek raw handle to the Python-side position */
185196
ret = PyObject_CallMethod(file, "tell", "");
186197
if (ret == NULL) {
187198
fclose(handle);
@@ -201,13 +212,22 @@ npy_PyFile_Dup(PyObject *file, char *mode)
201212
* Close the dup-ed file handle, and seek the Python one to the current position
202213
*/
203214
static NPY_INLINE int
204-
npy_PyFile_DupClose(PyObject *file, FILE* handle)
215+
npy_PyFile_DupClose(PyObject *file, FILE* handle, Py_ssize_t orig_pos)
205216
{
206217
PyObject *ret;
207218
Py_ssize_t position;
219+
208220
position = npy_ftell(handle);
221+
222+
/* Restore original file handle position, in order to not confuse
223+
Python-side data structures */
224+
npy_fseek(handle, orig_pos, SEEK_SET);
225+
226+
/* Flush before closing to sync the raw file handle position */
227+
fflush(handle);
209228
fclose(handle);
210229

230+
/* Seek Python-side handle to the raw handle position */
211231
ret = PyObject_CallMethod(file, "seek", NPY_SSIZE_T_PYFMT "i", position, 0);
212232
if (ret == NULL) {
213233
return -1;
@@ -230,8 +250,8 @@ npy_PyFile_Check(PyObject *file)
230250

231251
#else
232252

233-
#define npy_PyFile_Dup(file, mode) PyFile_AsFile(file)
234-
#define npy_PyFile_DupClose(file, handle) (0)
253+
#define npy_PyFile_Dup(file, mode, orig_pos_p) PyFile_AsFile(file)
254+
#define npy_PyFile_DupClose(file, handle, orig_pos) (0)
235255
#define npy_PyFile_Check PyFile_Check
236256

237257
#endif

‎numpy/core/src/multiarray/methods.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
566566
FILE *fd;
567567
char *sep = "";
568568
char *format = "";
569+
Py_ssize_t orig_pos;
569570
static char *kwlist[] = {"file", "sep", "format", NULL};
570571

571572
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|ss", kwlist,
@@ -587,7 +588,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
587588
own = 0;
588589
}
589590

590-
fd = npy_PyFile_Dup(file, "wb");
591+
fd = npy_PyFile_Dup(file, "wb", &orig_pos);
591592
if (fd == NULL) {
592593
PyErr_SetString(PyExc_IOError,
593594
"first argument must be a string or open file");
@@ -596,7 +597,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
596597
if (PyArray_ToFile(self, fd, sep, format) < 0) {
597598
goto fail;
598599
}
599-
if (npy_PyFile_DupClose(file, fd) < 0) {
600+
if (npy_PyFile_DupClose(file, fd, orig_pos) < 0) {
600601
goto fail;
601602
}
602603
if (own && npy_PyFile_CloseFile(file) < 0) {

‎numpy/core/src/multiarray/multiarraymodule.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1972,6 +1972,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
19721972
static char *kwlist[] = {"file", "dtype", "count", "sep", NULL};
19731973
PyArray_Descr *type = NULL;
19741974
int own;
1975+
Py_ssize_t orig_pos;
19751976
FILE *fp;
19761977

19771978
if (!PyArg_ParseTupleAndKeywords(args, keywds,
@@ -1991,7 +1992,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
19911992
Py_INCREF(file);
19921993
own = 0;
19931994
}
1994-
fp = npy_PyFile_Dup(file, "rb");
1995+
fp = npy_PyFile_Dup(file, "rb", &orig_pos);
19951996
if (fp == NULL) {
19961997
PyErr_SetString(PyExc_IOError,
19971998
"first argument must be an open file");
@@ -2003,7 +2004,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
20032004
}
20042005
ret = PyArray_FromFile(fp, type, (npy_intp) nin, sep);
20052006

2006-
if (npy_PyFile_DupClose(file, fp) < 0) {
2007+
if (npy_PyFile_DupClose(file, fp, orig_pos) < 0) {
20072008
goto fail;
20082009
}
20092010
if (own && npy_PyFile_CloseFile(file) < 0) {

‎numpy/core/tests/test_multiarray.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import os
66
import warnings
77
import operator
8+
import io
89
if sys.version_info[0] >= 3:
910
import builtins
1011
else:
@@ -2369,6 +2370,58 @@ def test_roundtrip_repr(self):
23692370
y = np.fromstring(s, sep="@")
23702371
assert_array_equal(x, y)
23712372

2373+
def test_file_position_after_fromfile(self):
2374+
# gh-4118
2375+
sizes = [io.DEFAULT_BUFFER_SIZE//8,
2376+
io.DEFAULT_BUFFER_SIZE,
2377+
io.DEFAULT_BUFFER_SIZE*8]
2378+
2379+
for size in sizes:
2380+
f = open(self.filename, 'wb')
2381+
f.seek(size-1)
2382+
f.write(b'\0')
2383+
f.close()
2384+
2385+
for mode in ['rb', 'r+b']:
2386+
err_msg = "%d %s" % (size, mode)
2387+
2388+
f = open(self.filename, mode)
2389+
f.read(2)
2390+
np.fromfile(f, dtype=np.float64, count=1)
2391+
pos = f.tell()
2392+
f.close()
2393+
assert_equal(pos, 10, err_msg=err_msg)
2394+
2395+
os.unlink(self.filename)
2396+
2397+
def test_file_position_after_tofile(self):
2398+
# gh-4118
2399+
sizes = [io.DEFAULT_BUFFER_SIZE//8,
2400+
io.DEFAULT_BUFFER_SIZE,
2401+
io.DEFAULT_BUFFER_SIZE*8]
2402+
2403+
for size in sizes:
2404+
err_msg = "%d" % (size,)
2405+
2406+
f = open(self.filename, 'wb')
2407+
f.seek(size-1)
2408+
f.write(b'\0')
2409+
f.seek(10)
2410+
f.write(b'12')
2411+
np.array([0], dtype=np.float64).tofile(f)
2412+
pos = f.tell()
2413+
f.close()
2414+
assert_equal(pos, 10 + 2 + 8, err_msg=err_msg)
2415+
2416+
f = open(self.filename, 'r+b')
2417+
f.read(2)
2418+
np.array([0], dtype=np.float64).tofile(f)
2419+
pos = f.tell()
2420+
f.close()
2421+
assert_equal(pos, 10, err_msg=err_msg)
2422+
2423+
os.unlink(self.filename)
2424+
23722425
def _check_from(self, s, value, **kw):
23732426
y = np.fromstring(asbytes(s), **kw)
23742427
assert_array_equal(y, value)

0 commit comments

Comments
 (0)
0