8000 ENH: Add file like support to np.core.records.fromfile (#2504) · sidhant007/numpy@645465c · GitHub
[go: up one dir, main page]
Skip to content

Commit 645465c

Browse files
sidhant007mattip
sidhant007
authored and
mattip
committed
ENH: Add file like support to np.core.records.fromfile (numpy#2504)
1 parent eec0aa2 commit 645465c

File tree

4 files changed

+36
-12
lines changed

4 files changed

+36
-12
lines changed

‎numpy/compat/py3k.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
compatibility with numpy while they transition their code to newer versions of
1111
Python.
1212
"""
13-
__all__ = ['bytes', 'asbytes', 'isfileobj', 'getexception', 'strchar',
13+
__all__ = ['bytes', 'asbytes', 'isfileobj', 'isfilelikeobj', 'getexception', 'strchar',
1414
'unicode', 'asunicode', 'asbytes_nested', 'asunicode_nested',
1515
'asstr', 'open_latin1', 'long', 'basestring', 'sixu',
1616
'integer_types', 'is_pathlib_path', 'npy_load_module', 'Path',
@@ -53,6 +53,9 @@ def asstr(s):
5353
def isfileobj(f):
5454
return isinstance(f, (io.FileIO, io.BufferedReader, io.BufferedWriter))
5555

56+
def isfilelikeobj(f):
57+
return hasattr(f, 'readinto')
58+
5659
def open_latin1(filename, mode='r'):
5760
return open(filename, mode=mode, encoding='iso-8859-1')
5861

‎numpy/compat/tests/test_compat.py

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
from os.path import join
22

3-
from numpy.compat import isfileobj
3+
from numpy.compat import isfileobj, isfilelikeobj
44
from numpy.testing import assert_
55
from numpy.testing import tempdir
66

7+
import io
8+
import gzip
79

810
def test_isfileobj():
911
with tempdir(prefix="numpy_test_compat_") as folder:
@@ -17,3 +19,17 @@ def test_isfileobj():
1719

1820
with open(filename, 'rb') as f:
1921
assert_(isfileobj(f))
22+
23+
def test_isfilelikeobj():
24+
with tempdir(prefix="numpy_test_compat_") as folder:
25+
filename = join(folder, 'a.bin')
26+
27+
with open(filename, 'wb') as f:
28+
assert_(isfilelikeobj(f))
29+
30+
with gzip.open(filename, 'rb') as f:
31+
assert_(isfilelikeobj(f))
32+
33+
assert_(isfilelikeobj(io.BytesIO()))
34+
35+
assert_(not isfilelikeobj(io.StringIO()))

‎numpy/core/records.py

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
from . import numeric as sb
4141
from . import numerictypes as nt
4242
from numpy.compat import (
43-
isfileobj, os_fspath, contextlib_nullcontext
43+
isfilelikeobj, os_fspath, contextlib_nullcontext
4444
)
4545
from numpy.core.overrides import set_module
4646
from .arrayprint import get_printoptions
@@ -847,12 +847,10 @@ def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None,
847847
return _array
848848

849849
def get_remaining_size(fd):
850-
try:
851-
fn = fd.fileno()
852-
except AttributeError:
853-
return os.path.getsize(fd.name) - fd.tell()
854-
st = os.fstat(fn)
855-
size = st.st_size - fd.tell()
850+
pos = fd.tell()
851+
fd.seek(0, 2)
852+
size = fd.tell() - pos
853+
fd.seek(pos, 0)
856854
return size
857855

858856
def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,
@@ -911,7 +909,7 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,
911909
elif isinstance(shape, int):
912910
shape = (shape,)
913911

914-
if isfileobj(fd):
912+
if isfilelikeobj(fd):
915913
# file already opened
916914
ctx = contextlib_nullcontext(fd)
917915
else:
@@ -1036,7 +1034,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
10361034
array('def', dtype='<U3')
10371035
"""
10381036

1039-
if ((isinstance(obj, (type(None), str)) or isfileobj(obj)) and
1037+
if ((isinstance(obj, (type(None), str)) or isfilelikeobj(obj)) and
10401038
formats is None and dtype is None):
10411039
raise ValueError("Must define formats (or dtype) if object is "
10421040
"None, string, or an open file")
@@ -1078,7 +1076,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
10781076
new = new.copy()
10791077
return new
10801078

1081-
elif isfileobj(obj):
1079+
elif isfilelikeobj(obj):
10821080
return fromfile(obj, dtype=dtype, shape=shape, offset=offset)
10831081

10841082
elif isinstance(obj, ndarray):

‎numpy/core/tests/test_records.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import collections.abc
22
import textwrap
3+
from io import BytesIO
34
from os import path
45
from pathlib import Path
56
import pytest
@@ -79,8 +80,14 @@ def test_recarray_fromfile(self):
7980
r1 = np.rec.fromfile(fd, formats='f8,i4,a5', shape=3, byteorder='big')
8081
fd.seek(2880 * 2)
8182
r2 = np.rec.array(fd, formats='f8,i4,a5', shape=3, byteorder='big')
83+
fd.seek(2880 * 2)
84+
bytes_array = BytesIO()
85+
bytes_array.write(fd.read())
86+
bytes_array.seek(0)
87+
r3 = np.rec.fromfile(bytes_array, formats='f8,i4,a5', shape=3, byteorder='big')
8288
fd.close()
8389
assert_equal(r1, r2)
90+
assert_equal(r2, r3)
8491

8592
def test_recarray_from_obj(self):
8693
count = 10

0 commit comments

Comments
 (0)
0