8000 ENH: Add support for file like objects to np.core.records.fromfile by sidhant007 · Pull Request #16675 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

ENH: Add support for file like objects to np.core.records.fromfile #16675

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 13, 2020
Merged

ENH: Add support for file like objects to np.core.records.fromfile #16675

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
645465c
ENH: Add file like support to np.core.records.fromfile (#2504)
sidhant007 Jun 24, 2020
107c029
Refactor: Remove isfilelikeobj function
sidhant007 Jun 29, 2020
09fdf11
Add comment for better explanation
sidhant007 Jul 3, 2020
c2727d0
DOC: add release note
mattip Aug 10, 2020
58d3bc9
DOC: intersphinx release note
rossbar Aug 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions doc/release/upcoming_changes/16675.improvement.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
`numpy.core.records.fromfile` now supports file-like objects
------------------------------------------------------------
`numpy.rec.fromfile` can now use file-like objects, for instance `BytesIO`
21 changes: 11 additions & 10 deletions numpy/core/records.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
from . import numeric as sb
from . import numerictypes as nt
from numpy.compat import (
isfileobj, os_fspath, contextlib_nullcontext
os_fspath, contextlib_nullcontext
)
from numpy.core.overrides import set_module
from .arrayprint import get_printoptions
Expand Down Expand Up @@ -847,13 +847,12 @@ def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None,
return _array

def get_remaining_size(fd):
pos = fd.tell()
try:
fn = fd.fileno()
except AttributeError:
return os.path.getsize(fd.name) - fd.tell()
st = os.fstat(fn)
size = st.st_size - fd.tell()
return size
fd.seek(0, 2)
return fd.tell() - pos
finally:
fd.seek(pos, 0)

def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,
names=None, titles=None, aligned=False, byteorder=None):
Expand Down Expand Up @@ -911,7 +910,9 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,
elif isinstance(shape, int):
shape = (shape,)

if isfileobj(fd):
if hasattr(fd, 'readinto'):
# GH issue 2504. fd supports io.RawIOBase or io.BufferedIOBase interface.
# Example of fd: gzip, BytesIO, BufferedReader
# file already opened
ctx = contextlib_nullcontext(fd)
else:
Expand Down Expand Up @@ -1036,7 +1037,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
array('def', dtype='<U3')
"""

if ((isinstance(obj, (type(None), str)) or isfileobj(obj)) and
if ((isinstance(obj, (type(None), str)) or hasattr(obj, 'readinto')) and
formats is None and dtype is None):
raise ValueError("Must define formats (or dtype) if object is "
"None, string, or an open file")
Expand Down Expand Up @@ -1078,7 +1079,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
new = new.copy()
return new

elif isfileobj(obj):
elif hasattr(obj, 'readinto'):
return fromfile(obj, dtype=dtype, shape=shape, offset=offset)

elif isinstance(obj, ndarray):
Expand Down
7 changes: 7 additions & 0 deletions numpy/core/tests/test_records.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import collections.abc
import textwrap
from io import BytesIO
from os import path
from pathlib import Path
import pytest
Expand Down Expand Up @@ -79,8 +80,14 @@ def test_recarray_fromfile(self):
r1 = np.rec.fromfile(fd, formats='f8,i4,a5', shape=3, byteorder='big')
fd.seek(2880 * 2)
r2 = np.rec.array(fd, formats='f8,i4,a5', shape=3, byteorder='big')
fd.seek(2880 * 2)
bytes_array = BytesIO()
bytes_array.write(fd.read())
bytes_array.seek(0)
r3 = np.rec.fromfile(bytes_array, formats='f8,i4,a5', shape=3, byteorder='big')
fd.close()
assert_equal(r1, r2)
assert_equal(r2, r3)

def test_recarray_from_obj(self):
count = 10
Expand Down
0