diff --git a/doc/release/1.16.0-notes.rst b/doc/release/1.16.0-notes.rst index 89a4623e19f5..bb2b1778233f 100644 --- a/doc/release/1.16.0-notes.rst +++ b/doc/release/1.16.0-notes.rst @@ -273,6 +273,13 @@ Speedup ``np.take`` for read-only arrays The implementation of ``np.take`` no longer makes an unnecessary copy of the source array when its ``writeable`` flag is set to ``False``. +Support path-like objects for more functions +-------------------------------------------- +The ``np.core.records.fromfile`` function now supports ``pathlib.Path`` +and other path-like objects in addition to a file object. Furthermore, the +``np.load`` function now also supports path-like objects when +using memory mapping (``mmap_mode`` keyword argument). + Changes ======= diff --git a/numpy/core/records.py b/numpy/core/records.py index a483871babf1..1b596e4de578 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -42,7 +42,7 @@ from . import numeric as sb from . import numerictypes as nt -from numpy.compat import isfileobj, bytes, long, unicode +from numpy.compat import isfileobj, bytes, long, unicode, os_fspath from .arrayprint import get_printoptions # All of the functions allow formats to be a dtype @@ -737,9 +737,9 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, names=None, titles=None, aligned=False, byteorder=None): """Create an array from binary file data - If file is a string then that file is opened, else it is assumed - to be a file object. The file object must support random access - (i.e. it must have tell and seek methods). + If file is a string or a path-like object then that file is opened, + else it is assumed to be a file object. The file object must + support random access (i.e. it must have tell and seek methods). >>> from tempfile import TemporaryFile >>> a = np.empty(10,dtype='f8,i4,a5') @@ -763,10 +763,14 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, elif isinstance(shape, (int, long)): shape = (shape,) - name = 0 - if isinstance(fd, str): + if isfileobj(fd): + # file already opened + name = 0 + else: + # open file + fd = open(os_fspath(fd), 'rb') name = 1 - fd = open(fd, 'rb') + if (offset > 0): fd.seek(offset, 1) size = get_remaining_size(fd) diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py index a77eef40405d..af6c86b9ec77 100644 --- a/numpy/core/tests/test_records.py +++ b/numpy/core/tests/test_records.py @@ -13,9 +13,10 @@ import pytest import numpy as np +from numpy.compat import Path from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_array_almost_equal, - assert_raises, assert_warns + assert_raises, assert_warns, temppath ) from numpy.core.numeric import pickle @@ -325,6 +326,24 @@ def test_zero_width_strings(self): assert_equal(rec['f1'], [b'', b'', b'']) +@pytest.mark.skipif(Path is None, reason="No pathlib.Path") +class TestPathUsage(object): + # Test that pathlib.Path can be used + def test_tofile_fromfile(self): + with temppath(suffix='.bin') as path: + path = Path(path) + a = np.empty(10, dtype='f8,i4,a5') + a[5] = (0.5,10,'abcde') + a.newbyteorder('<') + with path.open("wb") as fd: + a.tofile(fd) + x = np.core.records.fromfile(path, + formats='f8,i4,a5', + shape=10, + byteorder='<') + assert_array_equal(x, a) + + class TestRecord(object): def setup(self): self.data = np.rec.fromrecords([(1, 2, 3), (4, 5, 6)], diff --git a/numpy/lib/format.py b/numpy/lib/format.py index e25868236d20..1ef3dca47ec5 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -161,7 +161,9 @@ import io import warnings from numpy.lib.utils import safe_eval -from numpy.compat import asbytes, asstr, isfileobj, long, basestring +from numpy.compat import ( + asbytes, asstr, isfileobj, long, os_fspath + ) from numpy.core.numeric import pickle @@ -706,7 +708,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, Parameters ---------- - filename : str + filename : str or path-like The name of the file on disk. This may *not* be a file-like object. mode : str, optional @@ -747,9 +749,9 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, memmap """ - if not isinstance(filename, basestring): - raise ValueError("Filename must be a string. Memmap cannot use" - " existing file handles.") + if isfileobj(filename): + raise ValueError("Filename must be a string or a path-like object." + " Memmap cannot use existing file handles.") if 'w' in mode: # We are creating the file, not reading it. @@ -767,7 +769,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, shape=shape, ) # If we got here, then it should be safe to create the file. - fp = open(filename, mode+'b') + fp = open(os_fspath(filename), mode+'b') try: used_ver = _write_array_header(fp, d, version) # this warning can be removed when 1.9 has aged enough @@ -779,7 +781,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, fp.close() else: # Read the header of the file first. - fp = open(filename, 'rb') + fp = open(os_fspath(filename), 'rb') try: version = read_magic(fp) _check_version(version) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 08800ff97f5b..b746937b9655 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -2295,11 +2295,35 @@ def test_loadtxt(self): assert_array_equal(x, a) def test_save_load(self): - # Test that pathlib.Path instances can be used with savez. + # Test that pathlib.Path instances can be used with save. + with temppath(suffix='.npy') as path: + path = Path(path) + a = np.array([[1, 2], [3, 4]], int) + np.save(path, a) + data = np.load(path) + assert_array_equal(data, a) + + def test_save_load_memmap(self): + # Test that pathlib.Path instances can be loaded mem-mapped. + with temppath(suffix='.npy') as path: + path = Path(path) + a = np.array([[1, 2], [3, 4]], int) + np.save(path, a) + data = np.load(path, mmap_mode='r') + assert_array_equal(data, a) + # close the mem-mapped file + del data + + def test_save_load_memmap_readwrite(self): + # Test that pathlib.Path instances can be written mem-mapped. with temppath(suffix='.npy') as path: path = Path(path) a = np.array([[1, 2], [3, 4]], int) np.save(path, a) + b = np.load(path, mmap_mode='r+') + a[0][0] = 5 + b[0][0] = 5 + del b # closes the file data = np.load(path) assert_array_equal(data, a)