-
-
Notifications
You must be signed in to change notification settings - Fork 11.1k
ENH: Improve support for pathlib.Path objects in load functions #11348
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
bd3309d
c4e5aa1
2e1bc67
07cc4b6
d2d1571
6e2db10
84a0b77
271f3e3
5ce8d95
05ddb68
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,9 +14,10 @@ | |
import pytest | ||
|
||
import numpy as np | ||
from numpy.compat import Path | ||
from numpy.testing import ( | ||
assert_, assert_equal, assert_array_equal, assert_array_almost_equal, | ||
assert_raises, assert_warns | ||
assert_raises, assert_warns, temppath | ||
) | ||
|
||
|
||
|
@@ -325,6 +326,24 @@ def test_zero_width_strings(self): | |
assert_equal(rec['f1'], [b'', b'', b'']) | ||
|
||
|
||
@pytest.mark.skipif(Path is None, reason="No pathlib.Path") | ||
class TestPathUsage(object): | ||
# Test that pathlib.Path can be used | ||
def test_tofile_fromfile(self): | ||
with temppath(suffix='.bin') as path: | ||
path = Path(path) | ||
a = np.empty(10, dtype='f8,i4,a5') | ||
a[5] = (0.5,10,'abcde') | ||
a.newbyteorder('<') | ||
with path.open("wb") as fd: | ||
a.tofile(fd) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
I wonder if this is contributing to #12330. Certainly this test has just started failing stochastically on shippable / ARM for 2.7 / 3.7, so it may not have been stable in the first place given how recently this was merged. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This wasn't the problem -- I'm preparing the patch now in the linked PR, which is related to testing precision and np.empty() as far as I can tell. |
||
x = np.core.records.fromfile(path, | ||
formats='f8,i4,a5', | ||
shape=10, | ||
byteorder='<') | ||
assert_array_equal(x, a) | ||
|
||
|
||
class TestRecord(object): | ||
def setup(self): | ||
self.data = np.rec.fromrecords([(1, 2, 3), (4, 5, 6)], | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -161,7 +161,9 @@ | |
import io | ||
import warnings | ||
from numpy.lib.utils import safe_eval | ||
from numpy.compat import asbytes, asstr, isfileobj, long, basestring | ||
from numpy.compat import ( | ||
asbytes, asstr, isfileobj, long, basestring, is_pathlib_path | ||
) | ||
|
||
if sys.version_info[0] >= 3: | ||
import pickle | ||
|
@@ -709,7 +711,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, | |
|
||
Parameters | ||
---------- | ||
filename : str | ||
filename : str or pathlib.Path instance | ||
The name of the file on disk. This may *not* be a file-like | ||
object. | ||
mode : str, optional | ||
|
@@ -750,9 +752,9 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, | |
memmap | ||
|
||
""" | ||
if not isinstance(filename, basestring): | ||
raise ValueError("Filename must be a string. Memmap cannot use" | ||
" existing file handles.") | ||
if not (isinstance(filename, basestring) or is_pathlib_path(filename)): | ||
raise ValueError("Filename must be a string or a pathlib.Path." | ||
" Memmap cannot use existing file handles.") | ||
|
||
if 'w' in mode: | ||
# We are creating the file, not reading it. | ||
|
@@ -770,7 +772,10 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, | |
shape=shape, | ||
) | ||
# If we got here, then it should be safe to create the file. | ||
fp = open(filename, mode+'b') | ||
if is_pathlib_path(filename): | ||
fp = filename.open(mode+'b') | ||
else: | ||
fp = open(filename, mode+'b') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Better as from numpy.compat import os_fspath
fp = open(os_fspath(filename), mode+'b') Since this handles |
||
try: | ||
used_ver = _write_array_header(fp, d, version) | ||
# this warning can be removed when 1.9 has aged enough | ||
|
@@ -782,7 +787,10 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, | |
fp.close() | ||
else: | ||
# Read the header of the file first. | ||
fp = open(filename, 'rb') | ||
if is_pathlib_path(filename): | ||
fp = filename.open('rb') | ||
else: | ||
fp = open(filename, 'rb') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same comment here |
||
try: | ||
version = read_magic(fp) | ||
_check_version(version) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2252,11 +2252,34 @@ def test_loadtxt(self): | |
assert_array_equal(x, a) | ||
|
||
def test_save_load(self): | ||
# Test that pathlib.Path instances can be used with savez. | ||
# Test that pathlib.Path instances can be used with save. | ||
with temppath(suffix='.npy') as path: | ||
path = Path(path) | ||
a = np.array([[1, 2], [3, 4]], int) | ||
np.save(path, a) | ||
data = np.load(path) | ||
assert_array_equal(data, a) | ||
|
||
def test_save_load_memmap(self): | ||
# Test that pathlib.Path instances can be loaded mem-mapped. | ||
with temppath(suffix='.npy') as path: | ||
path = Path(path) | ||
a = np.array([[1, 2], [3, 4]], int) | ||
np.save(path, a) | ||
data = np.load(path, mmap_mode='r') | ||
assert_array_equal(data, a) | ||
del data | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What's the rationale for this line? A comment might be nice There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is required such that numpy closes the mem-mapped file. Otherwise you will get weird errors on Windows. |
||
|
||
def test_save_load_memmap_readwrite(self): | ||
# Test that pathlib.Path instances can be written mem-mapped. | ||
with temppath(suffix='.npy') as path: | ||
path = Path(path) | ||
a = np.array([[1, 2], [3, 4]], int) | ||
np.save(path, a) | ||
b = np.load(path, mmap_mode='r+') | ||
a[0][0] = 5 | ||
b[0][0] = 5 | ||
del b # closes the file | ||
data = np.load(path) | ||
assert_array_equal(data, a) | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
curious, what happens when fd is a unicode string on python2?