8000 ENH: Save to ZIP files without using temporary files. · numpy/numpy@52c1ef6 · GitHub
[go: up one dir, main page]

Skip to content

Commit 52c1ef6

Browse files
ENH: Save to ZIP files without using temporary files.
Since Python 3.6 it is possible to write directly to a ZIP file, without creating temporary files.
1 parent 56a29d2 commit 52c1ef6

File tree

2 files changed

+38
-22
lines changed

2 files changed

+38
-22
lines changed

doc/release/1.14.0-notes.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,11 @@ common cache line size. This makes ``npy`` files easier to use in
263263
programs which open them with ``mmap``, especially on Linux where an
264264
``mmap`` offset must be a multiple of the page size.
265265

266+
NPZ files now can be written without using temporary files
267+
----------------------------------------------------------
268+
In Python 3.6+ ``numpy.savez`` and ``numpy.savez_compressed`` now write
269+
directly to a ZIP file, without creating intermediate temporary files.
270+
266271
Better support for empty structured and string types
267272
----------------------------------------------------
268273
Structured types can contain zero fields, and string dtypes can contain zero

numpy/lib/npyio.py

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -661,8 +661,6 @@ def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None):
661661
# Import is postponed to here since zipfile depends on gzip, an optional
662662
# component of the so-called standard library.
663663
import zipfile
664-
# Import deferred for startup time improvement
665-
import tempfile
666664

667665
if isinstance(file, basestring):
668666
if not file.endswith('.npz'):
@@ -686,31 +684,44 @@ def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None):
686684

687685
zipf = zipfile_factory(file, mode="w", compression=compression)
688686

689-
# Stage arrays in a temporary file on disk, before writing to zip.
690-
691-
# Since target file might be big enough to exceed capacity of a global
692-
# temporary directory, create temp file side-by-side with the target file.
693-
file_dir, file_prefix = os.path.split(file) if _is_string_like(file) else (None, 'tmp')
694-
fd, tmpfile = tempfile.mkstemp(prefix=file_prefix, dir=file_dir, suffix='-numpy.npy')
695-
os.close(fd)
696-
try:
687+
if sys.version_info >= (3, 6):
688+
# Since Python 3.6 it is possible to write directly to a ZIP file.
697689
for key, val in namedict.items():
698690
fname = key + '.npy'
699-
fid = open(tmpfile, 'wb')
700-
try:
701-
format.write_array(fid, np.asanyarray(val),
691+
val = np.asanyarray(val)
692+
force_zip64 = val.nbytes >= 2**30
693+
with zipf.open(fname, 'w', force_zip64=force_zip64) as fid:
694+
format.write_array(fid, val,
702695
allow_pickle=allow_pickle,
703696
pickle_kwargs=pickle_kwargs)
704-
fid.close()
705-
fid = None
706-
zipf.write(tmpfile, arcname=fname)
707-
except IOError as exc:
708-
raise IOError("Failed to write to %s: %s" % (tmpfile, exc))
709-
finally:
710-
if fid:
697+
else:
698+
# Stage arrays in a temporary file on disk, before writing to zip.
699+
700+
# Import deferred for startup time improvement
701+
import tempfile
702+
# Since target file might be big enough to exceed capacity of a global
703+
# temporary directory, create temp file side-by-side with the target file.
704+
file_dir, file_prefix = os.path.split(file) if _is_string_like(file) else (None, 'tmp')
705+
fd, tmpfile = tempfile.mkstemp(prefix=file_prefix, dir=file_dir, suffix='-numpy.npy')
706+
os.close(fd)
707+
try:
708+
for key, val in namedict.items():
709+
fname = key + '.npy'
710+
fid = open(tmpfile, 'wb')
711+
try:
712+
format.write_array(fid, np.asanyarray(val),
713+
allow_pickle=allow_pickle,
714+
pickle_kwargs=pickle_kwargs)
711715
fid.close()
712-
finally:
713-
os.remove(tmpfile)
716+
fid = None
717+
zipf.write(tmpfile, arcname=fname)
718+
except IOError as exc:
719+
raise IOError("Failed to write to %s: %s" % (tmpfile, exc))
720+
finally:
721+
if fid:
722+
fid.close()
723+
finally:
724+
os.remove(tmpfile)
714725

715726
zipf.close()
716727

0 commit comments

Comments
 (0)
0