8000 Fixtureize and parametrize h5py copy tests. (#417) · trinetta/zarr-python@f6dba5a · GitHub
[go: up one dir, main page]
Skip to content

Commit f6dba5a

Browse files
QuLogicQuLogic
authored
Fixtureize and parametrize h5py copy tests. (zarr-developers#417)
* Parametrize h5py copy tests. * Avoid one more atexit in copy tests.
1 parent 0c530c3 commit f6dba5a

File tree

1 file changed

+56
-133
lines changed

1 file changed

+56
-133
lines changed

‎zarr/tests/test_convenience.py

Lines changed: 56 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import atexit
2-
import os
32
import tempfile
43
import unittest
54
from numbers import Integral
@@ -456,67 +455,67 @@ def test_copy_all():
456455
assert destination_group.subgroup.attrs["info"] == "sub attrs"
457456

458457

459-
# noinspection PyAttributeOutsideInit
460-
class TestCopy(unittest.TestCase):
461-
462-
def __init__(self, *args, **kwargs):
463-
super().__init__(*args, **kwargs)
464-
self.source_h5py = False
465-
self.dest_h5py = False
466-
self.new_source = group
467-
self.new_dest = group
468-
469-
def setUp(self):
470-
source = self.new_source()
471-
foo = source.create_group('foo')
472-
foo.attrs['experiment'] = 'weird science'
473-
baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,))
474-
baz.attrs['units'] = 'metres'
475-
if self.source_h5py:
476-
extra_kws = dict(compression='gzip', compression_opts=3, fillvalue=84,
477-
shuffle=True, fletcher32=True)
458+
class TestCopy:
459+
@pytest.fixture(params=[False, True], ids=['zarr', 'hdf5'])
460+
def source(self, request, tmpdir):
461+
def prep_source(source):
462+
foo = source.create_group('foo')
463+
foo.attrs['experiment'] = 'weird science'
464+
baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,))
465+
baz.attrs['units'] = 'metres'
466+
if request.param:
467+
extra_kws = dict(compression='gzip', compression_opts=3, fillvalue=84,
468+
shuffle=True, fletcher32=True)
469+
else:
470+
extra_kws = dict(compressor=Zlib(3), order='F', fill_value=42, filters=[Adler32()])
471+
source.create_dataset('spam', data=np.arange(100, 200).reshape(20, 5),
472+
chunks=(10, 2), dtype='i2', **extra_kws)
473+
return source
474+
475+
if request.param:
476+
h5py = pytest.importorskip('h5py')
477+
fn = tmpdir.join('source.h5')
478+
with h5py.File(str(fn), mode='w') as h5f:
479+
yield prep_source(h5f)
478480
else:
479-
extra_kws = dict(compressor=Zlib(3), order='F', fill_value=42,
480-
filters=[Adler32()])
481-
source.create_dataset('spam', data=np.arange(100, 200).reshape(20, 5),
482-
chunks=(10, 2), dtype='i2', **extra_kws)
483-
self.source = source
484-
485-
def test_copy_array(self):
486-
source = self.source
487-
dest = self.new_dest()
481+
yield prep_source(group())
482+
483+
@pytest.fixture(params=[False, True], ids=['zarr', 'hdf5'])
484+
def dest(self, request, tmpdir):
485+
if request.param:
486+
h5py = pytest.importorskip('h5py')
487+
fn = tmpdir.join('dest.h5')
488+
with h5py.File(str(fn), mode='w') as h5f:
489+
yield h5f
490+
else:
491+
yield group()
488492

493+
def test_copy_array(self, source, dest):
489494
# copy array with default options
490495
copy(source['foo/bar/baz'], dest)
491496
check_copied_array(source['foo/bar/baz'], dest['baz'])
492497
copy(source['spam'], dest)
493498
check_copied_array(source['spam'], dest['spam'])
494499

495-
def test_copy_bad_dest(self):
496-
source = self.source
497-
500+
def test_copy_bad_dest(self, source, dest):
498501
# try to copy to an array, dest must be a group
499-
dest = self.new_dest().create_dataset('eggs', shape=(100,))
502+
dest = dest.create_dataset('eggs', shape=(100,))
500503
with pytest.raises(ValueError):
501504
copy(source['foo/bar/baz'], dest)
502505

503-
def test_copy_array_name(self):
504-
source = self.source
505-
dest = self.new_dest()
506-
506+
def test_copy_array_name(self, source, dest):
507507
# copy array with name
508508
copy(source['foo/bar/baz'], dest, name='qux')
509509
assert 'baz' not in dest
510510
check_copied_array(source['foo/bar/baz'], dest['qux'])
511511

512-
def test_copy_array_create_options(self):
513-
source = self.source
514-
dest = self.new_dest()
512+
def test_copy_array_create_options(self, source, dest):
513+
dest_h5py = dest.__module__.startswith('h5py.')
515514

516515
# copy array, provide creation options
517516
compressor = Zlib(9)
518517
create_kws = dict(chunks=(10,))
519-
if self.dest_h5py:
518+
if dest_h5py:
520519
create_kws.update(compression='gzip', compression_opts=9,
521520
shuffle=True, fletcher32=True, fillvalue=42)
522521
else:
@@ -526,10 +525,7 @@ def test_copy_array_create_options(self):
526525
check_copied_array(source['foo/bar/baz'], dest['baz'],
527526
without_attrs=True, expect_props=create_kws)
528527

529-
def test_copy_array_exists_array(self):
530-
source = self.source
531-
dest = self.new_dest()
532-
528+
def test_copy_array_exists_array(self, source, dest):
533529
# copy array, dest array in the way
534530
dest.create_dataset('baz', shape=(10,))
535531

@@ -554,10 +550,7 @@ def test_copy_array_exists_array(self):
554550
with pytest.raises(ValueError):
555551
copy(source['foo/bar/baz'], dest, if_exists='foobar')
556552

557-
def test_copy_array_exists_group(self):
558-
source = self.source
559-
dest = self.new_dest()
560-
553+
def test_copy_array_exists_group(self, source, dest):
561554
# copy array, dest group in the way
562555
dest.create_group('baz')
563556

@@ -577,13 +570,13 @@ def test_copy_array_exists_group(self):
577570
copy(source['foo/bar/baz'], dest, if_exists='replace')
578571
check_copied_array(source['foo/bar/baz'], dest['baz'])
579572

580-
def test_copy_array_skip_initialized(self):
581-
source = self.source
582-
dest = self.new_dest()
573+
def test_copy_array_skip_initialized(self, source, dest):
574+
dest_h5py = dest.__module__.startswith('h5py.')
575+
583576
dest.create_dataset('baz', shape=(100,), chunks=(10,), dtype='i8')
584577
assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:])
585578

586-
if self.dest_h5py:
579+
if dest_h5py:
587580
with pytest.raises(ValueError):
588581
# not available with copy to h5py
589582
copy(source['foo/bar/baz'], dest, if_exists='skip_initialized')
@@ -599,55 +592,37 @@ def test_copy_array_skip_initialized(self):
599592
assert_array_equal(np.arange(100, 200), dest['baz'][:])
600593
assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:])
601594

602-
def test_copy_group(self):
603-
source = self.source
604-
dest = self.new_dest()
605-
595+
def test_copy_group(self, source, dest):
606596
# copy group, default options
607597
copy(source['foo'], dest)
608598
check_copied_group(source['foo'], dest['foo'])
609599

610-
def test_copy_group_no_name(self):
611-
source = self.source
612-
dest = self.new_dest()
613-
600+
def test_copy_group_no_name(self, source, dest):
614601
with pytest.raises(TypeError):
615602
# need a name if copy root
616603
copy(source, dest)
617604

618605
copy(source, dest, name='root')
619606
check_copied_group(source, dest['root'])
620607

621-
def test_copy_group_options(self):
622-
source = self.source
623-
dest = self.new_dest()
624-
608+
def test_copy_group_options(self, source, dest):
625609
# copy group, non-default options
626610
copy(source['foo'], dest, name='qux', without_attrs=True)
627611
assert 'foo' not in dest
628612
check_copied_group(source['foo'], dest['qux'], without_attrs=True)
629613

630-
def test_copy_group_shallow(self):
631-
source = self.source
632-
dest = self.new_dest()
633-
614+
def test_copy_group_shallow(self, source, dest):
634615
# copy group, shallow
635616
copy(source, dest, name='eggs', shallow=True)
636617
check_copied_group(source, dest['eggs'], shallow=True)
637618

638-
def test_copy_group_exists_group(self):
639-
source = self.source
640-
dest = self.new_dest()
641-
619+
def test_copy_group_exists_group(self, source, dest):
642620
# copy group, dest groups exist
643621
dest.create_group('foo/bar')
644622
copy(source['foo'], dest)
645623
check_copied_group(source['foo'], dest['foo'])
646624

647-
def test_copy_group_exists_array(self):
648-
source = self.source
649-
dest = self.new_dest()
650-
625+
def test_copy_group_exists_array(self, source, dest):
651626
# copy group, dest array in the way
652627
dest.create_dataset('foo/bar', shape=(10,))
653628

@@ -6 F438 67,10 +642,7 @@ def test_copy_group_exists_array(self):
667642
copy(source['foo'], dest, if_exists='replace')
668643
check_copied_group(source['foo'], dest['foo'])
669644

670-
def test_copy_group_dry_run(self):
671-
source = self.source
672-
dest = self.new_dest()
673-
645+
def test_copy_group_dry_run(self, source, dest):
674646
# dry run, empty destination
675647
n_copied, n_skipped, n_bytes_copied = \
676648
copy(source['foo'], dest, dry_run=True, return_stats=True)
@@ -710,67 +682,18 @@ def test_copy_group_dry_run(self):
710682
assert 0 == n_bytes_copied
711683
assert_array_equal(baz, dest['foo/bar/baz'])
712684

713-
def test_logging(self):
714-
source = self.source
715-
dest = self.new_dest()
716-
685+
def test_logging(self, source, dest, tmpdir):
717686
# callable log
718687
copy(source['foo'], dest, dry_run=True, log=print)
719688

720689
# file name
721-
fn = tempfile.mktemp()
722-
atexit.register(os.remove, fn)
690+
fn = str(tmpdir.join('log_name'))
723691
copy(source['foo'], dest, dry_run=True, log=fn)
724692

725693
# file
726-
with tempfile.TemporaryFile(mode='w') as f:
694+
with tmpdir.join('log_file').open(mode='w') as f:
727695
copy(source['foo'], dest, dry_run=True, log=f)
728696

729697
# bad option
730698
with pytest.raises(TypeError):
731699
copy(source['foo'], dest, dry_run=True, log=True)
732-
733-
734-
try:
735-
import h5py
736-
except ImportError: # pragma: no cover
737-
h5py = None
738-
739-
740-
def temp_h5f():
741-
h5py = pytest.importorskip("h5py")
742-
fn = tempfile.mktemp()
743-
atexit.register(os.remove, fn)
744-
h5f = h5py.File(fn, mode='w')
745-
atexit.register(lambda v: v.close(), h5f)
746-
return h5f
747-
748-
749-
class TestCopyHDF5ToZarr(TestCopy):
750-
751-
def __init__(self, *args, **kwargs):
752-
super().__init__(*args, **kwargs)
753-
self.source_h5py = True
754-
self.dest_h5py = False
755-
self.new_source = temp_h5f
756-
self.new_dest = group
757-
758-
759-
class TestCopyZarrToHDF5(TestCopy):
760-
761-
def __init__(self, *args, **kwargs):
762-
super().__init__(*args, **kwargs)
763-
self.source_h5py = False
764-
self.dest_h5py = True
765-
self.new_source = group
766-
self.new_dest = temp_h5f
767-
768-
769-
class TestCopyHDF5ToHDF5(TestCopy):
770-
771-
def __init__(self, *args, **kwargs):
772-
super().__init__(*args, **kwargs)
773-
self.source_h5py = True
774-
self.dest_h5py = True
775-
self.new_source = temp_h5f
776-
self.new_dest = temp_h5f

0 commit comments

Comments
 (0)
0