8000 Introduce a base store Class · zarr-developers/zarr-python@05cab29 · GitHub
[go: up one dir, main page]
Skip to content

Commit 05cab29

Browse files
CarreauCarreau
committed
Introduce a base store Class
Unconditionally close store in tests. All the tested stores should now have a `close()` method we can call and will be no-op if the stores do not need closing. Turn UserWarnings into errors And turn then back into only warnings into relevant tests. This ensure that we are not using deprecated functionalities, except when testing for it. initially based on 318eddcd
1 parent 480d21a commit 05cab29

15 files changed

+1898
-1366
lines changed

‎docs/tutorial.rst

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ print some diagnostics, e.g.::
176176
Read-only : False
177177
Compressor : Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE,
178178
: blocksize=0)
179-
Store type : builtins.dict
179+
Store type : zarr.storage.KVStore
180180
No. bytes : 400000000 (381.5M)
181181
No. bytes stored : 3379344 (3.2M)
182182
Storage ratio : 118.4
@@ -268,7 +268,7 @@ Here is an example using a delta filter with the Blosc compressor::
268268
Read-only : False
269269
Filter [0] : Delta(dtype='<i4')
270270
Compressor : Blosc(cname='zstd', clevel=1, shuffle=SHUFFLE, blocksize=0)
271-
Store type : builtins.dict
271+
Store type : zarr.storage.KVStore
272272
No. bytes : 400000000 (381.5M)
273273
No. bytes stored : 1290562 (1.2M)
274274
Storage ratio : 309.9
@@ -795,8 +795,10 @@ Here is an example using S3Map to read an array created previously::
795795
Order : C
796796
Read-only : False
797797
Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
798-
Store type : fsspec.mapping.FSMap
798+
Store type : zarr.storage.KVStore
799799
No. bytes : 21
800+
No. bytes stored : 382
801+
Storage ratio : 0.1
800802
Chunks initialized : 3/3
801803
>>> z[:]
802804
array([b'H', b'e', b'l', b'l', b'o', b' ', b'f', b'r', b'o', b'm', b' ',
@@ -1231,7 +1233,7 @@ ratios, depending on the correlation structure within the data. E.g.::
12311233
Order : C
12321234
Read-only : False
12331235
Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
1234-
Store type : builtins.dict
1236+
Store type : zarr.storage.KVStore
12351237
No. bytes : 400000000 (381.5M)
12361238
No. bytes stored : 6696010 (6.4M)
12371239
Storage ratio : 59.7
@@ -1245,7 +1247,7 @@ ratios, depending on the correlation structure within the data. E.g.::
12451247
Order : F
12461248
Read-only : False
12471249
Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
1248-
Store type : builtins.dict
1250+
Store type : zarr.storage.KVStore
12491251
No. bytes : 400000000 (381.5M)
12501252
No. bytes stored : 4684636 (4.5M)
12511253
Storage ratio : 85.4

‎mypy.ini

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
[mypy]
2-
python_version = 3.6
2+
python_version = 3.8
33
ignore_missing_imports = True
44
follow_imports = silent

‎pytest.ini

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,6 @@ doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS IGNORE_EXCEPTION_DETAIL
33
addopts = --durations=10
44
filterwarnings =
55
error::DeprecationWarning:zarr.*
6+
error::UserWarning:zarr.*
67
ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning
8+
ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning

‎zarr/convenience.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from zarr.hierarchy import group as _create_group
1313
from zarr.hierarchy import open_group
1414
from zarr.meta import json_dumps, json_loads
15-
from zarr.storage import contains_array, contains_group
15+
from zarr.storage import contains_array, contains_group, Store
1616
from zarr.util import TreeViewer, buffer_size, normalize_storage_path
1717

1818

@@ -72,8 +72,8 @@ def open(store=None, mode='a', **kwargs):
7272

7373
path = kwargs.get('path', None)
7474
# handle polymorphic store arg
75-
clobber = mode == 'w'
76-
store = normalize_store_arg(store, clobber=clobber)
75+
clobber = mode == "w"
76+
store: Store = normalize_store_arg(store, clobber=clobber)
7777
path = normalize_storage_path(path)
7878

7979
if mode in {'w', 'w-', 'x'}:
@@ -97,7 +97,7 @@ def open(store=None, mode='a', **kwargs):
9797
raise PathNotFoundError(path)
9898

9999

100-
def save_array(store, arr, **kwargs):
100+
def save_array(store: Store, arr, **kwargs):
101101
"""Convenience function to save a NumPy array to the local file system, following a
102102
similar API to the NumPy save() function.
103103

‎zarr/core.py

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,39 @@
99
import numpy as np
1010
from numcodecs.compat import ensure_bytes, ensure_ndarray
1111

12+
from collections.abc import MutableMapping
13+
1214
from zarr.attrs import Attributes
1315
from zarr.codecs import AsType, get_codec
1416
from zarr.errors import ArrayNotFoundError, ReadOnlyError
15-
from zarr.indexing import (BasicIndexer, CoordinateIndexer, MaskIndexer,
16-
OIndex, OrthogonalIndexer, VIndex, check_fields,
17-
check_no_multi_fields, ensure_tuple,
18-
err_too_many_indices, is_contiguous_selection,
19-
is_scalar, pop_fields)
17+
from zarr.indexing import (
18+
BasicIndexer,
19+
CoordinateIndexer,
20+
MaskIndexer,
21+
OIndex,
22+
OrthogonalIndexer,
23+
VIndex,
24+
check_fields,
25+
check_no_multi_fields,
26+
ensure_tuple,
27+
err_too_many_indices,
28+
is_contiguous_selection,
29+
is_scalar,
30+
pop_fields,
31+
)
2032
from zarr.meta import decode_array_metadata, encode_array_metadata
21-
from zarr.storage import array_meta_key, attrs_key, getsize, listdir
22-
from zarr.util import (InfoReporter, check_array_shape, human_readable_size,
23-
is_total_slice, nolock, normalize_chunks,
24-
normalize_resize_args, normalize_shape,
25-
normalize_storage_path)
33+
from zarr.storage import array_meta_key, attrs_key, getsize, listdir, Store
34+
from zarr.util import (
35+
InfoReporter,
36+
check_array_shape,
37+
human_readable_size,
38+
is_total_slice,
39+
nolock,
40+
normalize_chunks,
41+
normalize_resize_args,
42+
normalize_shape,
43+
normalize_storage_path,
44+
)
2645

2746

2847
# noinspection PyUnresolvedReferences
@@ -107,6 +126,9 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
107126
# N.B., expect at this point store is fully initialized with all
108127
# configuration metadata fully specified and normalized
109128

129+
store = Store._ensure_store(store)
130+
chunk_store = Store._ensure_store(chunk_store)
131+
110132
self._store = store
111133
self._chunk_store = chunk_store
112134
self._path = normalize_storage_path(path)
@@ -1821,7 +1843,7 @@ def _encode_chunk(self, chunk):
18211843
cdata = chunk
18221844

18231845
# ensure in-memory data is immutable and easy to compare
1824-
if isinstance(self.chunk_store, dict):
1846+
if isinstance(self.chunk_store, MutableMapping):
18251847
cdata = ensure_bytes(cdata)
18261848

18271849
return cdata
@@ -1854,10 +1876,10 @@ def info(self):
18541876
Order : C
18551877
Read-only : False
18561878
Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
1857-
Store type : builtins.dict
1879+
Store type : zarr.storage.KVStore
18581880
No. bytes : 4000000 (3.8M)
1859-
No. bytes stored : ...
1860-
Storage ratio : ...
1881+
No. bytes stored : 320
1882+
Storage ratio : 12500.0
18611883
Chunks initialized : 0/10
18621884
18631885
"""

‎zarr/creation.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import numpy as np
44
from numcodecs.registry import codec_registry
5+
from collections.abc import MutableMapping
56

67
from zarr.core import Array
78
from zarr.errors import (
@@ -10,9 +11,18 @@
1011
ContainsGroupError,
1112
)
1213
from zarr.n5 import N5Store
13-
from zarr.storage import (DirectoryStore, ZipStore, contains_array,
14-
contains_group, default_compressor, init_array,
15-
normalize_storage_path, FSStore)
14+
from zarr.storage import (
15+
DirectoryStore,
16+
ZipStore,
17+
KVStore,
18+
contains_array,
19+
contains_group,
20+
default_compressor,
21+
init_array,
22+
normalize_storage_path,
23+
FSStore,
24+
Store,
25+
)
1626

1727

1828
def create(shape, chunks=True, dtype=None, compressor='default',
@@ -129,7 +139,9 @@ def create(shape, chunks=True, dtype=None, compressor='default',
129139
return z
130140

131141

132-
def normalize_store_arg(store, clobber=False, default=dict, storage_options=None):
142+
def normalize_store_arg(
143+
store, clobber=False, default=dict, storage_options=None
144+
) -> Store:
133145
if store is None:
134146
return default()
135147
elif isinstance(store, str):
@@ -145,6 +157,8 @@ def normalize_store_arg(store, clobber=False, default=dict, storage_options=None
145157
else:
146158
return DirectoryStore(store)
147159
else:
160+
if not isinstance(store, Store) and isinstance(store, MutableMapping):
161+
store = KVStore(store)
148162
return store
149163

150164

‎zarr/hierarchy.py

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,26 @@
1515
ReadOnlyError,
1616
)
1717
from zarr.meta import decode_group_metadata
18-
from zarr.storage import (MemoryStore, attrs_key, contains_array,
19-
contains_group, group_meta_key, init_group, listdir,
20-
rename, rmdir)
21-
from zarr.util import (InfoReporter, TreeViewer, is_valid_python_name, nolock,
22-
normalize_shape, normalize_storage_path)
18+
from zarr.storage import (
19+
MemoryStore,
20+
attrs_key,
21+
contains_array,
22+
contains_group,
23+
group_meta_key,
24+
init_group,
25+
listdir,
26+
rename,
27+
rmdir,
28+
Store,
29+
)
30+
from zarr.util import (
31+
InfoReporter,
32+
TreeViewer,
33+
is_valid_python_name,
34+
nolock,
35+
normalize_shape,
36+
normalize_storage_path,
37+
)
2338

2439

2540
class Group(MutableMapping):
@@ -94,8 +109,19 @@ class Group(MutableMapping):
94109
95110
"""
96111

97-
def __init__(self, store, path=None, read_only=False, chunk_store=None,
98-
cache_attrs=True, synchronizer=None):
112+
def __init__(
113+
self,
114+
store,
115+
path=None,
116+
read_only=False,
117+
chunk_store=None,
118+
cache_attrs=True,
119+
synchronizer=None,
120+
):
121+
# assert hasattr(store, "listdir")
122+
123+
store = Store._ensure_store(store)
124+
chunk_store = Store._ensure_store(chunk_store)
99125
self._store = store
100126
self._chunk_store = chunk_store
101127
self._path = normalize_storage_path(path)
@@ -237,11 +263,8 @@ def __enter__(self):
237263
return self
238264

239265
def __exit__(self, exc_type, exc_val, exc_tb):
240-
"""If the underlying Store has a ``close`` method, call it."""
241-
try:
242-
self.store.close()
243-
except AttributeError:
244-
pass
266+
"""If the underlying Store should always heave a ``close`` method, call it."""
267+
self.store.close()
245268

246269
def info_items(self):
247270

0 commit comments

Comments
 (0)
0