From bb8ab0fa1f847eb95a545cd17af1c6ba51e69f65 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:42:07 -0800 Subject: [PATCH 001/160] chore: update pre-commit hooks (#2660) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.8.2 → v0.8.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.8.2...v0.8.6) - [github.com/pre-commit/mirrors-mypy: v1.13.0 → v1.14.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.13.0...v1.14.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea1cd4dbab..a9b4c8f444 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ ci: default_stages: [pre-commit, pre-push] repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.2 + rev: v0.8.6 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -22,7 +22,7 @@ repos: - id: check-yaml - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.13.0 + rev: v1.14.1 hooks: - id: mypy files: src|tests From f9c20243d207835d15d7091e05e8ec82a265b7d1 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Tue, 7 Jan 2025 08:48:44 +0100 Subject: [PATCH 002/160] Zstd: Don't persist the checksum param if false (#2655) --- src/zarr/core/metadata/v2.py | 8 +++++++- tests/test_metadata/test_v2.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index b95433068a..29cf15a119 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -116,7 +116,13 @@ def _json_convert( else: return o.descr if isinstance(o, numcodecs.abc.Codec): - return o.get_config() + codec_config = o.get_config() + + # Hotfix for https://github.com/zarr-developers/zarr-python/issues/2647 + if codec_config["id"] == "zstd" and not codec_config.get("checksum", False): + codec_config.pop("checksum", None) + + return codec_config if np.isscalar(o): out: Any if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 69dbd4645b..5a5bf5f73a 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -9,6 +9,7 @@ import zarr.api.asynchronous import zarr.storage from zarr.core.buffer import cpu +from zarr.core.buffer.core import default_buffer_prototype from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV2Metadata from zarr.core.metadata.v2 import parse_zarr_format @@ -282,3 +283,18 @@ def test_from_dict_extra_fields() -> None: order="C", ) assert result == expected + + +def test_zstd_checksum() -> None: + arr = zarr.create_array( + {}, + shape=(10,), + chunks=(10,), + dtype="int32", + compressors={"id": "zstd", "level": 5, "checksum": False}, + zarr_format=2, + ) + metadata = json.loads( + arr.metadata.to_buffer_dict(default_buffer_prototype())[".zarray"].to_bytes() + ) + assert "checksum" not in metadata["compressor"] From bc5877be4f61895a29fd811882e188f84fa3f8f2 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 7 Jan 2025 15:31:51 +0100 Subject: [PATCH 003/160] Feat/concurrent members (#2519) * feat: add wrapperstore * feat: add latencystore * rename noisysetter -> noisygetter * rename _wrapped to _store * loggingstore inherits from wrapperstore * initial commit * working members traversal * bolt concurrent members implementation onto async group * update scratch file * use metadata / node builders for v3 node creation * fix key/name handling in recursion * add latency-based test * add latency-based concurrency tests for group.members * improve comments for test * add concurrency limit * add test for concurrency limiting * docstrings * remove function that was only calling itself * docstrings * relax timing requirement for concurrency test * Update src/zarr/core/group.py Co-authored-by: Deepak Cherian * exists_ok -> overwrite * simplify group_members_perf test, just require that the duration is less than the number of groups * latency * update test docstring * remove vestigial test --------- Co-authored-by: Deepak Cherian --- src/zarr/api/asynchronous.py | 1 - src/zarr/core/array.py | 3 +- src/zarr/core/group.py | 328 ++++++++++++++++++++++++++--------- src/zarr/storage/_logging.py | 8 +- tests/test_group.py | 67 +++++++ 5 files changed, 318 insertions(+), 89 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 2e98a43f94..37a5b76bba 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -188,7 +188,6 @@ async def consolidate_metadata( group.store_path.store._check_writable() members_metadata = {k: v.metadata async for k, v in group.members(max_depth=None)} - # While consolidating, we want to be explicit about when child groups # are empty by inserting an empty dict for consolidated_metadata.metadata for k, v in members_metadata.items(): diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 915158cb5a..e0aad8b6ad 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1995,10 +1995,11 @@ def path(self) -> str: @property def name(self) -> str: + """Array name following h5py convention.""" return self._async_array.name @property - def basename(self) -> str | None: + def basename(self) -> str: """Final component of name.""" return self._async_array.basename diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index ebdc63364e..82970e4b7f 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -31,7 +31,7 @@ create_array, ) from zarr.core.attributes import Attributes -from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.core.common import ( JSON, ZARR_JSON, @@ -662,6 +662,7 @@ async def getitem( """ store_path = self.store_path / key logger.debug("key=%s, store_path=%s", key, store_path) + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata # Consolidated metadata lets us avoid some I/O operations so try that first. if self.metadata.consolidated_metadata is not None: @@ -678,12 +679,9 @@ async def getitem( raise KeyError(key) else: zarr_json = json.loads(zarr_json_bytes.to_bytes()) - if zarr_json["node_type"] == "group": - return type(self).from_dict(store_path, zarr_json) - elif zarr_json["node_type"] == "array": - return AsyncArray.from_dict(store_path, zarr_json) - else: - raise ValueError(f"unexpected node_type: {zarr_json['node_type']}") + metadata = _build_metadata_v3(zarr_json) + return _build_node_v3(metadata, store_path) + elif self.metadata.zarr_format == 2: # Q: how do we like optimistically fetching .zgroup, .zarray, and .zattrs? # This guarantees that we will always make at least one extra request to the store @@ -698,21 +696,19 @@ async def getitem( # unpack the zarray, if this is None then we must be opening a group zarray = json.loads(zarray_bytes.to_bytes()) if zarray_bytes else None + zgroup = json.loads(zgroup_bytes.to_bytes()) if zgroup_bytes else None # unpack the zattrs, this can be None if no attrs were written zattrs = json.loads(zattrs_bytes.to_bytes()) if zattrs_bytes is not None else {} if zarray is not None: - # TODO: update this once the V2 array support is part of the primary array class - zarr_json = {**zarray, "attributes": zattrs} - return AsyncArray.from_dict(store_path, zarr_json) + metadata = _build_metadata_v2(zarray, zattrs) + return _build_node_v2(metadata=metadata, store_path=store_path) else: - zgroup = ( - json.loads(zgroup_bytes.to_bytes()) - if zgroup_bytes is not None - else {"zarr_format": self.metadata.zarr_format} - ) - zarr_json = {**zgroup, "attributes": zattrs} - return type(self).from_dict(store_path, zarr_json) + # this is just for mypy + if TYPE_CHECKING: + assert zgroup is not None + metadata = _build_metadata_v2(zgroup, zattrs) + return _build_node_v2(metadata=metadata, store_path=store_path) else: raise ValueError(f"unexpected zarr_format: {self.metadata.zarr_format}") @@ -1346,18 +1342,50 @@ async def members( """ if max_depth is not None and max_depth < 0: raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") - async for item in self._members(max_depth=max_depth, current_depth=0): + async for item in self._members(max_depth=max_depth): yield item - async def _members( - self, max_depth: int | None, current_depth: int - ) -> AsyncGenerator[ + def _members_consolidated( + self, max_depth: int | None, prefix: str = "" + ) -> Generator[ tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], None, ]: + consolidated_metadata = self.metadata.consolidated_metadata + + do_recursion = max_depth is None or max_depth > 0 + + # we kind of just want the top-level keys. + if consolidated_metadata is not None: + for key in consolidated_metadata.metadata: + obj = self._getitem_consolidated( + self.store_path, key, prefix=self.name + ) # Metadata -> Group/Array + key = f"{prefix}/{key}".lstrip("/") + yield key, obj + + if do_recursion and isinstance(obj, AsyncGroup): + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + yield from obj._members_consolidated(new_depth, prefix=key) + + async def _members( + self, max_depth: int | None + ) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None + ]: + skip_keys: tuple[str, ...] + if self.metadata.zarr_format == 2: + skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") + elif self.metadata.zarr_format == 3: + skip_keys = ("zarr.json",) + else: + raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") + if self.metadata.consolidated_metadata is not None: - # we should be able to do members without any additional I/O - members = self._members_consolidated(max_depth, current_depth) + members = self._members_consolidated(max_depth=max_depth) for member in members: yield member return @@ -1371,66 +1399,12 @@ async def _members( ) raise ValueError(msg) - # would be nice to make these special keys accessible programmatically, - # and scoped to specific zarr versions - # especially true for `.zmetadata` which is configurable - _skip_keys = ("zarr.json", ".zgroup", ".zattrs", ".zmetadata") - - # hmm lots of I/O and logic interleaved here. - # We *could* have an async gen over self.metadata.consolidated_metadata.metadata.keys() - # and plug in here. `getitem` will skip I/O. - # Kinda a shame to have all the asyncio task overhead though, when it isn't needed. - - async for key in self.store_path.store.list_dir(self.store_path.path): - if key in _skip_keys: - continue - try: - obj = await self.getitem(key) - yield (key, obj) - - if ( - ((max_depth is None) or (current_depth < max_depth)) - and hasattr(obj.metadata, "node_type") - and obj.metadata.node_type == "group" - ): - # the assert is just for mypy to know that `obj.metadata.node_type` - # implies an AsyncGroup, not an AsyncArray - assert isinstance(obj, AsyncGroup) - async for child_key, val in obj._members( - max_depth=max_depth, current_depth=current_depth + 1 - ): - yield f"{key}/{child_key}", val - except KeyError: - # keyerror is raised when `key` names an object (in the object storage sense), - # as opposed to a prefix, in the store under the prefix associated with this group - # in which case `key` cannot be the name of a sub-array or sub-group. - warnings.warn( - f"Object at {key} is not recognized as a component of a Zarr hierarchy.", - UserWarning, - stacklevel=1, - ) - - def _members_consolidated( - self, max_depth: int | None, current_depth: int, prefix: str = "" - ) -> Generator[ - tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], - None, - ]: - consolidated_metadata = self.metadata.consolidated_metadata - - # we kind of just want the top-level keys. - if consolidated_metadata is not None: - for key in consolidated_metadata.metadata: - obj = self._getitem_consolidated( - self.store_path, key, prefix=self.name - ) # Metadata -> Group/Array - key = f"{prefix}/{key}".lstrip("/") - yield key, obj - - if ((max_depth is None) or (current_depth < max_depth)) and isinstance( - obj, AsyncGroup - ): - yield from obj._members_consolidated(max_depth, current_depth + 1, prefix=key) + # enforce a concurrency limit by passing a semaphore to all the recursive functions + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + async for member in _iter_members_deep( + self, max_depth=max_depth, skip_keys=skip_keys, semaphore=semaphore + ): + yield member async def keys(self) -> AsyncGenerator[str, None]: """Iterate over member names.""" @@ -2783,3 +2757,191 @@ def array( ) ) ) + + +async def _getitem_semaphore( + node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None +) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup: + """ + Combine node.getitem with an optional semaphore. If the semaphore parameter is an + asyncio.Semaphore instance, then the getitem operation is performed inside an async context + manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked + without a context manager. + """ + if semaphore is not None: + async with semaphore: + return await node.getitem(key) + else: + return await node.getitem(key) + + +async def _iter_members( + node: AsyncGroup, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None, +) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None +]: + """ + Iterate over the arrays and groups contained in a group. + + Parameters + ---------- + node : AsyncGroup + The group to traverse. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + + Yields + ------ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup] + """ + + # retrieve keys from storage + keys = [key async for key in node.store.list_dir(node.path)] + keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) + + node_tasks = tuple( + asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) + for key in keys_filtered + ) + + for fetched_node_coro in asyncio.as_completed(node_tasks): + try: + fetched_node = await fetched_node_coro + except KeyError as e: + # keyerror is raised when `key` names an object (in the object storage sense), + # as opposed to a prefix, in the store under the prefix associated with this group + # in which case `key` cannot be the name of a sub-array or sub-group. + warnings.warn( + f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", + UserWarning, + stacklevel=1, + ) + continue + match fetched_node: + case AsyncArray() | AsyncGroup(): + yield fetched_node.basename, fetched_node + case _: + raise ValueError(f"Unexpected type: {type(fetched_node)}") + + +async def _iter_members_deep( + group: AsyncGroup, + *, + max_depth: int | None, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None = None, +) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None +]: + """ + Iterate over the arrays and groups contained in a group, and optionally the + arrays and groups contained in those groups. + + Parameters + ---------- + group : AsyncGroup + The group to traverse. + max_depth : int | None + The maximum depth of recursion. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + + Yields + ------ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup] + """ + + to_recurse = {} + do_recursion = max_depth is None or max_depth > 0 + + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): + yield name, node + if isinstance(node, AsyncGroup) and do_recursion: + to_recurse[name] = _iter_members_deep( + node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore + ) + + for prefix, subgroup_iter in to_recurse.items(): + async for name, node in subgroup_iter: + key = f"{prefix}/{name}".lstrip("/") + yield key, node + + +def _resolve_metadata_v2( + blobs: tuple[str | bytes | bytearray, str | bytes | bytearray], +) -> ArrayV2Metadata | GroupMetadata: + zarr_metadata = json.loads(blobs[0]) + attrs = json.loads(blobs[1]) + if "shape" in zarr_metadata: + return ArrayV2Metadata.from_dict(zarr_metadata | {"attrs": attrs}) + else: + return GroupMetadata.from_dict(zarr_metadata | {"attrs": attrs}) + + +def _build_metadata_v3(zarr_json: dict[str, Any]) -> ArrayV3Metadata | GroupMetadata: + """ + Take a dict and convert it into the correct metadata type. + """ + if "node_type" not in zarr_json: + raise KeyError("missing `node_type` key in metadata document.") + match zarr_json: + case {"node_type": "array"}: + return ArrayV3Metadata.from_dict(zarr_json) + case {"node_type": "group"}: + return GroupMetadata.from_dict(zarr_json) + case _: + raise ValueError("invalid value for `node_type` key in metadata document") + + +def _build_metadata_v2( + zarr_json: dict[str, Any], attrs_json: dict[str, Any] +) -> ArrayV2Metadata | GroupMetadata: + """ + Take a dict and convert it into the correct metadata type. + """ + match zarr_json: + case {"shape": _}: + return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}) + case _: + return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}) + + +def _build_node_v3( + metadata: ArrayV3Metadata | GroupMetadata, store_path: StorePath +) -> AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Take a metadata object and return a node (AsyncArray or AsyncGroup). + """ + match metadata: + case ArrayV3Metadata(): + return AsyncArray(metadata, store_path=store_path) + case GroupMetadata(): + return AsyncGroup(metadata, store_path=store_path) + case _: + raise ValueError(f"Unexpected metadata type: {type(metadata)}") + + +def _build_node_v2( + metadata: ArrayV2Metadata | GroupMetadata, store_path: StorePath +) -> AsyncArray[ArrayV2Metadata] | AsyncGroup: + """ + Take a metadata object and return a node (AsyncArray or AsyncGroup). + """ + + match metadata: + case ArrayV2Metadata(): + return AsyncArray(metadata, store_path=store_path) + case GroupMetadata(): + return AsyncGroup(metadata, store_path=store_path) + case _: + raise ValueError(f"Unexpected metadata type: {type(metadata)}") diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py index 450913e9d3..45ddeef40c 100644 --- a/src/zarr/storage/_logging.py +++ b/src/zarr/storage/_logging.py @@ -11,7 +11,7 @@ from zarr.storage._wrapper import WrapperStore if TYPE_CHECKING: - from collections.abc import AsyncIterator, Generator, Iterable + from collections.abc import AsyncGenerator, Generator, Iterable from zarr.abc.store import ByteRangeRequest from zarr.core.buffer import Buffer, BufferPrototype @@ -205,19 +205,19 @@ async def set_partial_values( with self.log(keys): return await self._store.set_partial_values(key_start_values=key_start_values) - async def list(self) -> AsyncIterator[str]: + async def list(self) -> AsyncGenerator[str, None]: # docstring inherited with self.log(): async for key in self._store.list(): yield key - async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited with self.log(prefix): async for key in self._store.list_prefix(prefix=prefix): yield key - async def list_dir(self, prefix: str) -> AsyncIterator[str]: + async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited with self.log(prefix): async for key in self._store.list_dir(prefix=prefix): diff --git a/tests/test_group.py b/tests/test_group.py index 19a9f9c9bb..c2a5f751f3 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -3,6 +3,7 @@ import contextlib import operator import pickle +import time import warnings from typing import TYPE_CHECKING, Any, Literal @@ -22,6 +23,7 @@ from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore, make_store_path +from zarr.testing.store import LatencyStore from .conftest import parse_store @@ -1440,6 +1442,71 @@ def test_delitem_removes_children(store: Store, zarr_format: ZarrFormat) -> None g1["0/0"] +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_members_performance(store: MemoryStore) -> None: + """ + Test that the execution time of Group.members is less than the number of members times the + latency for accessing each member. + """ + get_latency = 0.1 + + # use the input store to create some groups + group_create = zarr.group(store=store) + num_groups = 10 + + # Create some groups + for i in range(num_groups): + group_create.create_group(f"group{i}") + + latency_store = LatencyStore(store, get_latency=get_latency) + # create a group with some latency on get operations + group_read = zarr.group(store=latency_store) + + # check how long it takes to iterate over the groups + # if .members is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + start = time.time() + _ = group_read.members() + elapsed = time.time() - start + + assert elapsed < (num_groups * get_latency) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_members_concurrency_limit(store: MemoryStore) -> None: + """ + Test that the execution time of Group.members can be constrained by the async concurrency + configuration setting. + """ + get_latency = 0.02 + + # use the input store to create some groups + group_create = zarr.group(store=store) + num_groups = 10 + + # Create some groups + for i in range(num_groups): + group_create.create_group(f"group{i}") + + latency_store = LatencyStore(store, get_latency=get_latency) + # create a group with some latency on get operations + group_read = zarr.group(store=latency_store) + + # check how long it takes to iterate over the groups + # if .members is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + from zarr.core.config import config + + with config.set({"async.concurrency": 1}): + start = time.time() + _ = group_read.members() + elapsed = time.time() - start + + assert elapsed > num_groups * get_latency + + @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_deprecated_compressor(store: Store) -> None: g = zarr.group(store=store, zarr_format=2) From 12f601258d7af950a853a9e7fbbdc32feae73901 Mon Sep 17 00:00:00 2001 From: Tom White Date: Tue, 7 Jan 2025 18:54:48 +0000 Subject: [PATCH 004/160] Fix `Group.array()` with `data` argument (#2668) --- src/zarr/core/group.py | 5 ++++- tests/test_group.py | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 82970e4b7f..79ab31112a 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -2729,6 +2729,8 @@ def array( Whether to overwrite an array with the same name in the store, if one exists. config : ArrayConfig or ArrayConfigLike, optional Runtime configuration for the array. + data : array_like + The data to fill the array with. Returns ------- @@ -2737,7 +2739,7 @@ def array( compressors = _parse_deprecated_compressor(compressor, compressors) return Array( self._sync( - self._async_group.create_array( + self._async_group.create_dataset( name=name, shape=shape, dtype=dtype, @@ -2754,6 +2756,7 @@ def array( overwrite=overwrite, storage_options=storage_options, config=config, + data=data, ) ) ) diff --git a/tests/test_group.py b/tests/test_group.py index c2a5f751f3..1d3563fe68 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -619,8 +619,7 @@ def test_group_create_array( array[:] = data elif method == "array": with pytest.warns(DeprecationWarning): - array = group.array(name="array", shape=shape, dtype=dtype) - array[:] = data + array = group.array(name="array", data=data, shape=shape, dtype=dtype) else: raise AssertionError From 29ef41dc9dd24cd96aaa294b442ea6fb892d5763 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 7 Jan 2025 16:04:00 -0800 Subject: [PATCH 005/160] Make make_store_path private (#2628) * Clean up public store API * chore: make_store_path is private --------- Co-authored-by: David Stansby --- src/zarr/api/asynchronous.py | 6 ++---- src/zarr/core/array.py | 5 +++-- src/zarr/core/group.py | 4 ++-- src/zarr/storage/__init__.py | 3 +-- tests/test_group.py | 3 ++- tests/test_store/test_core.py | 3 ++- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 37a5b76bba..8eba4fc152 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -27,16 +27,14 @@ from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v2 import _default_compressor, _default_filters from zarr.errors import NodeTypeValidationError -from zarr.storage import ( - StoreLike, - make_store_path, -) +from zarr.storage._common import make_store_path if TYPE_CHECKING: from collections.abc import Iterable from zarr.abc.codec import Codec from zarr.core.chunk_key_encodings import ChunkKeyEncoding + from zarr.storage import StoreLike # TODO: this type could use some more thought ArrayLike = AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array | npt.NDArray[Any] diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index e0aad8b6ad..ea29a6fc48 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -112,8 +112,8 @@ _parse_bytes_bytes_codec, get_pipeline_class, ) -from zarr.storage import StoreLike, make_store_path -from zarr.storage._common import StorePath, ensure_no_existing_node +from zarr.storage import StoreLike +from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path if TYPE_CHECKING: from collections.abc import Iterator, Sequence @@ -122,6 +122,7 @@ from zarr.abc.codec import CodecPipeline from zarr.codecs.sharding import ShardingCodecIndexLocation from zarr.core.group import AsyncGroup + from zarr.storage import StoreLike # Array and AsyncArray are defined in the base ``zarr`` namespace diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 79ab31112a..57d9c5cd8d 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -50,8 +50,8 @@ from zarr.core.metadata.v3 import V3JsonEncoder from zarr.core.sync import SyncMixin, sync from zarr.errors import MetadataValidationError -from zarr.storage import StoreLike, StorePath, make_store_path -from zarr.storage._common import ensure_no_existing_node +from zarr.storage import StoreLike, StorePath +from zarr.storage._common import ensure_no_existing_node, make_store_path if TYPE_CHECKING: from collections.abc import AsyncGenerator, Generator, Iterable, Iterator diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index c092ade03e..649857f773 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -3,7 +3,7 @@ from types import ModuleType from typing import Any -from zarr.storage._common import StoreLike, StorePath, make_store_path +from zarr.storage._common import StoreLike, StorePath from zarr.storage._fsspec import FsspecStore from zarr.storage._local import LocalStore from zarr.storage._logging import LoggingStore @@ -21,7 +21,6 @@ "StorePath", "WrapperStore", "ZipStore", - "make_store_path", ] diff --git a/tests/test_group.py b/tests/test_group.py index 1d3563fe68..788e81e603 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -22,7 +22,8 @@ from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError -from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore, make_store_path +from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore +from zarr.storage._common import make_store_path from zarr.testing.store import LatencyStore from .conftest import parse_store diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 5ab299442d..7806f3ecef 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -5,7 +5,8 @@ from _pytest.compat import LEGACY_PATH from zarr.core.common import AccessModeLiteral -from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath, make_store_path +from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath +from zarr.storage._common import make_store_path from zarr.storage._utils import normalize_path From 8bb0b3457bc31925e2ad0e737f1b29de9da74cbf Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 8 Jan 2025 00:24:06 -0800 Subject: [PATCH 006/160] add known bugs to work in progress section of the v3 migration guide (#2670) --- docs/user-guide/v3_migration.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/user-guide/v3_migration.rst b/docs/user-guide/v3_migration.rst index d90b87a897..66fcca6d19 100644 --- a/docs/user-guide/v3_migration.rst +++ b/docs/user-guide/v3_migration.rst @@ -206,3 +206,5 @@ of Zarr-Python, please open (or comment on) a * Object dtypes (:issue:`2617`) * Ragged arrays (:issue:`2618`) * Groups and Arrays do not implement ``__enter__`` and ``__exit__`` protocols (:issue:`2619`) + * Big Endian dtypes (:issue:`2324`) + * Default filters for object dtypes for Zarr format 2 arrays (:issue:`2627`) From eb2542498e93613e85c9555dcd2ccc606378fd57 Mon Sep 17 00:00:00 2001 From: Will Moore Date: Wed, 8 Jan 2025 10:26:30 +0000 Subject: [PATCH 007/160] Fix json indent (#2546) * Fix usage of config json_indent in V3JsonEncoder * Add test for json_indent * parametrize json indent * Add None to indent test parameters * ruff fix * other ruff fixes * Update src/zarr/core/metadata/v3.py Co-authored-by: Joe Hamman * Use explicit json encoder args * Add types * Update byte counts for tests --------- Co-authored-by: Joe Hamman Co-authored-by: Deepak Cherian --- docs/user-guide/arrays.rst | 4 ++-- docs/user-guide/groups.rst | 4 ++-- docs/user-guide/performance.rst | 4 ++-- src/zarr/core/metadata/v3.py | 28 +++++++++++++++++++++++++--- tests/test_array.py | 25 ++++++++++++------------- tests/test_metadata/test_v3.py | 11 ++++++++++- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst index ba85ce1cda..ae2c4b47eb 100644 --- a/docs/user-guide/arrays.rst +++ b/docs/user-guide/arrays.rst @@ -209,7 +209,7 @@ prints additional diagnostics, e.g.:: Serializer : BytesCodec(endian=) Compressors : (BloscCodec(typesize=4, cname=, clevel=3, shuffle=, blocksize=0),) No. bytes : 400000000 (381.5M) - No. bytes stored : 9696302 + No. bytes stored : 9696520 Storage ratio : 41.3 Chunks Initialized : 100 @@ -611,7 +611,7 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za Serializer : BytesCodec(endian=) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 100000000 (95.4M) - No. bytes stored : 3981060 + No. bytes stored : 3981552 Storage ratio : 25.1 Shards Initialized : 100 diff --git a/docs/user-guide/groups.rst b/docs/user-guide/groups.rst index da5f393246..1e72df3478 100644 --- a/docs/user-guide/groups.rst +++ b/docs/user-guide/groups.rst @@ -113,8 +113,8 @@ property. E.g.:: Serializer : BytesCodec(endian=) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 8000000 (7.6M) - No. bytes stored : 1432 - Storage ratio : 5586.6 + No. bytes stored : 1614 + Storage ratio : 4956.6 Chunks Initialized : 0 >>> baz.info Type : Array diff --git a/docs/user-guide/performance.rst b/docs/user-guide/performance.rst index 265bef8efe..42d830780f 100644 --- a/docs/user-guide/performance.rst +++ b/docs/user-guide/performance.rst @@ -131,7 +131,7 @@ ratios, depending on the correlation structure within the data. E.g.:: Serializer : BytesCodec(endian=) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 400000000 (381.5M) - No. bytes stored : 342588717 + No. bytes stored : 342588911 Storage ratio : 1.2 Chunks Initialized : 100 >>> with zarr.config.set({'array.order': 'F'}): @@ -150,7 +150,7 @@ ratios, depending on the correlation structure within the data. E.g.:: Serializer : BytesCodec(endian=) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 400000000 (381.5M) - No. bytes stored : 342588717 + No. bytes stored : 342588911 Storage ratio : 1.2 Chunks Initialized : 100 diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 13a275a6a1..ab62508c80 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -7,6 +7,7 @@ from zarr.core.buffer.core import default_buffer_prototype if TYPE_CHECKING: + from collections.abc import Callable from typing import Self from zarr.core.buffer import Buffer, BufferPrototype @@ -143,9 +144,30 @@ def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]: class V3JsonEncoder(json.JSONEncoder): - def __init__(self, *args: Any, **kwargs: Any) -> None: - self.indent = kwargs.pop("indent", config.get("json_indent")) - super().__init__(*args, **kwargs) + def __init__( + self, + *, + skipkeys: bool = False, + ensure_ascii: bool = True, + check_circular: bool = True, + allow_nan: bool = True, + sort_keys: bool = False, + indent: int | None = None, + separators: tuple[str, str] | None = None, + default: Callable[[object], object] | None = None, + ) -> None: + if indent is None: + indent = config.get("json_indent") + super().__init__( + skipkeys=skipkeys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + sort_keys=sort_keys, + indent=indent, + separators=separators, + default=default, + ) def default(self, o: object) -> Any: if isinstance(o, np.dtype): diff --git a/tests/test_array.py b/tests/test_array.py index 410b2e58d0..6600424147 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -399,13 +399,13 @@ async def test_chunks_initialized() -> None: def test_nbytes_stored() -> None: arr = zarr.create(shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec()]) result = arr.nbytes_stored() - assert result == 366 # the size of the metadata document. This is a fragile test. + assert result == 502 # the size of the metadata document. This is a fragile test. arr[:50] = 1 result = arr.nbytes_stored() - assert result == 566 # the size with 5 chunks filled. + assert result == 702 # the size with 5 chunks filled. arr[50:] = 2 result = arr.nbytes_stored() - assert result == 766 # the size with all chunks filled. + assert result == 902 # the size with all chunks filled. async def test_nbytes_stored_async() -> None: @@ -413,13 +413,13 @@ async def test_nbytes_stored_async() -> None: shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec()] ) result = await arr.nbytes_stored() - assert result == 366 # the size of the metadata document. This is a fragile test. + assert result == 502 # the size of the metadata document. This is a fragile test. await arr.setitem(slice(50), 1) result = await arr.nbytes_stored() - assert result == 566 # the size with 5 chunks filled. + assert result == 702 # the size with 5 chunks filled. await arr.setitem(slice(50, 100), 2) result = await arr.nbytes_stored() - assert result == 766 # the size with all chunks filled. + assert result == 902 # the size with all chunks filled. def test_default_fill_values() -> None: @@ -537,7 +537,7 @@ def test_info_complete(self, chunks: tuple[int, int], shards: tuple[int, int] | _serializer=BytesCodec(), _count_bytes=512, _count_chunks_initialized=0, - _count_bytes_stored=373 if shards is None else 578, # the metadata? + _count_bytes_stored=521 if shards is None else 982, # the metadata? ) assert result == expected @@ -545,11 +545,11 @@ def test_info_complete(self, chunks: tuple[int, int], shards: tuple[int, int] | result = arr.info_complete() if shards is None: expected = dataclasses.replace( - expected, _count_chunks_initialized=4, _count_bytes_stored=501 + expected, _count_chunks_initialized=4, _count_bytes_stored=649 ) else: expected = dataclasses.replace( - expected, _count_chunks_initialized=1, _count_bytes_stored=774 + expected, _count_chunks_initialized=1, _count_bytes_stored=1178 ) assert result == expected @@ -624,7 +624,7 @@ async def test_info_complete_async( _serializer=BytesCodec(), _count_bytes=512, _count_chunks_initialized=0, - _count_bytes_stored=373 if shards is None else 578, # the metadata? + _count_bytes_stored=521 if shards is None else 982, # the metadata? ) assert result == expected @@ -632,13 +632,12 @@ async def test_info_complete_async( result = await arr.info_complete() if shards is None: expected = dataclasses.replace( - expected, _count_chunks_initialized=4, _count_bytes_stored=501 + expected, _count_chunks_initialized=4, _count_bytes_stored=553 ) else: expected = dataclasses.replace( - expected, _count_chunks_initialized=1, _count_bytes_stored=774 + expected, _count_chunks_initialized=1, _count_bytes_stored=1178 ) - assert result == expected @pytest.mark.parametrize("store", ["memory"], indirect=True) diff --git a/tests/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py index ef527f42ef..a47cbf43bb 100644 --- a/tests/test_metadata/test_v3.py +++ b/tests/test_metadata/test_v3.py @@ -10,7 +10,8 @@ from zarr.codecs.bytes import BytesCodec from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding -from zarr.core.group import parse_node_type +from zarr.core.config import config +from zarr.core.group import GroupMetadata, parse_node_type from zarr.core.metadata.v3 import ( ArrayV3Metadata, DataType, @@ -304,6 +305,14 @@ def test_metadata_to_dict( assert observed == expected +@pytest.mark.parametrize("indent", [2, 4, None]) +def test_json_indent(indent: int): + with config.set({"json_indent": indent}): + m = GroupMetadata() + d = m.to_buffer_dict(default_buffer_prototype())["zarr.json"].to_bytes() + assert d == json.dumps(json.loads(d), indent=indent).encode() + + # @pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) # @pytest.mark.parametrize("precision", ["ns", "D"]) # async def test_datetime_metadata(fill_value: int, precision: str) -> None: From 0c1aad5782d1c9e3668dbf773cde877e70e64ac6 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 8 Jan 2025 05:51:05 -0800 Subject: [PATCH 008/160] fix: threadpool configuration (#2671) --- src/zarr/core/sync.py | 8 +++++--- tests/test_sync.py | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index f7d4529478..6a2de855e8 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -54,9 +54,7 @@ def _get_executor() -> ThreadPoolExecutor: global _executor if not _executor: max_workers = config.get("threading.max_workers", None) - print(max_workers) - # if max_workers is not None and max_workers > 0: - # raise ValueError(max_workers) + logger.debug("Creating Zarr ThreadPoolExecutor with max_workers=%s", max_workers) _executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="zarr_pool") _get_loop().set_default_executor(_executor) return _executor @@ -118,6 +116,9 @@ def sync( # NB: if the loop is not running *yet*, it is OK to submit work # and we will wait for it loop = _get_loop() + if _executor is None and config.get("threading.max_workers", None) is not None: + # trigger executor creation and attach to loop + _ = _get_executor() if not isinstance(loop, asyncio.AbstractEventLoop): raise TypeError(f"loop cannot be of type {type(loop)}") if loop.is_closed(): @@ -153,6 +154,7 @@ def _get_loop() -> asyncio.AbstractEventLoop: # repeat the check just in case the loop got filled between the # previous two calls from another thread if loop[0] is None: + logger.debug("Creating Zarr event loop") new_loop = asyncio.new_event_loop() loop[0] = new_loop iothread[0] = threading.Thread(target=new_loop.run_forever, name="zarr_io") diff --git a/tests/test_sync.py b/tests/test_sync.py index b0a6ecffd0..e0002fc5a7 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -12,6 +12,7 @@ _get_lock, _get_loop, cleanup_resources, + loop, sync, ) from zarr.storage import MemoryStore @@ -148,11 +149,20 @@ def test_open_positional_args_deprecate(): @pytest.mark.parametrize("workers", [None, 1, 2]) -def test_get_executor(clean_state, workers) -> None: +def test_threadpool_executor(clean_state, workers: int | None) -> None: with zarr.config.set({"threading.max_workers": workers}): - e = _get_executor() - if workers is not None and workers != 0: - assert e._max_workers == workers + _ = zarr.zeros(shape=(1,)) # trigger executor creation + assert loop != [None] # confirm loop was created + if workers is None: + # confirm no executor was created if no workers were specified + # (this is the default behavior) + assert loop[0]._default_executor is None + else: + # confirm executor was created and attached to loop as the default executor + # note: python doesn't have a direct way to get the default executor so we + # use the private attribute + assert _get_executor() is loop[0]._default_executor + assert _get_executor()._max_workers == workers def test_cleanup_resources_idempotent() -> None: From bc26199ccb0d0e4b75dbc07fb8ab598027941823 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 8 Jan 2025 09:22:38 -0800 Subject: [PATCH 009/160] api: hide zarr.core from api docs (#2669) * api: hide zarr.core from api docs * dont link to zarr.config doc module --- docs/conf.py | 2 +- docs/user-guide/config.rst | 2 +- src/zarr/core/__init__.py | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2a93e61d3e..8410b9b0b3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,7 +71,7 @@ def skip_submodules( ) -> bool: # Skip documenting zarr.codecs submodules # codecs are documented in the main zarr.codecs namespace - if what == "module" and name.startswith("zarr.codecs."): + if what == "module" and name.startswith("zarr.codecs.") or name.startswith("zarr.core"): skip = True return skip diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst index a17bce9d99..871291b72b 100644 --- a/docs/user-guide/config.rst +++ b/docs/user-guide/config.rst @@ -3,7 +3,7 @@ Runtime configuration ===================== -:mod:`zarr.config ` is responsible for managing the configuration of zarr and +``zarr.config`` is responsible for managing the configuration of zarr and is based on the `donfig `_ Python library. Configuration values can be set using code like the following:: diff --git a/src/zarr/core/__init__.py b/src/zarr/core/__init__.py index cbacfe3422..03a108dbbf 100644 --- a/src/zarr/core/__init__.py +++ b/src/zarr/core/__init__.py @@ -1,3 +1,8 @@ +""" +The ``zarr.core`` module is considered private API and should not be imported +directly by 3rd-party code. +""" + from __future__ import annotations from zarr.core.buffer import Buffer, NDBuffer # noqa: F401 From 22ebded93aa88ae1e5f87f6711fa7057ca2e8478 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 8 Jan 2025 17:38:43 +0000 Subject: [PATCH 010/160] Clean up release notes in preparation for v3 (#2634) --- docs/conf.py | 4 +- docs/developers/contributing.rst | 2 +- docs/developers/index.rst | 1 - docs/developers/release.rst | 2334 ------------------------------ docs/index.rst | 2 +- docs/release-notes.rst | 16 + docs/user-guide/v3_migration.rst | 2 + 7 files changed, 22 insertions(+), 2339 deletions(-) delete mode 100644 docs/developers/release.rst create mode 100644 docs/release-notes.rst diff --git a/docs/conf.py b/docs/conf.py index 8410b9b0b3..22d24c3515 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -105,10 +105,10 @@ def skip_submodules( "license": "https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt", "tutorial": "user-guide", "getting-started": "quickstart", - "release": "developers/release.html", "roadmap": "developers/roadmap.html", "installation": "user-guide/installation.html", - "api": "api/zarr/index" + "api": "api/zarr/index", + "release": "release-notes" } # The language for content autogenerated by Sphinx. Refer to documentation diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index 4358230eff..31cf80bed6 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -213,7 +213,7 @@ and functions are included in the API documentation, under the ``docs/api`` fold using the `autodoc `_ extension to sphinx. Any new features or important usage information should be included in the user-guide (``docs/user-guide``). Any changes should also be included in the release -notes (``docs/developers/release.rst``). +notes (``docs/release-notes.rst``). The documentation can be built locally by running:: diff --git a/docs/developers/index.rst b/docs/developers/index.rst index 3feb0aff71..4bccb3a469 100644 --- a/docs/developers/index.rst +++ b/docs/developers/index.rst @@ -6,5 +6,4 @@ Developer's Guide :maxdepth: 1 contributing - release roadmap diff --git a/docs/developers/release.rst b/docs/developers/release.rst deleted file mode 100644 index ce15c68f4a..0000000000 --- a/docs/developers/release.rst +++ /dev/null @@ -1,2334 +0,0 @@ -Release notes -============= - -.. - # Copy the warning statement _under_ the latest release version - # and unindent for pre-releases. - - .. warning:: - Pre-release! Use :command:`pip install --pre zarr` to evaluate this release. - -.. - # Unindent the section between releases in order - # to document your changes. On releases it will be - # re-indented so that it does not show up in the notes. - -.. note:: - Zarr-Python 2.18.* is expected be the final release in the 2.* series. Work on Zarr-Python 3.0 is underway. - See `GH1777 `_ for more details on the upcoming - 3.0 release. - -.. release_3.0.0-beta: - -3.0.0-beta series ------------------ - -.. warning:: - Zarr-Python 3.0.0-beta is a pre-release of the upcoming 3.0 release. This release is not feature complete or - expected to be ready for production applications. - -.. note:: - The complete release notes for 3.0 have not been added to this document yet. See the - `3.0.0-beta `_ release on GitHub - for a record of changes included in this release. - -Dependency Changes -~~~~~~~~~~~~~~~~~~ - -* fsspec was moved from a required dependency to an optional one. Users should install - fsspec and any relevant implementations (e.g. s3fs) before using the ``RemoteStore``. - By :user:`Joe Hamman ` :issue:`2391`. - -* ``RemoteStore`` was renamed to ``FsspecStore``. - By :user:`Joe Hamman ` :issue:`2557`. - -.. release_3.0.0-alpha: - -3.0.0-alpha series ------------------- - -.. warning:: - Zarr-Python 3.0.0-alpha is a pre-release of the upcoming 3.0 release. This release is not feature complete or - expected to be ready for production applications. - -.. note:: - The complete release notes for 3.0 have not been added to this document yet. See the - `3.0.0-alpha `_ release on GitHub - for a record of changes included in this release. - -Enhancements -~~~~~~~~~~~~ - -* Implement listing of the sub-arrays and sub-groups for a V3 ``Group``. - By :user:`Davis Bennett ` :issue:`1726`. - -* Bootstrap v3 branch with zarrita. - By :user:`Joe Hamman ` :issue:`1584`. - -* Extensible codecs for V3. - By :user:`Norman Rzepka ` :issue:`1588`. - -* Don't import from tests. - By :user:`Davis Bennett ` :issue:`1601`. - -* Listable V3 Stores. - By :user:`Joe Hamman ` :issue:`1634`. - -* Codecs without array metadata. - By :user:`Norman Rzepka ` :issue:`1632`. - -* fix sync group class methods. - By :user:`Joe Hamman ` :issue:`1652`. - -* implement eq for LocalStore. - By :user:`Charoula Kyriakides ` :issue:`1792`. - -* V3 reorg. - By :user:`Joe Hamman ` :issue:`1809`. - -* [v3] Sync with futures. - By :user:`Davis Bennett ` :issue:`1804`. - -* implement group.members. - By :user:`Davis Bennett ` :issue:`1726`. - -* Remove implicit groups. - By :user:`Joe Hamman ` :issue:`1827`. - -* feature(store): ``list_*`` -> AsyncGenerators. - By :user:`Joe Hamman ` :issue:`1844`. - -* Test codec entrypoints. - By :user:`Norman Rzepka ` :issue:`1835`. - -* Remove extra v3 sync module. - By :user:`Max Jones ` :issue:`1856`. - -* Use donfig for V3 configuration. - By :user:`Max Jones ` :issue:`1655`. - -* groundwork for V3 group tests. - By :user:`Davis Bennett ` :issue:`1743`. - -* [v3] First step to generalizes ndarray and bytes. - By :user:`Mads R. B. Kristensen ` :issue:`1826`. - -* Reworked codec pipelines. - By :user:`Norman Rzepka ` :issue:`1670`. - -* Followup on codecs. - By :user:`Norman Rzepka ` :issue:`1889`. - -* Protocols for Buffer and NDBuffer. - By :user:`Mads R. B. Kristensen ` :issue:`1899`. - -* [V3] Expand store tests. - By :user:`Davis Bennett ` :issue:`1900`. - -* [v3] Feature: Store open mode. - By :user:`Joe Hamman ` :issue:`1911`. - -* fix(types): Group.info -> NotImplementedError. - By :user:`Joe Hamman ` :issue:`1936`. - -* feature(typing): add py.typed file to package root. - By :user:`Joe Hamman ` :issue:`1935`. - -* Support all indexing variants. - By :user:`Norman Rzepka ` :issue:`1917`. - -* Feature: group and array name properties. - By :user:`Joe Hamman ` :issue:`1940`. - -* implement .chunks on v3 arrays. - By :user:`Ryan Abernathey ` :issue:`1929`. - -* Fixes bug in transpose. - By :user:`Norman Rzepka ` :issue:`1949`. - -* Buffer Prototype Argument. - By :user:`Mads R. B. Kristensen ` :issue:`1910`. - -* Feature: Top level V3 API. - By :user:`Joe Hamman ` :issue:`1884`. - -* Basic working FsspecStore. - By :user:`Martin Durant `; :issue:`1785`. - -Typing -~~~~~~ - -* Resolve Mypy errors in v3 branch. - By :user:`Daniel Jahn ` :issue:`1692`. - -* Allow dmypy to be run on v3 branch. - By :user:`David Stansby ` :issue:`1780`. - -* Remove unused typing ignore comments. - By :user:`David Stansby ` :issue:`1781`. - -* Check untyped defs on v3. - By :user:`David Stansby ` :issue:`1784`. - -* [v3] Enable some more strict mypy options. - By :user:`David Stansby ` :issue:`1793`. - -* [v3] Disallow generic Any typing. - By :user:`David Stansby ` :issue:`1794`. - -* Disallow incomplete type definitions. - By :user:`David Stansby ` :issue:`1814`. - -* Disallow untyped calls. - By :user:`David Stansby ` :issue:`1811`. - -* Fix some untyped calls. - By :user:`David Stansby ` :issue:`1865`. - -* Disallow untyped defs. - By :user:`David Stansby ` :issue:`1834`. - -* Add more typing to zarr.group. - By :user:`David Stansby ` :issue:`1870`. - -* Fix any generics in zarr.array. - By :user:`David Stansby ` :issue:`1861`. - -* Remove some unused mypy overrides. - By :user:`David Stansby ` :issue:`1894`. - -* Finish typing zarr.metadata. - By :user:`David Stansby ` :issue:`1880`. - -* Disallow implicit re-exports. - By :user:`David Stansby ` :issue:`1908`. - -* Make typing strict. - By :user:`David Stansby ` :issue:`1879`. - -* Enable extra mypy error codes. - By :user:`David Stansby ` :issue:`1909`. - -* Enable warn_unreachable for mypy. - By :user:`David Stansby ` :issue:`1937`. - -* Fix final typing errors. - By :user:`David Stansby ` :issue:`1939`. - -Maintenance -~~~~~~~~~~~ - -* Remedy a situation where ``zarr-python`` was importing ``DummyStorageTransformer`` from the test suite. - The dependency relationship is now reversed: the test suite imports this class from ``zarr-python``. - By :user:`Davis Bennett ` :issue:`1601`. - -* [V3] Update minimum supported Python and Numpy versions. - By :user:`Joe Hamman ` :issue:`1638` - -* use src layout and use hatch for packaging. - By :user:`Davis Bennett ` :issue:`1592`. - -* temporarily disable mypy in v3 directory. - By :user:`Joe Hamman ` :issue:`1649`. - -* create hatch test env. - By :user:`Ryan Abernathey ` :issue:`1650`. - -* removed unused environments and workflows. - By :user:`Ryan Abernathey ` :issue:`1651`. - -* Add env variables to sprint setup instructions. - By :user:`Max Jones ` :issue:`1654`. - -* Add test matrix for V3. - By :user:`Max Jones ` :issue:`1656`. - -* Remove attrs. - By :user:`Davis Bennett ` :issue:`1660`. - -* Specify hatch envs using GitHub actions matrix for v3 tests. - By :user:`Max Jones ` :issue:`1728`. - -* black -> ruff format + cleanup. - By :user:`Saransh Chopra ` :issue:`1639`. - -* Remove old v3. - By :user:`Davis Bennett ` :issue:`1742`. - -* V3 update pre commit. - By :user:`Joe Hamman ` :issue:`1808`. - -* remove windows testing on v3 branch. - By :user:`Joe Hamman ` :issue:`1817`. - -* fix: add mypy to test dependencies. - By :user:`Davis Bennett ` :issue:`1789`. - -* chore(ci): add numpy 2 release candidate to test matrix. - By :user:`Joe Hamman ` :issue:`1828`. - -* fix dependencies. - By :user:`Norman Rzepka ` :issue:`1840`. - -* Add pytest to mypy dependencies. - By :user:`David Stansby ` :issue:`1846`. - -* chore(pre-commit): update pre-commit versions and remove attrs dep mypy section. - By :user:`Joe Hamman ` :issue:`1848`. - -* Enable some ruff rules (RUF) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1869`. - -* Configure Ruff to apply flake8-bugbear/isort/pyupgrade. - By :user:`Norman Rzepka ` :issue:`1890`. - -* chore(ci): remove mypy from test action in favor of pre-commit action. - By :user:`Joe Hamman ` :issue:`1887`. - -* Enable ruff/flake8-raise rules (RSE) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1872`. - -* Apply assorted ruff/refurb rules (FURB). - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1873`. - -* Enable ruff/flake8-implicit-str-concat rules (ISC) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1868`. - -* Add numpy to mypy pre-commit check env. - By :user:`David Stansby ` :issue:`1893`. - -* remove fixture files from src. - By :user:`Davis Bennett ` :issue:`1897`. - -* Fix list of packages in mypy pre-commit environment. - By :user:`David Stansby ` :issue:`1907`. - -* Run sphinx directly on readthedocs. - By :user:`David Stansby ` :issue:`1919`. - -* Apply preview ruff rules. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1942`. - -* Enable and apply ruff rule RUF009. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1941`. - -Documentation -~~~~~~~~~~~~~ - -* Specify docs hatch env for v3 branch. - By :user:`Max Jones ` :issue:`1655`. - -* Development installation/contributing docs updates. - By :user:`Alden Keefe Sampson ` :issue:`1643`. - -* chore: update project settings per scientific python repo-review. - By :user:`Joe Hamman ` :issue:`1863`. - -* doc: update release notes for 3.0.0.alpha. - By :user:`Joe Hamman ` :issue:`1959`. - -.. _release_2.18.3: - -2.18.3 ------- - -Enhancements -~~~~~~~~~~~~ -* Added support for creating a copy of data when converting a `zarr.Array` - to a numpy array. - By :user:`David Stansby ` (:issue:`2106`) and - :user:`Joe Hamman ` (:issue:`2123`). - -Maintenance -~~~~~~~~~~~ -* Removed support for Python 3.9. - By :user:`David Stansby ` (:issue:`2074`). - -* Fix a regression when using orthogonal indexing with a scalar. - By :user:`Deepak Cherian ` :issue:`1931` - -* Added compatibility with NumPy 2.1. - By :user:`David Stansby ` - -* Bump minimum NumPy version to 1.24. - :user:`Joe Hamman ` (:issue:`2127`). - -Deprecations -~~~~~~~~~~~~ - -* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These - stores are slated to be removed in Zarr Python 3.0. - By :user:`Joe Hamman ` :issue:`2085`. - -.. _release_2.18.2: - -2.18.2 ------- - -Enhancements -~~~~~~~~~~~~ - -* Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` - -.. _release_2.18.1: - -2.18.1 ------- - -Maintenance -~~~~~~~~~~~ -* Fix a regression when getting or setting a single value from arrays with size-1 chunks. - By :user:`Deepak Cherian ` :issue:`1874` - -.. _release_2.18.0: - -2.18.0 ------- - -Enhancements -~~~~~~~~~~~~ -* Performance improvement for reading and writing chunks if any of the dimensions is size 1. - By :user:`Deepak Cherian ` :issue:`1730`. - -Maintenance -~~~~~~~~~~~ -* Enable ruff/bugbear rules (B) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. - -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. - By :user:`Joe Hamman ` :issue:`1842`. - -Deprecations -~~~~~~~~~~~~ - -* Deprecate experimental v3 support by issuing a `FutureWarning`. - Also updated docs to warn about using the experimental v3 version. - By :user:`Joe Hamman ` :issue:`1802` and :issue:`1807`. - -* Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, - :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, - and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. - By :user:`Joe Hamman ` :issue:`1801`. - -.. _release_2.17.2: - -2.17.2 ------- - -Enhancements -~~~~~~~~~~~~ - -* [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` - and handling any error that may arise. - By :user:`Deepak Cherian ` :issue:`1741`. - -* [v3] Reuse the downloaded array metadata when creating an ``Array``. - By :user:`Deepak Cherian ` :issue:`1734`. - -* Optimize ``Array.info`` so that it calls `getsize` only once. - By :user:`Deepak Cherian ` :issue:`1733`. - -* Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. - By :user:`Deepak Cherian ` :issue:`1716`. - -* FSStore now raises rather than return bad data. - By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. - -* Avoid redundant ``__contains__``. - By :user:`Deepak Cherian ` :issue:`1739`. - -Docs -~~~~ - -* Fix link to GCSMap in ``tutorial.rst``. - By :user:`Daniel Jahn ` :issue:`1689`. - -* Endorse `SPEC0000 `_ and state version support policy in ``installation.rst``. - By :user:`Sanket Verma ` :issue:`1665`. - -* Migrate v1 and v2 specification to `Zarr-Specs `_. - By :user:`Sanket Verma ` :issue:`1582`. - -Maintenance -~~~~~~~~~~~ - -* Add CI test environment for Python 3.12 - By :user:`Joe Hamman ` :issue:`1719`. - -* Bump minimum supported NumPy version to 1.23 (per spec 0000) - By :user:`Joe Hamman ` :issue:`1719`. - -* Minor fixes: Using ``is`` instead of ``type`` and removing unnecessary ``None``. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1737`. - -* Fix tests failure related to Pytest 8. - By :user:`David Stansby ` :issue:`1714`. - -.. _release_2.17.1: - -2.17.1 ------- - -Enhancements -~~~~~~~~~~~~ - -* Change occurrences of % and format() to f-strings. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1423`. - -* Proper argument for numpy.reshape. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1425`. - -* Add typing to dimension separator arguments. - By :user:`David Stansby ` :issue:`1620`. - -Docs -~~~~ - -* ZIP related tweaks. - By :user:`Davis Bennett ` :issue:`1641`. - -Maintenance -~~~~~~~~~~~ - -* Update config.yml with Zulip. - By :user:`Josh Moore `. - -* Replace Gitter with the new Zulip Chat link. - By :user:`Sanket Verma ` :issue:`1685`. - -* Fix RTD build. - By :user:`Sanket Verma ` :issue:`1694`. - -.. _release_2.17.0: - -2.17.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Added type hints to ``zarr.creation.create()``. - By :user:`David Stansby ` :issue:`1536`. - -* Pyodide support: Don't require fasteners on Emscripten. - By :user:`Hood Chatham ` :issue:`1663`. - -Docs -~~~~ - -* Minor correction and changes in documentation. - By :user:`Sanket Verma ` :issue:`1509`. - -* Fix typo in documentation. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1554` - -* The documentation build now fails if there are any warnings. - By :user:`David Stansby ` :issue:`1548`. - -* Add links to ``numcodecs`` docs in the tutorial. - By :user:`David Stansby ` :issue:`1535`. - -* Enable offline formats for documentation builds. - By :user:`Sanket Verma ` :issue:`1551`. - -* Minor tweak to advanced indexing tutorial examples. - By :user:`Ross Barnowski ` :issue:`1550`. - -* Automatically document array members using sphinx-automodapi. - By :user:`David Stansby ` :issue:`1547`. - -* Add a markdown file documenting the current and former core-developer team. - By :user:`Joe Hamman ` :issue:`1628`. - -* Add Norman Rzepka to core-dev team. - By :user:`Joe Hamman ` :issue:`1630`. - -* Added section about accessing ZIP archives on s3. - By :user:`Jeff Peck ` :issue:`1613`, :issue:`1615`, and :user:`Davis Bennett ` :issue:`1641`. - -* Add V3 roadmap and design document. - By :user:`Joe Hamman ` :issue:`1583`. - -Maintenance -~~~~~~~~~~~ - -* Drop Python 3.8 and NumPy 1.20 - By :user:`Josh Moore `; :issue:`1557`. - -* Cache result of ``FSStore._fsspec_installed()``. - By :user:`Janick Martinez Esturo ` :issue:`1581`. - -* Extend copyright notice to 2023. - By :user:`Jack Kelly ` :issue:`1528`. - -* Change occurrence of ``io.open()`` into ``open()``. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1421`. - -* Preserve ``dimension_separator`` when resizing arrays. - By :user:`Ziwen Liu ` :issue:`1533`. - -* Initialise some sets in tests with set literals instead of list literals. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1534`. - -* Allow ``black`` code formatter to be run with any Python version. - By :user:`David Stansby ` :issue:`1549`. - -* Remove ``sphinx-rtd-theme`` dependency from ``pyproject.toml``. - By :user:`Sanket Verma ` :issue:`1563`. - -* Remove ``CODE_OF_CONDUCT.md`` file from the Zarr-Python repository. - By :user:`Sanket Verma ` :issue:`1572`. - -* Bump version of black in pre-commit. - By :user:`David Stansby ` :issue:`1559`. - -* Use list comprehension where applicable. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1555`. - -* Use format specification mini-language to format string. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1558`. - -* Single startswith() call instead of multiple ones. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1556`. - -* Move codespell options around. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1196`. - -* Remove unused mypy ignore comments. - By :user:`David Stansby ` :issue:`1602`. - -.. _release_2.16.1: - -2.16.1 ------- - -Maintenance -~~~~~~~~~~~ - -* Require ``setuptools_scm`` version ``1.5.4``\+ - By :user:`John A. Kirkham ` :issue:`1477`. - -* Add ``docs`` requirements to ``pyproject.toml`` - By :user:`John A. Kirkham ` :issue:`1494`. - -* Fixed caching issue in ``LRUStoreCache``. - By :user:`Mads R. B. Kristensen ` :issue:`1499`. - -.. _release_2.16.0: - -2.16.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Allow for partial codec specification in V3 array metadata. - By :user:`Joe Hamman ` :issue:`1443`. - -* Add ``__contains__`` method to ``KVStore``. - By :user:`Christoph Gohlke ` :issue:`1454`. - -* **Block Indexing**: Implemented blockwise (chunk blocks) indexing to ``zarr.Array``. - By :user:`Altay Sansal ` :issue:`1428` - -Maintenance -~~~~~~~~~~~ - -* Refactor the core array tests to reduce code duplication. - By :user:`Davis Bennett ` :issue:`1462`. - -* Style the codebase with ``ruff`` and ``black``. - By :user:`Davis Bennett ` :issue:`1459` - -* Ensure that chunks is tuple of ints upon array creation. - By :user:`Philipp Hanslovsky ` :issue:`1461` - -.. _release_2.15.0: - -2.15.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Implement more extensive fallback of getitem/setitem for orthogonal indexing. - By :user:`Andreas Albert ` :issue:`1029`. - -* Getitems supports ``meta_array``. - By :user:`Mads R. B. Kristensen ` :issue:`1131`. - -* ``open_array()`` now takes the ``meta_array`` argument. - By :user:`Mads R. B. Kristensen ` :issue:`1396`. - -Maintenance -~~~~~~~~~~~ - -* Remove ``codecov`` from GitHub actions. - By :user:`John A. Kirkham ` :issue:`1391`. - -* Replace ``np.product`` with ``np.prod`` due to deprecation. - By :user:`James Bourbeau ` :issue:`1405`. - -* Activate Py 3.11 builds. - By :user:`Joe Hamman ` :issue:`1415`. - -Documentation -~~~~~~~~~~~~~ - -* Add API reference for V3 Implementation in the docs. - By :user:`Sanket Verma ` :issue:`1345`. - -Bug fixes -~~~~~~~~~ - -* Fix the conda-forge error. Read :issue:`1347` for detailed info. - By :user:`Josh Moore ` :issue:`1364` and :issue:`1367`. - -* Fix ``ReadOnlyError`` when opening V3 store via fsspec reference file system. - By :user:`Joe Hamman ` :issue:`1383`. - -* Fix ``normalize_fill_value`` for structured arrays. - By :user:`Alan Du ` :issue:`1397`. - -.. _release_2.14.2: - -2.14.2 ------- - -Bug fixes -~~~~~~~~~ - -* Ensure ``zarr.group`` uses writeable mode to fix issue with :issue:`1304`. - By :user:`Brandur Thorgrimsson ` :issue:`1354`. - -.. _release_2.14.1: - -2.14.1 ------- - -Documentation -~~~~~~~~~~~~~ - -* Fix API links. - By :user:`Josh Moore ` :issue:`1346`. - -* Fix unit tests which prevented the conda-forge release. - By :user:`Josh Moore ` :issue:`1348`. - -.. _release_2.14.0: - -2.14.0 ------- - -Major changes -~~~~~~~~~~~~~ - -* Improve Zarr V3 support, adding partial store read/write and storage transformers. - Add new features from the `v3 spec `_: - - * storage transformers - * `get_partial_values` and `set_partial_values` - * efficient `get_partial_values` implementation for `FSStoreV3` - * sharding storage transformer - - By :user:`Jonathan Striebel `; :issue:`1096`, :issue:`1111`. - -* N5 nows supports Blosc. - Remove warnings emitted when using N5Store or N5FSStore with a blosc-compressed array. - By :user:`Davis Bennett `; :issue:`1331`. - -Bug fixes -~~~~~~~~~ - -* Allow reading utf-8 encoded json files - By :user:`Nathan Zimmerberg ` :issue:`1308`. - -* Ensure contiguous data is give to ``FSStore``. Only copying if needed. - By :user:`Mads R. B. Kristensen ` :issue:`1285`. - -* NestedDirectoryStore.listdir now returns chunk keys with the correct '/' dimension_separator. - By :user:`Brett Graham ` :issue:`1334`. - -* N5Store/N5FSStore dtype returns zarr Stores readable dtype. - By :user:`Marwan Zouinkhi ` :issue:`1339`. - -.. _release_2.13.6: - -2.13.6 ------- - -Maintenance -~~~~~~~~~~~ - -* Bump gh-action-pypi-publish to 1.6.4. - By :user:`Josh Moore ` :issue:`1320`. - -.. _release_2.13.5: - -2.13.5 ------- - -Bug fixes -~~~~~~~~~ - -* Ensure ``zarr.create`` uses writeable mode to fix issue with :issue:`1304`. - By :user:`James Bourbeau ` :issue:`1309`. - -.. _release_2.13.4: - -2.13.4 ------- - -Appreciation -~~~~~~~~~~~~~ - -Special thanks to Outreachy participants for contributing to most of the -maintenance PRs. Please read the blog post summarising the contribution phase -and welcoming new Outreachy interns: -https://zarr.dev/blog/welcoming-outreachy-2022-interns/ - - -Enhancements -~~~~~~~~~~~~ - -* Handle fsspec.FSMap using FSStore store. - By :user:`Rafal Wojdyla ` :issue:`1304`. - -Bug fixes -~~~~~~~~~ - -* Fix bug that caused double counting of groups in ``groups()`` and ``group_keys()`` methods with V3 stores. - By :user:`Ryan Abernathey ` :issue:`1228`. - -* Remove unnecessary calling of `contains_array` for key that ended in `.array.json`. - By :user:`Joe Hamman ` :issue:`1149`. - -* Fix bug that caused double counting of groups in ``groups()`` and ``group_keys()`` - methods with V3 stores. - By :user:`Ryan Abernathey ` :issue:`1228`. - -Documentation -~~~~~~~~~~~~~ - -* Fix minor indexing errors in tutorial and specification examples of documentation. - By :user:`Kola Babalola ` :issue:`1277`. - -* Add `requirements_rtfd.txt` in `contributing.rst`. - By :user:`AWA BRANDON AWA ` :issue:`1243`. - -* Add documentation for find/findall using visit. - By :user:`Weddy Gikunda ` :issue:`1241`. - -* Refresh of the main landing page. - By :user:`Josh Moore ` :issue:`1173`. - -Maintenance -~~~~~~~~~~~ - -* Migrate to ``pyproject.toml`` and remove redundant infrastructure. - By :user:`Saransh Chopra ` :issue:`1158`. - -* Require ``setuptools`` 64.0.0+ - By :user:`Saransh Chopra ` :issue:`1193`. - -* Pin action versions (pypi-publish, setup-miniconda) for dependabot - By :user:`Saransh Chopra ` :issue:`1205`. - -* Remove ``tox`` support - By :user:`Saransh Chopra ` :issue:`1219`. - -* Add workflow to label PRs with "needs release notes". - By :user:`Saransh Chopra ` :issue:`1239`. - -* Simplify if/else statement. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1227`. - -* Get coverage up to 100%. - By :user:`John Kirkham ` :issue:`1264`. - -* Migrate coverage to ``pyproject.toml``. - By :user:`John Kirkham ` :issue:`1250`. - -* Use ``conda-incubator/setup-miniconda@v2.2.0``. - By :user:`John Kirkham ` :issue:`1263`. - -* Delete unused files. - By :user:`John Kirkham ` :issue:`1251`. - -* Skip labeller for bot PRs. - By :user:`Saransh Chopra ` :issue:`1271`. - -* Restore Flake8 configuration. - By :user:`John Kirkham ` :issue:`1249`. - -* Add missing newline at EOF. - By :user:`Dimitri Papadopoulos` :issue:`1253`. - -* Add `license_files` to `pyproject.toml`. - By :user:`John Kirkham ` :issue:`1247`. - -* Adding `pyupgrade` suggestions. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1225`. - -* Fixed some linting errors. - By :user:`Weddy Gikunda ` :issue:`1226`. - -* Added the link to main website in readthedocs sidebar. - By :user:`Stephanie_nkwatoh ` :issue:`1216`. - -* Remove redundant wheel dependency in `pyproject.toml`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1233`. - -* Turned on `isloated_build` in `tox.ini` file. - By :user:`AWA BRANDON AWA ` :issue:`1210`. - -* Fixed `flake8` alert and avoid duplication of `Zarr Developers`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1203`. - -* Bump to NumPy 1.20+ in `environment.yml`. - By :user:`John Kirkham ` :issue:`1201`. - -* Bump to NumPy 1.20 in `pyproject.toml`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1192`. - -* Remove LGTM (`.lgtm.yml`) configuration file. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1191`. - -* Codespell will skip `fixture` in pre-commit. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1197`. - -* Add msgpack in `requirements_rtfd.txt`. - By :user:`Emmanuel Bolarinwa ` :issue:`1188`. - -* Added license to docs fixed a typo from `_spec_v2` to `_spec_v3`. - By :user:`AWA BRANDON AWA ` :issue:`1182`. - -* Fixed installation link in `README.md`. - By :user:`AWA BRANDON AWA ` :issue:`1177`. - -* Fixed typos in `installation.rst` and `release.rst`. - By :user:`Chizoba Nweke ` :issue:`1178`. - -* Set `docs/conf.py` language to `en`. - By :user:`AWA BRANDON AWA ` :issue:`1174`. - -* Added `installation.rst` to the docs. - By :user:`AWA BRANDON AWA ` :issue:`1170`. - -* Adjustment of year to `2015-2018` to `2015-2022` in the docs. - By :user:`Emmanuel Bolarinwa ` :issue:`1165`. - -* Updated `Forking the repository` section in `contributing.rst`. - By :user:`AWA BRANDON AWA ` :issue:`1171`. - -* Updated GitHub actions. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1134`. - -* Update web links: `http:// → https://`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1313`. - -.. _release_2.13.3: - -2.13.3 ------- - -* Improve performance of slice selections with steps by omitting chunks with no relevant - data. - By :user:`Richard Shaw ` :issue:`843`. - -.. _release_2.13.2: - -2.13.2 ------- - -* Fix test failure on conda-forge builds (again). - By :user:`Josh Moore `; see - `zarr-feedstock#65 `_. - -.. _release_2.13.1: - -2.13.1 ------- - -* Fix test failure on conda-forge builds. - By :user:`Josh Moore `; see - `zarr-feedstock#65 `_. - -.. _release_2.13.0: - -2.13.0 ------- - -Major changes -~~~~~~~~~~~~~ - -* **Support of alternative array classes** by introducing a new argument, - meta_array, that specifies the type/class of the underlying array. The - meta_array argument can be any class instance that can be used as the like - argument in NumPy (see `NEP 35 - `_). - enabling support for CuPy through, for example, the creation of a CuPy CPU - compressor. - By :user:`Mads R. B. Kristensen ` :issue:`934`. - -* **Remove support for Python 3.7** in concert with NumPy dependency. - By :user:`Davis Bennett ` :issue:`1067`. - -* **Zarr v3: add support for the default root path** rather than requiring - that all API users pass an explicit path. - By :user:`Gregory R. Lee ` :issue:`1085`, :issue:`1142`. - - -Bug fixes -~~~~~~~~~ - -* Remove/relax erroneous "meta" path check (**regression**). - By :user:`Gregory R. Lee ` :issue:`1123`. - -* Cast all attribute keys to strings (and issue deprecation warning). - By :user:`Mattia Almansi ` :issue:`1066`. - -* Fix bug in N5 storage that prevented arrays located in the root of the hierarchy from - bearing the `n5` keyword. Along with fixing this bug, new tests were added for N5 routines - that had previously been excluded from testing, and type annotations were added to the N5 codebase. - By :user:`Davis Bennett ` :issue:`1092`. - -* Fix bug in LRUEStoreCache in which the current size wasn't reset on invalidation. - By :user:`BGCMHou ` and :user:`Josh Moore ` :issue:`1076`, :issue:`1077`. - -* Remove erroneous check that disallowed array keys starting with "meta". - By :user:`Gregory R. Lee ` :issue:`1105`. - -Documentation -~~~~~~~~~~~~~ - -* Typo fixes to close quotes. By :user:`Pavithra Eswaramoorthy ` - -* Added copy button to documentation. - By :user:`Altay Sansal ` :issue:`1124`. - -Maintenance -~~~~~~~~~~~ - -* Simplify release docs. - By :user:`Josh Moore ` :issue:`1119`. - -* Pin werkzeug to prevent test hangs. - By :user:`Davis Bennett ` :issue:`1098`. - -* Fix a few DeepSource.io alerts - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1080`. - -* Fix URLs. - By :user:`Dimitri Papadopoulos Orfanos `, :issue:`1074`. - -* Fix spelling. - By :user:`Dimitri Papadopoulos Orfanos `, :issue:`1073`. - -* Update GitHub issue templates with `YAML` format. - By :user:`Saransh Chopra ` :issue:`1079`. - -* Remove option to return None from _ensure_store. - By :user:`Gregory Lee ` :issue:`1068`. - -* Fix a typo of "integers". - By :user:`Richard Scott ` :issue:`1056`. - -.. _release_2.12.0: - -2.12.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* **Add support for reading and writing Zarr V3.** The new `zarr._store.v3` - package has the necessary classes and functions for evaluating Zarr V3. - Since the format is not yet finalized, the classes and functions are not - automatically imported into the regular `zarr` name space. Setting the - `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them. - By :user:`Gregory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` - as well as by :user:`Josh Moore ` :issue:`1032`. - -* **Create FSStore from an existing fsspec filesystem**. If you have created - an fsspec filesystem outside of Zarr, you can now pass it as a keyword - argument to ``FSStore``. - By :user:`Ryan Abernathey `; :issue:`911`. - -* Add numpy encoder class for json.dumps - By :user:`Eric Prestat `; :issue:`933`. - -* Appending performance improvement to Zarr arrays, e.g., when writing to S3. - By :user:`hailiangzhang `; :issue:`1014`. - -* Add number encoder for ``json.dumps`` to support numpy integers in - ``chunks`` arguments. By :user:`Eric Prestat ` :issue:`697`. - -Bug fixes -~~~~~~~~~ - -* Fix bug that made it impossible to create an ``FSStore`` on unlistable filesystems - (e.g. some HTTP servers). - By :user:`Ryan Abernathey `; :issue:`993`. - - -Documentation -~~~~~~~~~~~~~ - -* Update resize doc to clarify surprising behavior. - By :user:`hailiangzhang `; :issue:`1022`. - -Maintenance -~~~~~~~~~~~ - -* Added Pre-commit configuration, incl. Yaml Check. - By :user:`Shivank Chaudhary `; :issue:`1015`, :issue:`1016`. - -* Fix URL to renamed file in Blosc repo. - By :user:`Andrew Thomas ` :issue:`1028`. - -* Activate Py 3.10 builds. - By :user:`Josh Moore ` :issue:`1027`. - -* Make all unignored zarr warnings errors. - By :user:`Josh Moore ` :issue:`1021`. - - -.. _release_2.11.3: - -2.11.3 ------- - -Bug fixes -~~~~~~~~~ - -* Fix missing case to fully revert change to default write_empty_chunks. - By :user:`Tom White `; :issue:`1005`. - - -.. _release_2.11.2: - -2.11.2 ------- - -Bug fixes -~~~~~~~~~ - -* Changes the default value of ``write_empty_chunks`` to ``True`` to prevent - unanticipated data losses when the data types do not have a proper default - value when empty chunks are read back in. - By :user:`Vyas Ramasubramani `; :issue:`965`, :issue:`1001`. - -.. _release_2.11.1: - -2.11.1 ------- - -Bug fixes -~~~~~~~~~ - -* Fix bug where indexing with a scalar numpy value returned a single-value array. - By :user:`Ben Jeffery ` :issue:`967`. - -* Removed `clobber` argument from `normalize_store_arg`. This enables to change - data within an opened consolidated group using mode `"r+"` (i.e region write). - By :user:`Tobias Kölling ` :issue:`975`. - -.. _release_2.11.0: - -2.11.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* **Sparse changes with performance impact!** One of the advantages of the Zarr - format is that it is sparse, which means that chunks with no data (more - precisely, with data equal to the fill value, which is usually 0) don't need - to be written to disk at all. They will simply be assumed to be empty at read - time. However, until this release, the Zarr library would write these empty - chunks to disk anyway. This changes in this version: a small performance - penalty at write time leads to significant speedups at read time and in - filesystem operations in the case of sparse arrays. To revert to the old - behavior, pass the argument ``write_empty_chunks=True`` to the array creation - function. By :user:`Juan Nunez-Iglesias `; :issue:`853` and - :user:`Davis Bennett `; :issue:`738`. - -* **Fancy indexing**. Zarr arrays now support NumPy-style fancy indexing with - arrays of integer coordinates. This is equivalent to using zarr.Array.vindex. - Mixing slices and integer arrays is not supported. - By :user:`Juan Nunez-Iglesias `; :issue:`725`. - -* **New base class**. This release of Zarr Python introduces a new - ``BaseStore`` class that all provided store classes implemented in Zarr - Python now inherit from. This is done as part of refactoring to enable future - support of the Zarr version 3 spec. Existing third-party stores that are a - MutableMapping (e.g. dict) can be converted to a new-style key/value store - inheriting from ``BaseStore`` by passing them as the argument to the new - ``zarr.storage.KVStore`` class. For backwards compatibility, various - higher-level array creation and convenience functions still accept plain - Python dicts or other mutable mappings for the ``store`` argument, but will - internally convert these to a ``KVStore``. - By :user:`Gregory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. - -* Allow to assign array ``fill_values`` and update metadata accordingly. - By :user:`Ryan Abernathey `, :issue:`662`. - -* Allow to update array fill_values - By :user:`Matthias Bussonnier ` :issue:`665`. - -Bug fixes -~~~~~~~~~ - -* Fix bug where the checksum of zipfiles is wrong - By :user:`Oren Watson ` :issue:`930`. - -* Fix consolidate_metadata with FSStore. - By :user:`Joe Hamman ` :issue:`916`. - -* Unguarded next inside generator. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`889`. - -Documentation -~~~~~~~~~~~~~ - -* Update docs creation of dev env. - By :user:`Ray Bell ` :issue:`921`. - -* Update docs to use ``python -m pytest``. - By :user:`Ray Bell ` :issue:`923`. - -* Fix versionadded tag in zarr.Array docstring. - By :user:`Juan Nunez-Iglesias ` :issue:`852`. - -* Doctest seem to be stricter now, updating tostring() to tobytes(). - By :user:`John Kirkham ` :issue:`907`. - -* Minor doc fix. - By :user:`Mads R. B. Kristensen ` :issue:`937`. - -Maintenance -~~~~~~~~~~~ - -* Upgrade MongoDB in test env. - By :user:`Joe Hamman ` :issue:`939`. - -* Pass dimension_separator on fixture generation. - By :user:`Josh Moore ` :issue:`858`. - -* Activate Python 3.9 in GitHub Actions. - By :user:`Josh Moore ` :issue:`859`. - -* Drop shortcut ``fsspec[s3]`` for dependency. - By :user:`Josh Moore ` :issue:`920`. - -* and a swath of code-linting improvements by :user:`Dimitri Papadopoulos Orfanos `: - - - Unnecessary comprehension (:issue:`899`) - - - Unnecessary ``None`` provided as default (:issue:`900`) - - - use an if ``expression`` instead of `and`/`or` (:issue:`888`) - - - Remove unnecessary literal (:issue:`891`) - - - Decorate a few method with `@staticmethod` (:issue:`885`) - - - Drop unneeded ``return`` (:issue:`884`) - - - Drop explicit ``object`` inheritance from ``class``-es (:issue:`886`) - - - Unnecessary comprehension (:issue:`883`) - - - Codespell configuration (:issue:`882`) - - - Fix typos found by codespell (:issue:`880`) - - - Proper C-style formatting for integer (:issue:`913`) - - - Add LGTM.com / DeepSource.io configuration files (:issue:`909`) - -.. _release_2.10.3: - -2.10.3 ------- - -Bug fixes -~~~~~~~~~ - -* N5 keywords now emit UserWarning instead of raising a ValueError. - By :user:`Boaz Mohar `; :issue:`860`. - -* blocks_to_decompress not used in read_part function. - By :user:`Boaz Mohar `; :issue:`861`. - -* defines blocksize for array, updates hexdigest values. - By :user:`Andrew Fulton `; :issue:`867`. - -* Fix test failure on Debian and conda-forge builds. - By :user:`Josh Moore `; :issue:`871`. - -.. _release_2.10.2: - -2.10.2 ------- - -Bug fixes -~~~~~~~~~ - -* Fix NestedDirectoryStore datasets without dimension_separator metadata. - By :user:`Josh Moore `; :issue:`850`. - -.. _release_2.10.1: - -2.10.1 ------- - -Bug fixes -~~~~~~~~~ - -* Fix regression by setting normalize_keys=False in fsstore constructor. - By :user:`Davis Bennett `; :issue:`842`. - -.. _release_2.10.0: - -2.10.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Add N5FSStore. - By :user:`Davis Bennett `; :issue:`793`. - -Bug fixes -~~~~~~~~~ - -* Ignore None dim_separators in save_array. - By :user:`Josh Moore `; :issue:`831`. - -.. _release_2.9.5: - -2.9.5 ------ - -Bug fixes -~~~~~~~~~ - -* Fix FSStore.listdir behavior for nested directories. - By :user:`Gregory Lee `; :issue:`802`. - -.. _release_2.9.4: - -2.9.4 ------ - -Bug fixes -~~~~~~~~~ - -* Fix structured arrays that contain objects - By :user: `Attila Bergou `; :issue: `806` - -.. _release_2.9.3: - -2.9.3 ------ - -Maintenance -~~~~~~~~~~~ - -* Mark the fact that some tests that require ``fsspec``, without compromising the code coverage score. - By :user:`Ben Williams `; :issue:`823`. - -* Only inspect alternate node type if desired isn't present. - By :user:`Trevor Manz `; :issue:`696`. - -.. _release_2.9.2: - -2.9.2 ------ - -Maintenance -~~~~~~~~~~~ - -* Correct conda-forge deployment of Zarr by fixing some Zarr tests. - By :user:`Ben Williams `; :issue:`821`. - -.. _release_2.9.1: - -2.9.1 ------ - -Maintenance -~~~~~~~~~~~ - -* Correct conda-forge deployment of Zarr. - By :user:`Josh Moore `; :issue:`819`. - -.. _release_2.9.0: - -2.9.0 ------ - -This release of Zarr Python is the first release of Zarr to not support Python 3.6. - -Enhancements -~~~~~~~~~~~~ - -* Update ABSStore for compatibility with newer `azure.storage.blob`. - By :user:`Tom Augspurger `; :issue:`759`. - -* Pathlib support. - By :user:`Chris Barnes `; :issue:`768`. - -Documentation -~~~~~~~~~~~~~ - -* Clarify that arbitrary key/value pairs are OK for attributes. - By :user:`Stephan Hoyer `; :issue:`751`. - -* Clarify how to manually convert a DirectoryStore to a ZipStore. - By :user:`pmav99 `; :issue:`763`. - -Bug fixes -~~~~~~~~~ - -* Fix dimension_separator support. - By :user:`Josh Moore `; :issue:`775`. - -* Extract ABSStore to zarr._storage.absstore. - By :user:`Josh Moore `; :issue:`781`. - -* avoid NumPy 1.21.0 due to https://github.com/numpy/numpy/issues/19325 - By :user:`Gregory Lee `; :issue:`791`. - -Maintenance -~~~~~~~~~~~ - -* Drop 3.6 builds. - By :user:`Josh Moore `; :issue:`774`, :issue:`778`. - -* Fix build with Sphinx 4. - By :user:`Elliott Sales de Andrade `; :issue:`799`. - -* TST: add missing assert in test_hexdigest. - By :user:`Gregory Lee `; :issue:`801`. - -.. _release_2.8.3: - -2.8.3 ------ - -Bug fixes -~~~~~~~~~ - -* FSStore: default to normalize_keys=False - By :user:`Josh Moore `; :issue:`755`. -* ABSStore: compatibility with ``azure.storage.python>=12`` - By :user:`Tom Augspurger `; :issue:`618` - - -.. _release_2.8.2: - -2.8.2 ------ - -Documentation -~~~~~~~~~~~~~ - -* Add section on rechunking to tutorial - By :user:`David Baddeley `; :issue:`730`. - -Bug fixes -~~~~~~~~~ - -* Expand FSStore tests and fix implementation issues - By :user:`Davis Bennett `; :issue:`709`. - -Maintenance -~~~~~~~~~~~ - -* Updated ipytree warning for jlab3 - By :user:`Ian Hunt-Isaak `; :issue:`721`. - -* b170a48a - (issue-728, copy-nested) Updated ipytree warning for jlab3 (#721) (3 weeks ago) -* Activate dependabot - By :user:`Josh Moore `; :issue:`734`. - -* Update Python classifiers (Zarr is stable!) - By :user:`Josh Moore `; :issue:`731`. - -.. _release_2.8.1: - -2.8.1 ------ - -Bug fixes -~~~~~~~~~ - -* raise an error if create_dataset's dimension_separator is inconsistent - By :user:`Gregory R. Lee `; :issue:`724`. - -.. _release_2.8.0: - -2.8.0 ------ - -V2 Specification Update -~~~~~~~~~~~~~~~~~~~~~~~ - -* Introduce optional dimension_separator .zarray key for nested chunks. - By :user:`Josh Moore `; :issue:`715`, :issue:`716`. - -.. _release_2.7.1: - -2.7.1 ------ - -Bug fixes -~~~~~~~~~ - -* Update Array to respect FSStore's key_separator (#718) - By :user:`Gregory R. Lee `; :issue:`718`. - -.. _release_2.7.0: - -2.7.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* Start stop for iterator (`islice()`) - By :user:`Sebastian Grill `; :issue:`621`. - -* Add capability to partially read and decompress chunks - By :user:`Andrew Fulton `; :issue:`667`. - -Bug fixes -~~~~~~~~~ - -* Make DirectoryStore __setitem__ resilient against antivirus file locking - By :user:`Eric Younkin `; :issue:`698`. - -* Compare test data's content generally - By :user:`John Kirkham `; :issue:`436`. - -* Fix dtype usage in zarr/meta.py - By :user:`Josh Moore `; :issue:`700`. - -* Fix FSStore key_seperator usage - By :user:`Josh Moore `; :issue:`669`. - -* Simplify text handling in DB Store - By :user:`John Kirkham `; :issue:`670`. - -* GitHub Actions migration - By :user:`Matthias Bussonnier `; - :issue:`641`, :issue:`671`, :issue:`674`, :issue:`676`, :issue:`677`, :issue:`678`, - :issue:`679`, :issue:`680`, :issue:`682`, :issue:`684`, :issue:`685`, :issue:`686`, - :issue:`687`, :issue:`695`, :issue:`706`. - -.. _release_2.6.1: - -2.6.1 ------ - -* Minor build fix - By :user:`Matthias Bussonnier `; :issue:`666`. - -.. _release_2.6.0: - -2.6.0 ------ - -This release of Zarr Python is the first release of Zarr to not support Python 3.5. - -* End Python 3.5 support. - By :user:`Chris Barnes `; :issue:`602`. - -* Fix ``open_group/open_array`` to allow opening of read-only store with - ``mode='r'`` :issue:`269` - -* Add `Array` tests for FSStore. - By :user:`Andrew Fulton `; :issue: `644`. - -* fix a bug in which ``attrs`` would not be copied on the root when using ``copy_all``; :issue:`613` - -* Fix ``FileNotFoundError`` with dask/s3fs :issue:`649` - -* Fix flaky fixture in test_storage.py :issue:`652` - -* Fix FSStore getitems fails with arrays that have a 0 length shape dimension :issue:`644` - -* Use async to fetch/write result concurrently when possible. :issue:`536`, See `this comment - `_ for some performance analysis - showing order of magnitude faster response in some benchmark. - -See `this link `_ -for the full list of closed and merged PR tagged with the 2.6 milestone. - -* Add ability to partially read and decompress arrays, see :issue:`667`. It is - only available to chunks stored using fsspec and using Blosc as a compressor. - - For certain analysis case when only a small portion of chunks is needed it can - be advantageous to only access and decompress part of the chunks. Doing - partial read and decompression add high latency to many of the operation so - should be used only when the subset of the data is small compared to the full - chunks and is stored contiguously (that is to say either last dimensions for C - layout, firsts for F). Pass ``partial_decompress=True`` as argument when - creating an ``Array``, or when using ``open_array``. No option exists yet to - apply partial read and decompress on a per-operation basis. - -.. _release_2.5.0: - -2.5.0 ------ - -This release will be the last to support Python 3.5, next version of Zarr will be Python 3.6+. - -* `DirectoryStore` now uses `os.scandir`, which should make listing large store - faster, :issue:`563` - -* Remove a few remaining Python 2-isms. - By :user:`Poruri Sai Rahul `; :issue:`393`. - -* Fix minor bug in `N5Store`. - By :user:`gsakkis`, :issue:`550`. - -* Improve error message in Jupyter when trying to use the ``ipytree`` widget - without ``ipytree`` installed. - By :user:`Zain Patel `; :issue:`537` - -* Add typing information to many of the core functions :issue:`589` - -* Explicitly close stores during testing. - By :user:`Elliott Sales de Andrade `; :issue:`442` - -* Many of the convenience functions to emit errors (``err_*`` from - ``zarr.errors`` have been replaced by ``ValueError`` subclasses. The corresponding - ``err_*`` function have been removed. :issue:`590`, :issue:`614`) - -* Improve consistency of terminology regarding arrays and datasets in the - documentation. - By :user:`Josh Moore `; :issue:`571`. - -* Added support for generic URL opening by ``fsspec``, where the URLs have the - form "protocol://[server]/path" or can be chained URls with "::" separators. - The additional argument ``storage_options`` is passed to the backend, see - the ``fsspec`` docs. - By :user:`Martin Durant `; :issue:`546` - -* Added support for fetching multiple items via ``getitems`` method of a - store, if it exists. This allows for concurrent fetching of data blocks - from stores that implement this; presently HTTP, S3, GCS. Currently only - applies to reading. - By :user:`Martin Durant `; :issue:`606` - -* Efficient iteration expanded with option to pass start and stop index via - ``array.islice``. - By :user:`Sebastian Grill `, :issue:`615`. - -.. _release_2.4.0: - -2.4.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* Add key normalization option for ``DirectoryStore``, ``NestedDirectoryStore``, - ``TempStore``, and ``N5Store``. - By :user:`James Bourbeau `; :issue:`459`. - -* Add ``recurse`` keyword to ``Group.array_keys`` and ``Group.arrays`` methods. - By :user:`James Bourbeau `; :issue:`458`. - -* Use uniform chunking for all dimensions when specifying ``chunks`` as an integer. - Also adds support for specifying ``-1`` to chunk across an entire dimension. - By :user:`James Bourbeau `; :issue:`456`. - -* Rename ``DictStore`` to ``MemoryStore``. - By :user:`James Bourbeau `; :issue:`455`. - -* Rewrite ``.tree()`` pretty representation to use ``ipytree``. - Allows it to work in both the Jupyter Notebook and JupyterLab. - By :user:`John Kirkham `; :issue:`450`. - -* Do not rename Blosc parameters in n5 backend and add `blocksize` parameter, - compatible with n5-blosc. By :user:`axtimwalde`, :issue:`485`. - -* Update ``DirectoryStore`` to create files with more permissive permissions. - By :user:`Eduardo Gonzalez ` and :user:`James Bourbeau `; :issue:`493` - -* Use ``math.ceil`` for scalars. - By :user:`John Kirkham `; :issue:`500`. - -* Ensure contiguous data using ``astype``. - By :user:`John Kirkham `; :issue:`513`. - -* Refactor out ``_tofile``/``_fromfile`` from ``DirectoryStore``. - By :user:`John Kirkham `; :issue:`503`. - -* Add ``__enter__``/``__exit__`` methods to ``Group`` for ``h5py.File`` compatibility. - By :user:`Chris Barnes `; :issue:`509`. - -Bug fixes -~~~~~~~~~ - -* Fix Sqlite Store Wrong Modification. - By :user:`Tommy Tran `; :issue:`440`. - -* Add intermediate step (using ``zipfile.ZipInfo`` object) to write - inside ``ZipStore`` to solve too restrictive permission issue. - By :user:`Raphael Dussin `; :issue:`505`. - -* Fix '/' prepend bug in ``ABSStore``. - By :user:`Shikhar Goenka `; :issue:`525`. - -Documentation -~~~~~~~~~~~~~ -* Fix hyperlink in ``README.md``. - By :user:`Anderson Banihirwe `; :issue:`531`. - -* Replace "nuimber" with "number". - By :user:`John Kirkham `; :issue:`512`. - -* Fix azure link rendering in tutorial. - By :user:`James Bourbeau `; :issue:`507`. - -* Update ``README`` file to be more detailed. - By :user:`Zain Patel `; :issue:`495`. - -* Import blosc from numcodecs in tutorial. - By :user:`James Bourbeau `; :issue:`491`. - -* Adds logo to docs. - By :user:`James Bourbeau `; :issue:`462`. - -* Fix N5 link in tutorial. - By :user:`James Bourbeau `; :issue:`480`. - -* Fix typo in code snippet. - By :user:`Joe Jevnik `; :issue:`461`. - -* Fix URLs to point to zarr-python - By :user:`John Kirkham `; :issue:`453`. - -Maintenance -~~~~~~~~~~~ - -* Add documentation build to CI. - By :user:`James Bourbeau `; :issue:`516`. - -* Use ``ensure_ndarray`` in a few more places. - By :user:`John Kirkham `; :issue:`506`. - -* Support Python 3.8. - By :user:`John Kirkham `; :issue:`499`. - -* Require Numcodecs 0.6.4+ to use text handling functionality from it. - By :user:`John Kirkham `; :issue:`497`. - -* Updates tests to use ``pytest.importorskip``. - By :user:`James Bourbeau `; :issue:`492` - -* Removed support for Python 2. - By :user:`jhamman`; :issue:`393`, :issue:`470`. - -* Upgrade dependencies in the test matrices and resolve a - compatibility issue with testing against the Azure Storage - Emulator. By :user:`alimanfoo`; :issue:`468`, :issue:`467`. - -* Use ``unittest.mock`` on Python 3. - By :user:`Elliott Sales de Andrade `; :issue:`426`. - -* Drop ``decode`` from ``ConsolidatedMetadataStore``. - By :user:`John Kirkham `; :issue:`452`. - - -.. _release_2.3.2: - -2.3.2 ------ - -Enhancements -~~~~~~~~~~~~ - -* Use ``scandir`` in ``DirectoryStore``'s ``getsize`` method. - By :user:`John Kirkham `; :issue:`431`. - -Bug fixes -~~~~~~~~~ - -* Add and use utility functions to simplify reading and writing JSON. - By :user:`John Kirkham `; :issue:`429`, :issue:`430`. - -* Fix ``collections``'s ``DeprecationWarning``\ s. - By :user:`John Kirkham `; :issue:`432`. - -* Fix tests on big endian machines. - By :user:`Elliott Sales de Andrade `; :issue:`427`. - - -.. _release_2.3.1: - -2.3.1 ------ - -Bug fixes -~~~~~~~~~ - -* Makes ``azure-storage-blob`` optional for testing. - By :user:`John Kirkham `; :issue:`419`, :issue:`420`. - - -.. _release_2.3.0: - -2.3.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* New storage backend, backed by Azure Blob Storage, class :class:`zarr.storage.ABSStore`. - All data is stored as block blobs. By :user:`Shikhar Goenka `, - :user:`Tim Crone ` and :user:`Zain Patel `; :issue:`345`. - -* Add "consolidated" metadata as an experimental feature: use - :func:`zarr.convenience.consolidate_metadata` to copy all metadata from the various - metadata keys within a dataset hierarchy under a single key, and - :func:`zarr.convenience.open_consolidated` to use this single key. This can greatly - cut down the number of calls to the storage backend, and so remove a lot of overhead - for reading remote data. - By :user:`Martin Durant `, :user:`Alistair Miles `, - :user:`Ryan Abernathey `, :issue:`268`, :issue:`332`, :issue:`338`. - -* Support has been added for structured arrays with sub-array shape and/or nested fields. By - :user:`Tarik Onalan `, :issue:`111`, :issue:`296`. - -* Adds the SQLite-backed :class:`zarr.storage.SQLiteStore` class enabling an - SQLite database to be used as the backing store for an array or group. - By :user:`John Kirkham `, :issue:`368`, :issue:`365`. - -* Efficient iteration over arrays by decompressing chunkwise. - By :user:`Jerome Kelleher `, :issue:`398`, :issue:`399`. - -* Adds the Redis-backed :class:`zarr.storage.RedisStore` class enabling a - Redis database to be used as the backing store for an array or group. - By :user:`Joe Hamman `, :issue:`299`, :issue:`372`. - -* Adds the MongoDB-backed :class:`zarr.storage.MongoDBStore` class enabling a - MongoDB database to be used as the backing store for an array or group. - By :user:`Noah D Brenowitz `, :user:`Joe Hamman `, - :issue:`299`, :issue:`372`, :issue:`401`. - -* **New storage class for N5 containers**. The :class:`zarr.n5.N5Store` has been - added, which uses :class:`zarr.storage.NestedDirectoryStore` to support - reading and writing from and to N5 containers. - By :user:`Jan Funke ` and :user:`John Kirkham `. - -Bug fixes -~~~~~~~~~ - -* The implementation of the :class:`zarr.storage.DirectoryStore` class has been modified to - ensure that writes are atomic and there are no race conditions where a chunk might appear - transiently missing during a write operation. By :user:`sbalmer `, :issue:`327`, - :issue:`263`. - -* Avoid raising in :class:`zarr.storage.DirectoryStore`'s ``__setitem__`` when file already exists. - By :user:`Justin Swaney `, :issue:`272`, :issue:`318`. - -* The required version of the `Numcodecs`_ package has been upgraded - to 0.6.2, which has enabled some code simplification and fixes a failing test involving - msgpack encoding. By :user:`John Kirkham `, :issue:`361`, :issue:`360`, :issue:`352`, - :issue:`355`, :issue:`324`. - -* Failing tests related to pickling/unpickling have been fixed. By :user:`Ryan Williams `, - :issue:`273`, :issue:`308`. - -* Corrects handling of ``NaT`` in ``datetime64`` and ``timedelta64`` in various - compressors (by :user:`John Kirkham `; :issue:`344`). - -* Ensure ``DictStore`` contains only ``bytes`` to facilitate comparisons and protect against writes. - By :user:`John Kirkham `, :issue:`350`. - -* Test and fix an issue (w.r.t. fill values) when storing complex data to ``Array``. - By :user:`John Kirkham `, :issue:`363`. - -* Always use a ``tuple`` when indexing a NumPy ``ndarray``. - By :user:`John Kirkham `, :issue:`376`. - -* Ensure when ``Array`` uses a ``dict``-based chunk store that it only contains - ``bytes`` to facilitate comparisons and protect against writes. Drop the copy - for the no filter/compressor case as this handles that case. - By :user:`John Kirkham `, :issue:`359`. - -Maintenance -~~~~~~~~~~~ - -* Simplify directory creation and removal in ``DirectoryStore.rename``. - By :user:`John Kirkham `, :issue:`249`. - -* CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and - upgrade all pinned package requirements. :user:`Alistair Miles `, :issue:`308`. - -* Start using pyup.io to maintain dependencies. - :user:`Alistair Miles `, :issue:`326`. - -* Configure flake8 line limit generally. - :user:`John Kirkham `, :issue:`335`. - -* Add missing coverage pragmas. - :user:`John Kirkham `, :issue:`343`, :issue:`355`. - -* Fix missing backslash in docs. - :user:`John Kirkham `, :issue:`254`, :issue:`353`. - -* Include tests for stores' ``popitem`` and ``pop`` methods. - By :user:`John Kirkham `, :issue:`378`, :issue:`380`. - -* Include tests for different compressors, endianness, and attributes. - By :user:`John Kirkham `, :issue:`378`, :issue:`380`. - -* Test validity of stores' contents. - By :user:`John Kirkham `, :issue:`359`, :issue:`408`. - - -.. _release_2.2.0: - -2.2.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* **Advanced indexing**. The ``Array`` class has several new methods and - properties that enable a selection of items in an array to be retrieved or - updated. See the :ref:`user-guide-indexing` tutorial section for more - information. There is also a `notebook - `_ - with extended examples and performance benchmarks. :issue:`78`, :issue:`89`, - :issue:`112`, :issue:`172`. - -* **New package for compressor and filter codecs**. The classes previously - defined in the :mod:`zarr.codecs` module have been factored out into a - separate package called `Numcodecs`_. The `Numcodecs`_ package also includes - several new codec classes not previously available in Zarr, including - compressor codecs for Zstd and LZ4. This change is backwards-compatible with - existing code, as all codec classes defined by Numcodecs are imported into the - :mod:`zarr.codecs` namespace. However, it is recommended to import codecs from - the new package, see the tutorial sections on :ref:`user-guide-compress` and - :ref:`user-guide-filters` for examples. With contributions by - :user:`John Kirkham `; :issue:`74`, :issue:`102`, :issue:`120`, - :issue:`123`, :issue:`139`. - -* **New storage class for DBM-style databases**. The - :class:`zarr.storage.DBMStore` class enables any DBM-style database such as gdbm, - ndbm or Berkeley DB, to be used as the backing store for an array or group. See the - tutorial section on :ref:`user-guide-storage` for some examples. :issue:`133`, - :issue:`186`. - -* **New storage class for LMDB databases**. The :class:`zarr.storage.LMDBStore` class - enables an LMDB "Lightning" database to be used as the backing store for an array or - group. :issue:`192`. - -* **New storage class using a nested directory structure for chunk files**. The - :class:`zarr.storage.NestedDirectoryStore` has been added, which is similar to - the existing :class:`zarr.storage.DirectoryStore` class but nests chunk files - for multidimensional arrays into sub-directories. :issue:`155`, :issue:`177`. - -* **New tree() method for printing hierarchies**. The ``Group`` class has a new - :func:`zarr.hierarchy.Group.tree` method which enables a tree representation of - a group hierarchy to be printed. Also provides an interactive tree - representation when used within a Jupyter notebook. See the - :ref:`user-guide-diagnostics` tutorial section for examples. By - :user:`John Kirkham `; :issue:`82`, :issue:`140`, :issue:`184`. - -* **Visitor API**. The ``Group`` class now implements the h5py visitor API, see - docs for the :func:`zarr.hierarchy.Group.visit`, - :func:`zarr.hierarchy.Group.visititems` and - :func:`zarr.hierarchy.Group.visitvalues` methods. By - :user:`John Kirkham `, :issue:`92`, :issue:`122`. - -* **Viewing an array as a different dtype**. The ``Array`` class has a new - :func:`zarr.Array.astype` method, which is a convenience that enables an - array to be viewed as a different dtype. By :user:`John Kirkham `, - :issue:`94`, :issue:`96`. - -* **New open(), save(), load() convenience functions**. The function - :func:`zarr.convenience.open` provides a convenient way to open a persistent - array or group, using either a ``DirectoryStore`` or ``ZipStore`` as the backing - store. The functions :func:`zarr.convenience.save` and - :func:`zarr.convenience.load` are also available and provide a convenient way to - save an entire NumPy array to disk and load back into memory later. See the - tutorial section :ref:`user-guide-persist` for examples. :issue:`104`, - :issue:`105`, :issue:`141`, :issue:`181`. - -* **IPython completions**. The ``Group`` class now implements ``__dir__()`` and - ``_ipython_key_completions_()`` which enables tab-completion for group members - to be used in any IPython interactive environment. :issue:`170`. - -* **New info property; changes to __repr__**. The ``Group`` and - ``Array`` classes have a new ``info`` property which can be used to print - diagnostic information, including compression ratio where available. See the - tutorial section on :ref:`user-guide-diagnostics` for examples. The string - representation (``__repr__``) of these classes has been simplified to ensure - it is cheap and quick to compute in all circumstances. :issue:`83`, - :issue:`115`, :issue:`132`, :issue:`148`. - -* **Chunk options**. When creating an array, ``chunks=False`` can be specified, - which will result in an array with a single chunk only. Alternatively, - ``chunks=True`` will trigger an automatic chunk shape guess. See - :ref:`user-guide-chunks` for more on the ``chunks`` parameter. :issue:`106`, - :issue:`107`, :issue:`183`. - -* **Zero-dimensional arrays** and are now supported; by - :user:`Prakhar Goel `, :issue:`154`, :issue:`161`. - -* **Arrays with one or more zero-length dimensions** are now fully supported; by - :user:`Prakhar Goel `, :issue:`150`, :issue:`154`, :issue:`160`. - -* **The .zattrs key is now optional** and will now only be created when the first - custom attribute is set; :issue:`121`, :issue:`200`. - -* **New Group.move() method** supports moving a sub-group or array to a different - location within the same hierarchy. By :user:`John Kirkham `, - :issue:`191`, :issue:`193`, :issue:`196`. - -* **ZipStore is now thread-safe**; :issue:`194`, :issue:`192`. - -* **New Array.hexdigest() method** computes an ``Array``'s hash with ``hashlib``. - By :user:`John Kirkham `, :issue:`98`, :issue:`203`. - -* **Improved support for object arrays**. In previous versions of Zarr, - creating an array with ``dtype=object`` was possible but could under certain - circumstances lead to unexpected errors and/or segmentation faults. To make it easier - to properly configure an object array, a new ``object_codec`` parameter has been - added to array creation functions. See the tutorial section on :ref:`user-guide-objects` - for more information and examples. Also, runtime checks have been added in both Zarr - and Numcodecs so that segmentation faults are no longer possible, even with a badly - configured array. This API change is backwards compatible and previous code that created - an object array and provided an object codec via the ``filters`` parameter will - continue to work, however a warning will be raised to encourage use of the - ``object_codec`` parameter. :issue:`208`, :issue:`212`. - -* **Added support for datetime64 and timedelta64 data types**; - :issue:`85`, :issue:`215`. - -* **Array and group attributes are now cached by default** to improve performance with - slow stores, e.g., stores accessing data via the network; :issue:`220`, :issue:`218`, - :issue:`204`. - -* **New LRUStoreCache class**. The class :class:`zarr.storage.LRUStoreCache` has been - added and provides a means to locally cache data in memory from a store that may be - slow, e.g., a store that retrieves data from a remote server via the network; - :issue:`223`. - -* **New copy functions**. The new functions :func:`zarr.convenience.copy` and - :func:`zarr.convenience.copy_all` provide a way to copy groups and/or arrays - between HDF5 and Zarr, or between two Zarr groups. The - :func:`zarr.convenience.copy_store` provides a more efficient way to copy - data directly between two Zarr stores. :issue:`87`, :issue:`113`, - :issue:`137`, :issue:`217`. - -Bug fixes -~~~~~~~~~ - -* Fixed bug where ``read_only`` keyword argument was ignored when creating an - array; :issue:`151`, :issue:`179`. - -* Fixed bugs when using a ``ZipStore`` opened in 'w' mode; :issue:`158`, - :issue:`182`. - -* Fill values can now be provided for fixed-length string arrays; :issue:`165`, - :issue:`176`. - -* Fixed a bug where the number of chunks initialized could be counted - incorrectly; :issue:`97`, :issue:`174`. - -* Fixed a bug related to the use of an ellipsis (...) in indexing statements; - :issue:`93`, :issue:`168`, :issue:`172`. - -* Fixed a bug preventing use of other integer types for indexing; :issue:`143`, - :issue:`147`. - -Documentation -~~~~~~~~~~~~~ - -* Some changes have been made to the Zarr Specification v2 document to clarify - ambiguities and add some missing information. These changes do not break compatibility - with any of the material as previously implemented, and so the changes have been made - in-place in the document without incrementing the document version number. See the - section on changes in the specification document for more information. -* A new :ref:`user-guide-indexing` section has been added to the tutorial. -* A new :ref:`user-guide-strings` section has been added to the tutorial - (:issue:`135`, :issue:`175`). -* The :ref:`user-guide-chunks` tutorial section has been reorganised and updated. -* The :ref:`user-guide-persist` and :ref:`user-guide-storage` tutorial sections have - been updated with new examples (:issue:`100`, :issue:`101`, :issue:`103`). -* A new tutorial section on :ref:`user-guide-pickle` has been added (:issue:`91`). -* A new tutorial section on :ref:`user-guide-datetime` has been added. -* A new tutorial section on :ref:`user-guide-diagnostics` has been added. -* The tutorial sections on :ref:`user-guide-sync` and :ref:`user-guide-tips-blosc` have been - updated to provide information about how to avoid program hangs when using the Blosc - compressor with multiple processes (:issue:`199`, :issue:`201`). - -Maintenance -~~~~~~~~~~~ - -* A data fixture has been included in the test suite to ensure data format - compatibility is maintained; :issue:`83`, :issue:`146`. -* The test suite has been migrated from nosetests to pytest; :issue:`189`, :issue:`225`. -* Various continuous integration updates and improvements; :issue:`118`, :issue:`124`, - :issue:`125`, :issue:`126`, :issue:`109`, :issue:`114`, :issue:`171`. -* Bump numcodecs dependency to 0.5.3, completely remove nose dependency, :issue:`237`. -* Fix compatibility issues with NumPy 1.14 regarding fill values for structured arrays, - :issue:`222`, :issue:`238`, :issue:`239`. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Code was contributed to this release by :user:`Alistair Miles `, :user:`John -Kirkham ` and :user:`Prakhar Goel `. - -Documentation was contributed to this release by :user:`Mamy Ratsimbazafy ` -and :user:`Charles Noyes `. - -Thank you to :user:`John Kirkham `, :user:`Stephan Hoyer `, -:user:`Francesc Alted `, and :user:`Matthew Rocklin ` for code -reviews and/or comments on pull requests. - -.. _release_2.1.4: - -2.1.4 ------ - -* Resolved an issue where calling ``hasattr`` on a ``Group`` object erroneously - returned a ``KeyError``. By :user:`Vincent Schut `; :issue:`88`, - :issue:`95`. - -.. _release_2.1.3: - -2.1.3 ------ - -* Resolved an issue with :func:`zarr.creation.array` where dtype was given as - None (:issue:`80`). - -.. _release_2.1.2: - -2.1.2 ------ - -* Resolved an issue when no compression is used and chunks are stored in memory - (:issue:`79`). - -.. _release_2.1.1: - -2.1.1 ------ - -Various minor improvements, including: ``Group`` objects support member access -via dot notation (``__getattr__``); fixed metadata caching for ``Array.shape`` -property and derivatives; added ``Array.ndim`` property; fixed -``Array.__array__`` method arguments; fixed bug in pickling ``Array`` state; -fixed bug in pickling ``ThreadSynchronizer``. - -.. _release_2.1.0: - -2.1.0 ------ - -* Group objects now support member deletion via ``del`` statement - (:issue:`65`). -* Added :class:`zarr.storage.TempStore` class for convenience to provide - storage via a temporary directory - (:issue:`59`). -* Fixed performance issues with :class:`zarr.storage.ZipStore` class - (:issue:`66`). -* The Blosc extension has been modified to return bytes instead of array - objects from compress and decompress function calls. This should - improve compatibility and also provides a small performance increase for - compressing high compression ratio data - (:issue:`55`). -* Added ``overwrite`` keyword argument to array and group creation methods - on the :class:`zarr.hierarchy.Group` class - (:issue:`71`). -* Added ``cache_metadata`` keyword argument to array creation methods. -* The functions :func:`zarr.creation.open_array` and - :func:`zarr.hierarchy.open_group` now accept any store as first argument - (:issue:`56`). - -.. _release_2.0.1: - -2.0.1 ------ - -The bundled Blosc library has been upgraded to version 1.11.1. - -.. _release_2.0.0: - -2.0.0 ------ - -Hierarchies -~~~~~~~~~~~ - -Support has been added for organizing arrays into hierarchies via groups. See -the tutorial section on :ref:`user-guide-groups` and the :mod:`zarr.hierarchy` -API docs for more information. - -Filters -~~~~~~~ - -Support has been added for configuring filters to preprocess chunk data prior -to compression. See the tutorial section on :ref:`user-guide-filters` and the -:mod:`zarr.codecs` API docs for more information. - -Other changes -~~~~~~~~~~~~~ - -To accommodate support for hierarchies and filters, the Zarr metadata format -has been modified. See the ``spec_v2`` for more information. To migrate an -array stored using Zarr version 1.x, use the :func:`zarr.storage.migrate_1to2` -function. - -The bundled Blosc library has been upgraded to version 1.11.0. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Thanks to :user:`Matthew Rocklin `, :user:`Stephan Hoyer ` and -:user:`Francesc Alted ` for contributions and comments. - -.. _release_1.1.0: - -1.1.0 ------ - -* The bundled Blosc library has been upgraded to version 1.10.0. The 'zstd' - internal compression library is now available within Blosc. See the tutorial - section on :ref:`user-guide-compress` for an example. -* When using the Blosc compressor, the default internal compression library - is now 'lz4'. -* The default number of internal threads for the Blosc compressor has been - increased to a maximum of 8 (previously 4). -* Added convenience functions :func:`zarr.blosc.list_compressors` and - :func:`zarr.blosc.get_nthreads`. - -.. _release_1.0.0: - -1.0.0 ------ - -This release includes a complete re-organization of the code base. The -major version number has been bumped to indicate that there have been -backwards-incompatible changes to the API and the on-disk storage -format. However, Zarr is still in an early stage of development, so -please do not take the version number as an indicator of maturity. - -Storage -~~~~~~~ - -The main motivation for re-organizing the code was to create an -abstraction layer between the core array logic and data storage (:issue:`21`). -In this release, any -object that implements the ``MutableMapping`` interface can be used as -an array store. See the tutorial sections on :ref:`user-guide-persist` -and :ref:`user-guide-storage`, the ``spec_v1``, and the -:mod:`zarr.storage` module documentation for more information. - -Please note also that the file organization and file name conventions -used when storing a Zarr array in a directory on the file system have -changed. Persistent Zarr arrays created using previous versions of the -software will not be compatible with this version. See the -:mod:`zarr.storage` API docs and the ``spec_v1`` for more -information. - -Compression -~~~~~~~~~~~ - -An abstraction layer has also been created between the core array -logic and the code for compressing and decompressing array -chunks. This release still bundles the c-blosc library and uses Blosc -as the default compressor, however other compressors including zlib, -BZ2 and LZMA are also now supported via the Python standard -library. New compressors can also be dynamically registered for use -with Zarr. See the tutorial sections on :ref:`user-guide-compress` and -:ref:`user-guide-tips-blosc`, the ``spec_v1``, and the -:mod:`zarr.compressors` module documentation for more information. - -Synchronization -~~~~~~~~~~~~~~~ - -The synchronization code has also been refactored to create a layer of -abstraction, enabling Zarr arrays to be used in parallel computations -with a number of alternative synchronization methods. For more -information see the tutorial section on :ref:`user-guide-sync` and the -:mod:`zarr.sync` module documentation. - -Changes to the Blosc extension -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -NumPy is no longer a build dependency for the :mod:`zarr.blosc` Cython -extension, so setup.py will run even if NumPy is not already -installed, and should automatically install NumPy as a runtime -dependency. Manual installation of NumPy prior to installing Zarr is -still recommended, however, as the automatic installation of NumPy may -fail or be sub-optimal on some platforms. - -Some optimizations have been made within the :mod:`zarr.blosc` -extension to avoid unnecessary memory copies, giving a ~10-20% -performance improvement for multi-threaded compression operations. - -The :mod:`zarr.blosc` extension now automatically detects whether it -is running within a single-threaded or multi-threaded program and -adapts its internal behaviour accordingly (:issue:`27`). There is no need for -the user to make any API calls to switch Blosc between contextual and -non-contextual (global lock) mode. See also the tutorial section on -:ref:`user-guide-tips-blosc`. - -Other changes -~~~~~~~~~~~~~ - -The internal code for managing chunks has been rewritten to be more -efficient. Now no state is maintained for chunks outside of the array -store, meaning that chunks do not carry any extra memory overhead not -accounted for by the store. This negates the need for the "lazy" -option present in the previous release, and this has been removed. - -The memory layout within chunks can now be set as either "C" -(row-major) or "F" (column-major), which can help to provide better -compression for some data (:issue:`7`). See the tutorial -section on :ref:`user-guide-chunks-order` for more information. - -A bug has been fixed within the ``__getitem__`` and ``__setitem__`` -machinery for slicing arrays, to properly handle getting and setting -partial slices. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Thanks to :user:`Matthew Rocklin `, :user:`Stephan Hoyer `, -:user:`Francesc Alted `, :user:`Anthony Scopatz ` and -:user:`Martin Durant ` for contributions and comments. - -.. _release_0.4.0: - -0.4.0 ------ - -See `v0.4.0 release notes on GitHub -`_. - -.. _release_0.3.0: - -0.3.0 ------ - -See `v0.3.0 release notes on GitHub -`_. - -.. _Numcodecs: https://numcodecs.readthedocs.io/ diff --git a/docs/index.rst b/docs/index.rst index 0dcfd7f90f..6ab07b0693 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,8 +11,8 @@ Zarr-Python quickstart user-guide/index API reference + release-notes developers/index - developers/release about **Version**: |version| diff --git a/docs/release-notes.rst b/docs/release-notes.rst new file mode 100644 index 0000000000..175bd21aa5 --- /dev/null +++ b/docs/release-notes.rst @@ -0,0 +1,16 @@ +Release notes +============= + +.. _release_3.0.0: + +3.0.0 +----- + +3.0.0 is a new major release of Zarr-Python, with many breaking changes. +See the :ref:`v3 migration guide` for a listing of what's changed. + +Normal release note service will resume with further releases in the 3.0.0 +series. + +Release notes for the zarr-python 2.x and 1.x releases can be found here: +https://zarr.readthedocs.io/en/support-v2/release.html diff --git a/docs/user-guide/v3_migration.rst b/docs/user-guide/v3_migration.rst index 66fcca6d19..bda1ae64ed 100644 --- a/docs/user-guide/v3_migration.rst +++ b/docs/user-guide/v3_migration.rst @@ -1,3 +1,5 @@ +.. _v3 migration guide: + 3.0 Migration Guide =================== From 0328656b09b1395daaaba309798fefd78459c2ee Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 9 Jan 2025 01:57:19 -0700 Subject: [PATCH 011/160] Use dataclasses for ByteRangeRequests (#2585) * Use TypedDicts for more literate ByteRangeRequests * Update utility function * fixes sharding * Ignore mypy errors * Fix offset in _normalize_byte_range_index * Update get_partial_values for FsspecStore * Re-add fs._cat_ranges argument * Simplify typing * Update _normalize to return start, stop * Use explicit range * Use dataclasses * Update typing * Update docstring * Rename ExplicitRange to ExplicitByteRequest * Rename OffsetRange to OffsetByteRequest * Rename SuffixRange to SuffixByteRequest * Use match; case instead of if; elif * Revert "Use match; case instead of if; elif" This reverts commit a7d35f876b1b628b3216da61ee26ba0f3a9d9cf8. * Update ByteRangeRequest to ByteRequest * Remove ByteRange definition from common * Rename ExplicitByteRequest to RangeByteRequest * Provide more informative error message --------- Co-authored-by: Norman Rzepka --- src/zarr/abc/store.py | 50 ++++++++++++++++---- src/zarr/codecs/sharding.py | 24 ++++++---- src/zarr/core/common.py | 1 - src/zarr/storage/_common.py | 10 ++-- src/zarr/storage/_fsspec.py | 81 ++++++++++++++++++++------------- src/zarr/storage/_local.py | 43 ++++++++--------- src/zarr/storage/_logging.py | 6 +-- src/zarr/storage/_memory.py | 14 +++--- src/zarr/storage/_utils.py | 36 +++++++-------- src/zarr/storage/_wrapper.py | 8 ++-- src/zarr/storage/_zip.py | 33 ++++++++------ src/zarr/testing/stateful.py | 5 +- src/zarr/testing/store.py | 40 ++++++++++------ src/zarr/testing/strategies.py | 12 +++-- tests/test_store/test_fsspec.py | 3 +- 15 files changed, 221 insertions(+), 145 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index bd0a7ad503..e6a5518a4b 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from asyncio import gather +from dataclasses import dataclass from itertools import starmap from typing import TYPE_CHECKING, Protocol, runtime_checkable @@ -19,7 +20,34 @@ __all__ = ["ByteGetter", "ByteSetter", "Store", "set_or_delete"] -ByteRangeRequest: TypeAlias = tuple[int | None, int | None] + +@dataclass +class RangeByteRequest: + """Request a specific byte range""" + + start: int + """The start of the byte range request (inclusive).""" + end: int + """The end of the byte range request (exclusive).""" + + +@dataclass +class OffsetByteRequest: + """Request all bytes starting from a given byte offset""" + + offset: int + """The byte offset for the offset range request.""" + + +@dataclass +class SuffixByteRequest: + """Request up to the last `n` bytes""" + + suffix: int + """The number of bytes from the suffix to request.""" + + +ByteRequest: TypeAlias = RangeByteRequest | OffsetByteRequest | SuffixByteRequest class Store(ABC): @@ -141,14 +169,20 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: """Retrieve the value associated with a given key. Parameters ---------- key : str - byte_range : tuple[int | None, int | None], optional + byte_range : ByteRequest, optional + + ByteRequest may be one of the following. If not provided, all data associated with the key is retrieved. + + - RangeByteRequest(int, int): Request a specific range of bytes in the form (start, end). The end is exclusive. If the given range is zero-length or starts after the end of the object, an error will be returned. Additionally, if the range ends after the end of the object, the entire remainder of the object will be returned. Otherwise, the exact requested range will be returned. + - OffsetByteRequest(int): Request all bytes starting from a given byte offset. This is equivalent to bytes={int}- as an HTTP header. + - SuffixByteRequest(int): Request the last int bytes. Note that here, int is the size of the request, not the byte offset. This is equivalent to bytes=-{int} as an HTTP header. Returns ------- @@ -160,7 +194,7 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: """Retrieve possibly partial values from given key_ranges. @@ -338,7 +372,7 @@ def close(self) -> None: self._is_open = False async def _get_many( - self, requests: Iterable[tuple[str, BufferPrototype, ByteRangeRequest | None]] + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] ) -> AsyncGenerator[tuple[str, Buffer | None], None]: """ Retrieve a collection of objects from storage. In general this method does not guarantee @@ -416,17 +450,17 @@ async def getsize_prefix(self, prefix: str) -> int: @runtime_checkable class ByteGetter(Protocol): async def get( - self, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: ... @runtime_checkable class ByteSetter(Protocol): async def get( - self, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: ... - async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -> None: ... + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: ... async def delete(self) -> None: ... diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index a01145b3b2..160a74e892 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -17,7 +17,13 @@ Codec, CodecPipeline, ) -from zarr.abc.store import ByteGetter, ByteRangeRequest, ByteSetter +from zarr.abc.store import ( + ByteGetter, + ByteRequest, + ByteSetter, + RangeByteRequest, + SuffixByteRequest, +) from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.core.array_spec import ArrayConfig, ArraySpec @@ -77,7 +83,7 @@ class _ShardingByteGetter(ByteGetter): chunk_coords: ChunkCoords async def get( - self, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: assert byte_range is None, "byte_range is not supported within shards" assert ( @@ -90,7 +96,7 @@ async def get( class _ShardingByteSetter(_ShardingByteGetter, ByteSetter): shard_dict: ShardMutableMapping - async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -> None: + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: assert byte_range is None, "byte_range is not supported within shards" self.shard_dict[self.chunk_coords] = value @@ -129,7 +135,7 @@ def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None: if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64): return None else: - return (int(chunk_start), int(chunk_len)) + return (int(chunk_start), int(chunk_start + chunk_len)) def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: slice | None) -> None: localized_chunk = self._localize_chunk(chunk_coords) @@ -203,7 +209,7 @@ def create_empty( def __getitem__(self, chunk_coords: ChunkCoords) -> Buffer: chunk_byte_slice = self.index.get_chunk_slice(chunk_coords) if chunk_byte_slice: - return self.buf[chunk_byte_slice[0] : (chunk_byte_slice[0] + chunk_byte_slice[1])] + return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]] raise KeyError def __len__(self) -> int: @@ -504,7 +510,8 @@ async def _decode_partial_single( chunk_byte_slice = shard_index.get_chunk_slice(chunk_coords) if chunk_byte_slice: chunk_bytes = await byte_getter.get( - prototype=chunk_spec.prototype, byte_range=chunk_byte_slice + prototype=chunk_spec.prototype, + byte_range=RangeByteRequest(chunk_byte_slice[0], chunk_byte_slice[1]), ) if chunk_bytes: shard_dict[chunk_coords] = chunk_bytes @@ -696,11 +703,12 @@ async def _load_shard_index_maybe( shard_index_size = self._shard_index_size(chunks_per_shard) if self.index_location == ShardingCodecIndexLocation.start: index_bytes = await byte_getter.get( - prototype=numpy_buffer_prototype(), byte_range=(0, shard_index_size) + prototype=numpy_buffer_prototype(), + byte_range=RangeByteRequest(0, shard_index_size), ) else: index_bytes = await byte_getter.get( - prototype=numpy_buffer_prototype(), byte_range=(-shard_index_size, None) + prototype=numpy_buffer_prototype(), byte_range=SuffixByteRequest(shard_index_size) ) if index_bytes is not None: return await self._decode_shard_index(index_bytes, chunks_per_shard) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 7205b8c206..ad3316b619 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -31,7 +31,6 @@ ZATTRS_JSON = ".zattrs" ZMETADATA_V2_JSON = ".zmetadata" -ByteRangeRequest = tuple[int | None, int | None] BytesLike = bytes | bytearray | memoryview ShapeLike = tuple[int, ...] | int ChunkCoords = tuple[int, ...] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 523e470671..6ab539bb0a 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZGROUP_JSON, AccessModeLiteral, ZarrFormat from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError @@ -102,7 +102,7 @@ async def open( async def get( self, prototype: BufferPrototype | None = None, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: """ Read bytes from the store. @@ -111,7 +111,7 @@ async def get( ---------- prototype : BufferPrototype, optional The buffer prototype to use when reading the bytes. - byte_range : ByteRangeRequest, optional + byte_range : ByteRequest, optional The range of bytes to read. Returns @@ -123,7 +123,7 @@ async def get( prototype = default_buffer_prototype() return await self.store.get(self.path, prototype=prototype, byte_range=byte_range) - async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -> None: + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: """ Write bytes to the store. @@ -131,7 +131,7 @@ async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) - ---------- value : Buffer The buffer to write. - byte_range : ByteRangeRequest, optional + byte_range : ByteRequest, optional The range of bytes to write. If None, the entire buffer is written. Raises diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 89d80320dd..99c8c778e7 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -3,7 +3,13 @@ import warnings from typing import TYPE_CHECKING, Any -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) from zarr.storage._common import _dereference_path if TYPE_CHECKING: @@ -199,7 +205,7 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if not self._is_open: @@ -207,23 +213,26 @@ async def get( path = _dereference_path(self.path, key) try: - if byte_range: - # fsspec uses start/end, not start/length - start, length = byte_range - if start is not None and length is not None: - end = start + length - elif length is not None: - end = length - else: - end = None - value = prototype.buffer.from_bytes( - await ( - self.fs._cat_file(path, start=byte_range[0], end=end) - if byte_range - else self.fs._cat_file(path) + if byte_range is None: + value = prototype.buffer.from_bytes(await self.fs._cat_file(path)) + elif isinstance(byte_range, RangeByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file( + path, + start=byte_range.start, + end=byte_range.end, + ) ) - ) - + elif isinstance(byte_range, OffsetByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file(path, start=byte_range.offset, end=None) + ) + elif isinstance(byte_range, SuffixByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file(path, start=-byte_range.suffix, end=None) + ) + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") except self.allowed_exceptions: return None except OSError as e: @@ -270,25 +279,35 @@ async def exists(self, key: str) -> bool: async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited if key_ranges: - paths, starts, stops = zip( - *( - ( - _dereference_path(self.path, k[0]), - k[1][0], - ((k[1][0] or 0) + k[1][1]) if k[1][1] is not None else None, - ) - for k in key_ranges - ), - strict=False, - ) + # _cat_ranges expects a list of paths, start, and end ranges, so we need to reformat each ByteRequest. + key_ranges = list(key_ranges) + paths: list[str] = [] + starts: list[int | None] = [] + stops: list[int | None] = [] + for key, byte_range in key_ranges: + paths.append(_dereference_path(self.path, key)) + if byte_range is None: + starts.append(None) + stops.append(None) + elif isinstance(byte_range, RangeByteRequest): + starts.append(byte_range.start) + stops.append(byte_range.end) + elif isinstance(byte_range, OffsetByteRequest): + starts.append(byte_range.offset) + stops.append(None) + elif isinstance(byte_range, SuffixByteRequest): + starts.append(-byte_range.suffix) + stops.append(None) + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") else: return [] # TODO: expectations for exceptions or missing keys? - res = await self.fs._cat_ranges(list(paths), starts, stops, on_error="return") + res = await self.fs._cat_ranges(paths, starts, stops, on_error="return") # the following is an s3-specific condition we probably don't want to leak res = [b"" if (isinstance(r, OSError) and "not satisfiable" in str(r)) else r for r in res] for r in res: diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index f4226792cb..5eaa85c592 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -7,7 +7,13 @@ from pathlib import Path from typing import TYPE_CHECKING -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) from zarr.core.buffer import Buffer from zarr.core.buffer.core import default_buffer_prototype from zarr.core.common import concurrent_map @@ -18,29 +24,20 @@ from zarr.core.buffer import BufferPrototype -def _get( - path: Path, prototype: BufferPrototype, byte_range: tuple[int | None, int | None] | None -) -> Buffer: - if byte_range is not None: - if byte_range[0] is None: - start = 0 - else: - start = byte_range[0] - - end = (start + byte_range[1]) if byte_range[1] is not None else None - else: +def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None) -> Buffer: + if byte_range is None: return prototype.buffer.from_bytes(path.read_bytes()) with path.open("rb") as f: size = f.seek(0, io.SEEK_END) - if start is not None: - if start >= 0: - f.seek(start) - else: - f.seek(max(0, size + start)) - if end is not None: - if end < 0: - end = size + end - return prototype.buffer.from_bytes(f.read(end - f.tell())) + if isinstance(byte_range, RangeByteRequest): + f.seek(byte_range.start) + return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) + elif isinstance(byte_range, OffsetByteRequest): + f.seek(byte_range.offset) + elif isinstance(byte_range, SuffixByteRequest): + f.seek(max(0, size - byte_range.suffix)) + else: + raise TypeError(f"Unexpected byte_range, got {byte_range}.") return prototype.buffer.from_bytes(f.read()) @@ -127,7 +124,7 @@ async def get( self, key: str, prototype: BufferPrototype | None = None, - byte_range: tuple[int | None, int | None] | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if prototype is None: @@ -145,7 +142,7 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited args = [] diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py index 45ddeef40c..5ca716df2c 100644 --- a/src/zarr/storage/_logging.py +++ b/src/zarr/storage/_logging.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator, Generator, Iterable - from zarr.abc.store import ByteRangeRequest + from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, BufferPrototype counter: defaultdict[str, int] @@ -161,7 +161,7 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: tuple[int | None, int | None] | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited with self.log(key): @@ -170,7 +170,7 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited keys = ",".join([k[0] for k in key_ranges]) diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index 1f8dd75768..d35ecbe33d 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -3,10 +3,10 @@ from logging import getLogger from typing import TYPE_CHECKING, Self -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, gpu from zarr.core.common import concurrent_map -from zarr.storage._utils import _normalize_interval_index +from zarr.storage._utils import _normalize_byte_range_index if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable, MutableMapping @@ -75,7 +75,7 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: tuple[int | None, int | None] | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if not self._is_open: @@ -83,20 +83,20 @@ async def get( assert isinstance(key, str) try: value = self._store_dict[key] - start, length = _normalize_interval_index(value, byte_range) - return prototype.buffer.from_buffer(value[start : start + length]) + start, stop = _normalize_byte_range_index(value, byte_range) + return prototype.buffer.from_buffer(value[start:stop]) except KeyError: return None async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited # All the key-ranges arguments goes with the same prototype - async def _get(key: str, byte_range: ByteRangeRequest) -> Buffer | None: + async def _get(key: str, byte_range: ByteRequest | None) -> Buffer | None: return await self.get(key, prototype=prototype, byte_range=byte_range) return await concurrent_map(key_ranges, _get, limit=None) diff --git a/src/zarr/storage/_utils.py b/src/zarr/storage/_utils.py index 7ba82b00fd..4fc3171eb8 100644 --- a/src/zarr/storage/_utils.py +++ b/src/zarr/storage/_utils.py @@ -4,7 +4,10 @@ from pathlib import Path from typing import TYPE_CHECKING +from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest + if TYPE_CHECKING: + from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer @@ -44,25 +47,22 @@ def normalize_path(path: str | bytes | Path | None) -> str: return result -def _normalize_interval_index( - data: Buffer, interval: tuple[int | None, int | None] | None -) -> tuple[int, int]: +def _normalize_byte_range_index(data: Buffer, byte_range: ByteRequest | None) -> tuple[int, int]: """ - Convert an implicit interval into an explicit start and length + Convert an ByteRequest into an explicit start and stop """ - if interval is None: + if byte_range is None: start = 0 - length = len(data) + stop = len(data) + 1 + elif isinstance(byte_range, RangeByteRequest): + start = byte_range.start + stop = byte_range.end + elif isinstance(byte_range, OffsetByteRequest): + start = byte_range.offset + stop = len(data) + 1 + elif isinstance(byte_range, SuffixByteRequest): + start = len(data) - byte_range.suffix + stop = len(data) + 1 else: - maybe_start, maybe_len = interval - if maybe_start is None: - start = 0 - else: - start = maybe_start - - if maybe_len is None: - length = len(data) - start - else: - length = maybe_len - - return (start, length) + raise ValueError(f"Unexpected byte_range, got {byte_range}.") + return (start, stop) diff --git a/src/zarr/storage/_wrapper.py b/src/zarr/storage/_wrapper.py index c160100084..255e965439 100644 --- a/src/zarr/storage/_wrapper.py +++ b/src/zarr/storage/_wrapper.py @@ -7,7 +7,7 @@ from types import TracebackType from typing import Any, Self - from zarr.abc.store import ByteRangeRequest + from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.common import BytesLike @@ -70,14 +70,14 @@ def __eq__(self, value: object) -> bool: return type(self) is type(value) and self._store.__eq__(value) async def get( - self, key: str, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: return await self._store.get(key, prototype, byte_range) async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: return await self._store.get_partial_values(prototype, key_ranges) @@ -133,7 +133,7 @@ def close(self) -> None: self._store.close() async def _get_many( - self, requests: Iterable[tuple[str, BufferPrototype, ByteRangeRequest | None]] + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] ) -> AsyncGenerator[tuple[str, Buffer | None], None]: async for req in self._store._get_many(requests): yield req diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index a186b3cf59..e808b80e4e 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -7,7 +7,13 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) from zarr.core.buffer import Buffer, BufferPrototype if TYPE_CHECKING: @@ -138,23 +144,24 @@ def _get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited try: with self._zf.open(key) as f: # will raise KeyError if byte_range is None: return prototype.buffer.from_bytes(f.read()) - start, length = byte_range - if start: - if start < 0: - start = f.seek(start, os.SEEK_END) + start - else: - start = f.seek(start, os.SEEK_SET) - if length: - return prototype.buffer.from_bytes(f.read(length)) + elif isinstance(byte_range, RangeByteRequest): + f.seek(byte_range.start) + return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) + size = f.seek(0, os.SEEK_END) + if isinstance(byte_range, OffsetByteRequest): + f.seek(byte_range.offset) + elif isinstance(byte_range, SuffixByteRequest): + f.seek(max(0, size - byte_range.suffix)) else: - return prototype.buffer.from_bytes(f.read()) + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + return prototype.buffer.from_bytes(f.read()) except KeyError: return None @@ -162,7 +169,7 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited assert isinstance(key, str) @@ -173,7 +180,7 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited out = [] diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py index cc0f220807..1a1ef0e3a3 100644 --- a/src/zarr/testing/stateful.py +++ b/src/zarr/testing/stateful.py @@ -355,9 +355,8 @@ def get_partial_values(self, data: DataObject) -> None: model_vals_ls = [] for key, byte_range in key_range: - start = byte_range[0] or 0 - step = byte_range[1] - stop = start + step if step is not None else None + start = byte_range.start + stop = byte_range.end model_vals_ls.append(self.model[key][start:stop]) assert all( diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index ada028c273..602d001693 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -9,15 +9,21 @@ if TYPE_CHECKING: from typing import Any - from zarr.abc.store import ByteRangeRequest + from zarr.abc.store import ByteRequest from zarr.core.buffer.core import BufferPrototype import pytest -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.core.sync import _collect_aiterator -from zarr.storage._utils import _normalize_interval_index +from zarr.storage._utils import _normalize_byte_range_index from zarr.testing.utils import assert_bytes_equal __all__ = ["StoreTests"] @@ -115,18 +121,18 @@ def test_store_supports_listing(self, store: S) -> None: @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) - @pytest.mark.parametrize("byte_range", [None, (0, None), (1, None), (1, 2), (None, 1)]) - async def test_get( - self, store: S, key: str, data: bytes, byte_range: tuple[int | None, int | None] | None - ) -> None: + @pytest.mark.parametrize( + "byte_range", [None, RangeByteRequest(1, 4), OffsetByteRequest(1), SuffixByteRequest(1)] + ) + async def test_get(self, store: S, key: str, data: bytes, byte_range: ByteRequest) -> None: """ Ensure that data can be read from the store using the store.get method. """ data_buf = self.buffer_cls.from_bytes(data) await self.set(store, key, data_buf) observed = await store.get(key, prototype=default_buffer_prototype(), byte_range=byte_range) - start, length = _normalize_interval_index(data_buf, interval=byte_range) - expected = data_buf[start : start + length] + start, stop = _normalize_byte_range_index(data_buf, byte_range=byte_range) + expected = data_buf[start:stop] assert_bytes_equal(observed, expected) async def test_get_many(self, store: S) -> None: @@ -179,13 +185,17 @@ async def test_set_many(self, store: S) -> None: "key_ranges", [ [], - [("zarr.json", (0, 1))], - [("c/0", (0, 1)), ("zarr.json", (0, None))], - [("c/0/0", (0, 1)), ("c/0/1", (None, 2)), ("c/0/2", (0, 3))], + [("zarr.json", RangeByteRequest(0, 2))], + [("c/0", RangeByteRequest(0, 2)), ("zarr.json", None)], + [ + ("c/0/0", RangeByteRequest(0, 2)), + ("c/0/1", SuffixByteRequest(2)), + ("c/0/2", OffsetByteRequest(2)), + ], ], ) async def test_get_partial_values( - self, store: S, key_ranges: list[tuple[str, tuple[int | None, int | None]]] + self, store: S, key_ranges: list[tuple[str, ByteRequest]] ) -> None: # put all of the data for key, _ in key_ranges: @@ -367,7 +377,7 @@ async def set(self, key: str, value: Buffer) -> None: await self._store.set(key, value) async def get( - self, key: str, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: """ Add latency to the ``get`` method. @@ -380,7 +390,7 @@ async def get( The key to get prototype : BufferPrototype The BufferPrototype to use. - byte_range : ByteRangeRequest, optional + byte_range : ByteRequest, optional An optional byte range. Returns diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 1bde01b8f9..b948651ce6 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -7,6 +7,7 @@ from hypothesis.strategies import SearchStrategy import zarr +from zarr.abc.store import RangeByteRequest from zarr.core.array import Array from zarr.core.common import ZarrFormat from zarr.core.sync import sync @@ -194,12 +195,13 @@ def key_ranges( Function to generate key_ranges strategy for get_partial_values() returns list strategy w/ form:: - [(key, (range_start, range_step)), - (key, (range_start, range_step)),...] + [(key, (range_start, range_end)), + (key, (range_start, range_end)),...] """ - byte_ranges = st.tuples( - st.none() | st.integers(min_value=0, max_value=max_size), - st.none() | st.integers(min_value=0, max_value=max_size), + byte_ranges = st.builds( + RangeByteRequest, + start=st.integers(min_value=0, max_value=max_size), + end=st.integers(min_value=0, max_value=max_size), ) key_tuple = st.tuples(keys, byte_ranges) return st.lists(key_tuple, min_size=1, max_size=10) diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index b307f2cdf4..2713a2969d 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -8,6 +8,7 @@ from botocore.session import Session import zarr.api.asynchronous +from zarr.abc.store import OffsetByteRequest from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.sync import _collect_aiterator, sync from zarr.storage import FsspecStore @@ -97,7 +98,7 @@ async def test_basic() -> None: assert await store.exists("foo") assert (await store.get("foo", prototype=default_buffer_prototype())).to_bytes() == data out = await store.get_partial_values( - prototype=default_buffer_prototype(), key_ranges=[("foo", (1, None))] + prototype=default_buffer_prototype(), key_ranges=[("foo", OffsetByteRequest(1))] ) assert out[0].to_bytes() == data[1:] From e10b69d72a2d00ffdfa39ac4f4195363fb8e16fd Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 9 Jan 2025 08:21:03 -0800 Subject: [PATCH 012/160] doc: add release announcement banner (#2677) --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 22d24c3515..75584566c6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -91,7 +91,7 @@ def skip_submodules( # General information about the project. project = "zarr" -copyright = "2024, Zarr Developers" +copyright = "2025, Zarr Developers" author = "Zarr Developers" version = get_version("zarr") @@ -181,6 +181,7 @@ def skip_submodules( ], "collapse_navigation": True, "navigation_with_keys": False, + "announcement": "Zarr-Python 3 is here! Check out the release announcement here.", } # Add any paths that contain custom themes here, relative to this directory. From 99a3576beebdb64b3f51ec0ea084aea6ebe74e96 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Fri, 10 Jan 2025 12:02:42 +0100 Subject: [PATCH 013/160] Fix: order for v2 arrays (#2679) * fixes order for v2 arrays * release notes --- docs/release-notes.rst | 14 ++++++++++++++ src/zarr/core/array.py | 18 +++++++++++++----- tests/test_v2.py | 38 +++++++++++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 175bd21aa5..6703b82d35 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,6 +1,20 @@ Release notes ============= +Unreleased +---------- + +New features +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ +* Fixes ``order`` argument for Zarr format 2 arrays. + By :user:`Norman Rzepka ` (:issue:`2679`). + +Behaviour changes +~~~~~~~~~~~~~~~~~ + .. _release_3.0.0: 3.0.0 diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index ea29a6fc48..6f67b612d5 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4,7 +4,7 @@ import warnings from asyncio import gather from collections.abc import Iterable -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from itertools import starmap from logging import getLogger from typing import ( @@ -1226,14 +1226,17 @@ async def _get_selection( fill_value=self.metadata.fill_value, ) if product(indexer.shape) > 0: + # need to use the order from the metadata for v2 + _config = self._config + if self.metadata.zarr_format == 2: + _config = replace(_config, order=self.metadata.order) + # reading chunks and decoding them await self.codec_pipeline.read( [ ( self.store_path / self.metadata.encode_chunk_key(chunk_coords), - self.metadata.get_chunk_spec( - chunk_coords, self._config, prototype=prototype - ), + self.metadata.get_chunk_spec(chunk_coords, _config, prototype=prototype), chunk_selection, out_selection, ) @@ -1350,12 +1353,17 @@ async def _set_selection( # Buffer and NDBuffer between components. value_buffer = prototype.nd_buffer.from_ndarray_like(value) + # need to use the order from the metadata for v2 + _config = self._config + if self.metadata.zarr_format == 2: + _config = replace(_config, order=self.metadata.order) + # merging with existing data and encoding chunks await self.codec_pipeline.write( [ ( self.store_path / self.metadata.encode_chunk_key(chunk_coords), - self.metadata.get_chunk_spec(chunk_coords, self._config, prototype), + self.metadata.get_chunk_spec(chunk_coords, _config, prototype), chunk_selection, out_selection, ) diff --git a/tests/test_v2.py b/tests/test_v2.py index 72127f4ede..9fe31956f8 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -12,6 +12,8 @@ import zarr.core.buffer import zarr.storage from zarr import config +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.sync import sync from zarr.storage import MemoryStore, StorePath @@ -166,36 +168,54 @@ def test_v2_filters_codecs(filters: Any, order: Literal["C", "F"]) -> None: @pytest.mark.parametrize("array_order", ["C", "F"]) @pytest.mark.parametrize("data_order", ["C", "F"]) -def test_v2_non_contiguous(array_order: Literal["C", "F"], data_order: Literal["C", "F"]) -> None: +@pytest.mark.parametrize("memory_order", ["C", "F"]) +def test_v2_non_contiguous( + array_order: Literal["C", "F"], data_order: Literal["C", "F"], memory_order: Literal["C", "F"] +) -> None: + store = MemoryStore() arr = zarr.create_array( - MemoryStore({}), + store, shape=(10, 8), chunks=(3, 3), fill_value=np.nan, dtype="float64", zarr_format=2, + filters=None, + compressors=None, overwrite=True, order=array_order, + config={"order": memory_order}, ) # Non-contiguous write a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=data_order) - arr[slice(6, 9, None), slice(3, 6, None)] = a[ - slice(6, 9, None), slice(3, 6, None) - ] # The slice on the RHS is important + arr[6:9, 3:6] = a[6:9, 3:6] # The slice on the RHS is important + np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6]) + np.testing.assert_array_equal( - arr[slice(6, 9, None), slice(3, 6, None)], a[slice(6, 9, None), slice(3, 6, None)] + a[6:9, 3:6], + np.frombuffer( + sync(store.get("2.1", default_buffer_prototype())).to_bytes(), dtype="float64" + ).reshape((3, 3), order=array_order), ) + if memory_order == "F": + assert (arr[6:9, 3:6]).flags.f_contiguous + else: + assert (arr[6:9, 3:6]).flags.c_contiguous + store = MemoryStore() arr = zarr.create_array( - MemoryStore({}), + store, shape=(10, 8), chunks=(3, 3), fill_value=np.nan, dtype="float64", zarr_format=2, + compressors=None, + filters=None, overwrite=True, order=array_order, + config={"order": memory_order}, ) # Contiguous write @@ -204,8 +224,8 @@ def test_v2_non_contiguous(array_order: Literal["C", "F"], data_order: Literal[" assert a.flags.f_contiguous else: assert a.flags.c_contiguous - arr[slice(6, 9, None), slice(3, 6, None)] = a - np.testing.assert_array_equal(arr[slice(6, 9, None), slice(3, 6, None)], a) + arr[6:9, 3:6] = a + np.testing.assert_array_equal(arr[6:9, 3:6], a) def test_default_compressor_deprecation_warning(): From 0e1fde44b2ff3904bbe88fc4d1424d61d769dfe2 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 10 Jan 2025 07:43:38 -0800 Subject: [PATCH 014/160] test: enable codecov in main test action (#2682) * test: enable codecov in main test action * output coverage report * add codecov.yml * add junit config * comment: false * skip checking TYPE_CHECKING blocks --- .github/workflows/test.yml | 7 ++++++- codecov.yml | 10 ++++++++++ pyproject.toml | 8 ++++---- 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 codecov.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5309ea4565..ea65c3f0e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,12 @@ jobs: hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env - name: Run Tests run: | - hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run + hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) test-upstream-and-min-deps: name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000000..83274aedec --- /dev/null +++ b/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + patch: + default: + target: auto + project: + default: + target: auto + threshold: 0.1 +comment: false diff --git a/pyproject.toml b/pyproject.toml index 05db0860a8..96b7ead74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,13 +103,13 @@ Homepage = "https://github.com/zarr-developers/zarr-python" [tool.coverage.report] exclude_lines = [ "pragma: no cover", + "if TYPE_CHECKING:", "pragma: ${PY_MAJOR_VERSION} no cover", '.*\.\.\.' # Ignore "..." lines ] [tool.coverage.run] omit = [ - "src/zarr/meta_v1.py", "bench/compress_normal.py", ] @@ -140,8 +140,8 @@ numpy = ["1.25", "2.1"] features = ["gpu"] [tool.hatch.envs.test.scripts] -run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=src" -run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=src" +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" @@ -170,7 +170,7 @@ numpy = ["1.25", "2.1"] version = ["minimal"] [tool.hatch.envs.gputest.scripts] -run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=src" +run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" From 678b2e89645df6a254dd3787a22328817f162a69 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 11 Jan 2025 10:11:22 +0000 Subject: [PATCH 015/160] Bootstrap release notes post v3 (#2680) --- docs/developers/contributing.rst | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index 31cf80bed6..71294826cb 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -329,10 +329,16 @@ Release procedure Most of the release process is now handled by GitHub workflow which should automatically push a release to PyPI if a tag is pushed. -Before releasing, make sure that all pull requests which will be -included in the release have been properly documented in -`docs/release.rst`. - +Pre-release +""""""""""" +1. Make sure that all pull requests which will be + included in the release have been properly documented in + :file:`docs/release-notes.rst`. +2. Rename the "Unreleased" section heading in :file:`docs/release-notes.rst` + to the version you are about to release. + +Releasing +""""""""" To make a new release, go to https://github.com/zarr-developers/zarr-python/releases and click "Draft a new release". Choose a version number prefixed @@ -355,5 +361,8 @@ https://readthedocs.io. Full releases will be available under pre-releases will be available under `/latest `_. -Also review and merge the https://github.com/conda-forge/zarr-feedstock -pull request that will be automatically generated. +Post-release +"""""""""""" + +- Review and merge the pull request on the `conda-forge feedstock `_ that will be automatically generated. +- Create a new "Unreleased" section in the release notes From b773b3e0d6f6c7252c7fc6c0c55c0157a461b468 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 13 Jan 2025 16:18:55 +0000 Subject: [PATCH 016/160] Don't put usernames in changelog (#2687) --- docs/release-notes.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 6703b82d35..47a0f9c2c2 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -9,8 +9,7 @@ New features Bug fixes ~~~~~~~~~ -* Fixes ``order`` argument for Zarr format 2 arrays. - By :user:`Norman Rzepka ` (:issue:`2679`). +* Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`). Behaviour changes ~~~~~~~~~~~~~~~~~ From b982224d85c4708cee77e7480484e634ecfb0d81 Mon Sep 17 00:00:00 2001 From: Brian Michell Date: Mon, 13 Jan 2025 15:43:13 -0600 Subject: [PATCH 017/160] Quickstart guide alignment with V3 API (#2697) * Use mandatory dtype * Fix compressors field --- docs/quickstart.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 2d0e8ecef8..d520554593 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -74,7 +74,7 @@ Zarr supports data compression and filters. For example, to use Blosc compressio ... "data/example-3.zarr", ... mode="w", shape=(100, 100), ... chunks=(10, 10), dtype="f4", - ... compressor=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.SHUFFLE) + ... compressors=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle) ... ) >>> z[:, :] = np.random.random((100, 100)) >>> @@ -101,7 +101,7 @@ Zarr allows you to create hierarchical groups, similar to directories:: >>> root = zarr.group("data/example-2.zarr") >>> foo = root.create_group(name="foo") >>> bar = root.create_array( - ... name="bar", shape=(100, 10), chunks=(10, 10) + ... name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" ... ) >>> spam = foo.create_array(name="spam", shape=(10,), dtype="i4") >>> @@ -112,6 +112,7 @@ Zarr allows you to create hierarchical groups, similar to directories:: >>> # print the hierarchy >>> root.tree() / + ├── bar (100, 10) float32 └── foo └── spam (10,) int32 @@ -130,7 +131,7 @@ using external libraries like `s3fs `_ or >>> import s3fs # doctest: +SKIP >>> - >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10)) # doctest: +SKIP + >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10), dtype="f4") # doctest: +SKIP >>> z[:, :] = np.random.random((100, 100)) # doctest: +SKIP A single-file store can also be created using the the :class:`zarr.storage.ZipStore`:: From 0220e45806ef42ad3f0b6f9f409dca3c336d658e Mon Sep 17 00:00:00 2001 From: Manuel Reis <16836000+mannreis@users.noreply.github.com> Date: Tue, 14 Jan 2025 11:23:11 +0100 Subject: [PATCH 018/160] Ensure backwards compatibility (#2695) * Ensure backwards compatibility * Add test that detects backwards compatibility break * Add test description Co-authored-by: Davis Bennett * Add fix to release notes * Minor improvement to release note --------- Co-authored-by: Davis Bennett Co-authored-by: David Stansby --- docs/release-notes.rst | 2 ++ src/zarr/core/group.py | 4 ++-- tests/test_metadata/test_consolidated.py | 26 +++++++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 47a0f9c2c2..bc9700d7fe 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -11,6 +11,8 @@ Bug fixes ~~~~~~~~~ * Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`). +* Fixes a bug that prevented reading Zarr format 2 data with consolidated metadata written using ``zarr-python`` version 2 (:issue:`2694`). + Behaviour changes ~~~~~~~~~~~~~~~~~ diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 57d9c5cd8d..775e8ac0bb 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -573,8 +573,8 @@ def _from_bytes_v2( v2_consolidated_metadata = json.loads(consolidated_metadata_bytes.to_bytes()) v2_consolidated_metadata = v2_consolidated_metadata["metadata"] # We already read zattrs and zgroup. Should we ignore these? - v2_consolidated_metadata.pop(".zattrs") - v2_consolidated_metadata.pop(".zgroup") + v2_consolidated_metadata.pop(".zattrs", None) + v2_consolidated_metadata.pop(".zgroup", None) consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 2731abada4..4065f35120 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -17,7 +17,7 @@ open, open_consolidated, ) -from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer import cpu, default_buffer_prototype from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV3Metadata from zarr.core.metadata.v2 import ArrayV2Metadata @@ -476,6 +476,30 @@ async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat): with pytest.raises(ValueError): await zarr.api.asynchronous.open_consolidated(store, zarr_format=None) + @pytest.fixture + async def v2_consolidated_metadata_empty_dataset( + self, memory_store: zarr.storage.MemoryStore + ) -> AsyncGroup: + zgroup_bytes = cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) + zmetadata_bytes = cpu.Buffer.from_bytes( + b'{"metadata":{".zgroup":{"zarr_format":2}},"zarr_consolidated_format":1}' + ) + return AsyncGroup._from_bytes_v2( + None, zgroup_bytes, zattrs_bytes=None, consolidated_metadata_bytes=zmetadata_bytes + ) + + async def test_consolidated_metadata_backwards_compatibility( + self, v2_consolidated_metadata_empty_dataset + ): + """ + Test that consolidated metadata handles a missing .zattrs key. This is necessary for backwards compatibility with zarr-python 2.x. See https://github.com/zarr-developers/zarr-python/issues/2694 + """ + store = zarr.storage.MemoryStore() + await zarr.api.asynchronous.open(store=store, zarr_format=2) + await zarr.api.asynchronous.consolidate_metadata(store) + result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2) + assert result.metadata == v2_consolidated_metadata_empty_dataset.metadata + async def test_consolidated_metadata_v2(self): store = zarr.storage.MemoryStore() g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2) From d4da552444d0192babb3ed8aaaa87a3ff465a2e3 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 14 Jan 2025 10:37:16 +0000 Subject: [PATCH 019/160] Speed up hypothesis tests (#2650) * Reduce number of examples in hypothesis tests * Change max tests to 50 * Update tests/test_store/test_stateful.py Co-authored-by: Deepak Cherian --------- Co-authored-by: Deepak Cherian --- tests/test_store/test_stateful.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py index ae10ca8d79..637eea882d 100644 --- a/tests/test_store/test_stateful.py +++ b/tests/test_store/test_stateful.py @@ -18,7 +18,7 @@ def mk_test_instance_sync() -> ZarrHierarchyStateMachine: pytest.skip(reason="ZipStore does not support delete") if isinstance(sync_store, MemoryStore): run_state_machine_as_test( - mk_test_instance_sync, settings=Settings(report_multiple_bugs=False) + mk_test_instance_sync, settings=Settings(report_multiple_bugs=False, max_examples=50) ) @@ -28,6 +28,11 @@ def mk_test_instance_sync() -> None: if isinstance(sync_store, ZipStore): pytest.skip(reason="ZipStore does not support delete") - if isinstance(sync_store, LocalStore): + elif isinstance(sync_store, LocalStore): pytest.skip(reason="This test has errors") - run_state_machine_as_test(mk_test_instance_sync, settings=Settings(report_multiple_bugs=True)) + elif isinstance(sync_store, MemoryStore): + run_state_machine_as_test(mk_test_instance_sync, settings=Settings(max_examples=50)) + else: + run_state_machine_as_test( + mk_test_instance_sync, settings=Settings(report_multiple_bugs=True) + ) From 168999ceff099799da9ead7c12f38aa9378a831f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 14 Jan 2025 10:53:37 +0000 Subject: [PATCH 020/160] Remove un-needed files from source distribution (#2686) Co-authored-by: Davis Bennett --- docs/release-notes.rst | 6 ++++++ pyproject.toml | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index bc9700d7fe..2e57328293 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -16,6 +16,12 @@ Bug fixes Behaviour changes ~~~~~~~~~~~~~~~~~ +Other +~~~~~ +* Removed some unnecessary files from the source distribution + to reduce its size. (:issue:`2686`) + + .. _release_3.0.0: 3.0.0 diff --git a/pyproject.toml b/pyproject.toml index 96b7ead74b..a88e43c51d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,13 @@ requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" +[tool.hatch.build.targets.sdist] +exclude = [ + "/.github", + "/bench", + "/docs", + "/notebooks" +] [project] name = "zarr" From 45146ca0985c6b8fc3e71a608d3ca6c3d8f8f5e7 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 14 Jan 2025 15:08:59 -0500 Subject: [PATCH 021/160] Ensure compressor=None results in no compression for V2 (#2709) * Ensure compressor=None results in no compression for V2 * rename argumnent * Update tests/test_v2.py Co-authored-by: Davis Bennett * fix * coverage * add release note * Update release note --------- Co-authored-by: Davis Bennett Co-authored-by: David Stansby --- docs/release-notes.rst | 2 ++ src/zarr/core/array.py | 23 ++++++++++++------ src/zarr/core/group.py | 13 ++++++---- tests/test_group.py | 14 ++--------- tests/test_metadata/test_consolidated.py | 4 +-- tests/test_v2.py | 31 ++++++++++++++++++++---- 6 files changed, 55 insertions(+), 32 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 2e57328293..ecd413510b 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -13,6 +13,8 @@ Bug fixes * Fixes a bug that prevented reading Zarr format 2 data with consolidated metadata written using ``zarr-python`` version 2 (:issue:`2694`). +* Ensure that compressor=None results in no compression when writing Zarr format 2 data (:issue:`2708`) + Behaviour changes ~~~~~~~~~~~~~~~~~ diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 6f67b612d5..0f1fb42f79 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4131,15 +4131,22 @@ def _parse_chunk_encoding_v3( def _parse_deprecated_compressor( - compressor: CompressorLike | None, compressors: CompressorsLike + compressor: CompressorLike | None, compressors: CompressorsLike, zarr_format: int = 3 ) -> CompressorsLike | None: - if compressor: + if compressor != "auto": if compressors != "auto": raise ValueError("Cannot specify both `compressor` and `compressors`.") - warn( - "The `compressor` argument is deprecated. Use `compressors` instead.", - category=UserWarning, - stacklevel=2, - ) - compressors = (compressor,) + if zarr_format == 3: + warn( + "The `compressor` argument is deprecated. Use `compressors` instead.", + category=UserWarning, + stacklevel=2, + ) + if compressor is None: + # "no compression" + compressors = () + else: + compressors = (compressor,) + elif zarr_format == 2 and compressor == compressors == "auto": + compressors = ({"id": "blosc"},) return compressors diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 775e8ac0bb..93e55da377 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1011,7 +1011,7 @@ async def create_array( shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", - compressor: CompressorLike = None, + compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = 0, order: MemoryOrder | None = None, @@ -1114,8 +1114,9 @@ async def create_array( AsyncArray """ - - compressors = _parse_deprecated_compressor(compressor, compressors) + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) return await create_array( store=self.store_path, name=name, @@ -2244,7 +2245,7 @@ def create_array( shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", - compressor: CompressorLike = None, + compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = 0, order: MemoryOrder | None = "C", @@ -2346,7 +2347,9 @@ def create_array( ------- AsyncArray """ - compressors = _parse_deprecated_compressor(compressor, compressors) + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) return Array( self._sync( self._async_group.create_array( diff --git a/tests/test_group.py b/tests/test_group.py index 788e81e603..144054605e 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -9,7 +9,7 @@ import numpy as np import pytest -from numcodecs import Zstd +from numcodecs import Blosc import zarr import zarr.api.asynchronous @@ -499,7 +499,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat "chunks": (1,), "order": "C", "filters": None, - "compressor": Zstd(level=0), + "compressor": Blosc(), "zarr_format": zarr_format, }, "subgroup": { @@ -1505,13 +1505,3 @@ def test_group_members_concurrency_limit(store: MemoryStore) -> None: elapsed = time.time() - start assert elapsed > num_groups * get_latency - - -@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) -def test_deprecated_compressor(store: Store) -> None: - g = zarr.group(store=store, zarr_format=2) - with pytest.warns(UserWarning, match="The `compressor` argument is deprecated.*"): - a = g.create_array( - "foo", shape=(100,), chunks=(10,), dtype="i4", compressor={"id": "blosc"} - ) - assert a.metadata.compressor.codec_id == "blosc" diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 4065f35120..c1ff2e130a 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from numcodecs import Zstd +from numcodecs import Blosc import zarr.api.asynchronous import zarr.api.synchronous @@ -522,7 +522,7 @@ async def test_consolidated_metadata_v2(self): attributes={"key": "a"}, chunks=(1,), fill_value=0, - compressor=Zstd(level=0), + compressor=Blosc(), order="C", ), "g1": GroupMetadata( diff --git a/tests/test_v2.py b/tests/test_v2.py index 9fe31956f8..b657af9c47 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -7,11 +7,13 @@ import pytest from numcodecs import Delta from numcodecs.blosc import Blosc +from numcodecs.zstd import Zstd import zarr import zarr.core.buffer import zarr.storage from zarr import config +from zarr.abc.store import Store from zarr.core.buffer.core import default_buffer_prototype from zarr.core.sync import sync from zarr.storage import MemoryStore, StorePath @@ -93,11 +95,7 @@ async def test_v2_encode_decode(dtype): store = zarr.storage.MemoryStore() g = zarr.group(store=store, zarr_format=2) g.create_array( - name="foo", - shape=(3,), - chunks=(3,), - dtype=dtype, - fill_value=b"X", + name="foo", shape=(3,), chunks=(3,), dtype=dtype, fill_value=b"X", compressor=None ) result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype()) @@ -166,6 +164,29 @@ def test_v2_filters_codecs(filters: Any, order: Literal["C", "F"]) -> None: np.testing.assert_array_equal(result, array_fixture) +@pytest.mark.filterwarnings("ignore") +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_create_array_defaults(store: Store): + """ + Test that passing compressor=None results in no compressor. Also test that the default value of the compressor + parameter does produce a compressor. + """ + g = zarr.open(store, mode="w", zarr_format=2) + arr = g.create_array("one", dtype="i8", shape=(1,), chunks=(1,), compressor=None) + assert arr._async_array.compressor is None + assert not (arr.filters) + arr = g.create_array("two", dtype="i8", shape=(1,), chunks=(1,)) + assert arr._async_array.compressor is not None + assert not (arr.filters) + arr = g.create_array("three", dtype="i8", shape=(1,), chunks=(1,), compressor=Zstd()) + assert arr._async_array.compressor is not None + assert not (arr.filters) + with pytest.raises(ValueError): + g.create_array( + "four", dtype="i8", shape=(1,), chunks=(1,), compressor=None, compressors=None + ) + + @pytest.mark.parametrize("array_order", ["C", "F"]) @pytest.mark.parametrize("data_order", ["C", "F"]) @pytest.mark.parametrize("memory_order", ["C", "F"]) From cef2b247c055a40535769d838b7ab43f2c1e451c Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:05:49 +0100 Subject: [PATCH 022/160] Update pull request template (#2717) --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a0d41f9841..723c995cef 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,7 +3,7 @@ TODO: * [ ] Add unit tests and/or doctests in docstrings * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions -* [ ] New/modified features documented in docs/tutorial.rst -* [ ] Changes documented in docs/release.rst +* [ ] New/modified features documented in `docs/user-guide/*.rst` +* [ ] Changes documented in `docs/release-notes.rst` * [ ] GitHub Actions have all passed * [ ] Test coverage is 100% (Codecov passes) From 280daac770cf4cef182897d85a290234f5558f20 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:25:02 +0100 Subject: [PATCH 023/160] Upgrade ruff to 0.9.1 (#2703) * Apply ruff rule RUF100 RUF100 Unused `noqa` directive (unused: `E402`) Otherwise, latest ruff 0.9.1 complains. * Enable ruff/flake8-implicit-str-concat rules ISC001 and ISC002 Starting with ruff 0.9.1, they are documented to be compatible with the ruff formatter, if they are both enabled. * Run `ruff format` * Upgrade ruff to 0.9.1 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 -- src/zarr/codecs/sharding.py | 6 +++--- src/zarr/core/indexing.py | 6 +++--- src/zarr/core/metadata/v3.py | 6 +++--- tests/test_properties.py | 8 ++++---- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9b4c8f444..28d1673652 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ ci: default_stages: [pre-commit, pre-push] repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.1 hooks: - id: ruff args: ["--fix", "--show-fixes"] diff --git a/pyproject.toml b/pyproject.toml index a88e43c51d..ebe99118d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -326,8 +326,6 @@ ignore = [ "Q003", "COM812", "COM819", - "ISC001", - "ISC002", ] [tool.ruff.lint.extend-per-file-ignores] diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 160a74e892..e8730c86dd 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -86,9 +86,9 @@ async def get( self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: assert byte_range is None, "byte_range is not supported within shards" - assert ( - prototype == default_buffer_prototype() - ), f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}" + assert prototype == default_buffer_prototype(), ( + f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}" + ) return self.shard_dict.get(self.chunk_coords) diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index ca227be094..f1226821ba 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -289,9 +289,9 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or def get_chunk_shape(chunk_grid: ChunkGrid) -> ChunkCoords: from zarr.core.chunk_grids import RegularChunkGrid - assert isinstance( - chunk_grid, RegularChunkGrid - ), "Only regular chunk grid is supported, currently." + assert isinstance(chunk_grid, RegularChunkGrid), ( + "Only regular chunk grid is supported, currently." + ) return chunk_grid.chunk_shape diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index ab62508c80..087dbd8bfc 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -373,9 +373,9 @@ def inner_codecs(self) -> tuple[Codec, ...]: def get_chunk_spec( self, _chunk_coords: ChunkCoords, array_config: ArrayConfig, prototype: BufferPrototype ) -> ArraySpec: - assert isinstance( - self.chunk_grid, RegularChunkGrid - ), "Currently, only regular chunk grid is supported" + assert isinstance(self.chunk_grid, RegularChunkGrid), ( + "Currently, only regular chunk grid is supported" + ) return ArraySpec( shape=self.chunk_grid.chunk_shape, dtype=self.dtype, diff --git a/tests/test_properties.py b/tests/test_properties.py index 678dcae89c..2e60c951dd 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -4,11 +4,11 @@ pytest.importorskip("hypothesis") -import hypothesis.extra.numpy as npst # noqa: E402 -import hypothesis.strategies as st # noqa: E402 -from hypothesis import given # noqa: E402 +import hypothesis.extra.numpy as npst +import hypothesis.strategies as st +from hypothesis import given -from zarr.testing.strategies import arrays, basic_indices, numpy_arrays, zarr_formats # noqa: E402 +from zarr.testing.strategies import arrays, basic_indices, numpy_arrays, zarr_formats @given(data=st.data(), zarr_format=zarr_formats) From c63e4cacf4c25bba7b62c7510d78940043713d08 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 16 Jan 2025 22:31:33 +0100 Subject: [PATCH 024/160] Multiple imports for an import name (#2723) --- docs/conf.py | 5 +---- src/zarr/core/array.py | 1 - src/zarr/core/group.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 75584566c6..d69309d432 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,15 +15,12 @@ import os import sys +from importlib.metadata import version as get_version from typing import Any import sphinx import sphinx.application -from importlib.metadata import version as get_version - -import sphinx - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 0f1fb42f79..632e8221b4 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -112,7 +112,6 @@ _parse_bytes_bytes_codec, get_pipeline_class, ) -from zarr.storage import StoreLike from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path if TYPE_CHECKING: diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 93e55da377..4760923e0b 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -31,7 +31,7 @@ create_array, ) from zarr.core.attributes import Attributes -from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ( JSON, ZARR_JSON, From da5b82cdc25a763a8511f95fd8bee317d6f438ba Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 16 Jan 2025 21:44:47 +0000 Subject: [PATCH 025/160] Clean up dependabot config (#2702) --- .github/dependabot.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a437a5c269..469b6a4d19 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,7 @@ --- version: 2 updates: - # Updates for v3 branch (the default branch) - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "daily" - groups: - actions: - patterns: - - "*" + # Updates for main - package-ecosystem: "github-actions" directory: "/" schedule: @@ -19,19 +11,19 @@ updates: patterns: - "*" - # Same updates, but for main branch + # Updates for support/v2 branch - package-ecosystem: "pip" directory: "/" - target-branch: "main" + target-branch: "support/v2" schedule: - interval: "daily" + interval: "weekly" groups: requirements: patterns: - "*" - package-ecosystem: "github-actions" directory: "/" - target-branch: "main" + target-branch: "support/v2" schedule: interval: "weekly" groups: From 3652611a9f57922218f4993669b831bdc75cc857 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 17 Jan 2025 00:31:03 +0000 Subject: [PATCH 026/160] Fix minimum fsspec test verseion (#2715) --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ebe99118d6..c49778f285 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -228,11 +228,12 @@ See Spec 0000 for details and drop schedule: https://scientific-python.org/specs """ python = "3.11" dependencies = [ + 'zarr[remote]', 'packaging==22.*', 'numpy==1.25.*', 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs - 'fsspec==2022.10.0', - 's3fs==2022.10.0', + 'fsspec==2023.10.0', + 's3fs==2023.10.0', 'universal_pathlib==0.0.22', 'typing_extensions==4.9.*', 'donfig==0.8.*', From 57d9456b310038b62c598ea384509d203d7ee60e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 17 Jan 2025 08:08:18 +0000 Subject: [PATCH 027/160] Fix release notes labelling (#2701) --- .github/labeler.yml | 2 +- .github/workflows/needs_release_notes.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index dbc3b95333..f186216099 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,2 +1,2 @@ needs release notes: -- all: ['!docs/release.rst'] +- all: ['!docs/release-notes.rst'] diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml index d81ee0bdc4..f37c6349d4 100644 --- a/.github/workflows/needs_release_notes.yml +++ b/.github/workflows/needs_release_notes.yml @@ -8,7 +8,7 @@ jobs: if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }} && ${{ github.event.pull_request.user.login != 'pre-commit-ci[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/labeler@main + - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: true From 2e302717d6370c1f0f873af3409ea97c9f46b8cc Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 17 Jan 2025 09:50:01 -0600 Subject: [PATCH 028/160] doc: fix codec reprs in doctests (#2727) --- docs/release-notes.rst | 5 +++++ docs/user-guide/arrays.rst | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index ecd413510b..191db58702 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -18,6 +18,11 @@ Bug fixes Behaviour changes ~~~~~~~~~~~~~~~~~ +Documentation +~~~~~~~~~~~~~ + +* Fix doctest failures related to numcodecs 0.15 (:issue:`2727`). + Other ~~~~~ * Removed some unnecessary files from the source distribution diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst index ae2c4b47eb..a62b2ea0fa 100644 --- a/docs/user-guide/arrays.rst +++ b/docs/user-guide/arrays.rst @@ -241,7 +241,7 @@ built-in delta filter:: >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000) >>> z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) >>> z.compressors - (_make_bytes_bytes_codec.._Codec(codec_name='numcodecs.lzma', codec_config={'id': 'lzma', 'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),) + (LZMA(codec_name='numcodecs.lzma', codec_config={'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),) The default compressor can be changed by setting the value of the using Zarr's :ref:`user-guide-config`, e.g.:: @@ -292,7 +292,7 @@ Here is an example using a delta filter with the Blosc compressor:: Order : C Read-only : False Store type : LocalStore - Filters : (_make_array_array_codec.._Codec(codec_name='numcodecs.delta', codec_config={'id': 'delta', 'dtype': 'int32'}),) + Filters : (Delta(codec_name='numcodecs.delta', codec_config={'dtype': 'int32'}),) Serializer : BytesCodec(endian=) Compressors : (BloscCodec(typesize=4, cname=, clevel=1, shuffle=, blocksize=0),) No. bytes : 400000000 (381.5M) From 501fa5b0f6595ae13b71c36c42f50957ba729efa Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 17 Jan 2025 10:04:22 -0600 Subject: [PATCH 029/160] doc: release notes for 3.0.1 (#2726) * doc: release notes for 3.0.1 * fixup * tidy --- docs/release-notes.rst | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 191db58702..2c4b658d7b 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,38 +1,46 @@ Release notes ============= -Unreleased ----------- - -New features -~~~~~~~~~~~~ +3.0.1 (Jan. 17, 2025) +--------------------- Bug fixes ~~~~~~~~~ * Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`). -* Fixes a bug that prevented reading Zarr format 2 data with consolidated metadata written using ``zarr-python`` version 2 (:issue:`2694`). +* Fixes a bug that prevented reading Zarr format 2 data with consolidated + metadata written using ``zarr-python`` version 2 (:issue:`2694`). -* Ensure that compressor=None results in no compression when writing Zarr format 2 data (:issue:`2708`) +* Ensure that compressor=None results in no compression when writing Zarr + format 2 data (:issue:`2708`). -Behaviour changes -~~~~~~~~~~~~~~~~~ +* Fix for empty consolidated metadata dataset: backwards compatibility with + Zarr-Python 2 (:issue:`2695`). Documentation ~~~~~~~~~~~~~ +* Add v3.0.0 release announcement banner (:issue:`2677`). + +* Quickstart guide alignment with V3 API (:issue:`2697`). * Fix doctest failures related to numcodecs 0.15 (:issue:`2727`). Other ~~~~~ * Removed some unnecessary files from the source distribution - to reduce its size. (:issue:`2686`) + to reduce its size. (:issue:`2686`). + +* Enable codecov in GitHub actions (:issue:`2682`). + +* Speed up hypothesis tests (:issue:`2650`). + +* Remove multiple imports for an import name (:issue:`2723`). .. _release_3.0.0: -3.0.0 ------ +3.0.0 (Jan. 9, 2025) +-------------------- 3.0.0 is a new major release of Zarr-Python, with many breaking changes. See the :ref:`v3 migration guide` for a listing of what's changed. From 31d377b954eb639e3f4d51b02945fdd111f1b52e Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 17 Jan 2025 11:04:32 -0600 Subject: [PATCH 030/160] doc: seed unreleased release notes (#2728) --- docs/release-notes.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 2c4b658d7b..3fdf8c847a 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,6 +1,21 @@ Release notes ============= +Unreleased +---------- + +Bug fixes +~~~~~~~~~ + +Features +~~~~~~~~ + +Documentation +~~~~~~~~~~~~~ + +Other +~~~~~ + 3.0.1 (Jan. 17, 2025) --------------------- From e9772ac74de53b439bb1632bc26170de2c69a427 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 20 Jan 2025 02:39:31 -0700 Subject: [PATCH 031/160] Add hatch command for html coverage report (#2721) * Add hatch command for html coverage report * Fix grammer * Use more informative alias in contrib guide --------- Co-authored-by: Davis Bennett --- docs/developers/contributing.rst | 10 +++++++--- pyproject.toml | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index 71294826cb..5582c6ae8f 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -98,7 +98,7 @@ you can do something like the following:: To verify that your development environment is working, you can run the unit tests for one of the test environments, e.g.:: - $ hatch env run --env test.py3.12-2.1-optional run + $ hatch env run --env test.py3.12-2.1-optional run-pytest Creating a branch ~~~~~~~~~~~~~~~~~ @@ -140,7 +140,7 @@ Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see `creating a development environment`_ above) and invoke:: - $ hatch env run --env test.py3.12-2.1-optional run + $ hatch env run --env test.py3.12-2.1-optional run-pytest All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is @@ -190,9 +190,13 @@ Both unit tests and docstring doctests are included when computing coverage. Run $ hatch env run --env test.py3.12-2.1-optional run-coverage -will automatically run the test suite with coverage and produce a coverage report. +will automatically run the test suite with coverage and produce a XML coverage report. This should be 100% before code can be accepted into the main code base. +You can also generate an HTML coverage report by running:: + + $ hatch env run --env test.py3.12-2.1-optional run-coverage-html + When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. diff --git a/pyproject.toml b/pyproject.toml index c49778f285..73f3b9faae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,7 +149,9 @@ features = ["gpu"] [tool.hatch.envs.test.scripts] run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" run = "run-coverage --no-cov" +run-pytest = "run" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" From a260ae9b5730ba3823c337a2f5991c0ccc7d6a04 Mon Sep 17 00:00:00 2001 From: Ilan Gold Date: Tue, 21 Jan 2025 21:53:39 +0100 Subject: [PATCH 032/160] (fix): structured arrays for v2 (#2681) --------- Co-authored-by: Martin Durant --- docs/release-notes.rst | 2 ++ docs/user-guide/config.rst | 1 + src/zarr/core/buffer/core.py | 4 +++- src/zarr/core/config.py | 1 + src/zarr/core/metadata/v2.py | 15 +++++++++++-- tests/test_config.py | 1 + tests/test_v2.py | 41 ++++++++++++++++++++++++++++++------ 7 files changed, 56 insertions(+), 9 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 3fdf8c847a..2276889cf6 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -7,6 +7,8 @@ Unreleased Bug fixes ~~~~~~~~~ +* Backwards compatibility for Zarr format 2 structured arrays (:issue:`2134`) + Features ~~~~~~~~ diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst index 871291b72b..3662f75dff 100644 --- a/docs/user-guide/config.rst +++ b/docs/user-guide/config.rst @@ -53,6 +53,7 @@ This is the current default configuration:: 'level': 0}}, 'v2_default_filters': {'bytes': [{'id': 'vlen-bytes'}], 'numeric': None, + 'raw': None, 'string': [{'id': 'vlen-utf8'}]}, 'v3_default_compressors': {'bytes': [{'configuration': {'checksum': False, 'level': 0}, diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 85a7351fc7..ccab103e0f 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -470,7 +470,9 @@ def all_equal(self, other: Any, equal_nan: bool = True) -> bool: # every single time we have to write data? _data, other = np.broadcast_arrays(self._data, other) return np.array_equal( - self._data, other, equal_nan=equal_nan if self._data.dtype.kind not in "USTO" else False + self._data, + other, + equal_nan=equal_nan if self._data.dtype.kind not in "USTOV" else False, ) def fill(self, value: Any) -> None: diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 7920d220a4..051e8c68e1 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -75,6 +75,7 @@ def reset(self) -> None: "numeric": None, "string": [{"id": "vlen-utf8"}], "bytes": [{"id": "vlen-bytes"}], + "raw": None, }, "v3_default_filters": {"numeric": [], "string": [], "bytes": []}, "v3_default_serializer": { diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 29cf15a119..192db5b203 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -193,7 +193,14 @@ def to_dict(self) -> dict[str, JSON]: zarray_dict["fill_value"] = fill_value _ = zarray_dict.pop("dtype") - zarray_dict["dtype"] = self.dtype.str + dtype_json: JSON + # In the case of zarr v2, the simplest i.e., '|VXX' dtype is represented as a string + dtype_descr = self.dtype.descr + if self.dtype.kind == "V" and dtype_descr[0][0] != "" and len(dtype_descr) != 0: + dtype_json = tuple(self.dtype.descr) + else: + dtype_json = self.dtype.str + zarray_dict["dtype"] = dtype_json return zarray_dict @@ -220,6 +227,8 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]: + if isinstance(data, list): # this is a valid _VoidDTypeLike check + data = [tuple(d) for d in data] return np.dtype(data) @@ -376,8 +385,10 @@ def _default_filters( dtype_key = "numeric" elif dtype.kind in "U": dtype_key = "string" - elif dtype.kind in "OSV": + elif dtype.kind in "OS": dtype_key = "bytes" + elif dtype.kind == "V": + dtype_key = "raw" else: raise ValueError(f"Unsupported dtype kind {dtype.kind}") diff --git a/tests/test_config.py b/tests/test_config.py index c552ace840..1a2453d646 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -61,6 +61,7 @@ def test_config_defaults_set() -> None: "numeric": None, "string": [{"id": "vlen-utf8"}], "bytes": [{"id": "vlen-bytes"}], + "raw": None, }, "v3_default_filters": {"numeric": [], "string": [], "bytes": []}, "v3_default_serializer": { diff --git a/tests/test_v2.py b/tests/test_v2.py index b657af9c47..4c689c8e64 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -84,8 +84,15 @@ def test_codec_pipeline() -> None: np.testing.assert_array_equal(result, expected) -@pytest.mark.parametrize("dtype", ["|S", "|V"]) -async def test_v2_encode_decode(dtype): +@pytest.mark.parametrize( + ("dtype", "expected_dtype", "fill_value", "fill_value_encoding"), + [ + ("|S", "|S0", b"X", "WA=="), + ("|V", "|V0", b"X", "WA=="), + ("|V10", "|V10", b"X", "WAAAAAAAAAAAAA=="), + ], +) +async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_encoding) -> None: with config.set( { "array.v2_default_filters.bytes": [{"id": "vlen-bytes"}], @@ -95,7 +102,7 @@ async def test_v2_encode_decode(dtype): store = zarr.storage.MemoryStore() g = zarr.group(store=store, zarr_format=2) g.create_array( - name="foo", shape=(3,), chunks=(3,), dtype=dtype, fill_value=b"X", compressor=None + name="foo", shape=(3,), chunks=(3,), dtype=dtype, fill_value=fill_value, compressor=None ) result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype()) @@ -105,9 +112,9 @@ async def test_v2_encode_decode(dtype): expected = { "chunks": [3], "compressor": None, - "dtype": f"{dtype}0", - "fill_value": "WA==", - "filters": [{"id": "vlen-bytes"}], + "dtype": expected_dtype, + "fill_value": fill_value_encoding, + "filters": [{"id": "vlen-bytes"}] if dtype == "|S" else None, "order": "C", "shape": [3], "zarr_format": 2, @@ -284,3 +291,25 @@ def test_default_filters_and_compressor(dtype_expected: Any) -> None: assert arr.metadata.compressor.codec_id == expected_compressor if expected_filter is not None: assert arr.metadata.filters[0].codec_id == expected_filter + + +@pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"]) +def test_structured_dtype_roundtrip(fill_value, tmp_path) -> None: + a = np.array( + [(b"aaa", 1, 4.2), (b"bbb", 2, 8.4), (b"ccc", 3, 12.6)], + dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], + ) + array_path = tmp_path / "data.zarr" + za = zarr.create( + shape=(3,), + store=array_path, + chunks=(2,), + fill_value=fill_value, + zarr_format=2, + dtype=a.dtype, + ) + if fill_value is not None: + assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() + za[...] = a + za = zarr.open_array(store=array_path) + assert (a == za[:]).all() From 2be9f360d74df28b15dc198f15191e54e3ac95e4 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 22 Jan 2025 11:19:25 +0000 Subject: [PATCH 033/160] Use towncrier for changelog generation (#2736) * Use towncrier for changelog generation * Generate unreleased changelog * Fix towncrier command * Fix issue links * Don't do unreleased changelog on tagged builds * Rename 1234.doc.rst to 2736.doc.rst * Change existing release notes entry to new format * Update contributing guide for new release notes system * Update pull request template for new changelog system --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/labeler.yml | 6 ++++-- .github/workflows/needs_release_notes.yml | 5 ++++- .pre-commit-config.yaml | 4 ++++ .readthedocs.yaml | 7 +++++++ changes/.gitignore | 1 + changes/2681.bugfix.rst | 1 + changes/2736.doc.rst | 2 ++ changes/README.md | 14 ++++++++++++++ docs/developers/contributing.rst | 14 ++++++-------- docs/release-notes.rst | 16 +--------------- pyproject.toml | 10 ++++++++++ 12 files changed, 55 insertions(+), 27 deletions(-) create mode 100644 changes/.gitignore create mode 100644 changes/2681.bugfix.rst create mode 100644 changes/2736.doc.rst create mode 100644 changes/README.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 723c995cef..9b64c97d0a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,6 +4,6 @@ TODO: * [ ] Add unit tests and/or doctests in docstrings * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions * [ ] New/modified features documented in `docs/user-guide/*.rst` -* [ ] Changes documented in `docs/release-notes.rst` +* [ ] Changes documented as a new file in `changes/` * [ ] GitHub Actions have all passed * [ ] Test coverage is 100% (Codecov passes) diff --git a/.github/labeler.yml b/.github/labeler.yml index f186216099..4dd680ee5a 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,2 +1,4 @@ -needs release notes: -- all: ['!docs/release-notes.rst'] +- needs release notes: + - all: + - changed-files: + - any-glob-to-any-file: 'changes/*.rst' diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml index f37c6349d4..7a6c5462b4 100644 --- a/.github/workflows/needs_release_notes.yml +++ b/.github/workflows/needs_release_notes.yml @@ -4,8 +4,11 @@ on: - pull_request_target jobs: - triage: + labeler: if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }} && ${{ github.event.pull_request.user.login != 'pre-commit-ci[bot]' }} + permissions: + contents: read + pull-requests: write runs-on: ubuntu-latest steps: - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28d1673652..908b0d5c28 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,3 +49,7 @@ repos: rev: v1.8.0 hooks: - id: numpydoc-validation + - repo: https://github.com/twisted/towncrier + rev: 23.11.0 + hooks: + - id: towncrier-check diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 32a3f0e4e1..6253a7196f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,6 +4,13 @@ build: os: ubuntu-22.04 tools: python: "3.12" + jobs: + pre_build: + - | + if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; + then + towncrier build --version Unreleased --yes; + fi sphinx: configuration: docs/conf.py diff --git a/changes/.gitignore b/changes/.gitignore new file mode 100644 index 0000000000..f935021a8f --- /dev/null +++ b/changes/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/changes/2681.bugfix.rst b/changes/2681.bugfix.rst new file mode 100644 index 0000000000..fa69f73e06 --- /dev/null +++ b/changes/2681.bugfix.rst @@ -0,0 +1 @@ +Added backwards compatibility for Zarr format 2 structured arrays. diff --git a/changes/2736.doc.rst b/changes/2736.doc.rst new file mode 100644 index 0000000000..0cfe264eb1 --- /dev/null +++ b/changes/2736.doc.rst @@ -0,0 +1,2 @@ +Changed the machinery for creating changelog entries. +Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. diff --git a/changes/README.md b/changes/README.md new file mode 100644 index 0000000000..74ed9f94a9 --- /dev/null +++ b/changes/README.md @@ -0,0 +1,14 @@ +Writing a changelog entry +------------------------- + +Please put a new file in this directory named `xxxx..rst`, where + +- `xxxx` is the pull request number associated with this entry +- `` is one of: + - feature + - bugfix + - doc + - removal + - misc + +Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-python`. diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index 5582c6ae8f..220e24eced 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -216,8 +216,8 @@ The documentation consists both of prose and API documentation. All user-facing and functions are included in the API documentation, under the ``docs/api`` folder using the `autodoc `_ extension to sphinx. Any new features or important usage information should be included in the -user-guide (``docs/user-guide``). Any changes should also be included in the release -notes (``docs/release-notes.rst``). +user-guide (``docs/user-guide``). Any changes should also be included as a new file in the +:file:`changes` directory. The documentation can be built locally by running:: @@ -335,11 +335,9 @@ Release procedure Pre-release """"""""""" -1. Make sure that all pull requests which will be - included in the release have been properly documented in - :file:`docs/release-notes.rst`. -2. Rename the "Unreleased" section heading in :file:`docs/release-notes.rst` - to the version you are about to release. +1. Make sure that all pull requests which will be included in the release + have been properly documented as changelog files in :file:`changes`. +2. Run ``towncrier build --version x.y.z`` to create the changelog. Releasing """"""""" @@ -352,7 +350,7 @@ appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`). Set the description of the release to:: - See release notes https://zarr.readthedocs.io/en/stable/release.html#release-0-0-0 + See release notes https://zarr.readthedocs.io/en/stable/release-notes.html#release-0-0-0 replacing the correct version numbers. For pre-release versions, the URL should omit the pre-release suffix, e.g. "a1" or "rc1". diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 2276889cf6..2943250c38 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,22 +1,8 @@ Release notes ============= -Unreleased ----------- +.. towncrier release notes start -Bug fixes -~~~~~~~~~ - -* Backwards compatibility for Zarr format 2 structured arrays (:issue:`2134`) - -Features -~~~~~~~~ - -Documentation -~~~~~~~~~~~~~ - -Other -~~~~~ 3.0.1 (Jan. 17, 2025) --------------------- diff --git a/pyproject.toml b/pyproject.toml index 73f3b9faae..e7a1c5c2c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ test = [ ] optional = ["rich", "universal-pathlib"] docs = [ + # Doc building 'sphinx==8.1.3', 'sphinx-autobuild>=2021.3.14', 'sphinx-autoapi==3.4.0', @@ -94,6 +95,9 @@ docs = [ 'sphinx-reredirects', 'pydata-sphinx-theme', 'numpydoc', + # Changelog generation + 'towncrier', + # Optional dependencies to run examples 'numcodecs[msgpack]', 'rich', 's3fs', @@ -415,3 +419,9 @@ checks = [ "PR05", "PR06", ] + +[tool.towncrier] +directory = 'changes' +filename = "docs/release-notes.rst" +underlines = ["-", "~", "^"] +issue_format = ":issue:`{issue}`" From 0c154c327e92a40648b3dd95e9e9d9c71d3d53a3 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Wed, 22 Jan 2025 15:16:27 -0600 Subject: [PATCH 034/160] Wrap sync fs for xarray.to_zarr (#2533) Co-authored-by: Martin Durant --- changes/2533.bigfix.rst | 1 + src/zarr/storage/_fsspec.py | 11 +++++++++++ tests/test_store/test_fsspec.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 changes/2533.bigfix.rst diff --git a/changes/2533.bigfix.rst b/changes/2533.bigfix.rst new file mode 100644 index 0000000000..dbcdf40e3c --- /dev/null +++ b/changes/2533.bigfix.rst @@ -0,0 +1 @@ +Wrap sync fsspec filesystems with AsyncFileSystemWrapper in xarray.to_zarr \ No newline at end of file diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 99c8c778e7..752d237400 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -172,6 +172,17 @@ def from_url( opts = {"asynchronous": True, **opts} fs, path = url_to_fs(url, **opts) + if not fs.async_impl: + try: + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + fs = AsyncFileSystemWrapper(fs) + except ImportError as e: + raise ImportError( + f"The filesystem for URL '{url}' is synchronous, and the required " + "AsyncFileSystemWrapper is not available. Upgrade fsspec to version " + "2024.12.0 or later to enable this functionality." + ) from e # fsspec is not consistent about removing the scheme from the path, so check and strip it here # https://github.com/fsspec/filesystem_spec/issues/1722 diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index 2713a2969d..a560ca02e8 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -6,6 +6,7 @@ import pytest from botocore.session import Session +from packaging.version import parse as parse_version import zarr.api.asynchronous from zarr.abc.store import OffsetByteRequest @@ -215,3 +216,31 @@ async def test_empty_nonexistent_path(self, store_kwargs) -> None: store_kwargs["path"] += "/abc" store = await self.store_cls.open(**store_kwargs) assert await store.is_empty("") + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_wrap_sync_filesystem(): + """The local fs is not async so we should expect it to be wrapped automatically""" + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + store = FsspecStore.from_url("local://test/path") + + assert isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_no_wrap_async_filesystem(): + """An async fs should not be wrapped automatically; fsspec's https filesystem is such an fs""" + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + store = FsspecStore.from_url("https://test/path") + + assert not isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl From 40da497735fca2d88bb0dbcf8e03257406ff98fc Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 24 Jan 2025 14:33:50 +0000 Subject: [PATCH 035/160] Fix label action (#2744) Co-authored-by: Davis Bennett --- .github/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 4dd680ee5a..59f905ffa6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,4 +1,4 @@ -- needs release notes: +needs release notes: - all: - changed-files: - any-glob-to-any-file: 'changes/*.rst' From ad99a67630da715edfb8a987c4b4691b52398be9 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 24 Jan 2025 17:07:55 +0000 Subject: [PATCH 036/160] Fix "needs release notes" labeler action (#2759) --- .github/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 59f905ffa6..f2529dfef5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,4 +1,4 @@ needs release notes: - all: - changed-files: - - any-glob-to-any-file: 'changes/*.rst' + - any-glob-to-any-file: '!changes/*.rst' From 9fd45451a9b4b0112b7dd9c180be5977d18570ac Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 25 Jan 2025 08:37:28 +0000 Subject: [PATCH 037/160] Fix pickling of ZipStore (#2762) * Fix pickling of ZipStore * Add changelog entry --- changes/2762.bugfix.rst | 2 ++ src/zarr/storage/_zip.py | 13 ++++++++----- src/zarr/testing/store.py | 5 +++-- 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 changes/2762.bugfix.rst diff --git a/changes/2762.bugfix.rst b/changes/2762.bugfix.rst new file mode 100644 index 0000000000..4995d33edd --- /dev/null +++ b/changes/2762.bugfix.rst @@ -0,0 +1,2 @@ +Fixed ZipStore to make sure the correct attributes are saved when instances are pickled. +This fixes a previous bug that prevent using ZipStore with a ProcessPoolExecutor. diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index e808b80e4e..5a8b51196a 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -107,11 +107,14 @@ def _sync_open(self) -> None: async def _open(self) -> None: self._sync_open() - def __getstate__(self) -> tuple[Path, ZipStoreAccessModeLiteral, int, bool]: - return self.path, self._zmode, self.compression, self.allowZip64 - - def __setstate__(self, state: Any) -> None: - self.path, self._zmode, self.compression, self.allowZip64 = state + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__ + for attr in ["_zf", "_lock"]: + state.pop(attr, None) + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__ = state self._is_open = False self._sync_open() diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 602d001693..7de88a2a80 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -76,8 +76,9 @@ def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None: assert store == store2 def test_serializable_store(self, store: S) -> None: - foo = pickle.dumps(store) - assert pickle.loads(foo) == store + new_store: S = pickle.loads(pickle.dumps(store)) + assert new_store == store + assert new_store.read_only == store.read_only def test_store_read_only(self, store: S) -> None: assert not store.read_only From 80aea2ae5af416de15d1e4b95c50f214ee0151e4 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Sat, 25 Jan 2025 01:47:30 -0700 Subject: [PATCH 038/160] Improve test coverage for storage classes (#2693) * Run Store tests on logging * Run store tests on wrapper * Add read only open tests to WrapperStore * Ignore new coverage files * Simplify wrapper tests * Fix __eq__ method in WrapperStore * Implement __repr__ for WrapperStore * Allow separate open and init kwargs * Add open class method to LoggingStore * Add __str__ to WrapperStore * Add repr test for LoggingStore * Fix __eq__ in LoggingStore * Test getsize for stores * Test for invalid ByteRequest * Use stdout rather than stderr as the default logging stream * Test default logging stream * Add test for getsize_prefix * Document buffer prototype parameter * Add test for invalid modes in StorePath.open() * Add test for contains_group * Add tests for contains_array * Test for invalid root type for LocalStore * Test LocalStore.get with default prototype * Test for invalid set buffer arguments * Test get and set on closed stores * Test using stores in a context manager * Specify abstract methods for StoreTests * Apply suggestions from code review Co-authored-by: Davis Bennett * Lint * Fix typing for LoggingStore Co-authored-by: Davis Bennett * Match specific Errors in tests Co-authored-by: Davis Bennett * Add docstring Co-authored-by: Davis Bennett * Parametrize tests Co-authored-by: Davis Bennett * Test for contains group/array at multiple heirarchies Co-authored-by: Davis Bennett * Update TypeError on GpuMemoryStore * Don't implement _is_open setter on wrapped stores * Update reprs for LoggingStore and WrapperStore * Test check_writeable and close for WrapperStore * Update pull request template (#2717) * Add release notes * Comprehensive changelog entry * Match error message * Apply suggestions from code review Co-authored-by: David Stansby * Update 2693.bugfix.rst --------- Co-authored-by: Davis Bennett Co-authored-by: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Co-authored-by: David Stansby Co-authored-by: Joe Hamman --- .gitignore | 1 + changes/2693.bugfix.rst | 13 +++ src/zarr/abc/store.py | 6 +- src/zarr/storage/_fsspec.py | 7 +- src/zarr/storage/_local.py | 6 +- src/zarr/storage/_logging.py | 26 ++++-- src/zarr/storage/_memory.py | 9 +- src/zarr/storage/_wrapper.py | 16 +++- src/zarr/storage/_zip.py | 10 ++- src/zarr/testing/store.py | 149 +++++++++++++++++++++++++------ tests/test_store/test_core.py | 57 +++++++++++- tests/test_store/test_local.py | 21 +++++ tests/test_store/test_logging.py | 74 ++++++++++++++- tests/test_store/test_wrapper.py | 62 ++++++++++++- 14 files changed, 406 insertions(+), 51 deletions(-) create mode 100644 changes/2693.bugfix.rst diff --git a/.gitignore b/.gitignore index 5663f62d04..1b2b63e651 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ src/zarr/_version.py data/* src/fixture/ fixture/ +junit.xml .DS_Store tests/.hypothesis diff --git a/changes/2693.bugfix.rst b/changes/2693.bugfix.rst new file mode 100644 index 0000000000..14b45a221e --- /dev/null +++ b/changes/2693.bugfix.rst @@ -0,0 +1,13 @@ +Implement open() for LoggingStore +LoggingStore is now a generic class. +Use stdout rather than stderr as the default stream for LoggingStore +Ensure that ZipStore is open before getting or setting any values +Update equality for LoggingStore and WrapperStore such that 'other' must also be a LoggingStore or WrapperStore respectively, rather than only checking the types of the stores they wrap. +Indicate StoreTest's `test_store_repr`, `test_store_supports_writes`, `test_store_supports_partial_writes`, and `test_store_supports_listing` need to be implemented using `@abstractmethod` rather than `NotImplementedError`. +Separate instantiating and opening a store in StoreTests +Test using Store as a context manager in StoreTests +Match the errors raised by read only stores in StoreTests +Test that a ValueError is raise for invalid byte range syntax in StoreTests +Test getsize() and getsize_prefix() in StoreTests +Test the error raised for invalid buffer arguments in StoreTests +Test that data can be written to a store that's not yet open using the store.set method in StoreTests diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index e6a5518a4b..96165f8ba0 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -176,10 +176,10 @@ async def get( Parameters ---------- key : str + prototype : BufferPrototype + The prototype of the output buffer. Stores may support a default buffer prototype. byte_range : ByteRequest, optional - ByteRequest may be one of the following. If not provided, all data associated with the key is retrieved. - - RangeByteRequest(int, int): Request a specific range of bytes in the form (start, end). The end is exclusive. If the given range is zero-length or starts after the end of the object, an error will be returned. Additionally, if the range ends after the end of the object, the entire remainder of the object will be returned. Otherwise, the exact requested range will be returned. - OffsetByteRequest(int): Request all bytes starting from a given byte offset. This is equivalent to bytes={int}- as an HTTP header. - SuffixByteRequest(int): Request the last int bytes. Note that here, int is the size of the request, not the byte offset. This is equivalent to bytes=-{int} as an HTTP header. @@ -200,6 +200,8 @@ async def get_partial_values( Parameters ---------- + prototype : BufferPrototype + The prototype of the output buffer. Stores may support a default buffer prototype. key_ranges : Iterable[tuple[str, tuple[int | None, int | None]]] Ordered set of key, range pairs, a key may occur multiple times with different ranges diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 752d237400..c30c9b601b 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -10,6 +10,7 @@ Store, SuffixByteRequest, ) +from zarr.core.buffer import Buffer from zarr.storage._common import _dereference_path if TYPE_CHECKING: @@ -17,7 +18,7 @@ from fsspec.asyn import AsyncFileSystem - from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.buffer import BufferPrototype from zarr.core.common import BytesLike @@ -264,6 +265,10 @@ async def set( if not self._is_open: await self._open() self._check_writable() + if not isinstance(value, Buffer): + raise TypeError( + f"FsspecStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) path = _dereference_path(self.path, key) # write data if byte_range: diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index 5eaa85c592..1defea26b4 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -96,7 +96,7 @@ def __init__(self, root: Path | str, *, read_only: bool = False) -> None: root = Path(root) if not isinstance(root, Path): raise TypeError( - f'"root" must be a string or Path instance. Got an object with type {type(root)} instead.' + f"'root' must be a string or Path instance. Got an instance of {type(root)} instead." ) self.root = root @@ -169,7 +169,9 @@ async def _set(self, key: str, value: Buffer, exclusive: bool = False) -> None: self._check_writable() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError("LocalStore.set(): `value` must a Buffer instance") + raise TypeError( + f"LocalStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) path = self.root / key await asyncio.to_thread(_put, path, value, start=None, exclusive=exclusive) diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py index 5ca716df2c..e9d6211588 100644 --- a/src/zarr/storage/_logging.py +++ b/src/zarr/storage/_logging.py @@ -2,10 +2,11 @@ import inspect import logging +import sys import time from collections import defaultdict from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self, TypeVar from zarr.abc.store import Store from zarr.storage._wrapper import WrapperStore @@ -18,8 +19,10 @@ counter: defaultdict[str, int] +T_Store = TypeVar("T_Store", bound=Store) -class LoggingStore(WrapperStore[Store]): + +class LoggingStore(WrapperStore[T_Store]): """ Store wrapper that logs all calls to the wrapped store. @@ -42,7 +45,7 @@ class LoggingStore(WrapperStore[Store]): def __init__( self, - store: Store, + store: T_Store, log_level: str = "DEBUG", log_handler: logging.Handler | None = None, ) -> None: @@ -67,7 +70,7 @@ def _configure_logger( def _default_handler(self) -> logging.Handler: """Define a default log handler""" - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stdout) handler.setLevel(self.log_level) handler.setFormatter( logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -94,6 +97,14 @@ def log(self, hint: Any = "") -> Generator[None, None, None]: end_time = time.time() self.logger.info("Finished %s [%.2f s]", op, end_time - start_time) + @classmethod + async def open(cls: type[Self], store_cls: type[T_Store], *args: Any, **kwargs: Any) -> Self: + log_level = kwargs.pop("log_level", "DEBUG") + log_handler = kwargs.pop("log_handler", None) + store = store_cls(*args, **kwargs) + await store._open() + return cls(store=store, log_level=log_level, log_handler=log_handler) + @property def supports_writes(self) -> bool: with self.log(): @@ -126,8 +137,7 @@ def _is_open(self) -> bool: @_is_open.setter def _is_open(self, value: bool) -> None: - with self.log(value): - self._store._is_open = value + raise NotImplementedError("LoggingStore must be opened via the `_open` method") async def _open(self) -> None: with self.log(): @@ -151,11 +161,11 @@ def __str__(self) -> str: return f"logging-{self._store}" def __repr__(self) -> str: - return f"LoggingStore({repr(self._store)!r})" + return f"LoggingStore({self._store.__class__.__name__}, '{self._store}')" def __eq__(self, other: object) -> bool: with self.log(other): - return self._store == other + return type(self) is type(other) and self._store.__eq__(other._store) # type: ignore[attr-defined] async def get( self, diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index d35ecbe33d..b37fc8d5c9 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -111,7 +111,9 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None await self._ensure_open() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError(f"Expected Buffer. Got {type(value)}.") + raise TypeError( + f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) if byte_range is not None: buf = self._store_dict[key] @@ -231,8 +233,9 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None self._check_writable() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError(f"Expected Buffer. Got {type(value)}.") - + raise TypeError( + f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) # Convert to gpu.Buffer gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value) await super().set(key, gpu_value, byte_range=byte_range) diff --git a/src/zarr/storage/_wrapper.py b/src/zarr/storage/_wrapper.py index 255e965439..349048e495 100644 --- a/src/zarr/storage/_wrapper.py +++ b/src/zarr/storage/_wrapper.py @@ -56,6 +56,14 @@ async def _ensure_open(self) -> None: async def is_empty(self, prefix: str) -> bool: return await self._store.is_empty(prefix) + @property + def _is_open(self) -> bool: + return self._store._is_open + + @_is_open.setter + def _is_open(self, value: bool) -> None: + raise NotImplementedError("WrapperStore must be opened via the `_open` method") + async def clear(self) -> None: return await self._store.clear() @@ -67,7 +75,13 @@ def _check_writable(self) -> None: return self._store._check_writable() def __eq__(self, value: object) -> bool: - return type(self) is type(value) and self._store.__eq__(value) + return type(self) is type(value) and self._store.__eq__(value._store) # type: ignore[attr-defined] + + def __str__(self) -> str: + return f"wrapping-{self._store}" + + def __repr__(self) -> str: + return f"WrapperStore({self._store.__class__.__name__}, '{self._store}')" async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index 5a8b51196a..bf8f9900b9 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -149,6 +149,8 @@ def _get( prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: + if not self._is_open: + self._sync_open() # docstring inherited try: with self._zf.open(key) as f: # will raise KeyError @@ -193,6 +195,8 @@ async def get_partial_values( return out def _set(self, key: str, value: Buffer) -> None: + if not self._is_open: + self._sync_open() # generally, this should be called inside a lock keyinfo = zipfile.ZipInfo(filename=key, date_time=time.localtime(time.time())[:6]) keyinfo.compress_type = self.compression @@ -206,9 +210,13 @@ def _set(self, key: str, value: Buffer) -> None: async def set(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() + if not self._is_open: + self._sync_open() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError("ZipStore.set(): `value` must a Buffer instance") + raise TypeError( + f"ZipStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) with self._lock: self._set(key, value) diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 7de88a2a80..1fe544d292 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -2,6 +2,7 @@ import asyncio import pickle +from abc import abstractmethod from typing import TYPE_CHECKING, Generic, TypeVar from zarr.storage import WrapperStore @@ -37,30 +38,53 @@ class StoreTests(Generic[S, B]): store_cls: type[S] buffer_cls: type[B] + @abstractmethod async def set(self, store: S, key: str, value: Buffer) -> None: """ Insert a value into a storage backend, with a specific key. This should not not use any store methods. Bypassing the store methods allows them to be tested. """ - raise NotImplementedError + ... + @abstractmethod async def get(self, store: S, key: str) -> Buffer: """ Retrieve a value from a storage backend, by key. This should not not use any store methods. Bypassing the store methods allows them to be tested. """ + ... - raise NotImplementedError - + @abstractmethod @pytest.fixture def store_kwargs(self) -> dict[str, Any]: - return {"read_only": False} + """Kwargs for instantiating a store""" + ... + + @abstractmethod + def test_store_repr(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_writes(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_partial_writes(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_listing(self, store: S) -> None: ... @pytest.fixture - async def store(self, store_kwargs: dict[str, Any]) -> Store: - return await self.store_cls.open(**store_kwargs) + def open_kwargs(self, store_kwargs: dict[str, Any]) -> dict[str, Any]: + return store_kwargs + + @pytest.fixture + async def store(self, open_kwargs: dict[str, Any]) -> Store: + return await self.store_cls.open(**open_kwargs) + + @pytest.fixture + async def store_not_open(self, store_kwargs: dict[str, Any]) -> Store: + return self.store_cls(**store_kwargs) def test_store_type(self, store: S) -> None: assert isinstance(store, Store) @@ -87,39 +111,38 @@ def test_store_read_only(self, store: S) -> None: store.read_only = False # type: ignore[misc] @pytest.mark.parametrize("read_only", [True, False]) - async def test_store_open_read_only( - self, store_kwargs: dict[str, Any], read_only: bool - ) -> None: - store_kwargs["read_only"] = read_only - store = await self.store_cls.open(**store_kwargs) + async def test_store_open_read_only(self, open_kwargs: dict[str, Any], read_only: bool) -> None: + open_kwargs["read_only"] = read_only + store = await self.store_cls.open(**open_kwargs) assert store._is_open assert store.read_only == read_only - async def test_read_only_store_raises(self, store_kwargs: dict[str, Any]) -> None: - kwargs = {**store_kwargs, "read_only": True} + async def test_store_context_manager(self, open_kwargs: dict[str, Any]) -> None: + # Test that the context manager closes the store + with await self.store_cls.open(**open_kwargs) as store: + assert store._is_open + # Test trying to open an already open store + with pytest.raises(ValueError, match="store is already open"): + await store._open() + assert not store._is_open + + async def test_read_only_store_raises(self, open_kwargs: dict[str, Any]) -> None: + kwargs = {**open_kwargs, "read_only": True} store = await self.store_cls.open(**kwargs) assert store.read_only # set - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): await store.set("foo", self.buffer_cls.from_bytes(b"bar")) # delete - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): await store.delete("foo") - def test_store_repr(self, store: S) -> None: - raise NotImplementedError - - def test_store_supports_writes(self, store: S) -> None: - raise NotImplementedError - - def test_store_supports_partial_writes(self, store: S) -> None: - raise NotImplementedError - - def test_store_supports_listing(self, store: S) -> None: - raise NotImplementedError - @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) @pytest.mark.parametrize( @@ -136,6 +159,26 @@ async def test_get(self, store: S, key: str, data: bytes, byte_range: ByteReques expected = data_buf[start:stop] assert_bytes_equal(observed, expected) + async def test_get_not_open(self, store_not_open: S) -> None: + """ + Ensure that data can be read from the store that isn't yet open using the store.get method. + """ + assert not store_not_open._is_open + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await self.set(store_not_open, key, data_buf) + observed = await store_not_open.get(key, prototype=default_buffer_prototype()) + assert_bytes_equal(observed, data_buf) + + async def test_get_raises(self, store: S) -> None: + """ + Ensure that a ValueError is raise for invalid byte range syntax + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + await self.set(store, "c/0", data_buf) + with pytest.raises((ValueError, TypeError), match=r"Unexpected byte_range, got.*"): + await store.get("c/0", prototype=default_buffer_prototype(), byte_range=(0, 2)) # type: ignore[arg-type] + async def test_get_many(self, store: S) -> None: """ Ensure that multiple keys can be retrieved at once with the _get_many method. @@ -158,6 +201,37 @@ async def test_get_many(self, store: S) -> None: expected_kvs = sorted(((k, b) for k, b in zip(keys, values, strict=False))) assert observed_kvs == expected_kvs + @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) + @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) + async def test_getsize(self, store: S, key: str, data: bytes) -> None: + """ + Test the result of store.getsize(). + """ + data_buf = self.buffer_cls.from_bytes(data) + expected = len(data_buf) + await self.set(store, key, data_buf) + observed = await store.getsize(key) + assert observed == expected + + async def test_getsize_prefix(self, store: S) -> None: + """ + Test the result of store.getsize_prefix(). + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + keys = ["c/0/0", "c/0/1", "c/1/0", "c/1/1"] + keys_values = [(k, data_buf) for k in keys] + await store._set_many(keys_values) + expected = len(data_buf) * len(keys) + observed = await store.getsize_prefix("c") + assert observed == expected + + async def test_getsize_raises(self, store: S) -> None: + """ + Test that getsize() raise a FileNotFoundError if the key doesn't exist. + """ + with pytest.raises(FileNotFoundError): + await store.getsize("c/1000") + @pytest.mark.parametrize("key", ["zarr.json", "c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) async def test_set(self, store: S, key: str, data: bytes) -> None: @@ -170,6 +244,17 @@ async def test_set(self, store: S, key: str, data: bytes) -> None: observed = await self.get(store, key) assert_bytes_equal(observed, data_buf) + async def test_set_not_open(self, store_not_open: S) -> None: + """ + Ensure that data can be written to the store that's not yet open using the store.set method. + """ + assert not store_not_open._is_open + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await store_not_open.set(key, data_buf) + observed = await self.get(store_not_open, key) + assert_bytes_equal(observed, data_buf) + async def test_set_many(self, store: S) -> None: """ Test that a dict of key : value pairs can be inserted into the store via the @@ -182,6 +267,16 @@ async def test_set_many(self, store: S) -> None: for k, v in store_dict.items(): assert (await self.get(store, k)).to_bytes() == v.to_bytes() + async def test_set_invalid_buffer(self, store: S) -> None: + """ + Ensure that set raises a Type or Value Error for invalid buffer arguments. + """ + with pytest.raises( + (ValueError, TypeError), + match=r"\S+\.set\(\): `value` must be a Buffer instance. Got an instance of instead.", + ): + await store.set("c/0", 0) # type: ignore[arg-type] + @pytest.mark.parametrize( "key_ranges", [ diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 7806f3ecef..726da06a52 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -4,12 +4,55 @@ import pytest from _pytest.compat import LEGACY_PATH -from zarr.core.common import AccessModeLiteral +from zarr import Group +from zarr.core.common import AccessModeLiteral, ZarrFormat from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath -from zarr.storage._common import make_store_path +from zarr.storage._common import contains_array, contains_group, make_store_path from zarr.storage._utils import normalize_path +@pytest.mark.parametrize("path", ["foo", "foo/bar"]) +@pytest.mark.parametrize("write_group", [True, False]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_contains_group( + local_store, path: str, write_group: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that the contains_group method correctly reports the existence of a group. + """ + root = Group.from_store(store=local_store, zarr_format=zarr_format) + if write_group: + root.create_group(path) + store_path = StorePath(local_store, path=path) + assert await contains_group(store_path, zarr_format=zarr_format) == write_group + + +@pytest.mark.parametrize("path", ["foo", "foo/bar"]) +@pytest.mark.parametrize("write_array", [True, False]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_contains_array( + local_store, path: str, write_array: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that the contains array method correctly reports the existence of an array. + """ + root = Group.from_store(store=local_store, zarr_format=zarr_format) + if write_array: + root.create_array(path, shape=(100,), chunks=(10,), dtype="i4") + store_path = StorePath(local_store, path=path) + assert await contains_array(store_path, zarr_format=zarr_format) == write_array + + +@pytest.mark.parametrize("func", [contains_array, contains_group]) +async def test_contains_invalid_format_raises(local_store, func: callable) -> None: + """ + Test contains_group and contains_array raise errors for invalid zarr_formats + """ + store_path = StorePath(local_store) + with pytest.raises(ValueError): + assert await func(store_path, zarr_format="3.0") + + @pytest.mark.parametrize("path", [None, "", "bar"]) async def test_make_store_path_none(path: str) -> None: """ @@ -56,10 +99,18 @@ async def test_make_store_path_store_path( assert Path(store_path.store.root) == Path(tmpdir) path_normalized = normalize_path(path) assert store_path.path == (store_like / path_normalized).path - assert store_path.read_only == ro +@pytest.mark.parametrize("modes", [(True, "w"), (False, "x")]) +async def test_store_path_invalid_mode_raises(tmpdir: LEGACY_PATH, modes: tuple) -> None: + """ + Test that ValueErrors are raise for invalid mode. + """ + with pytest.raises(ValueError): + await StorePath.open(LocalStore(str(tmpdir), read_only=modes[0]), path=None, mode=modes[1]) + + async def test_make_store_path_invalid() -> None: """ Test that invalid types raise TypeError diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py index 22597a2c3f..d9d941c6f0 100644 --- a/tests/test_store/test_local.py +++ b/tests/test_store/test_local.py @@ -8,6 +8,7 @@ from zarr.core.buffer import Buffer, cpu from zarr.storage import LocalStore from zarr.testing.store import StoreTests +from zarr.testing.utils import assert_bytes_equal if TYPE_CHECKING: import pathlib @@ -53,3 +54,23 @@ def test_creates_new_directory(self, tmp_path: pathlib.Path): store = self.store_cls(root=target) zarr.group(store=store) + + def test_invalid_root_raises(self): + """ + Test that a TypeError is raised when a non-str/Path type is used for the `root` argument + """ + with pytest.raises( + TypeError, + match=r"'root' must be a string or Path instance. Got an instance of instead.", + ): + LocalStore(root=0) + + async def test_get_with_prototype_default(self, store: LocalStore): + """ + Ensure that data can be read via ``store.get`` if the prototype keyword argument is unspecified, i.e. set to ``None``. + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await self.set(store, key, data_buf) + observed = await store.get(key, prototype=None) + assert_bytes_equal(observed, data_buf) diff --git a/tests/test_store/test_logging.py b/tests/test_store/test_logging.py index b32a214db5..1a89dca874 100644 --- a/tests/test_store/test_logging.py +++ b/tests/test_store/test_logging.py @@ -1,17 +1,87 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING import pytest import zarr -from zarr.core.buffer import default_buffer_prototype -from zarr.storage import LoggingStore +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.storage import LocalStore, LoggingStore +from zarr.testing.store import StoreTests if TYPE_CHECKING: + from _pytest.compat import LEGACY_PATH + from zarr.abc.store import Store +class TestLoggingStore(StoreTests[LoggingStore, cpu.Buffer]): + store_cls = LoggingStore + buffer_cls = cpu.Buffer + + async def get(self, store: LoggingStore, key: str) -> Buffer: + return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) + + async def set(self, store: LoggingStore, key: str, value: Buffer) -> None: + parent = (store._store.root / key).parent + if not parent.exists(): + parent.mkdir(parents=True) + (store._store.root / key).write_bytes(value.to_bytes()) + + @pytest.fixture + def store_kwargs(self, tmpdir: LEGACY_PATH) -> dict[str, str]: + return {"store": LocalStore(str(tmpdir)), "log_level": "DEBUG"} + + @pytest.fixture + def open_kwargs(self, tmpdir) -> dict[str, str]: + return {"store_cls": LocalStore, "root": str(tmpdir), "log_level": "DEBUG"} + + @pytest.fixture + def store(self, store_kwargs: str | dict[str, Buffer] | None) -> LoggingStore: + return self.store_cls(**store_kwargs) + + def test_store_supports_writes(self, store: LoggingStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: LoggingStore) -> None: + assert store.supports_partial_writes + + def test_store_supports_listing(self, store: LoggingStore) -> None: + assert store.supports_listing + + def test_store_repr(self, store: LoggingStore) -> None: + assert f"{store!r}" == f"LoggingStore(LocalStore, 'file://{store._store.root.as_posix()}')" + + def test_store_str(self, store: LoggingStore) -> None: + assert str(store) == f"logging-file://{store._store.root.as_posix()}" + + async def test_default_handler(self, local_store, capsys) -> None: + # Store and then remove existing handlers to enter default handler code path + handlers = logging.getLogger().handlers[:] + for h in handlers: + logging.getLogger().removeHandler(h) + # Test logs are sent to stdout + wrapped = LoggingStore(store=local_store) + buffer = default_buffer_prototype().buffer + res = await wrapped.set("foo/bar/c/0", buffer.from_bytes(b"\x01\x02\x03\x04")) + assert res is None + captured = capsys.readouterr() + assert len(captured) == 2 + assert "Calling LocalStore.set" in captured.out + assert "Finished LocalStore.set" in captured.out + # Restore handlers + for h in handlers: + logging.getLogger().addHandler(h) + + def test_is_open_setter_raises(self, store: LoggingStore) -> None: + "Test that a user cannot change `_is_open` without opening the underlying store." + with pytest.raises( + NotImplementedError, match="LoggingStore must be opened via the `_open` method" + ): + store._is_open = True + + @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) async def test_logging_store(store: Store, caplog) -> None: wrapped = LoggingStore(store=store, log_level="DEBUG") diff --git a/tests/test_store/test_wrapper.py b/tests/test_store/test_wrapper.py index 489bcd5a7a..7e933548b3 100644 --- a/tests/test_store/test_wrapper.py +++ b/tests/test_store/test_wrapper.py @@ -5,13 +5,73 @@ import pytest from zarr.core.buffer.cpu import Buffer, buffer_prototype -from zarr.storage import WrapperStore +from zarr.storage import LocalStore, WrapperStore +from zarr.testing.store import StoreTests if TYPE_CHECKING: + from _pytest.compat import LEGACY_PATH + from zarr.abc.store import Store from zarr.core.buffer.core import BufferPrototype +class TestWrapperStore(StoreTests[WrapperStore, Buffer]): + store_cls = WrapperStore + buffer_cls = Buffer + + async def get(self, store: WrapperStore, key: str) -> Buffer: + return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) + + async def set(self, store: WrapperStore, key: str, value: Buffer) -> None: + parent = (store._store.root / key).parent + if not parent.exists(): + parent.mkdir(parents=True) + (store._store.root / key).write_bytes(value.to_bytes()) + + @pytest.fixture + def store_kwargs(self, tmpdir: LEGACY_PATH) -> dict[str, str]: + return {"store": LocalStore(str(tmpdir))} + + @pytest.fixture + def open_kwargs(self, tmpdir) -> dict[str, str]: + return {"store_cls": LocalStore, "root": str(tmpdir)} + + def test_store_supports_writes(self, store: WrapperStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: WrapperStore) -> None: + assert store.supports_partial_writes + + def test_store_supports_listing(self, store: WrapperStore) -> None: + assert store.supports_listing + + def test_store_repr(self, store: WrapperStore) -> None: + assert f"{store!r}" == f"WrapperStore(LocalStore, 'file://{store._store.root.as_posix()}')" + + def test_store_str(self, store: WrapperStore) -> None: + assert str(store) == f"wrapping-file://{store._store.root.as_posix()}" + + def test_check_writeable(self, store: WrapperStore) -> None: + """ + Test _check_writeable() runs without errors. + """ + store._check_writable() + + def test_close(self, store: WrapperStore) -> None: + "Test store can be closed" + store.close() + assert not store._is_open + + def test_is_open_setter_raises(self, store: WrapperStore) -> None: + """ + Test that a user cannot change `_is_open` without opening the underlying store. + """ + with pytest.raises( + NotImplementedError, match="WrapperStore must be opened via the `_open` method" + ): + store._is_open = True + + @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) async def test_wrapped_set(store: Store, capsys: pytest.CaptureFixture[str]) -> None: # define a class that prints when it sets From 458299857141a5470ba3956d8a1607f52ac33857 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:06:39 +0000 Subject: [PATCH 039/160] Bump pypa/gh-action-pypi-publish from 1.12.3 to 1.12.4 in the actions group (#2770) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.12.3 to 1.12.4 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.12.3...v1.12.4) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 1b23260c2e..c8903aa779 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -55,7 +55,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.12.3 + - uses: pypa/gh-action-pypi-publish@v1.12.4 with: user: __token__ password: ${{ secrets.pypi_password }} From fb37ff0942c7417a7767cc4d537de7173a9b3329 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 28 Jan 2025 09:53:53 +0000 Subject: [PATCH 040/160] Upload coverage after GPU tests (#2767) Co-authored-by: Davis Bennett --- .github/workflows/gpu_test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index b13da7d36f..c7056a2c4b 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -64,3 +64,9 @@ jobs: - name: Run Tests run: | hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage + + - name: Upload coverage + uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) From b5016c020ad5e2691b025506e490ad3abb0fa7c5 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 28 Jan 2025 16:53:24 +0000 Subject: [PATCH 041/160] Make botocore an optional test dependency (#2768) * Make botocore an optional test dependency * Add release notes --- changes/2768.bugfix.1.rst | 1 + changes/2768.bugfix.2.rst | 2 ++ pyproject.toml | 4 +++- tests/test_store/test_fsspec.py | 4 ++-- 4 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 changes/2768.bugfix.1.rst create mode 100644 changes/2768.bugfix.2.rst diff --git a/changes/2768.bugfix.1.rst b/changes/2768.bugfix.1.rst new file mode 100644 index 0000000000..b732b742ef --- /dev/null +++ b/changes/2768.bugfix.1.rst @@ -0,0 +1 @@ +Updated the optional test dependencies to include ``botocore`` and ``fsspec``. diff --git a/changes/2768.bugfix.2.rst b/changes/2768.bugfix.2.rst new file mode 100644 index 0000000000..1bf5fb8a85 --- /dev/null +++ b/changes/2768.bugfix.2.rst @@ -0,0 +1,2 @@ +Fixed the fsspec tests to skip if ``botocore`` is not installed. +Previously they would have failed with an import error. diff --git a/pyproject.toml b/pyproject.toml index e7a1c5c2c4..8d73485dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,10 +73,12 @@ test = [ "coverage", "pytest", "pytest-cov", + 'zarr[remote]', + "botocore", "s3fs", + "moto[s3,server]", "pytest-asyncio", "pytest-accept", - "moto[s3,server]", "requests", "rich", "mypy", diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index a560ca02e8..929de37869 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING import pytest -from botocore.session import Session from packaging.version import parse as parse_version import zarr.api.asynchronous @@ -26,6 +25,7 @@ requests = pytest.importorskip("requests") moto_server = pytest.importorskip("moto.moto_server.threaded_moto_server") moto = pytest.importorskip("moto") +botocore = pytest.importorskip("botocore") # ### amended from s3fs ### # test_bucket_name = "test" @@ -52,7 +52,7 @@ def s3_base() -> Generator[None, None, None]: def get_boto3_client() -> botocore.client.BaseClient: # NB: we use the sync botocore client for setup - session = Session() + session = botocore.session.Session() return session.create_client("s3", endpoint_url=endpoint_url) From 0c895d18aa69b1d90866f2249e93986022ddc5e7 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 28 Jan 2025 17:07:14 +0000 Subject: [PATCH 042/160] Some release note improvements (#2775) * Fix release note filename * Split big release note into lots of smaller ones * Fix code syntax --- changes/{2533.bigfix.rst => 2533.bugfix.rst} | 0 changes/2693.bugfix.1.rst | 1 + changes/2693.bugfix.2.rst | 1 + changes/2693.bugfix.3.rst | 1 + changes/2693.bugfix.4.rst | 1 + changes/2693.bugfix.rst | 13 ------------- changes/2693.feature.1.rst | 1 + changes/2693.feature.2.rst | 1 + changes/2693.feature.3.rst | 3 +++ changes/2693.feature.4.rst | 1 + changes/2693.feature.5.rst | 1 + changes/2693.feature.6.rst | 1 + changes/2693.feature.7.rst | 1 + changes/2693.feature.8.rst | 1 + changes/2693.feature.9.rst | 1 + 15 files changed, 15 insertions(+), 13 deletions(-) rename changes/{2533.bigfix.rst => 2533.bugfix.rst} (100%) create mode 100644 changes/2693.bugfix.1.rst create mode 100644 changes/2693.bugfix.2.rst create mode 100644 changes/2693.bugfix.3.rst create mode 100644 changes/2693.bugfix.4.rst delete mode 100644 changes/2693.bugfix.rst create mode 100644 changes/2693.feature.1.rst create mode 100644 changes/2693.feature.2.rst create mode 100644 changes/2693.feature.3.rst create mode 100644 changes/2693.feature.4.rst create mode 100644 changes/2693.feature.5.rst create mode 100644 changes/2693.feature.6.rst create mode 100644 changes/2693.feature.7.rst create mode 100644 changes/2693.feature.8.rst create mode 100644 changes/2693.feature.9.rst diff --git a/changes/2533.bigfix.rst b/changes/2533.bugfix.rst similarity index 100% rename from changes/2533.bigfix.rst rename to changes/2533.bugfix.rst diff --git a/changes/2693.bugfix.1.rst b/changes/2693.bugfix.1.rst new file mode 100644 index 0000000000..a46ad9d28d --- /dev/null +++ b/changes/2693.bugfix.1.rst @@ -0,0 +1 @@ +Match the errors raised by read only stores in StoreTests. diff --git a/changes/2693.bugfix.2.rst b/changes/2693.bugfix.2.rst new file mode 100644 index 0000000000..972ba0b27e --- /dev/null +++ b/changes/2693.bugfix.2.rst @@ -0,0 +1 @@ +Use stdout rather than stderr as the default stream for LoggingStore. diff --git a/changes/2693.bugfix.3.rst b/changes/2693.bugfix.3.rst new file mode 100644 index 0000000000..a6eed34c48 --- /dev/null +++ b/changes/2693.bugfix.3.rst @@ -0,0 +1 @@ +Ensure that ZipStore is open before getting or setting any values. diff --git a/changes/2693.bugfix.4.rst b/changes/2693.bugfix.4.rst new file mode 100644 index 0000000000..002f078345 --- /dev/null +++ b/changes/2693.bugfix.4.rst @@ -0,0 +1 @@ +Update equality for LoggingStore and WrapperStore such that 'other' must also be a LoggingStore or WrapperStore respectively, rather than only checking the types of the stores they wrap. diff --git a/changes/2693.bugfix.rst b/changes/2693.bugfix.rst deleted file mode 100644 index 14b45a221e..0000000000 --- a/changes/2693.bugfix.rst +++ /dev/null @@ -1,13 +0,0 @@ -Implement open() for LoggingStore -LoggingStore is now a generic class. -Use stdout rather than stderr as the default stream for LoggingStore -Ensure that ZipStore is open before getting or setting any values -Update equality for LoggingStore and WrapperStore such that 'other' must also be a LoggingStore or WrapperStore respectively, rather than only checking the types of the stores they wrap. -Indicate StoreTest's `test_store_repr`, `test_store_supports_writes`, `test_store_supports_partial_writes`, and `test_store_supports_listing` need to be implemented using `@abstractmethod` rather than `NotImplementedError`. -Separate instantiating and opening a store in StoreTests -Test using Store as a context manager in StoreTests -Match the errors raised by read only stores in StoreTests -Test that a ValueError is raise for invalid byte range syntax in StoreTests -Test getsize() and getsize_prefix() in StoreTests -Test the error raised for invalid buffer arguments in StoreTests -Test that data can be written to a store that's not yet open using the store.set method in StoreTests diff --git a/changes/2693.feature.1.rst b/changes/2693.feature.1.rst new file mode 100644 index 0000000000..faf54d4d37 --- /dev/null +++ b/changes/2693.feature.1.rst @@ -0,0 +1 @@ +Implemented open() for LoggingStore. diff --git a/changes/2693.feature.2.rst b/changes/2693.feature.2.rst new file mode 100644 index 0000000000..091ce4754e --- /dev/null +++ b/changes/2693.feature.2.rst @@ -0,0 +1 @@ +LoggingStore is now a generic class. diff --git a/changes/2693.feature.3.rst b/changes/2693.feature.3.rst new file mode 100644 index 0000000000..06200e3010 --- /dev/null +++ b/changes/2693.feature.3.rst @@ -0,0 +1,3 @@ +Change StoreTest's ``test_store_repr``, ``test_store_supports_writes``, +``test_store_supports_partial_writes``, and ``test_store_supports_listing`` +to to be implemented using ``@abstractmethod``, rather raising ``NotImplementedError``. diff --git a/changes/2693.feature.4.rst b/changes/2693.feature.4.rst new file mode 100644 index 0000000000..c69ec87cc5 --- /dev/null +++ b/changes/2693.feature.4.rst @@ -0,0 +1 @@ +Separate instantiating and opening a store in StoreTests. diff --git a/changes/2693.feature.5.rst b/changes/2693.feature.5.rst new file mode 100644 index 0000000000..b9dde46f67 --- /dev/null +++ b/changes/2693.feature.5.rst @@ -0,0 +1 @@ +Add a test for using Stores as a context managers in StoreTests. diff --git a/changes/2693.feature.6.rst b/changes/2693.feature.6.rst new file mode 100644 index 0000000000..3dd6c79c8d --- /dev/null +++ b/changes/2693.feature.6.rst @@ -0,0 +1 @@ +Test that a ValueError is raised for invalid byte range syntax in StoreTests. diff --git a/changes/2693.feature.7.rst b/changes/2693.feature.7.rst new file mode 100644 index 0000000000..dfa346391a --- /dev/null +++ b/changes/2693.feature.7.rst @@ -0,0 +1 @@ +Test getsize() and getsize_prefix() in StoreTests. diff --git a/changes/2693.feature.8.rst b/changes/2693.feature.8.rst new file mode 100644 index 0000000000..3a9d4043a2 --- /dev/null +++ b/changes/2693.feature.8.rst @@ -0,0 +1 @@ +Test the error raised for invalid buffer arguments in StoreTests. diff --git a/changes/2693.feature.9.rst b/changes/2693.feature.9.rst new file mode 100644 index 0000000000..8362e89b17 --- /dev/null +++ b/changes/2693.feature.9.rst @@ -0,0 +1 @@ +Test that data can be written to a store that's not yet open using the store.set method in StoreTests From e602aa1d19f26bb06669994231e524c55bcecbeb Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 29 Jan 2025 23:22:55 +0100 Subject: [PATCH 043/160] Use ChunkKeyEncodingLike type alias (#2763) * make proper ChunkKeyEncodingLike type alias, and use it * model the not-required-ness of the separator * add cast * changelog --- changes/2763.chore.rst | 3 +++ src/zarr/core/array.py | 14 +++++++------- src/zarr/core/chunk_key_encodings.py | 19 +++++++++++++------ src/zarr/core/metadata/v3.py | 4 ++-- 4 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 changes/2763.chore.rst diff --git a/changes/2763.chore.rst b/changes/2763.chore.rst new file mode 100644 index 0000000000..f36c63c289 --- /dev/null +++ b/changes/2763.chore.rst @@ -0,0 +1,3 @@ +Created a type alias ``ChunkKeyEncodingLike`` to model the union of ``ChunkKeyEncoding`` instances and the dict form of the +parameters of those instances. ``ChunkKeyEncodingLike`` should be used by high-level functions to provide a convenient +way for creating ``ChunkKeyEncoding`` objects. \ No newline at end of file diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 632e8221b4..6b68d1a0ac 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -412,7 +412,7 @@ async def create( # v3 only chunk_shape: ShapeLike | None = None, chunk_key_encoding: ( - ChunkKeyEncoding + ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None @@ -453,7 +453,7 @@ async def create( The shape of the array's chunks Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. If not specified, default are guessed based on the shape and dtype. - chunk_key_encoding : ChunkKeyEncoding, optional + chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. Default is ``("default", "/")``. @@ -553,7 +553,7 @@ async def _create( # v3 only chunk_shape: ShapeLike | None = None, chunk_key_encoding: ( - ChunkKeyEncoding + ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None @@ -671,7 +671,7 @@ async def _create_v3( config: ArrayConfig, fill_value: Any | None = None, chunk_key_encoding: ( - ChunkKeyEncoding + ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None @@ -1708,7 +1708,7 @@ def create( The shape of the Array's chunks. Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. If not specified, default are guessed based on the shape and dtype. - chunk_key_encoding : ChunkKeyEncoding, optional + chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. Default is ``("default", "/")``. @@ -3756,7 +3756,7 @@ async def create_array( order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, @@ -3834,7 +3834,7 @@ async def create_array( The zarr format to use when saving. attributes : dict, optional Attributes for the array. - chunk_key_encoding : ChunkKeyEncoding, optional + chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. diff --git a/src/zarr/core/chunk_key_encodings.py b/src/zarr/core/chunk_key_encodings.py index 95ce9108f3..103472c3b4 100644 --- a/src/zarr/core/chunk_key_encodings.py +++ b/src/zarr/core/chunk_key_encodings.py @@ -2,7 +2,10 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Literal, TypedDict, cast +from typing import TYPE_CHECKING, Literal, TypeAlias, TypedDict, cast + +if TYPE_CHECKING: + from typing import NotRequired from zarr.abc.metadata import Metadata from zarr.core.common import ( @@ -20,9 +23,9 @@ def parse_separator(data: JSON) -> SeparatorLiteral: return cast(SeparatorLiteral, data) -class ChunkKeyEncodingLike(TypedDict): +class ChunkKeyEncodingParams(TypedDict): name: Literal["v2", "default"] - separator: SeparatorLiteral + separator: NotRequired[SeparatorLiteral] @dataclass(frozen=True) @@ -36,9 +39,7 @@ def __init__(self, *, separator: SeparatorLiteral) -> None: object.__setattr__(self, "separator", separator_parsed) @classmethod - def from_dict( - cls, data: dict[str, JSON] | ChunkKeyEncoding | ChunkKeyEncodingLike - ) -> ChunkKeyEncoding: + def from_dict(cls, data: dict[str, JSON] | ChunkKeyEncodingLike) -> ChunkKeyEncoding: if isinstance(data, ChunkKeyEncoding): return data @@ -46,6 +47,9 @@ def from_dict( if "name" in data and "separator" in data: data = {"name": data["name"], "configuration": {"separator": data["separator"]}} + # TODO: remove this cast when we are statically typing the JSON metadata completely. + data = cast(dict[str, JSON], data) + # configuration is optional for chunk key encodings name_parsed, config_parsed = parse_named_configuration(data, require_configuration=False) if name_parsed == "default": @@ -73,6 +77,9 @@ def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: pass +ChunkKeyEncodingLike: TypeAlias = ChunkKeyEncodingParams | ChunkKeyEncoding + + @dataclass(frozen=True) class DefaultChunkKeyEncoding(ChunkKeyEncoding): name: Literal["default"] = "default" diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 087dbd8bfc..9154762648 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -27,7 +27,7 @@ from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid -from zarr.core.chunk_key_encodings import ChunkKeyEncoding +from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike from zarr.core.common import ( JSON, ZARR_JSON, @@ -253,7 +253,7 @@ def __init__( shape: Iterable[int], data_type: npt.DTypeLike | DataType, chunk_grid: dict[str, JSON] | ChunkGrid, - chunk_key_encoding: dict[str, JSON] | ChunkKeyEncoding, + chunk_key_encoding: ChunkKeyEncodingLike, fill_value: Any, codecs: Iterable[Codec | dict[str, JSON]], attributes: dict[str, JSON] | None, From fc08f31a0b4210c9d9b2071373c5ea7d55b66287 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 30 Jan 2025 00:28:02 +0100 Subject: [PATCH 044/160] add init_array, and data kwarg for create_array (#2761) * add init_array, and data kwarg for create_array * clean up some type hints and docstrings * add release notes * add tests for synchronous create_array * Update src/zarr/api/synchronous.py Co-authored-by: Deepak Cherian * error if shape / dtype and data are provided --------- Co-authored-by: Deepak Cherian --- changes/2761.feature.1.rst | 5 + src/zarr/api/synchronous.py | 19 +- src/zarr/core/array.py | 416 +++++++++++++++++++++++++++++------- tests/test_array.py | 70 +++++- 4 files changed, 419 insertions(+), 91 deletions(-) create mode 100644 changes/2761.feature.1.rst diff --git a/changes/2761.feature.1.rst b/changes/2761.feature.1.rst new file mode 100644 index 0000000000..387de1a380 --- /dev/null +++ b/changes/2761.feature.1.rst @@ -0,0 +1,5 @@ +Adds a new function ``init_array`` for initializing an array in storage, and refactors ``create_array`` +to use ``init_array``. ``create_array`` takes two a new parameters: ``data``, an optional array-like object, and ``write_data``, a bool which defaults to ``True``. +If ``data`` is given to ``create_array``, then the ``dtype`` and ``shape`` attributes of ``data`` are used to define the +corresponding attributes of the resulting Zarr array. Additionally, if ``data`` given and ``write_data`` is ``True``, +then the values in ``data`` will be written to the newly created array. \ No newline at end of file diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index f8bee9fcef..305446ec97 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from collections.abc import Iterable + import numpy as np import numpy.typing as npt from zarr.abc.codec import Codec @@ -744,8 +745,9 @@ def create_array( store: str | StoreLike, *, name: str | None = None, - shape: ShapeLike, - dtype: npt.DTypeLike, + shape: ShapeLike | None = None, + dtype: npt.DTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunkCoords | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", @@ -772,10 +774,14 @@ def create_array( name : str or None, optional The name of the array within the store. If ``name`` is ``None``, the array will be located at the root of the store. - shape : ChunkCoords - Shape of the array. - dtype : npt.DTypeLike - Data type of the array. + shape : ChunkCoords, optional + Shape of the array. Can be ``None`` if ``data`` is provided. + dtype : npt.DTypeLike, optional + Data type of the array. Can be ``None`` if ``data`` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``, + or ``None``. chunks : ChunkCoords, optional Chunk shape of the array. If not specified, default are guessed based on the shape and dtype. @@ -874,6 +880,7 @@ def create_array( name=name, shape=shape, dtype=dtype, + data=data, chunks=chunks, shards=shards, filters=filters, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 6b68d1a0ac..4c444a81fa 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -660,6 +660,48 @@ async def _create( return result + @staticmethod + def _create_metadata_v3( + shape: ShapeLike, + dtype: np.dtype[Any], + chunk_shape: ChunkCoords, + fill_value: Any | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: Iterable[str] | None = None, + attributes: dict[str, JSON] | None = None, + ) -> ArrayV3Metadata: + """ + Create an instance of ArrayV3Metadata. + """ + + shape = parse_shapelike(shape) + codecs = list(codecs) if codecs is not None else _get_default_codecs(np.dtype(dtype)) + chunk_key_encoding_parsed: ChunkKeyEncodingLike + if chunk_key_encoding is None: + chunk_key_encoding_parsed = {"name": "default", "separator": "/"} + else: + chunk_key_encoding_parsed = chunk_key_encoding + + if dtype.kind in "UTS": + warn( + f"The dtype `{dtype}` is currently not part in the Zarr format 3 specification. It " + "may not be supported by other zarr implementations and may change in the future.", + category=UserWarning, + stacklevel=2, + ) + chunk_grid_parsed = RegularChunkGrid(chunk_shape=chunk_shape) + return ArrayV3Metadata( + shape=shape, + data_type=dtype, + chunk_grid=chunk_grid_parsed, + chunk_key_encoding=chunk_key_encoding_parsed, + fill_value=fill_value, + codecs=codecs, + dimension_names=tuple(dimension_names) if dimension_names else None, + attributes=attributes or {}, + ) + @classmethod async def _create_v3( cls, @@ -689,13 +731,6 @@ async def _create_v3( else: await ensure_no_existing_node(store_path, zarr_format=3) - shape = parse_shapelike(shape) - codecs = list(codecs) if codecs is not None else _get_default_codecs(np.dtype(dtype)) - - if chunk_key_encoding is None: - chunk_key_encoding = ("default", "/") - assert chunk_key_encoding is not None - if isinstance(chunk_key_encoding, tuple): chunk_key_encoding = ( V2ChunkKeyEncoding(separator=chunk_key_encoding[1]) @@ -703,29 +738,58 @@ async def _create_v3( else DefaultChunkKeyEncoding(separator=chunk_key_encoding[1]) ) - if dtype.kind in "UTS": - warn( - f"The dtype `{dtype}` is currently not part in the Zarr format 3 specification. It " - "may not be supported by other zarr implementations and may change in the future.", - category=UserWarning, - stacklevel=2, - ) - - metadata = ArrayV3Metadata( + metadata = cls._create_metadata_v3( shape=shape, - data_type=dtype, - chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), - chunk_key_encoding=chunk_key_encoding, + dtype=dtype, + chunk_shape=chunk_shape, fill_value=fill_value, + chunk_key_encoding=chunk_key_encoding, codecs=codecs, - dimension_names=tuple(dimension_names) if dimension_names else None, - attributes=attributes or {}, + dimension_names=dimension_names, + attributes=attributes, ) array = cls(metadata=metadata, store_path=store_path, config=config) await array._save_metadata(metadata, ensure_parents=True) return array + @staticmethod + def _create_metadata_v2( + shape: ChunkCoords, + dtype: np.dtype[Any], + chunks: ChunkCoords, + order: MemoryOrder, + dimension_separator: Literal[".", "/"] | None = None, + fill_value: float | None = None, + filters: Iterable[dict[str, JSON] | numcodecs.abc.Codec] | None = None, + compressor: dict[str, JSON] | numcodecs.abc.Codec | None = None, + attributes: dict[str, JSON] | None = None, + ) -> ArrayV2Metadata: + if dimension_separator is None: + dimension_separator = "." + + dtype = parse_dtype(dtype, zarr_format=2) + + # inject VLenUTF8 for str dtype if not already present + if np.issubdtype(dtype, np.str_): + filters = filters or [] + from numcodecs.vlen import VLenUTF8 + + if not any(isinstance(x, VLenUTF8) or x["id"] == "vlen-utf8" for x in filters): + filters = list(filters) + [VLenUTF8()] + + return ArrayV2Metadata( + shape=shape, + dtype=np.dtype(dtype), + chunks=chunks, + order=order, + dimension_separator=dimension_separator, + fill_value=fill_value, + compressor=compressor, + filters=filters, + attributes=attributes, + ) + @classmethod async def _create_v2( cls, @@ -751,30 +815,18 @@ async def _create_v2( else: await ensure_no_existing_node(store_path, zarr_format=2) - if dimension_separator is None: - dimension_separator = "." - - dtype = parse_dtype(dtype, zarr_format=2) - - # inject VLenUTF8 for str dtype if not already present - if np.issubdtype(dtype, np.str_): - filters = filters or [] - from numcodecs.vlen import VLenUTF8 - - if not any(isinstance(x, VLenUTF8) or x["id"] == "vlen-utf8" for x in filters): - filters = list(filters) + [VLenUTF8()] - - metadata = ArrayV2Metadata( + metadata = cls._create_metadata_v2( shape=shape, - dtype=np.dtype(dtype), + dtype=dtype, chunks=chunks, order=order, dimension_separator=dimension_separator, fill_value=fill_value, - compressor=compressor, filters=filters, + compressor=compressor, attributes=attributes, ) + array = cls(metadata=metadata, store_path=store_path, config=config) await array._save_metadata(metadata, ensure_parents=True) return array @@ -3741,10 +3793,9 @@ class ShardsConfigParam(TypedDict): ShardsLike: TypeAlias = ChunkCoords | ShardsConfigParam | Literal["auto"] -async def create_array( - store: str | StoreLike, +async def init_array( *, - name: str | None = None, + store_path: StorePath, shape: ShapeLike, dtype: npt.DTypeLike, chunks: ChunkCoords | Literal["auto"] = "auto", @@ -3758,19 +3809,14 @@ async def create_array( attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: Iterable[str] | None = None, - storage_options: dict[str, Any] | None = None, overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, -) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an array. +) -> ArrayV3Metadata | ArrayV2Metadata: + """Create and persist an array metadata document. Parameters ---------- - store : str or Store - Store or path to directory in file system or name of zip file. - name : str or None, optional - The name of the array within the store. If ``name`` is ``None``, the array will be located - at the root of the store. + store_path : StorePath + StorePath instance. The path attribute is the name of the array to initialize. shape : ChunkCoords Shape of the array. dtype : npt.DTypeLike @@ -3841,30 +3887,13 @@ async def create_array( dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. - config : ArrayConfig or ArrayConfigLike, optional - Runtime configuration for the array. Returns ------- - AsyncArray - The array. - - Examples - -------- - >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') - >>> async_arr = await zarr.api.asynchronous.create_array( - >>> store=store, - >>> shape=(100,100), - >>> chunks=(10,10), - >>> dtype='i4', - >>> fill_value=0) - + ArrayV3Metadata | ArrayV2Metadata + The array metadata document. """ if zarr_format is None: @@ -3872,20 +3901,25 @@ async def create_array( from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation - mode: Literal["a"] = "a" dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) - config_parsed = parse_array_config(config) shape_parsed = parse_shapelike(shape) chunk_key_encoding_parsed = _parse_chunk_key_encoding( chunk_key_encoding, zarr_format=zarr_format ) - store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + + if overwrite: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + shard_shape_parsed, chunk_shape_parsed = _auto_partition( array_shape=shape_parsed, shard_shape=shards, chunk_shape=chunks, dtype=dtype_parsed ) chunks_out: tuple[int, ...] - result: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] - + meta: ArrayV2Metadata | ArrayV3Metadata if zarr_format == 2: if shard_shape_parsed is not None: msg = ( @@ -3908,8 +3942,7 @@ async def create_array( else: order_parsed = order - result = await AsyncArray._create_v2( - store_path=store_path, + meta = AsyncArray._create_metadata_v2( shape=shape_parsed, dtype=dtype_parsed, chunks=chunk_shape_parsed, @@ -3919,8 +3952,6 @@ async def create_array( filters=filters_parsed, compressor=compressor_parsed, attributes=attributes, - overwrite=overwrite, - config=config_parsed, ) else: array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( @@ -3951,25 +3982,199 @@ async def create_array( chunks_out = chunk_shape_parsed codecs_out = sub_codecs - result = await AsyncArray._create_v3( - store_path=store_path, + meta = AsyncArray._create_metadata_v3( shape=shape_parsed, dtype=dtype_parsed, fill_value=fill_value, - attributes=attributes, chunk_shape=chunks_out, chunk_key_encoding=chunk_key_encoding_parsed, codecs=codecs_out, dimension_names=dimension_names, - overwrite=overwrite, - config=config_parsed, + attributes=attributes, ) + # save the metadata to disk + # TODO: make this easier -- it should be a simple function call that takes a {key: buffer} + coros = ( + (store_path / key).set(value) + for key, value in meta.to_buffer_dict(default_buffer_prototype()).items() + ) + await gather(*coros) + return meta + + +async def create_array( + store: str | StoreLike, + *, + name: str | None = None, + shape: ShapeLike | None = None, + dtype: npt.DTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: Iterable[str] | None = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfig | ArrayConfigLike | None = None, + write_data: bool = True, +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + shape : ChunkCoords, optional + Shape of the array. Can be ``None`` if ``data`` is provided. + dtype : npt.DTypeLike | None + Data type of the array. Can be ``None`` if ``data`` is provided. + data : Array-like data to use for initializing the array. If this parameter is provided, the + ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``, + or ``None``. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the ``data`` parameter + then ``write_data`` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If ``write_data`` is ``False``, then the + array will be left empty. + + Returns + ------- + AsyncArray + The array. + + Examples + -------- + >>> import zarr + >>> store = zarr.storage.MemoryStore(mode='w') + >>> async_arr = await zarr.api.asynchronous.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='i4', + >>> fill_value=0) + + """ + mode: Literal["a"] = "a" + config_parsed = parse_array_config(config) + store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + + data_parsed, shape_parsed, dtype_parsed = _parse_data_params( + data=data, shape=shape, dtype=dtype + ) + meta = await init_array( + store_path=store_path, + shape=shape_parsed, + dtype=dtype_parsed, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + overwrite=overwrite, + ) + + result = AsyncArray(metadata=meta, store_path=store_path, config=config_parsed) + if write_data is True and data_parsed is not None: + await result._set_selection( + BasicIndexer(..., shape=result.shape, chunk_grid=result.metadata.chunk_grid), + data_parsed, + prototype=default_buffer_prototype(), + ) return result def _parse_chunk_key_encoding( - data: ChunkKeyEncoding | ChunkKeyEncodingLike | None, zarr_format: ZarrFormat + data: ChunkKeyEncodingLike | None, zarr_format: ZarrFormat ) -> ChunkKeyEncoding: """ Take an implicit specification of a chunk key encoding and parse it into a ChunkKeyEncoding object. @@ -4149,3 +4354,48 @@ def _parse_deprecated_compressor( elif zarr_format == 2 and compressor == compressors == "auto": compressors = ({"id": "blosc"},) return compressors + + +def _parse_data_params( + *, + data: np.ndarray[Any, np.dtype[Any]] | None, + shape: ShapeLike | None, + dtype: npt.DTypeLike | None, +) -> tuple[np.ndarray[Any, np.dtype[Any]] | None, ShapeLike, npt.DTypeLike]: + """ + Ensure an array-like ``data`` parameter is consistent with the ``dtype`` and ``shape`` + parameters. + """ + if data is None: + if shape is None: + msg = ( + "The data parameter was set to None, but shape was not specified. " + "Either provide a value for data, or specify shape." + ) + raise ValueError(msg) + shape_out = shape + if dtype is None: + msg = ( + "The data parameter was set to None, but dtype was not specified." + "Either provide an array-like value for data, or specify dtype." + ) + raise ValueError(msg) + dtype_out = dtype + else: + if shape is not None: + msg = ( + "The data parameter was used, but the shape parameter was also " + "used. This is an error. Either use the data parameter, or the shape parameter, " + "but not both." + ) + raise ValueError(msg) + shape_out = data.shape + if dtype is not None: + msg = ( + "The data parameter was used, but the dtype parameter was also " + "used. This is an error. Either use the data parameter, or the dtype parameter, " + "but not both." + ) + raise ValueError(msg) + dtype_out = data.dtype + return data, shape_out, dtype_out diff --git a/tests/test_array.py b/tests/test_array.py index 6600424147..abc048423c 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -11,7 +11,9 @@ import pytest import zarr.api.asynchronous +import zarr.api.synchronous as sync_api from zarr import Array, AsyncArray, Group +from zarr.abc.store import Store from zarr.codecs import ( BytesCodec, GzipCodec, @@ -36,14 +38,15 @@ from zarr.core.chunk_grids import _auto_partition from zarr.core.common import JSON, MemoryOrder, ZarrFormat from zarr.core.group import AsyncGroup -from zarr.core.indexing import ceildiv -from zarr.core.metadata.v3 import DataType +from zarr.core.indexing import BasicIndexer, ceildiv +from zarr.core.metadata.v3 import ArrayV3Metadata, DataType from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.storage import LocalStore, MemoryStore, StorePath if TYPE_CHECKING: from zarr.core.array_spec import ArrayConfigLike + from zarr.core.metadata.v2 import ArrayV2Metadata @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @@ -1257,6 +1260,69 @@ async def test_create_array_v2_no_shards(store: MemoryStore) -> None: ) +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("impl", ["sync", "async"]) +async def test_create_array_data(impl: Literal["sync", "async"], store: Store) -> None: + """ + Test that we can invoke ``create_array`` with a ``data`` parameter. + """ + data = np.arange(10) + name = "foo" + arr: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array + if impl == "sync": + arr = sync_api.create_array(store, name=name, data=data) + stored = arr[:] + elif impl == "async": + arr = await create_array(store, name=name, data=data, zarr_format=3) + stored = await arr._get_selection( + BasicIndexer(..., shape=arr.shape, chunk_grid=arr.metadata.chunk_grid), + prototype=default_buffer_prototype(), + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + assert np.array_equal(stored, data) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +async def test_create_array_data_invalid_params(store: Store) -> None: + """ + Test that failing to specify data AND shape / dtype results in a ValueError + """ + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype=None) + + # we catch shape=None first, so specifying a dtype should raise the same exception as before + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype="uint8") + + with pytest.raises(ValueError, match="dtype was not specified"): + await create_array(store, data=None, shape=(10, 10)) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +async def test_create_array_data_ignored_params(store: Store) -> None: + """ + Test that specify data AND shape AND dtype results in a warning + """ + data = np.arange(10) + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=None, overwrite=True) + + # we catch shape first, so specifying a dtype should raise the same warning as before + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=data.dtype, overwrite=True) + + with pytest.raises( + ValueError, match="The data parameter was used, but the dtype parameter was also used." + ): + await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) + + async def test_scalar_array() -> None: arr = zarr.array(1.5) assert arr[...] == 1.5 From 15d92601d23458bc0fc278c17a90e1e18de09e5a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 30 Jan 2025 15:57:42 -0700 Subject: [PATCH 045/160] Optimize is_total_slice for chunksize == 1 (#2782) --- changes/2782.bugfix.rst | 1 + src/zarr/core/indexing.py | 11 +++++++---- tests/test_array.py | 9 +++++++++ tests/test_indexing.py | 6 ++++++ 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 changes/2782.bugfix.rst diff --git a/changes/2782.bugfix.rst b/changes/2782.bugfix.rst new file mode 100644 index 0000000000..baea970f51 --- /dev/null +++ b/changes/2782.bugfix.rst @@ -0,0 +1 @@ +Optimize full chunk writes diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index f1226821ba..733b2464ac 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -1373,10 +1373,13 @@ def is_total_slice(item: Selection, shape: ChunkCoords) -> bool: item = (item,) if isinstance(item, tuple): return all( - isinstance(dim_sel, slice) - and ( - (dim_sel == slice(None)) - or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) + (isinstance(dim_sel, int) and dim_len == 1) + or ( + isinstance(dim_sel, slice) + and ( + (dim_sel == slice(None)) + or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) + ) ) for dim_sel, dim_len in zip(item, shape, strict=False) ) diff --git a/tests/test_array.py b/tests/test_array.py index abc048423c..80ff8444fc 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -5,6 +5,7 @@ import re from itertools import accumulate from typing import TYPE_CHECKING, Any, Literal +from unittest import mock import numcodecs import numpy as np @@ -1328,3 +1329,11 @@ async def test_scalar_array() -> None: assert arr[...] == 1.5 assert arr[()] == 1.5 assert arr.shape == () + + +async def test_orthogonal_set_total_slice() -> None: + """Ensure that a whole chunk overwrite does not read chunks""" + store = MemoryStore() + array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1) + with mock.patch("zarr.storage.MemoryStore.get", side_effect=ValueError): + array[0, slice(4, 10)] = np.arange(6) diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 30d0d75f22..932c32f1ae 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -19,6 +19,7 @@ OrthogonalSelection, Selection, _iter_grid, + is_total_slice, make_slice_selection, normalize_integer_selection, oindex, @@ -1953,3 +1954,8 @@ def test_vectorized_indexing_incompatible_shape(store) -> None: ) with pytest.raises(ValueError, match="Attempting to set"): arr[np.array([1, 2]), np.array([1, 2])] = np.array([[-1, -2], [-3, -4]]) + + +def test_is_total_slice(): + assert is_total_slice((0, slice(4, 6)), (1, 2)) + assert is_total_slice((slice(0, 1, None), slice(4, 6)), (1, 2)) From cab29d21dd7ef88a2ccaab0749faa59db345c86e Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 31 Jan 2025 08:08:38 -0800 Subject: [PATCH 046/160] prepare changelog for 3.0.2 release (#2783) * prepare changelog for 3.0.2 release * fixup * Update docs/release-notes.rst Co-authored-by: Davis Bennett * fixup --------- Co-authored-by: Davis Bennett --- changes/2533.bugfix.rst | 1 - changes/2681.bugfix.rst | 1 - changes/2693.bugfix.1.rst | 1 - changes/2693.bugfix.2.rst | 1 - changes/2693.bugfix.3.rst | 1 - changes/2693.bugfix.4.rst | 1 - changes/2693.feature.1.rst | 1 - changes/2693.feature.2.rst | 1 - changes/2693.feature.3.rst | 3 --- changes/2693.feature.4.rst | 1 - changes/2693.feature.5.rst | 1 - changes/2693.feature.6.rst | 1 - changes/2693.feature.7.rst | 1 - changes/2693.feature.8.rst | 1 - changes/2693.feature.9.rst | 1 - changes/2736.doc.rst | 2 -- changes/2761.feature.1.rst | 5 ---- changes/2762.bugfix.rst | 2 -- changes/2763.chore.rst | 3 --- changes/2768.bugfix.1.rst | 1 - changes/2768.bugfix.2.rst | 2 -- changes/2782.bugfix.rst | 1 - docs/release-notes.rst | 54 ++++++++++++++++++++++++++++++++++++++ 23 files changed, 54 insertions(+), 33 deletions(-) delete mode 100644 changes/2533.bugfix.rst delete mode 100644 changes/2681.bugfix.rst delete mode 100644 changes/2693.bugfix.1.rst delete mode 100644 changes/2693.bugfix.2.rst delete mode 100644 changes/2693.bugfix.3.rst delete mode 100644 changes/2693.bugfix.4.rst delete mode 100644 changes/2693.feature.1.rst delete mode 100644 changes/2693.feature.2.rst delete mode 100644 changes/2693.feature.3.rst delete mode 100644 changes/2693.feature.4.rst delete mode 100644 changes/2693.feature.5.rst delete mode 100644 changes/2693.feature.6.rst delete mode 100644 changes/2693.feature.7.rst delete mode 100644 changes/2693.feature.8.rst delete mode 100644 changes/2693.feature.9.rst delete mode 100644 changes/2736.doc.rst delete mode 100644 changes/2761.feature.1.rst delete mode 100644 changes/2762.bugfix.rst delete mode 100644 changes/2763.chore.rst delete mode 100644 changes/2768.bugfix.1.rst delete mode 100644 changes/2768.bugfix.2.rst delete mode 100644 changes/2782.bugfix.rst diff --git a/changes/2533.bugfix.rst b/changes/2533.bugfix.rst deleted file mode 100644 index dbcdf40e3c..0000000000 --- a/changes/2533.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Wrap sync fsspec filesystems with AsyncFileSystemWrapper in xarray.to_zarr \ No newline at end of file diff --git a/changes/2681.bugfix.rst b/changes/2681.bugfix.rst deleted file mode 100644 index fa69f73e06..0000000000 --- a/changes/2681.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Added backwards compatibility for Zarr format 2 structured arrays. diff --git a/changes/2693.bugfix.1.rst b/changes/2693.bugfix.1.rst deleted file mode 100644 index a46ad9d28d..0000000000 --- a/changes/2693.bugfix.1.rst +++ /dev/null @@ -1 +0,0 @@ -Match the errors raised by read only stores in StoreTests. diff --git a/changes/2693.bugfix.2.rst b/changes/2693.bugfix.2.rst deleted file mode 100644 index 972ba0b27e..0000000000 --- a/changes/2693.bugfix.2.rst +++ /dev/null @@ -1 +0,0 @@ -Use stdout rather than stderr as the default stream for LoggingStore. diff --git a/changes/2693.bugfix.3.rst b/changes/2693.bugfix.3.rst deleted file mode 100644 index a6eed34c48..0000000000 --- a/changes/2693.bugfix.3.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure that ZipStore is open before getting or setting any values. diff --git a/changes/2693.bugfix.4.rst b/changes/2693.bugfix.4.rst deleted file mode 100644 index 002f078345..0000000000 --- a/changes/2693.bugfix.4.rst +++ /dev/null @@ -1 +0,0 @@ -Update equality for LoggingStore and WrapperStore such that 'other' must also be a LoggingStore or WrapperStore respectively, rather than only checking the types of the stores they wrap. diff --git a/changes/2693.feature.1.rst b/changes/2693.feature.1.rst deleted file mode 100644 index faf54d4d37..0000000000 --- a/changes/2693.feature.1.rst +++ /dev/null @@ -1 +0,0 @@ -Implemented open() for LoggingStore. diff --git a/changes/2693.feature.2.rst b/changes/2693.feature.2.rst deleted file mode 100644 index 091ce4754e..0000000000 --- a/changes/2693.feature.2.rst +++ /dev/null @@ -1 +0,0 @@ -LoggingStore is now a generic class. diff --git a/changes/2693.feature.3.rst b/changes/2693.feature.3.rst deleted file mode 100644 index 06200e3010..0000000000 --- a/changes/2693.feature.3.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change StoreTest's ``test_store_repr``, ``test_store_supports_writes``, -``test_store_supports_partial_writes``, and ``test_store_supports_listing`` -to to be implemented using ``@abstractmethod``, rather raising ``NotImplementedError``. diff --git a/changes/2693.feature.4.rst b/changes/2693.feature.4.rst deleted file mode 100644 index c69ec87cc5..0000000000 --- a/changes/2693.feature.4.rst +++ /dev/null @@ -1 +0,0 @@ -Separate instantiating and opening a store in StoreTests. diff --git a/changes/2693.feature.5.rst b/changes/2693.feature.5.rst deleted file mode 100644 index b9dde46f67..0000000000 --- a/changes/2693.feature.5.rst +++ /dev/null @@ -1 +0,0 @@ -Add a test for using Stores as a context managers in StoreTests. diff --git a/changes/2693.feature.6.rst b/changes/2693.feature.6.rst deleted file mode 100644 index 3dd6c79c8d..0000000000 --- a/changes/2693.feature.6.rst +++ /dev/null @@ -1 +0,0 @@ -Test that a ValueError is raised for invalid byte range syntax in StoreTests. diff --git a/changes/2693.feature.7.rst b/changes/2693.feature.7.rst deleted file mode 100644 index dfa346391a..0000000000 --- a/changes/2693.feature.7.rst +++ /dev/null @@ -1 +0,0 @@ -Test getsize() and getsize_prefix() in StoreTests. diff --git a/changes/2693.feature.8.rst b/changes/2693.feature.8.rst deleted file mode 100644 index 3a9d4043a2..0000000000 --- a/changes/2693.feature.8.rst +++ /dev/null @@ -1 +0,0 @@ -Test the error raised for invalid buffer arguments in StoreTests. diff --git a/changes/2693.feature.9.rst b/changes/2693.feature.9.rst deleted file mode 100644 index 8362e89b17..0000000000 --- a/changes/2693.feature.9.rst +++ /dev/null @@ -1 +0,0 @@ -Test that data can be written to a store that's not yet open using the store.set method in StoreTests diff --git a/changes/2736.doc.rst b/changes/2736.doc.rst deleted file mode 100644 index 0cfe264eb1..0000000000 --- a/changes/2736.doc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changed the machinery for creating changelog entries. -Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. diff --git a/changes/2761.feature.1.rst b/changes/2761.feature.1.rst deleted file mode 100644 index 387de1a380..0000000000 --- a/changes/2761.feature.1.rst +++ /dev/null @@ -1,5 +0,0 @@ -Adds a new function ``init_array`` for initializing an array in storage, and refactors ``create_array`` -to use ``init_array``. ``create_array`` takes two a new parameters: ``data``, an optional array-like object, and ``write_data``, a bool which defaults to ``True``. -If ``data`` is given to ``create_array``, then the ``dtype`` and ``shape`` attributes of ``data`` are used to define the -corresponding attributes of the resulting Zarr array. Additionally, if ``data`` given and ``write_data`` is ``True``, -then the values in ``data`` will be written to the newly created array. \ No newline at end of file diff --git a/changes/2762.bugfix.rst b/changes/2762.bugfix.rst deleted file mode 100644 index 4995d33edd..0000000000 --- a/changes/2762.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed ZipStore to make sure the correct attributes are saved when instances are pickled. -This fixes a previous bug that prevent using ZipStore with a ProcessPoolExecutor. diff --git a/changes/2763.chore.rst b/changes/2763.chore.rst deleted file mode 100644 index f36c63c289..0000000000 --- a/changes/2763.chore.rst +++ /dev/null @@ -1,3 +0,0 @@ -Created a type alias ``ChunkKeyEncodingLike`` to model the union of ``ChunkKeyEncoding`` instances and the dict form of the -parameters of those instances. ``ChunkKeyEncodingLike`` should be used by high-level functions to provide a convenient -way for creating ``ChunkKeyEncoding`` objects. \ No newline at end of file diff --git a/changes/2768.bugfix.1.rst b/changes/2768.bugfix.1.rst deleted file mode 100644 index b732b742ef..0000000000 --- a/changes/2768.bugfix.1.rst +++ /dev/null @@ -1 +0,0 @@ -Updated the optional test dependencies to include ``botocore`` and ``fsspec``. diff --git a/changes/2768.bugfix.2.rst b/changes/2768.bugfix.2.rst deleted file mode 100644 index 1bf5fb8a85..0000000000 --- a/changes/2768.bugfix.2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed the fsspec tests to skip if ``botocore`` is not installed. -Previously they would have failed with an import error. diff --git a/changes/2782.bugfix.rst b/changes/2782.bugfix.rst deleted file mode 100644 index baea970f51..0000000000 --- a/changes/2782.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize full chunk writes diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 2943250c38..08c64eb899 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -3,6 +3,60 @@ Release notes .. towncrier release notes start +3.0.2 (2025-01-31) +------------------ + +Features +~~~~~~~~ + +- Test ``getsize()`` and ``getsize_prefix()`` in ``StoreTests``. (:issue:`2693`) +- Test that a ``ValueError`` is raised for invalid byte range syntax in ``StoreTests``. (:issue:`2693`) +- Separate instantiating and opening a store in ``StoreTests``. (:issue:`2693`) +- Add a test for using Stores as a context managers in ``StoreTests``. (:issue:`2693`) +- Implemented ``LogingStore.open()``. (:issue:`2693`) +- ``LoggingStore`` is now a generic class. (:issue:`2693`) +- Change StoreTest's ``test_store_repr``, ``test_store_supports_writes``, + ``test_store_supports_partial_writes``, and ``test_store_supports_listing`` + to to be implemented using ``@abstractmethod``, rather raising ``NotImplementedError``. (:issue:`2693`) +- Test the error raised for invalid buffer arguments in ``StoreTests``. (:issue:`2693`) +- Test that data can be written to a store that's not yet open using the store.set method in ``StoreTests``. (:issue:`2693`) +- Adds a new function ``init_array`` for initializing an array in storage, and refactors ``create_array`` + to use ``init_array``. ``create_array`` takes two new parameters: ``data``, an optional array-like object, and ``write_data``, a bool which defaults to ``True``. + If ``data`` is given to ``create_array``, then the ``dtype`` and ``shape`` attributes of ``data`` are used to define the + corresponding attributes of the resulting Zarr array. Additionally, if ``data`` given and ``write_data`` is ``True``, + then the values in ``data`` will be written to the newly created array. (:issue:`2761`) + + +Bugfixes +~~~~~~~~ + +- Wrap sync fsspec filesystems with ``AsyncFileSystemWrapper``. (:issue:`2533`) +- Added backwards compatibility for Zarr format 2 structured arrays. (:issue:`2681`) +- Update equality for ``LoggingStore`` and ``WrapperStore`` such that 'other' must also be a ``LoggingStore`` or ``WrapperStore`` respectively, rather than only checking the types of the stores they wrap. (:issue:`2693`) +- Ensure that ``ZipStore`` is open before getting or setting any values. (:issue:`2693`) +- Use stdout rather than stderr as the default stream for ``LoggingStore``. (:issue:`2693`) +- Match the errors raised by read only stores in ``StoreTests``. (:issue:`2693`) +- Fixed ``ZipStore`` to make sure the correct attributes are saved when instances are pickled. + This fixes a previous bug that prevent using ``ZipStore`` with a ``ProcessPoolExecutor``. (:issue:`2762`) +- Updated the optional test dependencies to include ``botocore`` and ``fsspec``. (:issue:`2768`) +- Fixed the fsspec tests to skip if ``botocore`` is not installed. + Previously they would have failed with an import error. (:issue:`2768`) +- Optimize full chunk writes. (:issue:`2782`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Changed the machinery for creating changelog entries. + Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. (:issue:`2736`) + +Other +~~~~~ + +- Created a type alias ``ChunkKeyEncodingLike`` to model the union of ``ChunkKeyEncoding`` instances and the dict form of the + parameters of those instances. ``ChunkKeyEncodingLike`` should be used by high-level functions to provide a convenient + way for creating ``ChunkKeyEncoding`` objects. (:issue:`2763`) + 3.0.1 (Jan. 17, 2025) --------------------- From 3c4ac0f492b24dd4b42f53ea6be0fb4cc8cd17a6 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Fri, 31 Jan 2025 11:22:36 -0600 Subject: [PATCH 047/160] Use all rather than any for labeler config (#2781) Co-authored-by: Davis Bennett --- .github/labeler.yml | 2 +- changes/2781.bugfix.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/2781.bugfix.rst diff --git a/.github/labeler.yml b/.github/labeler.yml index f2529dfef5..2e96aefee1 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,4 +1,4 @@ needs release notes: - all: - changed-files: - - any-glob-to-any-file: '!changes/*.rst' + - all-glob-to-all-file: '!changes/*.rst' diff --git a/changes/2781.bugfix.rst b/changes/2781.bugfix.rst new file mode 100644 index 0000000000..3673eeece7 --- /dev/null +++ b/changes/2781.bugfix.rst @@ -0,0 +1 @@ +Enable automatic removal of `needs release notes` with labeler action \ No newline at end of file From 87557e39fbb5b067466f9d5625fcdffb5af26caa Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Fri, 31 Jan 2025 11:44:56 -0600 Subject: [PATCH 048/160] Pluralize glob and file in labeler.yml (#2785) --- .github/labeler.yml | 2 +- changes/2785.bugfix.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/2785.bugfix.rst diff --git a/.github/labeler.yml b/.github/labeler.yml index 2e96aefee1..ede89c9d35 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,4 +1,4 @@ needs release notes: - all: - changed-files: - - all-glob-to-all-file: '!changes/*.rst' + - all-globs-to-all-files: '!changes/*.rst' diff --git a/changes/2785.bugfix.rst b/changes/2785.bugfix.rst new file mode 100644 index 0000000000..3f2b3111ea --- /dev/null +++ b/changes/2785.bugfix.rst @@ -0,0 +1 @@ +Use the proper label config \ No newline at end of file From cd1f33d943580819a1efb10dadb91d1fcf6666a9 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 4 Feb 2025 22:26:35 +0100 Subject: [PATCH 049/160] pin astroid to <4 to avoid docs build failures (#2796) * add file that does nothing * remove dummy file and pin astroid * changelog * Add astroid to ignore list for codespell * Fix codespell ignore --------- Co-authored-by: David Stansby --- changes/2796.chore.rst | 1 + pyproject.toml | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changes/2796.chore.rst diff --git a/changes/2796.chore.rst b/changes/2796.chore.rst new file mode 100644 index 0000000000..0ef1b8d31e --- /dev/null +++ b/changes/2796.chore.rst @@ -0,0 +1 @@ +The docs environment is now built with ``astroid`` pinned to a version less than 4. This allows the docs to build in CI. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8d73485dac..6fbcd1991e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,7 @@ docs = [ 'numcodecs[msgpack]', 'rich', 's3fs', + 'astroid<4' ] @@ -427,3 +428,6 @@ directory = 'changes' filename = "docs/release-notes.rst" underlines = ["-", "~", "^"] issue_format = ":issue:`{issue}`" + +[tool.codespell] +ignore-words-list = "astroid" From bb2c4fed22c44607a03c1c0ac8572df8fcef1924 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 09:58:40 +0000 Subject: [PATCH 050/160] chore: update pre-commit hooks (#2791) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.9.1 → v0.9.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.1...v0.9.4) - [github.com/codespell-project/codespell: v2.3.0 → v2.4.1](https://github.com/codespell-project/codespell/compare/v2.3.0...v2.4.1) - [github.com/scientific-python/cookie: 2024.08.19 → 2025.01.22](https://github.com/scientific-python/cookie/compare/2024.08.19...2025.01.22) - [github.com/twisted/towncrier: 23.11.0 → 24.8.0](https://github.com/twisted/towncrier/compare/23.11.0...24.8.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 908b0d5c28..94d5342486 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,13 +6,13 @@ ci: default_stages: [pre-commit, pre-push] repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.1 + rev: v0.9.4 hooks: - id: ruff args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell args: ["-L", "fo,ihs,kake,te", "-S", "fixture"] @@ -37,7 +37,7 @@ repos: # Tests - pytest - repo: https://github.com/scientific-python/cookie - rev: 2024.08.19 + rev: 2025.01.22 hooks: - id: sp-repo-review - repo: https://github.com/pre-commit/pygrep-hooks @@ -50,6 +50,6 @@ repos: hooks: - id: numpydoc-validation - repo: https://github.com/twisted/towncrier - rev: 23.11.0 + rev: 24.8.0 hooks: - id: towncrier-check From ab5925b3a18a562261761308fbf7dbe168dfa4b0 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Wed, 5 Feb 2025 06:25:42 -0600 Subject: [PATCH 051/160] Use removeprefix rather than replace to avoid separator deletion (#2778) * Use removeprefix rather than replace to avoid separator deletion * Test list behavior with empty paths --------- Co-authored-by: Joe Hamman Co-authored-by: Davis Bennett --- changes/2778.bugfix.rst | 1 + src/zarr/storage/_fsspec.py | 2 +- src/zarr/testing/store.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 changes/2778.bugfix.rst diff --git a/changes/2778.bugfix.rst b/changes/2778.bugfix.rst new file mode 100644 index 0000000000..2968c4441c --- /dev/null +++ b/changes/2778.bugfix.rst @@ -0,0 +1 @@ +Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list` \ No newline at end of file diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index c30c9b601b..92c14fcc76 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -341,7 +341,7 @@ async def set_partial_values( async def list(self) -> AsyncIterator[str]: # docstring inherited allfiles = await self.fs._find(self.path, detail=False, withdirs=False) - for onefile in (a.replace(self.path + "/", "") for a in allfiles): + for onefile in (a.removeprefix(self.path + "/") for a in allfiles): yield onefile async def list_dir(self, prefix: str) -> AsyncIterator[str]: diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 1fe544d292..00427f6a0e 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -400,6 +400,37 @@ async def test_list_prefix(self, store: S) -> None: expected = tuple(sorted(expected)) assert observed == expected + async def test_list_empty_path(self, store: S) -> None: + """ + Verify that list and list_prefix work correctly when path is an empty string, + i.e. no unwanted replacement occurs. + """ + data = self.buffer_cls.from_bytes(b"") + store_dict = { + "foo/bar/zarr.json": data, + "foo/bar/c/1": data, + "foo/baz/c/0": data, + } + await store._set_many(store_dict.items()) + + # Test list() + observed_list = await _collect_aiterator(store.list()) + observed_list_sorted = sorted(observed_list) + expected_list_sorted = sorted(store_dict.keys()) + assert observed_list_sorted == expected_list_sorted + + # Test list_prefix() with an empty prefix + observed_prefix_empty = await _collect_aiterator(store.list_prefix("")) + observed_prefix_empty_sorted = sorted(observed_prefix_empty) + expected_prefix_empty_sorted = sorted(store_dict.keys()) + assert observed_prefix_empty_sorted == expected_prefix_empty_sorted + + # Test list_prefix() with a non-empty prefix + observed_prefix = await _collect_aiterator(store.list_prefix("foo/bar/")) + observed_prefix_sorted = sorted(observed_prefix) + expected_prefix_sorted = sorted(k for k in store_dict if k.startswith("foo/bar/")) + assert observed_prefix_sorted == expected_prefix_sorted + async def test_list_dir(self, store: S) -> None: root = "foo" store_dict = { From a52048ddb2d5d069c3404e7457439a9ecb5e40c3 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Wed, 5 Feb 2025 11:25:46 -0600 Subject: [PATCH 052/160] Fix UTF generation for numpy in property-based tests (#2801) * Fix UTF generation for numpy in property-based tests * Add changelog entry --------- Co-authored-by: Deepak Cherian --- changes/2801.bugfix.rst | 1 + src/zarr/testing/strategies.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 changes/2801.bugfix.rst diff --git a/changes/2801.bugfix.rst b/changes/2801.bugfix.rst new file mode 100644 index 0000000000..294934aacf --- /dev/null +++ b/changes/2801.bugfix.rst @@ -0,0 +1 @@ +Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index b948651ce6..81bb482b06 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -51,6 +51,21 @@ def v2_dtypes() -> st.SearchStrategy[np.dtype]: ) +def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]: + """Generate UTF-8-safe text constrained to max_len of dtype.""" + # account for utf-32 encoding (i.e. 4 bytes/character) + max_len = max(1, dtype.itemsize // 4) + + return st.text( + alphabet=st.characters( + blacklist_categories=["Cs"], # Avoid *technically allowed* surrogates + min_codepoint=32, + ), + min_size=1, + max_size=max_len, + ) + + # From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names # 1. must not be the empty string ("") # 2. must not include the character "/" @@ -86,7 +101,12 @@ def numpy_arrays( Generate numpy arrays that can be saved in the provided Zarr format. """ zarr_format = draw(zarr_formats) - return draw(npst.arrays(dtype=v3_dtypes() if zarr_format == 3 else v2_dtypes(), shape=shapes)) + dtype = draw(v3_dtypes() if zarr_format == 3 else v2_dtypes()) + if np.issubdtype(dtype, np.str_): + safe_unicode_strings = safe_unicode_for_dtype(dtype) + return draw(npst.arrays(dtype=dtype, shape=shapes, elements=safe_unicode_strings)) + + return draw(npst.arrays(dtype=dtype, shape=shapes)) @st.composite # type: ignore[misc] From 57181c965f9e563df8d0ae56ab3f8c715eeb2412 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 7 Feb 2025 12:04:20 -0700 Subject: [PATCH 053/160] Strengthen pickling test and fix ZipStore __getstate__() (#2807) * Strenghten serialization test * Fix ZipStore serialization * Add changelog bug fix --- changes/2807.bugfix.rst | 1 + src/zarr/storage/_zip.py | 3 ++- src/zarr/testing/store.py | 8 +++++++- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changes/2807.bugfix.rst diff --git a/changes/2807.bugfix.rst b/changes/2807.bugfix.rst new file mode 100644 index 0000000000..ae0eb2f6ac --- /dev/null +++ b/changes/2807.bugfix.rst @@ -0,0 +1 @@ +Fix pickling for ZipStore diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index bf8f9900b9..51bb702c27 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -108,7 +108,8 @@ async def _open(self) -> None: self._sync_open() def __getstate__(self) -> dict[str, Any]: - state = self.__dict__ + # We need a copy to not modify the state of the original store + state = self.__dict__.copy() for attr in ["_zf", "_lock"]: state.pop(attr, None) return state diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 00427f6a0e..112f6261e9 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -99,10 +99,16 @@ def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None: store2 = self.store_cls(**store_kwargs) assert store == store2 - def test_serializable_store(self, store: S) -> None: + async def test_serializable_store(self, store: S) -> None: new_store: S = pickle.loads(pickle.dumps(store)) assert new_store == store assert new_store.read_only == store.read_only + # quickly roundtrip data to a key to test that new store works + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "foo" + await store.set(key, data_buf) + observed = await store.get(key, prototype=default_buffer_prototype()) + assert_bytes_equal(observed, data_buf) def test_store_read_only(self, store: S) -> None: assert not store.read_only From 037adf63edeb57cea6b7eca622d9bd7a61f03c60 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 7 Feb 2025 13:30:13 -0700 Subject: [PATCH 054/160] Enable stateful tests for LocalStore (#2804) * Restrict stateful tests to slow hypothesis CI workflow. * cleanuip * Strategy updates * Fix LocalStore test * add changes * add coverage to hypothesis workflow * review comments * Review comments --- .github/workflows/hypothesis.yaml | 6 ++++++ changes/2804.feature.rst | 1 + pyproject.toml | 5 +++-- src/zarr/storage/_local.py | 13 +++++++++++++ src/zarr/testing/stateful.py | 13 +++++++++++++ src/zarr/testing/strategies.py | 11 ++++++++--- tests/conftest.py | 18 ++++++++++++++++++ tests/test_store/test_stateful.py | 26 ++++++++++++-------------- 8 files changed, 74 insertions(+), 19 deletions(-) create mode 100644 changes/2804.feature.rst diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 1029063ef4..0a320de00b 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -69,6 +69,12 @@ jobs: path: .hypothesis/ key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) + - name: Generate and publish the report if: | failure() diff --git a/changes/2804.feature.rst b/changes/2804.feature.rst new file mode 100644 index 0000000000..5a707752a0 --- /dev/null +++ b/changes/2804.feature.rst @@ -0,0 +1 @@ +:py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient. diff --git a/pyproject.toml b/pyproject.toml index 6fbcd1991e..ab285ff7ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ run = "run-coverage --no-cov" run-pytest = "run" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.doctest] @@ -398,7 +398,8 @@ filterwarnings = [ "ignore:.*is currently not part in the Zarr format 3 specification.*:UserWarning", ] markers = [ - "gpu: mark a test as requiring CuPy and GPU" + "gpu: mark a test as requiring CuPy and GPU", + "slow_hypothesis: slow hypothesis tests", ] [tool.repo-review] diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index 1defea26b4..b20a601bed 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -208,6 +208,19 @@ async def delete(self, key: str) -> None: else: await asyncio.to_thread(path.unlink, True) # Q: we may want to raise if path is missing + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + self._check_writable() + path = self.root / prefix + if path.is_dir(): + shutil.rmtree(path) + elif path.is_file(): + raise ValueError(f"delete_dir was passed a {prefix=!r} that is a file") + else: + # Non-existent directory + # This path is tested by test_group:test_create_creates_parents for one + pass + async def exists(self, key: str) -> bool: # docstring inherited path = self.root / key diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py index 1a1ef0e3a3..3e8dbcdf04 100644 --- a/src/zarr/testing/stateful.py +++ b/src/zarr/testing/stateful.py @@ -68,6 +68,9 @@ def can_add(self, path: str) -> bool: # -------------------- store operations ----------------------- @rule(name=node_names, data=st.data()) def add_group(self, name: str, data: DataObject) -> None: + # Handle possible case-insensitive file systems (e.g. MacOS) + if isinstance(self.store, LocalStore): + name = name.lower() if self.all_groups: parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Group parent") else: @@ -90,6 +93,9 @@ def add_array( name: str, array_and_chunks: tuple[np.ndarray[Any, Any], tuple[int, ...]], ) -> None: + # Handle possible case-insensitive file systems (e.g. MacOS) + if isinstance(self.store, LocalStore): + name = name.lower() array, chunks = array_and_chunks fill_value = data.draw(npst.from_dtype(array.dtype)) if self.all_groups: @@ -135,6 +141,7 @@ def add_array( # self.model.rename(from_group, new_path) # self.repo.store.rename(from_group, new_path) + @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.all_arrays) >= 1) @rule(data=st.data()) def delete_array_using_del(self, data: DataObject) -> None: @@ -149,6 +156,7 @@ def delete_array_using_del(self, data: DataObject) -> None: del group[array_name] self.all_arrays.remove(array_path) + @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.all_groups) >= 2) # fixme don't delete root @rule(data=st.data()) def delete_group_using_del(self, data: DataObject) -> None: @@ -284,6 +292,10 @@ def supports_partial_writes(self) -> bool: def supports_writes(self) -> bool: return self.store.supports_writes + @property + def supports_deletes(self) -> bool: + return self.store.supports_deletes + class ZarrStoreStateMachine(RuleBasedStateMachine): """ " @@ -366,6 +378,7 @@ def get_partial_values(self, data: DataObject) -> None: model_vals_ls, ) + @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.model.keys()) > 0) @rule(data=st.data()) def delete(self, data: DataObject) -> None: diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 81bb482b06..5722f3c99e 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,3 +1,4 @@ +import sys from typing import Any import hypothesis.extra.numpy as npst @@ -209,7 +210,7 @@ def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs: Any) -> Any: def key_ranges( - keys: SearchStrategy = node_names, max_size: int | None = None + keys: SearchStrategy = node_names, max_size: int = sys.maxsize ) -> SearchStrategy[list[int]]: """ Function to generate key_ranges strategy for get_partial_values() @@ -218,10 +219,14 @@ def key_ranges( [(key, (range_start, range_end)), (key, (range_start, range_end)),...] """ + + def make_request(start: int, length: int) -> RangeByteRequest: + return RangeByteRequest(start, end=min(start + length, max_size)) + byte_ranges = st.builds( - RangeByteRequest, + make_request, start=st.integers(min_value=0, max_value=max_size), - end=st.integers(min_value=0, max_value=max_size), + length=st.integers(min_value=0, max_value=max_size), ) key_tuple = st.tuples(keys, byte_ranges) return st.lists(key_tuple, min_size=1, max_size=10) diff --git a/tests/conftest.py b/tests/conftest.py index e9cd2b8120..9be675cb20 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -147,6 +147,24 @@ def zarr_format(request: pytest.FixtureRequest) -> ZarrFormat: raise ValueError(msg) +def pytest_addoption(parser: Any) -> None: + parser.addoption( + "--run-slow-hypothesis", + action="store_true", + default=False, + help="run slow hypothesis tests", + ) + + +def pytest_collection_modifyitems(config: Any, items: Any) -> None: + if config.getoption("--run-slow-hypothesis"): + return + skip_slow_hyp = pytest.mark.skip(reason="need --run-slow-hypothesis option to run") + for item in items: + if "slow_hypothesis" in item.keywords: + item.add_marker(skip_slow_hyp) + + settings.register_profile( "ci", max_examples=1000, diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py index 637eea882d..63df814ac9 100644 --- a/tests/test_store/test_stateful.py +++ b/tests/test_store/test_stateful.py @@ -1,14 +1,15 @@ # Stateful tests for arbitrary Zarr stores. import pytest from hypothesis.stateful import ( - Settings, run_state_machine_as_test, ) from zarr.abc.store import Store -from zarr.storage import LocalStore, MemoryStore, ZipStore +from zarr.storage import LocalStore, ZipStore from zarr.testing.stateful import ZarrHierarchyStateMachine, ZarrStoreStateMachine +pytestmark = pytest.mark.slow_hypothesis + def test_zarr_hierarchy(sync_store: Store): def mk_test_instance_sync() -> ZarrHierarchyStateMachine: @@ -16,10 +17,8 @@ def mk_test_instance_sync() -> ZarrHierarchyStateMachine: if isinstance(sync_store, ZipStore): pytest.skip(reason="ZipStore does not support delete") - if isinstance(sync_store, MemoryStore): - run_state_machine_as_test( - mk_test_instance_sync, settings=Settings(report_multiple_bugs=False, max_examples=50) - ) + + run_state_machine_as_test(mk_test_instance_sync) def test_zarr_store(sync_store: Store) -> None: @@ -28,11 +27,10 @@ def mk_test_instance_sync() -> None: if isinstance(sync_store, ZipStore): pytest.skip(reason="ZipStore does not support delete") - elif isinstance(sync_store, LocalStore): - pytest.skip(reason="This test has errors") - elif isinstance(sync_store, MemoryStore): - run_state_machine_as_test(mk_test_instance_sync, settings=Settings(max_examples=50)) - else: - run_state_machine_as_test( - mk_test_instance_sync, settings=Settings(report_multiple_bugs=True) - ) + + if isinstance(sync_store, LocalStore): + # This test uses arbitrary keys, which are passed to `set` and `delete`. + # It assumes that `set` and `delete` are the only two operations that modify state. + # But LocalStore, directories can hang around even after a key is delete-d. + pytest.skip(reason="Test isn't suitable for LocalStore.") + run_state_machine_as_test(mk_test_instance_sync) From 1823a090ec565fe2e29d198cabe1aae222669d82 Mon Sep 17 00:00:00 2001 From: Ilan Gold Date: Mon, 10 Feb 2025 17:05:03 +0100 Subject: [PATCH 055/160] (fix): ensure zip directory store compares key to prefix correctly (#2758) * (fix): ensure zip directory store compares key to prefix correctly * (chore): add test for externally zipped zarr store * (fix): no need for async test * (chore): rel note --------- Co-authored-by: Deepak Cherian --- changes/2758.bugfix.rst | 1 + src/zarr/storage/_zip.py | 2 +- tests/test_store/test_zip.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 changes/2758.bugfix.rst diff --git a/changes/2758.bugfix.rst b/changes/2758.bugfix.rst new file mode 100644 index 0000000000..6b80f8a626 --- /dev/null +++ b/changes/2758.bugfix.rst @@ -0,0 +1 @@ +Fix zip-store path checking for stores with directories listed as files. \ No newline at end of file diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index 51bb702c27..bbfe6c67aa 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -283,7 +283,7 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: yield key else: for key in keys: - if key.startswith(prefix + "/") and key != prefix: + if key.startswith(prefix + "/") and key.strip("/") != prefix: k = key.removeprefix(prefix + "/").split("/")[0] if k not in seen: seen.add(k) diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index a83327d99a..839656108b 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shutil import tempfile import zipfile from typing import TYPE_CHECKING @@ -14,6 +15,7 @@ from zarr.testing.store import StoreTests if TYPE_CHECKING: + from pathlib import Path from typing import Any @@ -111,3 +113,15 @@ async def test_zip_open_mode_translation( kws = {**store_kwargs, "mode": zip_mode} store = await self.store_cls.open(**kws) assert store.read_only == read_only + + def test_externally_zipped_store(self, tmp_path: Path) -> None: + # See: https://github.com/zarr-developers/zarr-python/issues/2757 + zarr_path = tmp_path / "foo.zarr" + root = zarr.open_group(store=zarr_path, mode="w") + root.require_group("foo") + root["foo"]["bar"] = np.array([1]) + shutil.make_archive(zarr_path, "zip", zarr_path) + zip_path = tmp_path / "foo.zarr.zip" + zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") + assert list(zipped.keys()) == list(root.keys()) + assert list(zipped["foo"].keys()) == list(root["foo"].keys()) From f4278a5aa37768ef0abdf7b2b0e89bfaf594f22b Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 10 Feb 2025 10:53:49 -0700 Subject: [PATCH 056/160] Regression test for codec overwriting. (#2811) * Regression test for codec overwriting. Fix is in upstream https://github.com/zarr-developers/numcodecs/pull/700 Closes #2800 * add note * fix skip --- .github/workflows/test.yml | 4 ++++ changes/2811.bugfix.rst | 1 + tests/test_array.py | 45 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 changes/2811.bugfix.rst diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea65c3f0e4..c85648e0ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 # grab all branches and tags - name: Set up Python uses: actions/setup-python@v5 with: @@ -82,6 +84,8 @@ jobs: dependency-set: upstream steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/changes/2811.bugfix.rst b/changes/2811.bugfix.rst new file mode 100644 index 0000000000..ef4e8eb7ed --- /dev/null +++ b/changes/2811.bugfix.rst @@ -0,0 +1 @@ +Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`. diff --git a/tests/test_array.py b/tests/test_array.py index 80ff8444fc..e458ba106e 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -10,6 +10,7 @@ import numcodecs import numpy as np import pytest +from packaging.version import Version import zarr.api.asynchronous import zarr.api.synchronous as sync_api @@ -1337,3 +1338,47 @@ async def test_orthogonal_set_total_slice() -> None: array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1) with mock.patch("zarr.storage.MemoryStore.get", side_effect=ValueError): array[0, slice(4, 10)] = np.arange(6) + + +@pytest.mark.skipif( + Version(numcodecs.__version__) < Version("0.15.1"), + reason="codec configuration is overwritten on older versions. GH2800", +) +def test_roundtrip_numcodecs() -> None: + store = MemoryStore() + + compressors = [ + {"name": "numcodecs.shuffle", "configuration": {"elementsize": 2}}, + {"name": "numcodecs.zlib", "configuration": {"level": 4}}, + ] + filters = [ + { + "name": "numcodecs.fixedscaleoffset", + "configuration": { + "scale": 100.0, + "offset": 0.0, + "dtype": " Date: Tue, 11 Feb 2025 12:47:13 +0100 Subject: [PATCH 057/160] Deterministic chunk padding (#2755) * test deterministic memory store * deterministic memory store * simplify test * document changes * Update src/zarr/core/buffer/cpu.py Co-authored-by: Joe Hamman * lint * handle fill_value==None * better test * improve changes documentation Co-authored-by: David Stansby * update docstrings * document changed `zarr.empty` * add notes to empty() and empty_like() --------- Co-authored-by: Norman Rzepka Co-authored-by: Joe Hamman Co-authored-by: David Stansby --- changes/2755.bugfix.rst | 3 +++ src/zarr/api/asynchronous.py | 12 ++++++++++-- src/zarr/api/synchronous.py | 12 ++++++++++-- src/zarr/core/buffer/cpu.py | 8 ++++---- src/zarr/core/group.py | 19 ++++++++++++++----- tests/test_store/test_memory.py | 26 ++++++++++++++++++++++++++ 6 files changed, 67 insertions(+), 13 deletions(-) create mode 100644 changes/2755.bugfix.rst diff --git a/changes/2755.bugfix.rst b/changes/2755.bugfix.rst new file mode 100644 index 0000000000..2555369544 --- /dev/null +++ b/changes/2755.bugfix.rst @@ -0,0 +1,3 @@ +The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the +specified fill value, or with zeros if no fill value is provided. +This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes. \ No newline at end of file diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 8eba4fc152..0584f19c3f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1065,7 +1065,8 @@ async def create( async def empty( shape: ChunkCoords, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty array. + """Create an empty array with the specified shape. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -1087,7 +1088,8 @@ async def empty( async def empty_like( a: ArrayLike, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty array like `a`. + """Create an empty array like `a`. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -1100,6 +1102,12 @@ async def empty_like( ------- Array The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. """ like_kwargs = _like_args(a, kwargs) return await empty(**like_kwargs) diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 305446ec97..fe68981cb9 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -902,7 +902,8 @@ def create_array( # TODO: add type annotations for kwargs def empty(shape: ChunkCoords, **kwargs: Any) -> Array: - """Create an empty array. + """Create an empty array with the specified shape. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -928,7 +929,8 @@ def empty(shape: ChunkCoords, **kwargs: Any) -> Array: # TODO: move ArrayLike to common module # TODO: add type annotations for kwargs def empty_like(a: ArrayLike, **kwargs: Any) -> Array: - """Create an empty array like another array. + """Create an empty array like another array. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -941,6 +943,12 @@ def empty_like(a: ArrayLike, **kwargs: Any) -> Array: ------- Array The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. """ return Array(sync(async_api.empty_like(a, **kwargs))) diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 5019075496..225adb6f5c 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -154,10 +154,10 @@ def create( order: Literal["C", "F"] = "C", fill_value: Any | None = None, ) -> Self: - ret = cls(np.empty(shape=tuple(shape), dtype=dtype, order=order)) - if fill_value is not None: - ret.fill(fill_value) - return ret + if fill_value is None: + return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order)) + else: + return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order)) @classmethod def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 4760923e0b..1f5d57c0ab 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1498,7 +1498,8 @@ async def tree(self, expand: bool | None = None, level: int | None = None) -> An async def empty( self, *, name: str, shape: ChunkCoords, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty array in this Group. + """Create an empty array with the specified shape in this Group. The contents will + be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -1515,7 +1516,6 @@ async def empty( retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - return await async_api.empty(shape=shape, store=self.store_path, path=name, **kwargs) async def zeros( @@ -1592,7 +1592,8 @@ async def full( async def empty_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty sub-array like `data`. + """Create an empty sub-array like `data`. The contents will be filled with + the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -2442,7 +2443,8 @@ def require_array(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array: @_deprecate_positional_args def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: - """Create an empty array in this Group. + """Create an empty array with the specified shape in this Group. The contents will be filled with + the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -2531,7 +2533,8 @@ def full( @_deprecate_positional_args def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array: - """Create an empty sub-array like `data`. + """Create an empty sub-array like `data`. The contents will be filled + with the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -2546,6 +2549,12 @@ def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> ------- Array The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. """ return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py index ba38889b52..f00d75a8f0 100644 --- a/tests/test_store/test_memory.py +++ b/tests/test_store/test_memory.py @@ -1,12 +1,19 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +import numpy as np import pytest +import zarr from zarr.core.buffer import Buffer, cpu, gpu from zarr.storage import GpuMemoryStore, MemoryStore from zarr.testing.store import StoreTests from zarr.testing.utils import gpu_test +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]): store_cls = MemoryStore @@ -46,6 +53,25 @@ def test_store_supports_partial_writes(self, store: MemoryStore) -> None: def test_list_prefix(self, store: MemoryStore) -> None: assert True + @pytest.mark.parametrize("dtype", ["uint8", "float32", "int64"]) + @pytest.mark.parametrize("zarr_format", [2, 3]) + async def test_deterministic_size( + self, store: MemoryStore, dtype, zarr_format: ZarrFormat + ) -> None: + a = zarr.empty( + store=store, + shape=(3,), + chunks=(1000,), + dtype=dtype, + zarr_format=zarr_format, + overwrite=True, + ) + a[...] = 1 + a.resize((1000,)) + + np.testing.assert_array_equal(a[:3], 1) + np.testing.assert_array_equal(a[3:], 0) + @gpu_test class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): From 2f8b88aa3cea8bb73d2906f6b9d64869d8c52e69 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 11 Feb 2025 17:17:53 +0100 Subject: [PATCH 058/160] Multiprocessing support (#2815) * add failing multiprocessing test * add hook to reset global vars after fork * parametrize multiprocessing test over different methods * guard execution of register_at_fork with a hasattr check * exempt runs-in-a-forked-process code from coverage * update literal type --- src/zarr/core/sync.py | 21 +++++++++++++++++++++ tests/test_array.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 6a2de855e8..2bb5f24802 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -3,6 +3,7 @@ import asyncio import atexit import logging +import os import threading from concurrent.futures import ThreadPoolExecutor, wait from typing import TYPE_CHECKING, TypeVar @@ -89,6 +90,26 @@ def cleanup_resources() -> None: atexit.register(cleanup_resources) +def reset_resources_after_fork() -> None: + """ + Ensure that global resources are reset after a fork. Without this function, + forked processes will retain invalid references to the parent process's resources. + """ + global loop, iothread, _executor + # These lines are excluded from coverage because this function only runs in a child process, + # which is not observed by the test coverage instrumentation. Despite the apparent lack of + # test coverage, this function should be adequately tested by any test that uses Zarr IO with + # multiprocessing. + loop[0] = None # pragma: no cover + iothread[0] = None # pragma: no cover + _executor = None # pragma: no cover + + +# this is only available on certain operating systems +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=reset_resources_after_fork) + + async def _runner(coro: Coroutine[Any, Any, T]) -> T | BaseException: """ Await a coroutine and return the result of running it. If awaiting the coroutine raises an diff --git a/tests/test_array.py b/tests/test_array.py index e458ba106e..1b84d1d061 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1,8 +1,10 @@ import dataclasses import json import math +import multiprocessing as mp import pickle import re +import sys from itertools import accumulate from typing import TYPE_CHECKING, Any, Literal from unittest import mock @@ -1382,3 +1384,39 @@ def test_roundtrip_numcodecs() -> None: metadata = root["test"].metadata.to_dict() expected = (*filters, BYTES_CODEC, *compressors) assert metadata["codecs"] == expected + + +def _index_array(arr: Array, index: Any) -> Any: + return arr[index] + + +@pytest.mark.parametrize( + "method", + [ + pytest.param( + "fork", + marks=pytest.mark.skipif( + sys.platform in ("win32", "darwin"), reason="fork not supported on Windows or OSX" + ), + ), + "spawn", + pytest.param( + "forkserver", + marks=pytest.mark.skipif( + sys.platform == "win32", reason="forkserver not supported on Windows" + ), + ), + ], +) +@pytest.mark.parametrize("store", ["local"], indirect=True) +def test_multiprocessing(store: Store, method: Literal["fork", "spawn", "forkserver"]) -> None: + """ + Test that arrays can be pickled and indexed in child processes + """ + data = np.arange(100) + arr = zarr.create_array(store=store, data=data) + ctx = mp.get_context(method) + pool = ctx.Pool() + + results = pool.starmap(_index_array, [(arr, slice(len(data)))]) + assert all(np.array_equal(r, data) for r in results) From c66f32b93f1dfe4b4e6deebb22a5ea45d40c5aa5 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Wed, 12 Feb 2025 20:38:04 +1100 Subject: [PATCH 059/160] fix: implicit fill value initialisation (#2799) * fix: implicit fill value initialisation - initialise empty chunks to the default fill value during writing - add default fill value for datetime, timedelta, structured data types * fmt * add "other" dtype test * changelog --------- Co-authored-by: Davis Bennett --- changes/2799.bugfix.rst | 1 + src/zarr/core/codec_pipeline.py | 34 ++++++++++++++++----------------- src/zarr/core/metadata/v2.py | 8 ++++++++ tests/test_v2.py | 19 ++++++++++++++++++ 4 files changed, 45 insertions(+), 17 deletions(-) create mode 100644 changes/2799.bugfix.rst diff --git a/changes/2799.bugfix.rst b/changes/2799.bugfix.rst new file mode 100644 index 0000000000..f22b7074bb --- /dev/null +++ b/changes/2799.bugfix.rst @@ -0,0 +1 @@ +Enitialise empty chunks to the default fill value during writing and add default fill values for datetime, timedelta, structured, and other (void* fixed size) data types \ No newline at end of file diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index 583ca01c5e..a35c5ca210 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -56,6 +56,19 @@ def resolve_batched(codec: Codec, chunk_specs: Iterable[ArraySpec]) -> Iterable[ return [codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] +def fill_value_or_default(chunk_spec: ArraySpec) -> Any: + fill_value = chunk_spec.fill_value + if fill_value is None: + # Zarr V2 allowed `fill_value` to be null in the metadata. + # Zarr V3 requires it to be set. This has already been + # validated when decoding the metadata, but we support reading + # Zarr V2 data and need to support the case where fill_value + # is None. + return _default_fill_value(dtype=chunk_spec.dtype) + else: + return fill_value + + @dataclass(frozen=True) class BatchedCodecPipeline(CodecPipeline): """Default codec pipeline. @@ -247,17 +260,7 @@ async def read_batch( if chunk_array is not None: out[out_selection] = chunk_array else: - fill_value = chunk_spec.fill_value - - if fill_value is None: - # Zarr V2 allowed `fill_value` to be null in the metadata. - # Zarr V3 requires it to be set. This has already been - # validated when decoding the metadata, but we support reading - # Zarr V2 data and need to support the case where fill_value - # is None. - fill_value = _default_fill_value(dtype=chunk_spec.dtype) - - out[out_selection] = fill_value + out[out_selection] = fill_value_or_default(chunk_spec) else: chunk_bytes_batch = await concurrent_map( [ @@ -284,10 +287,7 @@ async def read_batch( tmp = tmp.squeeze(axis=drop_axes) out[out_selection] = tmp else: - fill_value = chunk_spec.fill_value - if fill_value is None: - fill_value = _default_fill_value(dtype=chunk_spec.dtype) - out[out_selection] = fill_value + out[out_selection] = fill_value_or_default(chunk_spec) def _merge_chunk_array( self, @@ -305,7 +305,7 @@ def _merge_chunk_array( shape=chunk_spec.shape, dtype=chunk_spec.dtype, order=chunk_spec.order, - fill_value=chunk_spec.fill_value, + fill_value=fill_value_or_default(chunk_spec), ) else: chunk_array = existing_chunk_array.copy() # make a writable copy @@ -394,7 +394,7 @@ async def _read_key( chunk_array_batch.append(None) # type: ignore[unreachable] else: if not chunk_spec.config.write_empty_chunks and chunk_array.all_equal( - chunk_spec.fill_value + fill_value_or_default(chunk_spec) ): chunk_array_batch.append(None) else: diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 192db5b203..25697c4545 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -349,6 +349,14 @@ def _default_fill_value(dtype: np.dtype[Any]) -> Any: return b"" elif dtype.kind in "UO": return "" + elif dtype.kind in "Mm": + return dtype.type("nat") + elif dtype.kind == "V": + if dtype.fields is not None: + default = tuple([_default_fill_value(field[0]) for field in dtype.fields.values()]) + return np.array([default], dtype=dtype) + else: + return np.zeros(1, dtype=dtype) else: return dtype.type(0) diff --git a/tests/test_v2.py b/tests/test_v2.py index 4c689c8e64..0a4487cfcc 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -313,3 +313,22 @@ def test_structured_dtype_roundtrip(fill_value, tmp_path) -> None: za[...] = a za = zarr.open_array(store=array_path) assert (a == za[:]).all() + + +@pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"]) +def test_other_dtype_roundtrip(fill_value, tmp_path) -> None: + a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7") + array_path = tmp_path / "data.zarr" + za = zarr.create( + shape=(3,), + store=array_path, + chunks=(2,), + fill_value=fill_value, + zarr_format=2, + dtype=a.dtype, + ) + if fill_value is not None: + assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() + za[...] = a + za = zarr.open_array(store=array_path) + assert (a == za[:]).all() From feeb08f4e49f6574d712fe5ceb42ce80ab6ceb3f Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 12 Feb 2025 07:50:50 -0700 Subject: [PATCH 060/160] Always skip reads when completely overwriting chunks (#2784) * Skip reads when completely overwriting boundary chunks Uses `slice(..., None)` to indicate that a `chunk_selection` ends at the boundary of the current chunk. Also does so for a last chunk that is shorter than the chunk size. `is_total_slice` now understands this convention, and correctly detects boundary chunks as total slices. Closes #757 * normalize in codec_pipeline * Revert "normalize in codec_pipeline" This reverts commit 234431cd6efb661c53e2a832a0e4ea4dca772c1b. * Partially Revert "Skip reads when completely overwriting boundary chunks" This reverts commit edbba372de50bf70eb79c7b1deecf4828eab7340. * Different approach * fix bug * add oindex property test * more complex oindex test * cleanup * more oindex * Add changelog entry * [revert] note * fix for numpy 1.25 --------- Co-authored-by: Davis Bennett --- changes/2784.feature.rst | 1 + src/zarr/abc/codec.py | 4 +-- src/zarr/codecs/sharding.py | 14 +++++--- src/zarr/core/array.py | 6 ++-- src/zarr/core/codec_pipeline.py | 60 ++++++++++++++++++-------------- src/zarr/core/indexing.py | 61 ++++++++++++--------------------- src/zarr/storage/_logging.py | 2 +- src/zarr/testing/strategies.py | 37 ++++++++++++++++++++ tests/test_array.py | 11 +++++- tests/test_indexing.py | 6 ---- tests/test_properties.py | 19 +++++++++- 11 files changed, 138 insertions(+), 83 deletions(-) create mode 100644 changes/2784.feature.rst diff --git a/changes/2784.feature.rst b/changes/2784.feature.rst new file mode 100644 index 0000000000..e3218e6df0 --- /dev/null +++ b/changes/2784.feature.rst @@ -0,0 +1 @@ +Avoid reading chunks during writes where possible. :issue:`757` diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index fabd042dbe..16400f5f4b 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -357,7 +357,7 @@ async def encode( @abstractmethod async def read( self, - batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -379,7 +379,7 @@ async def read( @abstractmethod async def write( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index e8730c86dd..459805d808 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -455,8 +455,9 @@ async def _decode_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], out, ) @@ -486,7 +487,7 @@ async def _decode_partial_single( ) indexed_chunks = list(indexer) - all_chunk_coords = {chunk_coords for chunk_coords, _, _ in indexed_chunks} + all_chunk_coords = {chunk_coords for chunk_coords, *_ in indexed_chunks} # reading bytes of all requested chunks shard_dict: ShardMapping = {} @@ -524,8 +525,9 @@ async def _decode_partial_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], out, ) @@ -558,8 +560,9 @@ async def _encode_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], shard_array, ) @@ -601,8 +604,9 @@ async def _encode_partial_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], shard_array, ) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 4c444a81fa..9e2fdf3733 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1290,8 +1290,9 @@ async def _get_selection( self.metadata.get_chunk_spec(chunk_coords, _config, prototype=prototype), chunk_selection, out_selection, + is_complete_chunk, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer ], out_buffer, drop_axes=indexer.drop_axes, @@ -1417,8 +1418,9 @@ async def _set_selection( self.metadata.get_chunk_spec(chunk_coords, _config, prototype), chunk_selection, out_selection, + is_complete_chunk, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer ], value_buffer, drop_axes=indexer.drop_axes, diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index a35c5ca210..0c53cda96c 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -16,7 +16,7 @@ ) from zarr.core.common import ChunkCoords, concurrent_map from zarr.core.config import config -from zarr.core.indexing import SelectorTuple, is_scalar, is_total_slice +from zarr.core.indexing import SelectorTuple, is_scalar from zarr.core.metadata.v2 import _default_fill_value from zarr.registry import register_pipeline @@ -243,7 +243,7 @@ async def encode_partial_batch( async def read_batch( self, - batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -251,10 +251,10 @@ async def read_batch( chunk_array_batch = await self.decode_partial_batch( [ (byte_getter, chunk_selection, chunk_spec) - for byte_getter, chunk_spec, chunk_selection, _ in batch_info + for byte_getter, chunk_spec, chunk_selection, *_ in batch_info ] ) - for chunk_array, (_, chunk_spec, _, out_selection) in zip( + for chunk_array, (_, chunk_spec, _, out_selection, _) in zip( chunk_array_batch, batch_info, strict=False ): if chunk_array is not None: @@ -263,22 +263,19 @@ async def read_batch( out[out_selection] = fill_value_or_default(chunk_spec) else: chunk_bytes_batch = await concurrent_map( - [ - (byte_getter, array_spec.prototype) - for byte_getter, array_spec, _, _ in batch_info - ], + [(byte_getter, array_spec.prototype) for byte_getter, array_spec, *_ in batch_info], lambda byte_getter, prototype: byte_getter.get(prototype), config.get("async.concurrency"), ) chunk_array_batch = await self.decode_batch( [ (chunk_bytes, chunk_spec) - for chunk_bytes, (_, chunk_spec, _, _) in zip( + for chunk_bytes, (_, chunk_spec, *_) in zip( chunk_bytes_batch, batch_info, strict=False ) ], ) - for chunk_array, (_, chunk_spec, chunk_selection, out_selection) in zip( + for chunk_array, (_, chunk_spec, chunk_selection, out_selection, _) in zip( chunk_array_batch, batch_info, strict=False ): if chunk_array is not None: @@ -296,9 +293,10 @@ def _merge_chunk_array( out_selection: SelectorTuple, chunk_spec: ArraySpec, chunk_selection: SelectorTuple, + is_complete_chunk: bool, drop_axes: tuple[int, ...], ) -> NDBuffer: - if is_total_slice(chunk_selection, chunk_spec.shape) and value.shape == chunk_spec.shape: + if is_complete_chunk and value.shape == chunk_spec.shape: return value if existing_chunk_array is None: chunk_array = chunk_spec.prototype.nd_buffer.create( @@ -327,7 +325,7 @@ def _merge_chunk_array( async def write_batch( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -337,14 +335,14 @@ async def write_batch( await self.encode_partial_batch( [ (byte_setter, value, chunk_selection, chunk_spec) - for byte_setter, chunk_spec, chunk_selection, out_selection in batch_info + for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info ], ) else: await self.encode_partial_batch( [ (byte_setter, value[out_selection], chunk_selection, chunk_spec) - for byte_setter, chunk_spec, chunk_selection, out_selection in batch_info + for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info ], ) @@ -361,10 +359,10 @@ async def _read_key( chunk_bytes_batch = await concurrent_map( [ ( - None if is_total_slice(chunk_selection, chunk_spec.shape) else byte_setter, + None if is_complete_chunk else byte_setter, chunk_spec.prototype, ) - for byte_setter, chunk_spec, chunk_selection, _ in batch_info + for byte_setter, chunk_spec, chunk_selection, _, is_complete_chunk in batch_info ], _read_key, config.get("async.concurrency"), @@ -372,7 +370,7 @@ async def _read_key( chunk_array_decoded = await self.decode_batch( [ (chunk_bytes, chunk_spec) - for chunk_bytes, (_, chunk_spec, _, _) in zip( + for chunk_bytes, (_, chunk_spec, *_) in zip( chunk_bytes_batch, batch_info, strict=False ) ], @@ -380,14 +378,24 @@ async def _read_key( chunk_array_merged = [ self._merge_chunk_array( - chunk_array, value, out_selection, chunk_spec, chunk_selection, drop_axes - ) - for chunk_array, (_, chunk_spec, chunk_selection, out_selection) in zip( - chunk_array_decoded, batch_info, strict=False + chunk_array, + value, + out_selection, + chunk_spec, + chunk_selection, + is_complete_chunk, + drop_axes, ) + for chunk_array, ( + _, + chunk_spec, + chunk_selection, + out_selection, + is_complete_chunk, + ) in zip(chunk_array_decoded, batch_info, strict=False) ] chunk_array_batch: list[NDBuffer | None] = [] - for chunk_array, (_, chunk_spec, _, _) in zip( + for chunk_array, (_, chunk_spec, *_) in zip( chunk_array_merged, batch_info, strict=False ): if chunk_array is None: @@ -403,7 +411,7 @@ async def _read_key( chunk_bytes_batch = await self.encode_batch( [ (chunk_array, chunk_spec) - for chunk_array, (_, chunk_spec, _, _) in zip( + for chunk_array, (_, chunk_spec, *_) in zip( chunk_array_batch, batch_info, strict=False ) ], @@ -418,7 +426,7 @@ async def _write_key(byte_setter: ByteSetter, chunk_bytes: Buffer | None) -> Non await concurrent_map( [ (byte_setter, chunk_bytes) - for chunk_bytes, (byte_setter, _, _, _) in zip( + for chunk_bytes, (byte_setter, *_) in zip( chunk_bytes_batch, batch_info, strict=False ) ], @@ -446,7 +454,7 @@ async def encode( async def read( self, - batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -461,7 +469,7 @@ async def read( async def write( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 733b2464ac..c197f6f397 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -321,12 +321,12 @@ class ChunkDimProjection(NamedTuple): Selection of items from chunk array. dim_out_sel Selection of items in target (output) array. - """ dim_chunk_ix: int dim_chunk_sel: Selector dim_out_sel: Selector | None + is_complete_chunk: bool @dataclass(frozen=True) @@ -346,7 +346,8 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: dim_offset = dim_chunk_ix * self.dim_chunk_len dim_chunk_sel = self.dim_sel - dim_offset dim_out_sel = None - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + is_complete_chunk = self.dim_chunk_len == 1 + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) @dataclass(frozen=True) @@ -420,7 +421,10 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: dim_out_sel = slice(dim_out_offset, dim_out_offset + dim_chunk_nitems) - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + is_complete_chunk = ( + dim_chunk_sel_start == 0 and (self.stop >= dim_limit) and self.step in [1, None] + ) + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) def check_selection_length(selection: SelectionNormalized, shape: ChunkCoords) -> None: @@ -493,12 +497,14 @@ class ChunkProjection(NamedTuple): Selection of items from chunk array. out_selection Selection of items in target (output) array. - + is_complete_chunk: + True if a complete chunk is indexed """ chunk_coords: ChunkCoords chunk_selection: tuple[Selector, ...] | npt.NDArray[np.intp] out_selection: tuple[Selector, ...] | npt.NDArray[np.intp] | slice + is_complete_chunk: bool def is_slice(s: Any) -> TypeGuard[slice]: @@ -574,8 +580,8 @@ def __iter__(self) -> Iterator[ChunkProjection]: out_selection = tuple( p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) @@ -643,8 +649,9 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] stop = self.chunk_nitems_cumsum[dim_chunk_ix] dim_out_sel = slice(start, stop) + is_complete_chunk = False # TODO - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) class Order(Enum): @@ -783,8 +790,8 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: # find region in chunk dim_offset = dim_chunk_ix * self.dim_chunk_len dim_chunk_sel = self.dim_sel[start:stop] - dim_offset - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + is_complete_chunk = False # TODO + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) def slice_to_range(s: slice, length: int) -> range: @@ -921,7 +928,8 @@ def __iter__(self) -> Iterator[ChunkProjection]: if not is_basic_selection(out_selection): out_selection = ix_(out_selection, self.shape) - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) @@ -1030,8 +1038,8 @@ def __iter__(self) -> Iterator[ChunkProjection]: out_selection = tuple( p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) @@ -1198,7 +1206,8 @@ def __iter__(self) -> Iterator[ChunkProjection]: for (dim_sel, dim_chunk_offset) in zip(self.selection, chunk_offsets, strict=True) ) - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = False # TODO + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) @@ -1361,32 +1370,6 @@ def c_order_iter(chunks_per_shard: ChunkCoords) -> Iterator[ChunkCoords]: return itertools.product(*(range(x) for x in chunks_per_shard)) -def is_total_slice(item: Selection, shape: ChunkCoords) -> bool: - """Determine whether `item` specifies a complete slice of array with the - given `shape`. Used to optimize __setitem__ operations on the Chunk - class.""" - - # N.B., assume shape is normalized - if item == slice(None): - return True - if isinstance(item, slice): - item = (item,) - if isinstance(item, tuple): - return all( - (isinstance(dim_sel, int) and dim_len == 1) - or ( - isinstance(dim_sel, slice) - and ( - (dim_sel == slice(None)) - or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) - ) - ) - for dim_sel, dim_len in zip(item, shape, strict=False) - ) - else: - raise TypeError(f"expected slice or tuple of slices, found {item!r}") - - def get_indexer( selection: SelectionWithFields, shape: ChunkCoords, chunk_grid: ChunkGrid ) -> Indexer: diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py index e9d6211588..5f1a97acd9 100644 --- a/src/zarr/storage/_logging.py +++ b/src/zarr/storage/_logging.py @@ -88,7 +88,7 @@ def log(self, hint: Any = "") -> Generator[None, None, None]: op = f"{type(self._store).__name__}.{method}" if hint: op = f"{op}({hint})" - self.logger.info("Calling %s", op) + self.logger.info(" Calling %s", op) start_time = time.time() try: self.counter[method] += 1 diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 5722f3c99e..0883d79bf0 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -209,6 +209,43 @@ def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs: Any) -> Any: ) +@st.composite # type: ignore[misc] +def orthogonal_indices( + draw: st.DrawFn, *, shape: tuple[int] +) -> tuple[tuple[np.ndarray[Any, Any], ...], tuple[np.ndarray[Any, Any], ...]]: + """ + Strategy that returns + (1) a tuple of integer arrays used for orthogonal indexing of Zarr arrays. + (2) an tuple of integer arrays that can be used for equivalent indexing of numpy arrays + """ + zindexer = [] + npindexer = [] + ndim = len(shape) + for axis, size in enumerate(shape): + val = draw( + npst.integer_array_indices( + shape=(size,), result_shape=npst.array_shapes(min_side=1, max_side=size, max_dims=1) + ) + | basic_indices(min_dims=1, shape=(size,), allow_ellipsis=False) + .map(lambda x: (x,) if not isinstance(x, tuple) else x) # bare ints, slices + .filter(lambda x: bool(x)) # skip empty tuple + ) + (idxr,) = val + if isinstance(idxr, int): + idxr = np.array([idxr]) + zindexer.append(idxr) + if isinstance(idxr, slice): + idxr = np.arange(*idxr.indices(size)) + elif isinstance(idxr, (tuple, int)): + idxr = np.array(idxr) + newshape = [1] * ndim + newshape[axis] = idxr.size + npindexer.append(idxr.reshape(newshape)) + + # casting the output of broadcast_arrays is needed for numpy 1.25 + return tuple(zindexer), tuple(np.broadcast_arrays(*npindexer)) + + def key_ranges( keys: SearchStrategy = node_names, max_size: int = sys.maxsize ) -> SearchStrategy[list[int]]: diff --git a/tests/test_array.py b/tests/test_array.py index 1b84d1d061..6aaf1072ba 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1338,9 +1338,18 @@ async def test_orthogonal_set_total_slice() -> None: """Ensure that a whole chunk overwrite does not read chunks""" store = MemoryStore() array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1) - with mock.patch("zarr.storage.MemoryStore.get", side_effect=ValueError): + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): array[0, slice(4, 10)] = np.arange(6) + array = zarr.create_array( + store, shape=(20, 21), chunks=(1, 2), dtype=int, fill_value=-1, overwrite=True + ) + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[0, :] = np.arange(21) + + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[:] = 1 + @pytest.mark.skipif( Version(numcodecs.__version__) < Version("0.15.1"), diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 932c32f1ae..30d0d75f22 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -19,7 +19,6 @@ OrthogonalSelection, Selection, _iter_grid, - is_total_slice, make_slice_selection, normalize_integer_selection, oindex, @@ -1954,8 +1953,3 @@ def test_vectorized_indexing_incompatible_shape(store) -> None: ) with pytest.raises(ValueError, match="Attempting to set"): arr[np.array([1, 2]), np.array([1, 2])] = np.array([[-1, -2], [-3, -4]]) - - -def test_is_total_slice(): - assert is_total_slice((0, slice(4, 6)), (1, 2)) - assert is_total_slice((slice(0, 1, None), slice(4, 6)), (1, 2)) diff --git a/tests/test_properties.py b/tests/test_properties.py index 2e60c951dd..cfa6a706d8 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -8,7 +8,13 @@ import hypothesis.strategies as st from hypothesis import given -from zarr.testing.strategies import arrays, basic_indices, numpy_arrays, zarr_formats +from zarr.testing.strategies import ( + arrays, + basic_indices, + numpy_arrays, + orthogonal_indices, + zarr_formats, +) @given(data=st.data(), zarr_format=zarr_formats) @@ -32,6 +38,17 @@ def test_basic_indexing(data: st.DataObject) -> None: assert_array_equal(nparray, zarray[:]) +@given(data=st.data()) +def test_oindex(data: st.DataObject) -> None: + # integer_array_indices can't handle 0-size dimensions. + zarray = data.draw(arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) + nparray = zarray[:] + + zindexer, npindexer = data.draw(orthogonal_indices(shape=nparray.shape)) + actual = zarray.oindex[zindexer] + assert_array_equal(nparray[npindexer], actual) + + @given(data=st.data()) def test_vindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. From 870265a0474eeadef32ed015f33da28e104c95eb Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Thu, 13 Feb 2025 22:06:05 +1100 Subject: [PATCH 061/160] fix: sharding codec with fancy indexing (#2817) * fix: sharding codec with fancy indexing * changelog * add a better test * proper fix * fix: ArrayOfIntOrBool typing * Revert "fix: ArrayOfIntOrBool typing" This reverts commit 1a30563a2b6d67c74357e14b091c144ab6befe46. * ignore typing error in test --------- Co-authored-by: Deepak Cherian --- changes/2817.bugfix.rst | 1 + src/zarr/codecs/sharding.py | 6 +++++- tests/test_array.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changes/2817.bugfix.rst diff --git a/changes/2817.bugfix.rst b/changes/2817.bugfix.rst new file mode 100644 index 0000000000..b1c0fa9220 --- /dev/null +++ b/changes/2817.bugfix.rst @@ -0,0 +1 @@ +Fix fancy indexing (e.g. arr[5, [0, 1]]) with the sharding codec \ No newline at end of file diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 459805d808..42b1313fac 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -531,7 +531,11 @@ async def _decode_partial_single( ], out, ) - return out + + if hasattr(indexer, "sel_shape"): + return out.reshape(indexer.sel_shape) + else: + return out async def _encode_single( self, diff --git a/tests/test_array.py b/tests/test_array.py index 6aaf1072ba..4838129561 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1429,3 +1429,18 @@ def test_multiprocessing(store: Store, method: Literal["fork", "spawn", "forkser results = pool.starmap(_index_array, [(arr, slice(len(data)))]) assert all(np.array_equal(r, data) for r in results) + + +async def test_sharding_coordinate_selection() -> None: + store = MemoryStore() + g = zarr.open_group(store, mode="w") + arr = g.create_array( + name="a", + shape=(2, 3, 4), + chunks=(1, 2, 2), + overwrite=True, + dtype=np.float32, + shards=(2, 4, 4), + ) + arr[:] = np.arange(2 * 3 * 4).reshape((2, 3, 4)) + assert (arr[1, [0, 1]] == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all() # type: ignore[index] From 99621ecf0b81400e323828111363fe21cf0c7592 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 13 Feb 2025 09:34:33 -0700 Subject: [PATCH 062/160] Add Array metadata strategy (#2813) * Add array_metadata strategy * Fix merge --- changes/2813.feature.rst | 1 + src/zarr/testing/strategies.py | 62 ++++++++++++++++++++++++++++++++-- tests/test_properties.py | 17 ++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 changes/2813.feature.rst diff --git a/changes/2813.feature.rst b/changes/2813.feature.rst new file mode 100644 index 0000000000..8a28f75082 --- /dev/null +++ b/changes/2813.feature.rst @@ -0,0 +1 @@ +Add `zarr.testing.strategies.array_metadata` to generate ArrayV2Metadata and ArrayV3Metadata instances. diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 0883d79bf0..84edb04c83 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,5 +1,5 @@ import sys -from typing import Any +from typing import Any, Literal import hypothesis.extra.numpy as npst import hypothesis.strategies as st @@ -8,9 +8,13 @@ from hypothesis.strategies import SearchStrategy import zarr -from zarr.abc.store import RangeByteRequest +from zarr.abc.store import RangeByteRequest, Store +from zarr.codecs.bytes import BytesCodec from zarr.core.array import Array +from zarr.core.chunk_grids import RegularChunkGrid +from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding from zarr.core.common import ZarrFormat +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync from zarr.storage import MemoryStore, StoreLike from zarr.storage._common import _dereference_path @@ -67,6 +71,11 @@ def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]: ) +def clear_store(x: Store) -> Store: + sync(x.clear()) + return x + + # From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names # 1. must not be the empty string ("") # 2. must not include the character "/" @@ -85,12 +94,59 @@ def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]: # st.builds will only call a new store constructor for different keyword arguments # i.e. stores.examples() will always return the same object per Store class. # So we map a clear to reset the store. -stores = st.builds(MemoryStore, st.just({})).map(lambda x: sync(x.clear())) +stores = st.builds(MemoryStore, st.just({})).map(clear_store) compressors = st.sampled_from([None, "default"]) zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([2, 3]) array_shapes = npst.array_shapes(max_dims=4, min_side=0) +@st.composite # type: ignore[misc] +def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[None | str] | None: + simple_text = st.text(zarr_key_chars, min_size=0) + return draw(st.none() | st.lists(st.none() | simple_text, min_size=ndim, max_size=ndim)) # type: ignore[no-any-return] + + +@st.composite # type: ignore[misc] +def array_metadata( + draw: st.DrawFn, + *, + array_shapes: st.SearchStrategy[tuple[int, ...]] = npst.array_shapes, + zarr_formats: st.SearchStrategy[Literal[2, 3]] = zarr_formats, + attributes: st.SearchStrategy[dict[str, Any]] = attrs, +) -> ArrayV2Metadata | ArrayV3Metadata: + zarr_format = draw(zarr_formats) + # separator = draw(st.sampled_from(['/', '\\'])) + shape = draw(array_shapes()) + ndim = len(shape) + chunk_shape = draw(array_shapes(min_dims=ndim, max_dims=ndim)) + dtype = draw(v3_dtypes()) + fill_value = draw(npst.from_dtype(dtype)) + if zarr_format == 2: + return ArrayV2Metadata( + shape=shape, + chunks=chunk_shape, + dtype=dtype, + fill_value=fill_value, + order=draw(st.sampled_from(["C", "F"])), + attributes=draw(attributes), + dimension_separator=draw(st.sampled_from([".", "/"])), + filters=None, + compressor=None, + ) + else: + return ArrayV3Metadata( + shape=shape, + data_type=dtype, + chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), + fill_value=fill_value, + attributes=draw(attributes), + dimension_names=draw(dimension_names(ndim=ndim)), + chunk_key_encoding=DefaultChunkKeyEncoding(separator="/"), # FIXME + codecs=[BytesCodec()], + storage_transformers=(), + ) + + @st.composite # type: ignore[misc] def numpy_arrays( draw: st.DrawFn, diff --git a/tests/test_properties.py b/tests/test_properties.py index cfa6a706d8..bf98f9d162 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -2,17 +2,23 @@ import pytest from numpy.testing import assert_array_equal +from zarr.core.buffer import default_buffer_prototype + pytest.importorskip("hypothesis") import hypothesis.extra.numpy as npst import hypothesis.strategies as st from hypothesis import given +from zarr.abc.store import Store +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.testing.strategies import ( + array_metadata, arrays, basic_indices, numpy_arrays, orthogonal_indices, + stores, zarr_formats, ) @@ -64,6 +70,17 @@ def test_vindex(data: st.DataObject) -> None: assert_array_equal(nparray[indexer], actual) +@given(store=stores, meta=array_metadata()) # type: ignore[misc] +async def test_roundtrip_array_metadata( + store: Store, meta: ArrayV2Metadata | ArrayV3Metadata +) -> None: + asdict = meta.to_buffer_dict(prototype=default_buffer_prototype()) + for key, expected in asdict.items(): + await store.set(f"0/{key}", expected) + actual = await store.get(f"0/{key}", prototype=default_buffer_prototype()) + assert actual == expected + + # @st.composite # def advanced_indices(draw, *, shape): # basic_idxr = draw( From 47003d7052ba0592b7b23924888380b2850b4741 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 14 Feb 2025 12:14:52 +0100 Subject: [PATCH 063/160] Redundant list comprehension can be replaced using generator (#2829) --- src/zarr/core/metadata/v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 25697c4545..3d292c81b4 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -353,7 +353,7 @@ def _default_fill_value(dtype: np.dtype[Any]) -> Any: return dtype.type("nat") elif dtype.kind == "V": if dtype.fields is not None: - default = tuple([_default_fill_value(field[0]) for field in dtype.fields.values()]) + default = tuple(_default_fill_value(field[0]) for field in dtype.fields.values()) return np.array([default], dtype=dtype) else: return np.zeros(1, dtype=dtype) From 24ef221fca1f34c7fe6b1655c66b3968bc926ee9 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 14 Feb 2025 09:47:54 -0600 Subject: [PATCH 064/160] Update and document GPU buffer handling (#2751) * Update GPU handling This updates how we handle GPU buffers. See the new docs page for a simple example. The basic idea, as discussed in ..., is to use host buffers for all metadata objects and device buffers for data. Zarr has two types of buffers: plain buffers (used for a stream of bytes) and ndbuffers (used for bytes that represent ndarrays). To make it easier for users, I've added a new config option `zarr.config.enable_gpu()` that can be used to update those both. If we need additional customizations in the future, we can add them here. * fixed doc * Fixup * changelog * doctest, skip * removed not gpu * assert that the type matches * Added changelog notes --------- Co-authored-by: Davis Bennett --- changes/2751.bugfix.rst | 1 + changes/2751.doc.rst | 1 + changes/2751.feature.rst | 1 + docs/developers/contributing.rst | 21 ++++++++++++++++++ docs/user-guide/config.rst | 1 + docs/user-guide/gpu.rst | 37 +++++++++++++++++++++++++++++++ docs/user-guide/index.rst | 1 + src/zarr/core/array.py | 16 ++++++++------ src/zarr/core/buffer/gpu.py | 7 ++++++ src/zarr/core/config.py | 13 ++++++++++- src/zarr/testing/utils.py | 2 +- tests/test_api.py | 38 ++++++++++++++++++++++++++++++++ 12 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 changes/2751.bugfix.rst create mode 100644 changes/2751.doc.rst create mode 100644 changes/2751.feature.rst create mode 100644 docs/user-guide/gpu.rst diff --git a/changes/2751.bugfix.rst b/changes/2751.bugfix.rst new file mode 100644 index 0000000000..6f737999cf --- /dev/null +++ b/changes/2751.bugfix.rst @@ -0,0 +1 @@ +Fixed bug with Zarr using device memory, instead of host memory, for storing metadata when using GPUs. \ No newline at end of file diff --git a/changes/2751.doc.rst b/changes/2751.doc.rst new file mode 100644 index 0000000000..19fbcbeea6 --- /dev/null +++ b/changes/2751.doc.rst @@ -0,0 +1 @@ +Added new user guide on :ref:`user-guide-gpu`. \ No newline at end of file diff --git a/changes/2751.feature.rst b/changes/2751.feature.rst new file mode 100644 index 0000000000..61d97479c6 --- /dev/null +++ b/changes/2751.feature.rst @@ -0,0 +1 @@ +Added :meth:`zarr.config.enable_gpu` to update Zarr's configuration to use GPUs. \ No newline at end of file diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index 220e24eced..de10fab2c6 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -230,6 +230,27 @@ during development at `http://0.0.0.0:8000/ `_. This can b $ hatch --env docs run serve +.. _changelog: + +Changelog +~~~~~~~~~ + +zarr-python uses `towncrier`_ to manage release notes. Most pull requests should +include at least one news fragment describing the changes. To add a release +note, you'll need the GitHub issue or pull request number and the type of your +change (``feature``, ``bugfix``, ``doc``, ``removal``, ``misc``). With that, run +```towncrier create``` with your development environment, which will prompt you +for the issue number, change type, and the news text:: + + towncrier create + +Alternatively, you can manually create the files in the ``changes`` directory +using the naming convention ``{issue-number}.{change-type}.rst``. + +See the `towncrier`_ docs for more. + +.. _towncrier: https://towncrier.readthedocs.io/en/stable/tutorial.html + Development best practices, policies and procedures --------------------------------------------------- diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst index 3662f75dff..91ffe50b91 100644 --- a/docs/user-guide/config.rst +++ b/docs/user-guide/config.rst @@ -32,6 +32,7 @@ Configuration options include the following: - Whether empty chunks are written to storage ``array.write_empty_chunks`` - Async and threading options, e.g. ``async.concurrency`` and ``threading.max_workers`` - Selections of implementations of codecs, codec pipelines and buffers +- Enabling GPU support with ``zarr.config.enable_gpu()``. See :ref:`user-guide-gpu` for more. For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations in the registry and then select them in the config. diff --git a/docs/user-guide/gpu.rst b/docs/user-guide/gpu.rst new file mode 100644 index 0000000000..4d3492f8bd --- /dev/null +++ b/docs/user-guide/gpu.rst @@ -0,0 +1,37 @@ +.. _user-guide-gpu: + +Using GPUs with Zarr +==================== + +Zarr can use GPUs to accelerate your workload by running +:meth:`zarr.config.enable_gpu`. + +.. note:: + + `zarr-python` currently supports reading the ndarray data into device (GPU) + memory as the final stage of the codec pipeline. Data will still be read into + or copied to host (CPU) memory for encoding and decoding. + + In the future, codecs will be available compressing and decompressing data on + the GPU, avoiding the need to move data between the host and device for + compression and decompression. + +Reading data into device memory +------------------------------- + +:meth:`zarr.config.enable_gpu` configures Zarr to use GPU memory for the data +buffers used internally by Zarr. + +.. code-block:: python + + >>> import zarr + >>> import cupy as cp # doctest: +SKIP + >>> zarr.config.enable_gpu() # doctest: +SKIP + >>> store = zarr.storage.MemoryStore() # doctest: +SKIP + >>> z = zarr.create_array( # doctest: +SKIP + ... store=store, shape=(100, 100), chunks=(10, 10), dtype="float32", + ... ) + >>> type(z[:10, :10]) # doctest: +SKIP + cupy.ndarray + +Note that the output type is a ``cupy.ndarray`` rather than a NumPy array. diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index a7bbd12453..c50713332b 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -23,6 +23,7 @@ Advanced Topics performance consolidated_metadata extending + gpu .. Coming soon diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 9e2fdf3733..4de24bab41 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -38,6 +38,7 @@ NDBuffer, default_buffer_prototype, ) +from zarr.core.buffer.cpu import buffer_prototype as cpu_buffer_prototype from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition, normalize_chunks from zarr.core.chunk_key_encodings import ( ChunkKeyEncoding, @@ -163,19 +164,20 @@ async def get_array_metadata( ) -> dict[str, JSON]: if zarr_format == 2: zarray_bytes, zattrs_bytes = await gather( - (store_path / ZARRAY_JSON).get(), (store_path / ZATTRS_JSON).get() + (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), ) if zarray_bytes is None: raise FileNotFoundError(store_path) elif zarr_format == 3: - zarr_json_bytes = await (store_path / ZARR_JSON).get() + zarr_json_bytes = await (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype) if zarr_json_bytes is None: raise FileNotFoundError(store_path) elif zarr_format is None: zarr_json_bytes, zarray_bytes, zattrs_bytes = await gather( - (store_path / ZARR_JSON).get(), - (store_path / ZARRAY_JSON).get(), - (store_path / ZATTRS_JSON).get(), + (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), ) if zarr_json_bytes is not None and zarray_bytes is not None: # warn and favor v3 @@ -1348,7 +1350,7 @@ async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = F """ Asynchronously save the array metadata. """ - to_save = metadata.to_buffer_dict(default_buffer_prototype()) + to_save = metadata.to_buffer_dict(cpu_buffer_prototype) awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] if ensure_parents: @@ -1360,7 +1362,7 @@ async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = F [ (parent.store_path / key).set_if_not_exists(value) for key, value in parent.metadata.to_buffer_dict( - default_buffer_prototype() + cpu_buffer_prototype ).items() ] ) diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index 6941c8897e..aac6792cff 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -13,6 +13,10 @@ from zarr.core.buffer import core from zarr.core.buffer.core import ArrayLike, BufferPrototype, NDArrayLike +from zarr.registry import ( + register_buffer, + register_ndbuffer, +) if TYPE_CHECKING: from collections.abc import Iterable @@ -215,3 +219,6 @@ def __setitem__(self, key: Any, value: Any) -> None: buffer_prototype = BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) + +register_buffer(Buffer) +register_ndbuffer(NDBuffer) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 051e8c68e1..c565cb0708 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -29,10 +29,13 @@ from __future__ import annotations -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast from donfig import Config as DConfig +if TYPE_CHECKING: + from donfig.config_obj import ConfigSet + class BadConfigError(ValueError): _msg = "bad Config: %r" @@ -56,6 +59,14 @@ def reset(self) -> None: self.clear() self.refresh() + def enable_gpu(self) -> ConfigSet: + """ + Configure Zarr to use GPUs where possible. + """ + return self.set( + {"buffer": "zarr.core.buffer.gpu.Buffer", "ndbuffer": "zarr.core.buffer.gpu.NDBuffer"} + ) + # The default configuration for zarr config = Config( diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index c7b6e7939c..0a93b93fdb 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -38,7 +38,7 @@ def has_cupy() -> bool: return False -T_Callable = TypeVar("T_Callable", bound=Callable[[], Coroutine[Any, Any, None]]) +T_Callable = TypeVar("T_Callable", bound=Callable[..., Coroutine[Any, Any, None] | None]) # Decorator for GPU tests diff --git a/tests/test_api.py b/tests/test_api.py index aacd558f2a..e9db33f6c5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -27,6 +27,7 @@ from zarr.errors import MetadataValidationError from zarr.storage import MemoryStore from zarr.storage._utils import normalize_path +from zarr.testing.utils import gpu_test def test_create(memory_store: Store) -> None: @@ -1121,3 +1122,40 @@ def test_open_array_with_mode_r_plus(store: Store) -> None: assert isinstance(z2, Array) assert (z2[:] == 1).all() z2[:] = 3 + + +@gpu_test +@pytest.mark.parametrize( + "store", + ["local", "memory", "zip"], + indirect=True, +) +@pytest.mark.parametrize("zarr_format", [None, 2, 3]) +def test_gpu_basic(store: Store, zarr_format: ZarrFormat | None) -> None: + import cupy as cp + + if zarr_format == 2: + # Without this, the zstd codec attempts to convert the cupy + # array to bytes. + compressors = None + else: + compressors = "auto" + + with zarr.config.enable_gpu(): + src = cp.random.uniform(size=(100, 100)) # allocate on the device + z = zarr.create_array( + store, + name="a", + shape=src.shape, + chunks=(10, 10), + dtype=src.dtype, + overwrite=True, + zarr_format=zarr_format, + compressors=compressors, + ) + z[:10, :10] = src[:10, :10] + + result = z[:10, :10] + # assert_array_equal doesn't check the type + assert isinstance(result, type(src)) + cp.testing.assert_array_equal(result, src[:10, :10]) From 3c25dacdd41e9a67f32c2cfac364292bb34d9370 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 14 Feb 2025 09:53:00 -0700 Subject: [PATCH 065/160] Add shards to array strategy (#2822) * Add shards to array strategy * Prioritize v3 over v2 in property tests --------- Co-authored-by: Davis Bennett --- changes/2822.feature.rst | 1 + src/zarr/testing/strategies.py | 55 +++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 17 deletions(-) create mode 100644 changes/2822.feature.rst diff --git a/changes/2822.feature.rst b/changes/2822.feature.rst new file mode 100644 index 0000000000..37b3bf1faf --- /dev/null +++ b/changes/2822.feature.rst @@ -0,0 +1 @@ +Add arbitrary `shards` to Hypothesis strategy for generating arrays. diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 84edb04c83..8847b49020 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -96,7 +96,7 @@ def clear_store(x: Store) -> Store: # So we map a clear to reset the store. stores = st.builds(MemoryStore, st.just({})).map(clear_store) compressors = st.sampled_from([None, "default"]) -zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([2, 3]) +zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([3, 2]) array_shapes = npst.array_shapes(max_dims=4, min_side=0) @@ -166,6 +166,32 @@ def numpy_arrays( return draw(npst.arrays(dtype=dtype, shape=shapes)) +@st.composite # type: ignore[misc] +def chunk_shapes(draw: st.DrawFn, *, shape: tuple[int, ...]) -> tuple[int, ...]: + # We want this strategy to shrink towards arrays with smaller number of chunks + # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks + numchunks = draw( + st.tuples(*[st.integers(min_value=0 if size == 0 else 1, max_value=size) for size in shape]) + ) + # 2. and now generate the chunks tuple + return tuple( + size // nchunks if nchunks > 0 else 0 + for size, nchunks in zip(shape, numchunks, strict=True) + ) + + +@st.composite # type: ignore[misc] +def shard_shapes( + draw: st.DrawFn, *, shape: tuple[int, ...], chunk_shape: tuple[int, ...] +) -> tuple[int, ...]: + # We want this strategy to shrink towards arrays with smaller number of shards + # shards must be an integral number of chunks + assert all(c != 0 for c in chunk_shape) + numchunks = tuple(s // c for s, c in zip(shape, chunk_shape, strict=True)) + multiples = tuple(draw(st.integers(min_value=1, max_value=nc)) for nc in numchunks) + return tuple(m * c for m, c in zip(multiples, chunk_shape, strict=True)) + + @st.composite # type: ignore[misc] def np_array_and_chunks( draw: st.DrawFn, *, arrays: st.SearchStrategy[np.ndarray] = numpy_arrays @@ -175,19 +201,7 @@ def np_array_and_chunks( Returns: a tuple of the array and a suitable random chunking for it. """ array = draw(arrays) - # We want this strategy to shrink towards arrays with smaller number of chunks - # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks - numchunks = draw( - st.tuples( - *[st.integers(min_value=0 if size == 0 else 1, max_value=size) for size in array.shape] - ) - ) - # 2. and now generate the chunks tuple - chunks = tuple( - size // nchunks if nchunks > 0 else 0 - for size, nchunks in zip(array.shape, numchunks, strict=True) - ) - return (array, chunks) + return (array, draw(chunk_shapes(shape=array.shape))) @st.composite # type: ignore[misc] @@ -210,7 +224,12 @@ def arrays( zarr_format = draw(zarr_formats) if arrays is None: arrays = numpy_arrays(shapes=shapes, zarr_formats=st.just(zarr_format)) - nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) + nparray = draw(arrays) + chunk_shape = draw(chunk_shapes(shape=nparray.shape)) + if zarr_format == 3 and all(c > 0 for c in chunk_shape): + shard_shape = draw(st.none() | shard_shapes(shape=nparray.shape, chunk_shape=chunk_shape)) + else: + shard_shape = None # test that None works too. fill_value = draw(st.one_of([st.none(), npst.from_dtype(nparray.dtype)])) # compressor = draw(compressors) @@ -223,7 +242,8 @@ def arrays( a = root.create_array( array_path, shape=nparray.shape, - chunks=chunks, + chunks=chunk_shape, + shards=shard_shape, dtype=nparray.dtype, attributes=attributes, # compressor=compressor, # FIXME @@ -236,7 +256,8 @@ def arrays( assert a.name is not None assert isinstance(root[array_path], Array) assert nparray.shape == a.shape - assert chunks == a.chunks + assert chunk_shape == a.chunks + assert shard_shape == a.shards assert array_path == a.path, (path, name, array_path, a.name, a.path) assert a.basename == name, (a.basename, name) assert dict(a.attrs) == expected_attrs From f8bc3153162e85ad41f62cdb6c693239868b85c6 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Fri, 14 Feb 2025 18:49:24 +0100 Subject: [PATCH 066/160] create_array creates explicit groups (#2795) * refactor create_array tests and add failing test for implicit -> explicit groups * clean up array config parsing, and modify init_array to create parent groups and return an asyncarray instead of metadata * typecheck tests * remove comment * release notes * add type: ignore statement * fix unbound local error in test * remove type:ignore * Add property test * fix test * Update tests/test_array.py --------- Co-authored-by: Deepak Cherian --- changes/2795.bugfix.rst | 1 + src/zarr/api/asynchronous.py | 6 +- src/zarr/api/synchronous.py | 10 +- src/zarr/core/array.py | 60 ++- src/zarr/core/array_spec.py | 11 +- tests/test_array.py | 735 +++++++++++++++++------------------ tests/test_properties.py | 19 + 7 files changed, 429 insertions(+), 413 deletions(-) create mode 100644 changes/2795.bugfix.rst diff --git a/changes/2795.bugfix.rst b/changes/2795.bugfix.rst new file mode 100644 index 0000000000..0ee6619c16 --- /dev/null +++ b/changes/2795.bugfix.rst @@ -0,0 +1 @@ +Alters the behavior of ``create_array`` to ensure that any groups implied by the array's name are created if they do not already exist. Also simplifies the type signature for any function that takes an ArrayConfig-like object. \ No newline at end of file diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 0584f19c3f..3a3d03bb71 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -10,7 +10,7 @@ from typing_extensions import deprecated from zarr.core.array import Array, AsyncArray, create_array, get_array_metadata -from zarr.core.array_spec import ArrayConfig, ArrayConfigLike +from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams from zarr.core.buffer import NDArrayLike from zarr.core.common import ( JSON, @@ -856,7 +856,7 @@ async def create( codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, **kwargs: Any, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Create an array. @@ -1018,7 +1018,7 @@ async def create( mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - config_dict: ArrayConfigLike = {} + config_dict: ArrayConfigParams = {} if write_empty_chunks is not None: if config is not None: diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index fe68981cb9..e1f92633cd 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -25,7 +25,7 @@ SerializerLike, ShardsLike, ) - from zarr.core.array_spec import ArrayConfig, ArrayConfigLike + from zarr.core.array_spec import ArrayConfigLike from zarr.core.buffer import NDArrayLike from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike from zarr.core.common import ( @@ -625,7 +625,7 @@ def create( codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, **kwargs: Any, ) -> Array: """Create an array. @@ -695,7 +695,7 @@ def create( storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. - config : ArrayConfig or ArrayConfigLike, optional + config : ArrayConfigLike, optional Runtime configuration of the array. If provided, will override the default values from `zarr.config.array`. @@ -761,7 +761,7 @@ def create_array( dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> Array: """Create an array. @@ -853,7 +853,7 @@ def create_array( Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. - config : ArrayConfig or ArrayConfigLike, optional + config : ArrayConfigLike, optional Runtime configuration for the array. Returns diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 4de24bab41..9c2f8a7260 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -221,7 +221,7 @@ class AsyncArray(Generic[T_ArrayMetadata]): The metadata of the array. store_path : StorePath The path to the Zarr store. - config : ArrayConfig, optional + config : ArrayConfigLike, optional The runtime configuration of the array, by default None. Attributes @@ -246,7 +246,7 @@ def __init__( self: AsyncArray[ArrayV2Metadata], metadata: ArrayV2Metadata | ArrayV2MetadataDict, store_path: StorePath, - config: ArrayConfig | None = None, + config: ArrayConfigLike | None = None, ) -> None: ... @overload @@ -254,14 +254,14 @@ def __init__( self: AsyncArray[ArrayV3Metadata], metadata: ArrayV3Metadata | ArrayV3MetadataDict, store_path: StorePath, - config: ArrayConfig | None = None, + config: ArrayConfigLike | None = None, ) -> None: ... def __init__( self, metadata: ArrayMetadata | ArrayMetadataDict, store_path: StorePath, - config: ArrayConfig | None = None, + config: ArrayConfigLike | None = None, ) -> None: if isinstance(metadata, dict): zarr_format = metadata["zarr_format"] @@ -275,12 +275,11 @@ def __init__( raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3") metadata_parsed = parse_array_metadata(metadata) - - config = ArrayConfig.from_dict({}) if config is None else config + config_parsed = parse_array_config(config) object.__setattr__(self, "metadata", metadata_parsed) object.__setattr__(self, "store_path", store_path) - object.__setattr__(self, "_config", config) + object.__setattr__(self, "_config", config_parsed) object.__setattr__(self, "codec_pipeline", create_codec_pipeline(metadata=metadata_parsed)) # this overload defines the function signature when zarr_format is 2 @@ -304,7 +303,7 @@ async def create( # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV2Metadata]: ... # this overload defines the function signature when zarr_format is 3 @@ -333,7 +332,7 @@ async def create( # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV3Metadata]: ... @overload @@ -361,7 +360,7 @@ async def create( # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV3Metadata]: ... @overload @@ -395,7 +394,7 @@ async def create( # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: ... @classmethod @@ -430,7 +429,7 @@ async def create( # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Method to create a new asynchronous array instance. @@ -508,7 +507,7 @@ async def create( Whether to raise an error if the store already exists (default is False). data : npt.ArrayLike, optional The data to be inserted into the array (default is None). - config : ArrayConfig or ArrayConfigLike, optional + config : ArrayConfigLike, optional Runtime configuration for the array. Returns @@ -571,7 +570,7 @@ async def _create( # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Method to create a new asynchronous array instance. See :func:`AsyncArray.create` for more details. @@ -1745,7 +1744,7 @@ def create( compressor: dict[str, JSON] | None = None, # runtime overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> Array: """Creates a new Array instance from an initialized store. @@ -1874,7 +1873,7 @@ def _create( compressor: dict[str, JSON] | None = None, # runtime overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> Array: """Creates a new Array instance from an initialized store. See :func:`Array.create` for more details. @@ -3814,7 +3813,8 @@ async def init_array( chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: Iterable[str] | None = None, overwrite: bool = False, -) -> ArrayV3Metadata | ArrayV2Metadata: + config: ArrayConfigLike | None, +) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: """Create and persist an array metadata document. Parameters @@ -3893,11 +3893,13 @@ async def init_array( Zarr format 3 only. Zarr format 2 arrays should not use this parameter. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfigLike or None, optional + Configuration for this array. Returns ------- - ArrayV3Metadata | ArrayV2Metadata - The array metadata document. + AsyncArray + The AsyncArray. """ if zarr_format is None: @@ -3997,14 +3999,9 @@ async def init_array( attributes=attributes, ) - # save the metadata to disk - # TODO: make this easier -- it should be a simple function call that takes a {key: buffer} - coros = ( - (store_path / key).set(value) - for key, value in meta.to_buffer_dict(default_buffer_prototype()).items() - ) - await gather(*coros) - return meta + arr = AsyncArray(metadata=meta, store_path=store_path, config=config) + await arr._save_metadata(meta, ensure_parents=True) + return arr async def create_array( @@ -4027,7 +4024,7 @@ async def create_array( dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, write_data: bool = True, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Create an array. @@ -4117,7 +4114,7 @@ async def create_array( Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. - config : ArrayConfig or ArrayConfigLike, optional + config : ArrayConfigLike, optional Runtime configuration for the array. write_data : bool If a pre-existing array-like object was provided to this function via the ``data`` parameter @@ -4143,13 +4140,12 @@ async def create_array( """ mode: Literal["a"] = "a" - config_parsed = parse_array_config(config) store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) data_parsed, shape_parsed, dtype_parsed = _parse_data_params( data=data, shape=shape, dtype=dtype ) - meta = await init_array( + result = await init_array( store_path=store_path, shape=shape_parsed, dtype=dtype_parsed, @@ -4165,9 +4161,9 @@ async def create_array( chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, overwrite=overwrite, + config=config, ) - result = AsyncArray(metadata=meta, store_path=store_path, config=config_parsed) if write_data is True and data_parsed is not None: await result._set_selection( BasicIndexer(..., shape=result.shape, chunk_grid=result.metadata.chunk_grid), diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py index b1a6a3cad0..59d3cc6b40 100644 --- a/src/zarr/core/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -21,7 +21,7 @@ from zarr.core.common import ChunkCoords -class ArrayConfigLike(TypedDict): +class ArrayConfigParams(TypedDict): """ A TypedDict model of the attributes of an ArrayConfig class, but with no required fields. This allows for partial construction of an ArrayConfig, with the assumption that the unset @@ -56,13 +56,13 @@ def __init__(self, order: MemoryOrder, write_empty_chunks: bool) -> None: object.__setattr__(self, "write_empty_chunks", write_empty_chunks_parsed) @classmethod - def from_dict(cls, data: ArrayConfigLike) -> Self: + def from_dict(cls, data: ArrayConfigParams) -> Self: """ Create an ArrayConfig from a dict. The keys of that dict are a subset of the attributes of the ArrayConfig class. Any keys missing from that dict will be set to the the values in the ``array`` namespace of ``zarr.config``. """ - kwargs_out: ArrayConfigLike = {} + kwargs_out: ArrayConfigParams = {} for f in fields(ArrayConfig): field_name = cast(Literal["order", "write_empty_chunks"], f.name) if field_name not in data: @@ -72,7 +72,10 @@ def from_dict(cls, data: ArrayConfigLike) -> Self: return cls(**kwargs_out) -def parse_array_config(data: ArrayConfig | ArrayConfigLike | None) -> ArrayConfig: +ArrayConfigLike = ArrayConfig | ArrayConfigParams + + +def parse_array_config(data: ArrayConfigLike | None) -> ArrayConfig: """ Convert various types of data to an ArrayConfig. """ diff --git a/tests/test_array.py b/tests/test_array.py index 4838129561..b81f966e20 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -829,66 +829,6 @@ def test_append_bad_shape(store: MemoryStore, zarr_format: ZarrFormat) -> None: z.append(b) -@pytest.mark.parametrize("order", ["C", "F", None]) -@pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_array_create_metadata_order_v2( - order: MemoryOrder | None, zarr_format: int, store: MemoryStore -) -> None: - """ - Test that the ``order`` attribute in zarr v2 array metadata is set correctly via the ``order`` - keyword argument to ``Array.create``. When ``order`` is ``None``, the value of the - ``array.order`` config is used. - """ - arr = zarr.create_array(store=store, shape=(2, 2), order=order, zarr_format=2, dtype="i4") - - expected = order or zarr.config.get("array.order") - assert arr.metadata.zarr_format == 2 # guard for mypy - assert arr.metadata.order == expected - - -@pytest.mark.parametrize("order_config", ["C", "F", None]) -@pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_array_create_order( - order_config: MemoryOrder | None, - zarr_format: ZarrFormat, - store: MemoryStore, -) -> None: - """ - Test that the arrays generated by array indexing have a memory order defined by the config order - value - """ - config: ArrayConfigLike = {} - if order_config is None: - config = {} - expected = zarr.config.get("array.order") - else: - config = {"order": order_config} - expected = order_config - - arr = zarr.create_array( - store=store, shape=(2, 2), zarr_format=zarr_format, dtype="i4", config=config - ) - - vals = np.asarray(arr) - if expected == "C": - assert vals.flags.c_contiguous - elif expected == "F": - assert vals.flags.f_contiguous - else: - raise AssertionError - - -@pytest.mark.parametrize("write_empty_chunks", [True, False]) -def test_write_empty_chunks_config(write_empty_chunks: bool) -> None: - """ - Test that the value of write_empty_chunks is sensitive to the global config when not set - explicitly - """ - with zarr.config.set({"array.write_empty_chunks": write_empty_chunks}): - arr = zarr.create_array({}, shape=(2, 2), dtype="i4") - assert arr._async_array._config.write_empty_chunks == write_empty_chunks - - @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("write_empty_chunks", [True, False]) @pytest.mark.parametrize("fill_value", [0, 5]) @@ -992,339 +932,396 @@ def test_auto_partition_auto_shards( assert auto_shards == expected_shards -def test_chunks_and_shards() -> None: - store = StorePath(MemoryStore()) - shape = (100, 100) - chunks = (5, 5) - shards = (10, 10) - - arr_v3 = zarr.create_array(store=store / "v3", shape=shape, chunks=chunks, dtype="i4") - assert arr_v3.chunks == chunks - assert arr_v3.shards is None - - arr_v3_sharding = zarr.create_array( - store=store / "v3_sharding", - shape=shape, - chunks=chunks, - shards=shards, - dtype="i4", - ) - assert arr_v3_sharding.chunks == chunks - assert arr_v3_sharding.shards == shards - - arr_v2 = zarr.create_array( - store=store / "v2", shape=shape, chunks=chunks, zarr_format=2, dtype="i4" - ) - assert arr_v2.chunks == chunks - assert arr_v2.shards is None - - -def test_create_array_default_fill_values() -> None: - a = zarr.create_array(MemoryStore(), shape=(5,), chunks=(5,), dtype=" None: - """ - Test that the default ``filters`` and ``compressors`` are removed when ``create_array`` is invoked. - """ +class TestCreateArray: + @staticmethod + def test_chunks_and_shards(store: Store) -> None: + spath = StorePath(store) + shape = (100, 100) + chunks = (5, 5) + shards = (10, 10) + + arr_v3 = zarr.create_array(store=spath / "v3", shape=shape, chunks=chunks, dtype="i4") + assert arr_v3.chunks == chunks + assert arr_v3.shards is None + + arr_v3_sharding = zarr.create_array( + store=spath / "v3_sharding", + shape=shape, + chunks=chunks, + shards=shards, + dtype="i4", + ) + assert arr_v3_sharding.chunks == chunks + assert arr_v3_sharding.shards == shards - # v2 - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=2, - compressors=empty_value, - filters=empty_value, + arr_v2 = zarr.create_array( + store=spath / "v2", shape=shape, chunks=chunks, zarr_format=2, dtype="i4" + ) + assert arr_v2.chunks == chunks + assert arr_v2.shards is None + + @staticmethod + @pytest.mark.parametrize( + ("dtype", "fill_value_expected"), [(" None: + a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype) + assert a.fill_value == fill_value_expected + + @staticmethod + @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + @pytest.mark.parametrize("empty_value", [None, ()]) + async def test_no_filters_compressors(store: MemoryStore, dtype: str, empty_value: Any) -> None: + """ + Test that the default ``filters`` and ``compressors`` are removed when ``create_array`` is invoked. + """ + + # v2 + arr = await create_array( + store=store, + dtype=dtype, + shape=(10,), + zarr_format=2, + compressors=empty_value, + filters=empty_value, + ) + # Test metadata explicitly + assert arr.metadata.zarr_format == 2 # guard for mypy + # The v2 metadata stores None and () separately + assert arr.metadata.filters == empty_value + # The v2 metadata does not allow tuple for compressor, therefore it is turned into None + assert arr.metadata.compressor is None + + assert arr.filters == () + assert arr.compressors == () + + # v3 + arr = await create_array( + store=store, + dtype=dtype, + shape=(10,), + compressors=empty_value, + filters=empty_value, + ) + assert arr.metadata.zarr_format == 3 # guard for mypy + if dtype == "str": + assert arr.metadata.codecs == (VLenUTF8Codec(),) + assert arr.serializer == VLenUTF8Codec() + else: + assert arr.metadata.codecs == (BytesCodec(),) + assert arr.serializer == BytesCodec() + + @staticmethod + @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + @pytest.mark.parametrize( + "compressors", + [ + "auto", + None, + (), + (ZstdCodec(level=3),), + (ZstdCodec(level=3), GzipCodec(level=0)), + ZstdCodec(level=3), + {"name": "zstd", "configuration": {"level": 3}}, + ({"name": "zstd", "configuration": {"level": 3}},), + ], ) - assert arr.metadata.zarr_format == 3 # guard for mypy - if dtype == "str": - assert arr.metadata.codecs == (VLenUTF8Codec(),) - assert arr.serializer == VLenUTF8Codec() - else: - assert arr.metadata.codecs == (BytesCodec(),) - assert arr.serializer == BytesCodec() - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) -@pytest.mark.parametrize( - "compressors", - [ - "auto", - None, - (), - (ZstdCodec(level=3),), - (ZstdCodec(level=3), GzipCodec(level=0)), - ZstdCodec(level=3), - {"name": "zstd", "configuration": {"level": 3}}, - ({"name": "zstd", "configuration": {"level": 3}},), - ], -) -@pytest.mark.parametrize( - "filters", - [ - "auto", - None, - (), - ( - TransposeCodec( - order=[ - 0, - ] + @pytest.mark.parametrize( + "filters", + [ + "auto", + None, + (), + ( + TransposeCodec( + order=[ + 0, + ] + ), ), - ), - ( - TransposeCodec( - order=[ - 0, - ] + ( + TransposeCodec( + order=[ + 0, + ] + ), + TransposeCodec( + order=[ + 0, + ] + ), ), TransposeCodec( order=[ 0, ] ), - ), - TransposeCodec( - order=[ - 0, - ] - ), - {"name": "transpose", "configuration": {"order": [0]}}, - ({"name": "transpose", "configuration": {"order": [0]}},), - ], -) -@pytest.mark.parametrize(("chunks", "shards"), [((6,), None), ((3,), (6,))]) -async def test_create_array_v3_chunk_encoding( - store: MemoryStore, - compressors: CompressorsLike, - filters: FiltersLike, - dtype: str, - chunks: tuple[int, ...], - shards: tuple[int, ...] | None, -) -> None: - """ - Test various possibilities for the compressors and filters parameter to create_array - """ - arr = await create_array( - store=store, - dtype=dtype, - shape=(12,), - chunks=chunks, - shards=shards, - zarr_format=3, - filters=filters, - compressors=compressors, - ) - filters_expected, _, compressors_expected = _parse_chunk_encoding_v3( - filters=filters, compressors=compressors, serializer="auto", dtype=np.dtype(dtype) - ) - assert arr.filters == filters_expected - assert arr.compressors == compressors_expected - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) -@pytest.mark.parametrize( - "compressors", - [ - "auto", - None, - numcodecs.Zstd(level=3), - (), - (numcodecs.Zstd(level=3),), - ], -) -@pytest.mark.parametrize( - "filters", ["auto", None, numcodecs.GZip(level=1), (numcodecs.GZip(level=1),)] -) -async def test_create_array_v2_chunk_encoding( - store: MemoryStore, compressors: CompressorsLike, filters: FiltersLike, dtype: str -) -> None: - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=2, - compressors=compressors, - filters=filters, - ) - filters_expected, compressor_expected = _parse_chunk_encoding_v2( - filters=filters, compressor=compressors, dtype=np.dtype(dtype) - ) - assert arr.metadata.zarr_format == 2 # guard for mypy - assert arr.metadata.compressor == compressor_expected - assert arr.metadata.filters == filters_expected - - # Normalize for property getters - compressor_expected = () if compressor_expected is None else (compressor_expected,) - filters_expected = () if filters_expected is None else filters_expected - - assert arr.compressors == compressor_expected - assert arr.filters == filters_expected - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) -async def test_create_array_v3_default_filters_compressors(store: MemoryStore, dtype: str) -> None: - """ - Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with - ``zarr_format`` = 3 and ``filters`` and ``compressors`` are not specified. - """ - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=3, - ) - expected_filters, expected_serializer, expected_compressors = _get_default_chunk_encoding_v3( - np_dtype=np.dtype(dtype) - ) - assert arr.filters == expected_filters - assert arr.serializer == expected_serializer - assert arr.compressors == expected_compressors - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) -async def test_create_array_v2_default_filters_compressors(store: MemoryStore, dtype: str) -> None: - """ - Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with - ``zarr_format`` = 2 and ``filters`` and ``compressors`` are not specified. - """ - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=2, + {"name": "transpose", "configuration": {"order": [0]}}, + ({"name": "transpose", "configuration": {"order": [0]}},), + ], ) - expected_filters, expected_compressors = _get_default_chunk_encoding_v2( - np_dtype=np.dtype(dtype) + @pytest.mark.parametrize(("chunks", "shards"), [((6,), None), ((3,), (6,))]) + async def test_v3_chunk_encoding( + store: MemoryStore, + compressors: CompressorsLike, + filters: FiltersLike, + dtype: str, + chunks: tuple[int, ...], + shards: tuple[int, ...] | None, + ) -> None: + """ + Test various possibilities for the compressors and filters parameter to create_array + """ + arr = await create_array( + store=store, + dtype=dtype, + shape=(12,), + chunks=chunks, + shards=shards, + zarr_format=3, + filters=filters, + compressors=compressors, + ) + filters_expected, _, compressors_expected = _parse_chunk_encoding_v3( + filters=filters, compressors=compressors, serializer="auto", dtype=np.dtype(dtype) + ) + assert arr.filters == filters_expected + assert arr.compressors == compressors_expected + + @staticmethod + @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + @pytest.mark.parametrize( + "compressors", + [ + "auto", + None, + numcodecs.Zstd(level=3), + (), + (numcodecs.Zstd(level=3),), + ], ) - assert arr.metadata.zarr_format == 2 # guard for mypy - assert arr.metadata.filters == expected_filters - assert arr.metadata.compressor == expected_compressors - - # Normalize for property getters - expected_filters = () if expected_filters is None else expected_filters - expected_compressors = () if expected_compressors is None else (expected_compressors,) - assert arr.filters == expected_filters - assert arr.compressors == expected_compressors - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -async def test_create_array_v2_no_shards(store: MemoryStore) -> None: - """ - Test that creating a Zarr v2 array with ``shard_shape`` set to a non-None value raises an error. - """ - msg = re.escape( - "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. Got `shard_shape=(5,)` instead." + @pytest.mark.parametrize( + "filters", ["auto", None, numcodecs.GZip(level=1), (numcodecs.GZip(level=1),)] ) - with pytest.raises(ValueError, match=msg): - _ = await create_array( + async def test_v2_chunk_encoding( + store: MemoryStore, compressors: CompressorsLike, filters: FiltersLike, dtype: str + ) -> None: + arr = await create_array( store=store, - dtype="uint8", + dtype=dtype, shape=(10,), - shards=(5,), zarr_format=2, + compressors=compressors, + filters=filters, ) - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("impl", ["sync", "async"]) -async def test_create_array_data(impl: Literal["sync", "async"], store: Store) -> None: - """ - Test that we can invoke ``create_array`` with a ``data`` parameter. - """ - data = np.arange(10) - name = "foo" - arr: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array - if impl == "sync": - arr = sync_api.create_array(store, name=name, data=data) - stored = arr[:] - elif impl == "async": - arr = await create_array(store, name=name, data=data, zarr_format=3) - stored = await arr._get_selection( - BasicIndexer(..., shape=arr.shape, chunk_grid=arr.metadata.chunk_grid), - prototype=default_buffer_prototype(), + filters_expected, compressor_expected = _parse_chunk_encoding_v2( + filters=filters, compressor=compressors, dtype=np.dtype(dtype) ) - else: - raise ValueError(f"Invalid impl: {impl}") + assert arr.metadata.zarr_format == 2 # guard for mypy + assert arr.metadata.compressor == compressor_expected + assert arr.metadata.filters == filters_expected - assert np.array_equal(stored, data) + # Normalize for property getters + compressor_expected = () if compressor_expected is None else (compressor_expected,) + filters_expected = () if filters_expected is None else filters_expected + assert arr.compressors == compressor_expected + assert arr.filters == filters_expected -@pytest.mark.parametrize("store", ["memory"], indirect=True) -async def test_create_array_data_invalid_params(store: Store) -> None: - """ - Test that failing to specify data AND shape / dtype results in a ValueError - """ - with pytest.raises(ValueError, match="shape was not specified"): - await create_array(store, data=None, shape=None, dtype=None) - - # we catch shape=None first, so specifying a dtype should raise the same exception as before - with pytest.raises(ValueError, match="shape was not specified"): - await create_array(store, data=None, shape=None, dtype="uint8") - - with pytest.raises(ValueError, match="dtype was not specified"): - await create_array(store, data=None, shape=(10, 10)) - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -async def test_create_array_data_ignored_params(store: Store) -> None: - """ - Test that specify data AND shape AND dtype results in a warning - """ - data = np.arange(10) - with pytest.raises( - ValueError, match="The data parameter was used, but the shape parameter was also used." - ): - await create_array(store, data=data, shape=data.shape, dtype=None, overwrite=True) + @staticmethod + @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + async def test_default_filters_compressors( + store: MemoryStore, dtype: str, zarr_format: ZarrFormat + ) -> None: + """ + Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with ``filters`` and ``compressors`` unspecified. + """ + arr = await create_array( + store=store, + dtype=dtype, + shape=(10,), + zarr_format=zarr_format, + ) + if zarr_format == 3: + expected_filters, expected_serializer, expected_compressors = ( + _get_default_chunk_encoding_v3(np_dtype=np.dtype(dtype)) + ) - # we catch shape first, so specifying a dtype should raise the same warning as before - with pytest.raises( - ValueError, match="The data parameter was used, but the shape parameter was also used." - ): - await create_array(store, data=data, shape=data.shape, dtype=data.dtype, overwrite=True) + elif zarr_format == 2: + default_filters, default_compressors = _get_default_chunk_encoding_v2( + np_dtype=np.dtype(dtype) + ) + if default_filters is None: + expected_filters = () + else: + expected_filters = default_filters + if default_compressors is None: + expected_compressors = () + else: + expected_compressors = (default_compressors,) + expected_serializer = None + else: + raise ValueError(f"Invalid zarr_format: {zarr_format}") + + assert arr.filters == expected_filters + assert arr.serializer == expected_serializer + assert arr.compressors == expected_compressors + + @staticmethod + async def test_v2_no_shards(store: Store) -> None: + """ + Test that creating a Zarr v2 array with ``shard_shape`` set to a non-None value raises an error. + """ + msg = re.escape( + "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. Got `shard_shape=(5,)` instead." + ) + with pytest.raises(ValueError, match=msg): + _ = await create_array( + store=store, + dtype="uint8", + shape=(10,), + shards=(5,), + zarr_format=2, + ) - with pytest.raises( - ValueError, match="The data parameter was used, but the dtype parameter was also used." - ): - await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) + @staticmethod + @pytest.mark.parametrize("impl", ["sync", "async"]) + async def test_with_data(impl: Literal["sync", "async"], store: Store) -> None: + """ + Test that we can invoke ``create_array`` with a ``data`` parameter. + """ + data = np.arange(10) + name = "foo" + arr: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array + if impl == "sync": + arr = sync_api.create_array(store, name=name, data=data) + stored = arr[:] + elif impl == "async": + arr = await create_array(store, name=name, data=data, zarr_format=3) + stored = await arr._get_selection( + BasicIndexer(..., shape=arr.shape, chunk_grid=arr.metadata.chunk_grid), + prototype=default_buffer_prototype(), + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + assert np.array_equal(stored, data) + + @staticmethod + async def test_with_data_invalid_params(store: Store) -> None: + """ + Test that failing to specify data AND shape / dtype results in a ValueError + """ + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype=None) + + # we catch shape=None first, so specifying a dtype should raise the same exception as before + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype="uint8") + + with pytest.raises(ValueError, match="dtype was not specified"): + await create_array(store, data=None, shape=(10, 10)) + + @staticmethod + async def test_data_ignored_params(store: Store) -> None: + """ + Test that specifying data AND shape AND dtype results in a ValueError + """ + data = np.arange(10) + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=None, overwrite=True) + + # we catch shape first, so specifying a dtype should raise the same warning as before + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=data.dtype, overwrite=True) + + with pytest.raises( + ValueError, match="The data parameter was used, but the dtype parameter was also used." + ): + await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) + + @staticmethod + @pytest.mark.parametrize("order_config", ["C", "F", None]) + def test_order( + order_config: MemoryOrder | None, + zarr_format: ZarrFormat, + store: MemoryStore, + ) -> None: + """ + Test that the arrays generated by array indexing have a memory order defined by the config order + value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly. + """ + config: ArrayConfigLike = {} + if order_config is None: + config = {} + expected = zarr.config.get("array.order") + else: + config = {"order": order_config} + expected = order_config + if zarr_format == 2: + arr = zarr.create_array( + store=store, + shape=(2, 2), + zarr_format=zarr_format, + dtype="i4", + order=expected, + config=config, + ) + # guard for type checking + assert arr.metadata.zarr_format == 2 + assert arr.metadata.order == expected + else: + arr = zarr.create_array( + store=store, shape=(2, 2), zarr_format=zarr_format, dtype="i4", config=config + ) + vals = np.asarray(arr) + if expected == "C": + assert vals.flags.c_contiguous + elif expected == "F": + assert vals.flags.f_contiguous + else: + raise AssertionError + + @staticmethod + @pytest.mark.parametrize("write_empty_chunks", [True, False]) + async def test_write_empty_chunks_config(write_empty_chunks: bool, store: Store) -> None: + """ + Test that the value of write_empty_chunks is sensitive to the global config when not set + explicitly + """ + with zarr.config.set({"array.write_empty_chunks": write_empty_chunks}): + arr = await create_array(store, shape=(2, 2), dtype="i4") + assert arr._config.write_empty_chunks == write_empty_chunks + + @staticmethod + @pytest.mark.parametrize("path", [None, "", "/", "/foo", "foo", "foo/bar"]) + async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) -> None: + arr = await create_array( + store, shape=(2, 2), dtype="i4", name=path, zarr_format=zarr_format + ) + if path is None: + expected_path = "" + elif path.startswith("/"): + expected_path = path.lstrip("/") + else: + expected_path = path + assert arr.path == expected_path + assert arr.name == "/" + expected_path + + # test that implicit groups were created + path_parts = expected_path.split("/") + if len(path_parts) > 1: + *parents, _ = ["", *accumulate(path_parts, lambda x, y: "/".join([x, y]))] # noqa: FLY002 + for parent_path in parents: + # this will raise if these groups were not created + _ = await zarr.api.asynchronous.open_group( + store=store, path=parent_path, mode="r", zarr_format=zarr_format + ) async def test_scalar_array() -> None: diff --git a/tests/test_properties.py b/tests/test_properties.py index bf98f9d162..71fbeeb839 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -12,6 +12,7 @@ from zarr.abc.store import Store from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.sync import sync from zarr.testing.strategies import ( array_metadata, arrays, @@ -30,6 +31,24 @@ def test_roundtrip(data: st.DataObject, zarr_format: int) -> None: assert_array_equal(nparray, zarray[:]) +@given(array=arrays()) +def test_array_creates_implicit_groups(array): + path = array.path + ancestry = path.split("/")[:-1] + for i in range(len(ancestry)): + parent = "/".join(ancestry[: i + 1]) + if array.metadata.zarr_format == 2: + assert ( + sync(array.store.get(f"{parent}/.zgroup", prototype=default_buffer_prototype())) + is not None + ) + elif array.metadata.zarr_format == 3: + assert ( + sync(array.store.get(f"{parent}/zarr.json", prototype=default_buffer_prototype())) + is not None + ) + + @given(data=st.data()) def test_basic_indexing(data: st.DataObject) -> None: zarray = data.draw(arrays()) From 23abb5b0cfb8b9565985e7e682a498405d630b34 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 14 Feb 2025 11:45:53 -0700 Subject: [PATCH 067/160] Add more setitem property tests (#2825) * Add more setitem property tests * comment out vindex setitem * xfail with shards --- tests/test_properties.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_properties.py b/tests/test_properties.py index 71fbeeb839..acecd44810 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,4 +1,3 @@ -import numpy as np import pytest from numpy.testing import assert_array_equal @@ -8,7 +7,7 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st -from hypothesis import given +from hypothesis import assume, given from zarr.abc.store import Store from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata @@ -57,7 +56,7 @@ def test_basic_indexing(data: st.DataObject) -> None: actual = zarray[indexer] assert_array_equal(nparray[indexer], actual) - new_data = np.ones_like(actual) + new_data = data.draw(npst.arrays(shape=st.just(actual.shape), dtype=nparray.dtype)) zarray[indexer] = new_data nparray[indexer] = new_data assert_array_equal(nparray, zarray[:]) @@ -73,6 +72,12 @@ def test_oindex(data: st.DataObject) -> None: actual = zarray.oindex[zindexer] assert_array_equal(nparray[npindexer], actual) + assume(zarray.shards is None) # GH2834 + new_data = data.draw(npst.arrays(shape=st.just(actual.shape), dtype=nparray.dtype)) + nparray[npindexer] = new_data + zarray.oindex[zindexer] = new_data + assert_array_equal(nparray, zarray[:]) + @given(data=st.data()) def test_vindex(data: st.DataObject) -> None: @@ -88,6 +93,14 @@ def test_vindex(data: st.DataObject) -> None: actual = zarray.vindex[indexer] assert_array_equal(nparray[indexer], actual) + # FIXME! + # when the indexer is such that a value gets overwritten multiple times, + # I think the output depends on chunking. + # new_data = data.draw(npst.arrays(shape=st.just(actual.shape), dtype=nparray.dtype)) + # nparray[indexer] = new_data + # zarray.vindex[indexer] = new_data + # assert_array_equal(nparray, zarray[:]) + @given(store=stores, meta=array_metadata()) # type: ignore[misc] async def test_roundtrip_array_metadata( From 48f7c9af3b49e3221c65c3447e869437c0fb7575 Mon Sep 17 00:00:00 2001 From: Cameron Arshadi <44130022+carshadi@users.noreply.github.com> Date: Fri, 14 Feb 2025 14:05:34 -0500 Subject: [PATCH 068/160] Feat: improves delete_dir for s3fs-backed FsspecStore (#2661) * Implement asynchronous directory deletion in FsspecStore - override Store.delete_dir default method, which deletes keys one by one, to support bulk deletion for fsspec implementations that support a list of paths in the fs._rm method. - This can greatly reduce the number of requests to S3, which reduces likelihood of running into throttling errors and improves delete performance. - Currently, only s3fs is supported. * Use async batched _rm() for FsspecStore.delete_dir() * Suppress allowed exceptions instead of try-except-pass * Adds note on possibly redundant condition in FsspecStore.delete_dir() * Fix: unpack allowed arguments list * Adds tests for FsspecStore.delete_dir * Update src/zarr/storage/_fsspec.py Co-authored-by: Joe Hamman * Remove supports_listing condition from FsspecStore.delete_dir * use f-string for url formatting * assert `store.fs.asynchronous` instead of `store.fs.async_impl` * updates release notes * remove unused import * Explicitly construct wrapped local filesystem for test --------- Co-authored-by: Joe Hamman Co-authored-by: Joe Hamman Co-authored-by: Deepak Cherian --- changes/2661.feature.1.rst | 1 + src/zarr/storage/_fsspec.py | 14 ++++++++++++++ tests/test_store/test_fsspec.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 changes/2661.feature.1.rst diff --git a/changes/2661.feature.1.rst b/changes/2661.feature.1.rst new file mode 100644 index 0000000000..5d0209c581 --- /dev/null +++ b/changes/2661.feature.1.rst @@ -0,0 +1 @@ +Improves performance of FsspecStore.delete_dir for remote filesystems supporting concurrent/batched deletes, e.g., s3fs. \ No newline at end of file diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 92c14fcc76..1cc7039e68 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from contextlib import suppress from typing import TYPE_CHECKING, Any from zarr.abc.store import ( @@ -286,6 +287,19 @@ async def delete(self, key: str) -> None: except self.allowed_exceptions: pass + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + if not self.supports_deletes: + raise NotImplementedError( + "This method is only available for stores that support deletes." + ) + self._check_writable() + + path_to_delete = _dereference_path(self.path, prefix) + + with suppress(*self.allowed_exceptions): + await self.fs._rm(path_to_delete, recursive=True) + async def exists(self, key: str) -> bool: # docstring inherited path = _dereference_path(self.path, key) diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index 929de37869..a710b9e22b 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -217,6 +217,14 @@ async def test_empty_nonexistent_path(self, store_kwargs) -> None: store = await self.store_cls.open(**store_kwargs) assert await store.is_empty("") + async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: + store.supports_deletes = False + with pytest.raises( + NotImplementedError, + match="This method is only available for stores that support deletes.", + ): + await store.delete_dir("test_prefix") + @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), @@ -244,3 +252,28 @@ def test_no_wrap_async_filesystem(): assert not isinstance(store.fs, AsyncFileSystemWrapper) assert store.fs.async_impl + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +async def test_delete_dir_wrapped_filesystem(tmpdir) -> None: + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + from fsspec.implementations.local import LocalFileSystem + + wrapped_fs = AsyncFileSystemWrapper(LocalFileSystem(auto_mkdir=True)) + store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmpdir}/test/path") + + assert isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.asynchronous + + await store.set("zarr.json", cpu.Buffer.from_bytes(b"root")) + await store.set("foo-bar/zarr.json", cpu.Buffer.from_bytes(b"root")) + await store.set("foo/zarr.json", cpu.Buffer.from_bytes(b"bar")) + await store.set("foo/c/0", cpu.Buffer.from_bytes(b"chunk")) + await store.delete_dir("foo") + assert await store.exists("zarr.json") + assert await store.exists("foo-bar/zarr.json") + assert not await store.exists("foo/zarr.json") + assert not await store.exists("foo/c/0") From 8ad4cd69335a32d85b0b1c380572596b1e9f26b7 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 14 Feb 2025 14:13:10 -0700 Subject: [PATCH 069/160] Run fsspec tests in min_deps env (#2835) * Run fsspec tests in min_deps env * Skip failing test --- .github/workflows/test.yml | 5 +++++ pyproject.toml | 19 ++++++++----------- tests/test_store/test_fsspec.py | 18 ++++++++++++++++-- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c85648e0ff..7d82a95662 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,6 +102,11 @@ jobs: - name: Run Tests run: | hatch env run --env ${{ matrix.dependency-set }} run + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) doctests: name: doctests diff --git a/pyproject.toml b/pyproject.toml index ab285ff7ff..0137927039 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,11 +212,7 @@ dependencies = [ 'typing_extensions @ git+https://github.com/python/typing_extensions', 'donfig @ git+https://github.com/pytroll/donfig', # test deps - 'hypothesis', - 'pytest', - 'pytest-cov', - 'pytest-asyncio', - 'moto[s3]', + 'zarr[test]', ] [tool.hatch.envs.upstream.env-vars] @@ -228,6 +224,9 @@ PIP_PRE = "1" run = "pytest --verbose" run-mypy = "mypy src" run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" list-env = "pip list" [tool.hatch.envs.min_deps] @@ -247,18 +246,16 @@ dependencies = [ 'typing_extensions==4.9.*', 'donfig==0.8.*', # test deps - 'hypothesis', - 'pytest', - 'pytest-cov', - 'pytest-asyncio', - 'moto[s3]', + 'zarr[test]', ] [tool.hatch.envs.min_deps.scripts] run = "pytest --verbose" run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" - +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" [tool.ruff] line-length = 100 diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index a710b9e22b..2e9620f29d 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -110,7 +110,12 @@ class TestFsspecStoreS3(StoreTests[FsspecStore, cpu.Buffer]): @pytest.fixture def store_kwargs(self, request) -> dict[str, str | bool]: - fs, path = fsspec.url_to_fs( + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + fs, path = url_to_fs( f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=True ) return {"fs": fs, "path": path} @@ -182,6 +187,10 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value-3"} + @pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.03.01"), + reason="Prior bug in from_upath", + ) def test_from_upath(self) -> None: upath = pytest.importorskip("upath") path = upath.UPath( @@ -204,7 +213,12 @@ def test_init_raises_if_path_has_scheme(self, store_kwargs) -> None: self.store_cls(**store_kwargs) def test_init_warns_if_fs_asynchronous_is_false(self) -> None: - fs, path = fsspec.url_to_fs( + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + fs, path = url_to_fs( f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=False ) store_kwargs = {"fs": fs, "path": path} From 5a36e175325c1f6e55b0638009b8aa7e5ab89a05 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 14 Feb 2025 16:28:39 -0700 Subject: [PATCH 070/160] Changelog for v3.0.3 (#2836) * Changelof for v3.0.3 * fix version --- changes/2661.feature.1.rst | 1 - changes/2751.bugfix.rst | 1 - changes/2751.doc.rst | 1 - changes/2751.feature.rst | 1 - changes/2755.bugfix.rst | 3 --- changes/2758.bugfix.rst | 1 - changes/2778.bugfix.rst | 1 - changes/2781.bugfix.rst | 1 - changes/2784.feature.rst | 1 - changes/2785.bugfix.rst | 1 - changes/2795.bugfix.rst | 1 - changes/2799.bugfix.rst | 1 - changes/2801.bugfix.rst | 1 - changes/2804.feature.rst | 1 - changes/2807.bugfix.rst | 1 - changes/2811.bugfix.rst | 1 - changes/2813.feature.rst | 1 - changes/2817.bugfix.rst | 1 - changes/2822.feature.rst | 1 - docs/release-notes.rst | 39 ++++++++++++++++++++++++++++++++++++++ 20 files changed, 39 insertions(+), 21 deletions(-) delete mode 100644 changes/2661.feature.1.rst delete mode 100644 changes/2751.bugfix.rst delete mode 100644 changes/2751.doc.rst delete mode 100644 changes/2751.feature.rst delete mode 100644 changes/2755.bugfix.rst delete mode 100644 changes/2758.bugfix.rst delete mode 100644 changes/2778.bugfix.rst delete mode 100644 changes/2781.bugfix.rst delete mode 100644 changes/2784.feature.rst delete mode 100644 changes/2785.bugfix.rst delete mode 100644 changes/2795.bugfix.rst delete mode 100644 changes/2799.bugfix.rst delete mode 100644 changes/2801.bugfix.rst delete mode 100644 changes/2804.feature.rst delete mode 100644 changes/2807.bugfix.rst delete mode 100644 changes/2811.bugfix.rst delete mode 100644 changes/2813.feature.rst delete mode 100644 changes/2817.bugfix.rst delete mode 100644 changes/2822.feature.rst diff --git a/changes/2661.feature.1.rst b/changes/2661.feature.1.rst deleted file mode 100644 index 5d0209c581..0000000000 --- a/changes/2661.feature.1.rst +++ /dev/null @@ -1 +0,0 @@ -Improves performance of FsspecStore.delete_dir for remote filesystems supporting concurrent/batched deletes, e.g., s3fs. \ No newline at end of file diff --git a/changes/2751.bugfix.rst b/changes/2751.bugfix.rst deleted file mode 100644 index 6f737999cf..0000000000 --- a/changes/2751.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug with Zarr using device memory, instead of host memory, for storing metadata when using GPUs. \ No newline at end of file diff --git a/changes/2751.doc.rst b/changes/2751.doc.rst deleted file mode 100644 index 19fbcbeea6..0000000000 --- a/changes/2751.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Added new user guide on :ref:`user-guide-gpu`. \ No newline at end of file diff --git a/changes/2751.feature.rst b/changes/2751.feature.rst deleted file mode 100644 index 61d97479c6..0000000000 --- a/changes/2751.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added :meth:`zarr.config.enable_gpu` to update Zarr's configuration to use GPUs. \ No newline at end of file diff --git a/changes/2755.bugfix.rst b/changes/2755.bugfix.rst deleted file mode 100644 index 2555369544..0000000000 --- a/changes/2755.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the -specified fill value, or with zeros if no fill value is provided. -This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes. \ No newline at end of file diff --git a/changes/2758.bugfix.rst b/changes/2758.bugfix.rst deleted file mode 100644 index 6b80f8a626..0000000000 --- a/changes/2758.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix zip-store path checking for stores with directories listed as files. \ No newline at end of file diff --git a/changes/2778.bugfix.rst b/changes/2778.bugfix.rst deleted file mode 100644 index 2968c4441c..0000000000 --- a/changes/2778.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list` \ No newline at end of file diff --git a/changes/2781.bugfix.rst b/changes/2781.bugfix.rst deleted file mode 100644 index 3673eeece7..0000000000 --- a/changes/2781.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Enable automatic removal of `needs release notes` with labeler action \ No newline at end of file diff --git a/changes/2784.feature.rst b/changes/2784.feature.rst deleted file mode 100644 index e3218e6df0..0000000000 --- a/changes/2784.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid reading chunks during writes where possible. :issue:`757` diff --git a/changes/2785.bugfix.rst b/changes/2785.bugfix.rst deleted file mode 100644 index 3f2b3111ea..0000000000 --- a/changes/2785.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Use the proper label config \ No newline at end of file diff --git a/changes/2795.bugfix.rst b/changes/2795.bugfix.rst deleted file mode 100644 index 0ee6619c16..0000000000 --- a/changes/2795.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Alters the behavior of ``create_array`` to ensure that any groups implied by the array's name are created if they do not already exist. Also simplifies the type signature for any function that takes an ArrayConfig-like object. \ No newline at end of file diff --git a/changes/2799.bugfix.rst b/changes/2799.bugfix.rst deleted file mode 100644 index f22b7074bb..0000000000 --- a/changes/2799.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Enitialise empty chunks to the default fill value during writing and add default fill values for datetime, timedelta, structured, and other (void* fixed size) data types \ No newline at end of file diff --git a/changes/2801.bugfix.rst b/changes/2801.bugfix.rst deleted file mode 100644 index 294934aacf..0000000000 --- a/changes/2801.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests diff --git a/changes/2804.feature.rst b/changes/2804.feature.rst deleted file mode 100644 index 5a707752a0..0000000000 --- a/changes/2804.feature.rst +++ /dev/null @@ -1 +0,0 @@ -:py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient. diff --git a/changes/2807.bugfix.rst b/changes/2807.bugfix.rst deleted file mode 100644 index ae0eb2f6ac..0000000000 --- a/changes/2807.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix pickling for ZipStore diff --git a/changes/2811.bugfix.rst b/changes/2811.bugfix.rst deleted file mode 100644 index ef4e8eb7ed..0000000000 --- a/changes/2811.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`. diff --git a/changes/2813.feature.rst b/changes/2813.feature.rst deleted file mode 100644 index 8a28f75082..0000000000 --- a/changes/2813.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add `zarr.testing.strategies.array_metadata` to generate ArrayV2Metadata and ArrayV3Metadata instances. diff --git a/changes/2817.bugfix.rst b/changes/2817.bugfix.rst deleted file mode 100644 index b1c0fa9220..0000000000 --- a/changes/2817.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix fancy indexing (e.g. arr[5, [0, 1]]) with the sharding codec \ No newline at end of file diff --git a/changes/2822.feature.rst b/changes/2822.feature.rst deleted file mode 100644 index 37b3bf1faf..0000000000 --- a/changes/2822.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add arbitrary `shards` to Hypothesis strategy for generating arrays. diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 08c64eb899..93466a0992 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -3,6 +3,45 @@ Release notes .. towncrier release notes start +3.0.3 (2025-02-14) +------------------ + +Features +~~~~~~~~ + +- Improves performance of FsspecStore.delete_dir for remote filesystems supporting concurrent/batched deletes, e.g., s3fs. (:issue:`2661`) +- Added :meth:`zarr.config.enable_gpu` to update Zarr's configuration to use GPUs. (:issue:`2751`) +- Avoid reading chunks during writes where possible. :issue:`757` (:issue:`2784`) +- :py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient. (:issue:`2804`) +- Add `zarr.testing.strategies.array_metadata` to generate ArrayV2Metadata and ArrayV3Metadata instances. (:issue:`2813`) +- Add arbitrary `shards` to Hypothesis strategy for generating arrays. (:issue:`2822`) + + +Bugfixes +~~~~~~~~ + +- Fixed bug with Zarr using device memory, instead of host memory, for storing metadata when using GPUs. (:issue:`2751`) +- The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the + specified fill value, or with zeros if no fill value is provided. + This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes. (:issue:`2755`) +- Fix zip-store path checking for stores with directories listed as files. (:issue:`2758`) +- Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list` (:issue:`2778`) +- Enable automatic removal of `needs release notes` with labeler action (:issue:`2781`) +- Use the proper label config (:issue:`2785`) +- Alters the behavior of ``create_array`` to ensure that any groups implied by the array's name are created if they do not already exist. Also simplifies the type signature for any function that takes an ArrayConfig-like object. (:issue:`2795`) +- Enitialise empty chunks to the default fill value during writing and add default fill values for datetime, timedelta, structured, and other (void* fixed size) data types (:issue:`2799`) +- Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests (:issue:`2801`) +- Fix pickling for ZipStore (:issue:`2807`) +- Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`. (:issue:`2811`) +- Fix fancy indexing (e.g. arr[5, [0, 1]]) with the sharding codec (:issue:`2817`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Added new user guide on :ref:`user-guide-gpu`. (:issue:`2751`) + + 3.0.2 (2025-01-31) ------------------ From e8bfb6484aeb543c21b9b0f6de2e0f0a034fc5d7 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 17 Feb 2025 17:07:51 +0100 Subject: [PATCH 071/160] Unnecessary lambda expression (#2828) --- src/zarr/testing/strategies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 8847b49020..0e25e44592 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -305,7 +305,7 @@ def orthogonal_indices( ) | basic_indices(min_dims=1, shape=(size,), allow_ellipsis=False) .map(lambda x: (x,) if not isinstance(x, tuple) else x) # bare ints, slices - .filter(lambda x: bool(x)) # skip empty tuple + .filter(bool) # skip empty tuple ) (idxr,) = val if isinstance(idxr, int): From 96c967786c42e235f4d07dfcdd2a3a72bcd508e1 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 20 Feb 2025 11:34:13 +0100 Subject: [PATCH 072/160] don't serialize empty tuples for v2 filters, and warn when reading such metadata (#2847) * don't serialize empty tuples for v2 filters, and warn when reading such metadata * release notes * alter text in comment * add skeleton of metadata spec compliance property test * do something for V3 --------- Co-authored-by: Deepak Cherian --- changes/2847.fix.rst | 1 + src/zarr/core/metadata/v2.py | 17 +++++++++++- tests/test_array.py | 47 +++++++++++++++------------------- tests/test_metadata/test_v2.py | 20 ++++++++++++++- tests/test_properties.py | 14 ++++++++++ 5 files changed, 71 insertions(+), 28 deletions(-) create mode 100644 changes/2847.fix.rst diff --git a/changes/2847.fix.rst b/changes/2847.fix.rst new file mode 100644 index 0000000000..148e191b98 --- /dev/null +++ b/changes/2847.fix.rst @@ -0,0 +1 @@ +Fixed a bug where ``ArrayV2Metadata`` could save ``filters`` as an empty array. \ No newline at end of file diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 3d292c81b4..823944e067 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import warnings from collections.abc import Iterable from enum import Enum from functools import cached_property @@ -178,6 +179,16 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: # handle the renames expected |= {"dtype", "chunks"} + # check if `filters` is an empty sequence; if so use None instead and raise a warning + if _data["filters"] is not None and len(_data["filters"]) == 0: + msg = ( + "Found an empty list of filters in the array metadata document. " + "This is contrary to the Zarr V2 specification, and will cause an error in the future. " + "Use None (or Null in a JSON document) instead of an empty list of filters." + ) + warnings.warn(msg, UserWarning, stacklevel=1) + _data["filters"] = None + _data = {k: v for k, v in _data.items() if k in expected} return cls(**_data) @@ -255,7 +266,11 @@ def parse_filters(data: object) -> tuple[numcodecs.abc.Codec, ...] | None: else: msg = f"Invalid filter at index {idx}. Expected a numcodecs.abc.Codec or a dict representation of numcodecs.abc.Codec. Got {type(val)} instead." raise TypeError(msg) - return tuple(out) + if len(out) == 0: + # Per the v2 spec, an empty tuple is not allowed -- use None to express "no filters" + return None + else: + return tuple(out) # take a single codec instance and wrap it in a tuple if isinstance(data, numcodecs.abc.Codec): return (data,) diff --git a/tests/test_array.py b/tests/test_array.py index b81f966e20..efcf8a6bf9 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -972,45 +972,40 @@ def test_default_fill_value(dtype: str, fill_value_expected: object, store: Stor @staticmethod @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) @pytest.mark.parametrize("empty_value", [None, ()]) - async def test_no_filters_compressors(store: MemoryStore, dtype: str, empty_value: Any) -> None: + async def test_no_filters_compressors( + store: MemoryStore, dtype: str, empty_value: object, zarr_format: ZarrFormat + ) -> None: """ Test that the default ``filters`` and ``compressors`` are removed when ``create_array`` is invoked. """ - # v2 arr = await create_array( store=store, dtype=dtype, shape=(10,), - zarr_format=2, + zarr_format=zarr_format, compressors=empty_value, filters=empty_value, ) # Test metadata explicitly - assert arr.metadata.zarr_format == 2 # guard for mypy - # The v2 metadata stores None and () separately - assert arr.metadata.filters == empty_value - # The v2 metadata does not allow tuple for compressor, therefore it is turned into None - assert arr.metadata.compressor is None - - assert arr.filters == () - assert arr.compressors == () - - # v3 - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - compressors=empty_value, - filters=empty_value, - ) - assert arr.metadata.zarr_format == 3 # guard for mypy - if dtype == "str": - assert arr.metadata.codecs == (VLenUTF8Codec(),) - assert arr.serializer == VLenUTF8Codec() + if zarr_format == 2: + assert arr.metadata.zarr_format == 2 # guard for mypy + # v2 spec requires that filters be either a collection with at least one filter, or None + assert arr.metadata.filters is None + # Compressor is a single element in v2 metadata; the absence of a compressor is encoded + # as None + assert arr.metadata.compressor is None + + assert arr.filters == () + assert arr.compressors == () else: - assert arr.metadata.codecs == (BytesCodec(),) - assert arr.serializer == BytesCodec() + assert arr.metadata.zarr_format == 3 # guard for mypy + if dtype == "str": + assert arr.metadata.codecs == (VLenUTF8Codec(),) + assert arr.serializer == VLenUTF8Codec() + else: + assert arr.metadata.codecs == (BytesCodec(),) + assert arr.serializer == BytesCodec() @staticmethod @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 5a5bf5f73a..4600a977d4 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -33,7 +33,7 @@ def test_parse_zarr_format_invalid(data: Any) -> None: @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) -@pytest.mark.parametrize("filters", [None, (), (numcodecs.GZip(),)]) +@pytest.mark.parametrize("filters", [None, (numcodecs.GZip(),)]) @pytest.mark.parametrize("compressor", [None, numcodecs.GZip()]) @pytest.mark.parametrize("fill_value", [None, 0, 1]) @pytest.mark.parametrize("order", ["C", "F"]) @@ -81,6 +81,24 @@ def test_metadata_to_dict( assert observed == expected +def test_filters_empty_tuple_warns() -> None: + metadata_dict = { + "zarr_format": 2, + "shape": (1,), + "chunks": (1,), + "dtype": "uint8", + "order": "C", + "compressor": None, + "filters": (), + "fill_value": 0, + } + with pytest.warns( + UserWarning, match="Found an empty list of filters in the array metadata document." + ): + meta = ArrayV2Metadata.from_dict(metadata_dict) + assert meta.filters is None + + class TestConsolidated: @pytest.fixture async def v2_consolidated_metadata( diff --git a/tests/test_properties.py b/tests/test_properties.py index acecd44810..5643cf3853 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -113,6 +113,20 @@ async def test_roundtrip_array_metadata( assert actual == expected +@given(store=stores, meta=array_metadata()) # type: ignore[misc] +def test_array_metadata_meets_spec(store: Store, meta: ArrayV2Metadata | ArrayV3Metadata) -> None: + # TODO: fill this out + asdict = meta.to_dict() + if isinstance(meta, ArrayV2Metadata): + assert asdict["filters"] != () + assert asdict["filters"] is None or isinstance(asdict["filters"], tuple) + assert asdict["zarr_format"] == 2 + elif isinstance(meta, ArrayV3Metadata): + assert asdict["zarr_format"] == 3 + else: + raise NotImplementedError + + # @st.composite # def advanced_indices(draw, *, shape): # basic_idxr = draw( From 8b59a38463dca7b3c908aa3db7d5321a80a3c285 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sun, 23 Feb 2025 10:29:09 -0700 Subject: [PATCH 073/160] Fix a bug when setting complete chunks (#2851) * Fix a bug when setting complete chunks Closes #2849 * much simpler test * add release note * Update strategy priorities: 1. Emphasize arrays of side > 1, 2. Emphasize indexing the last chunk for both setitem & getitem * Use short node names * bug fix * Add scalar tests * [revert] * Add unit test * one more test * Add xfails * switch to skip, XPASS is not allwoed * Fix test * cleaniup --------- Co-authored-by: Davis Bennett --- changes/2851.bugfix.rst | 1 + src/zarr/core/codec_pipeline.py | 25 +++++----- src/zarr/storage/_fsspec.py | 2 +- src/zarr/testing/stateful.py | 8 +-- src/zarr/testing/strategies.py | 86 +++++++++++++++++++++++++++------ tests/test_indexing.py | 31 ++++++++++++ tests/test_properties.py | 16 ++++-- 7 files changed, 133 insertions(+), 36 deletions(-) create mode 100644 changes/2851.bugfix.rst diff --git a/changes/2851.bugfix.rst b/changes/2851.bugfix.rst new file mode 100644 index 0000000000..977f683847 --- /dev/null +++ b/changes/2851.bugfix.rst @@ -0,0 +1 @@ +Fix a bug when setting values of a smaller last chunk. diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index 0c53cda96c..628a7e0487 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -296,17 +296,6 @@ def _merge_chunk_array( is_complete_chunk: bool, drop_axes: tuple[int, ...], ) -> NDBuffer: - if is_complete_chunk and value.shape == chunk_spec.shape: - return value - if existing_chunk_array is None: - chunk_array = chunk_spec.prototype.nd_buffer.create( - shape=chunk_spec.shape, - dtype=chunk_spec.dtype, - order=chunk_spec.order, - fill_value=fill_value_or_default(chunk_spec), - ) - else: - chunk_array = existing_chunk_array.copy() # make a writable copy if chunk_selection == () or is_scalar(value.as_ndarray_like(), chunk_spec.dtype): chunk_value = value else: @@ -320,6 +309,20 @@ def _merge_chunk_array( for idx in range(chunk_spec.ndim) ) chunk_value = chunk_value[item] + if is_complete_chunk and chunk_value.shape == chunk_spec.shape: + # TODO: For the last chunk, we could have is_complete_chunk=True + # that is smaller than the chunk_spec.shape but this throws + # an error in the _decode_single + return chunk_value + if existing_chunk_array is None: + chunk_array = chunk_spec.prototype.nd_buffer.create( + shape=chunk_spec.shape, + dtype=chunk_spec.dtype, + order=chunk_spec.order, + fill_value=fill_value_or_default(chunk_spec), + ) + else: + chunk_array = existing_chunk_array.copy() # make a writable copy chunk_array[chunk_selection] = chunk_value return chunk_array diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 1cc7039e68..a4730a93d9 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -178,7 +178,7 @@ def from_url( try: from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper - fs = AsyncFileSystemWrapper(fs) + fs = AsyncFileSystemWrapper(fs, asynchronous=True) except ImportError as e: raise ImportError( f"The filesystem for URL '{url}' is synchronous, and the required " diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py index 3e8dbcdf04..ede83201ae 100644 --- a/src/zarr/testing/stateful.py +++ b/src/zarr/testing/stateful.py @@ -325,7 +325,7 @@ def __init__(self, store: Store) -> None: def init_store(self) -> None: self.store.clear() - @rule(key=zarr_keys, data=st.binary(min_size=0, max_size=MAX_BINARY_SIZE)) + @rule(key=zarr_keys(), data=st.binary(min_size=0, max_size=MAX_BINARY_SIZE)) def set(self, key: str, data: DataObject) -> None: note(f"(set) Setting {key!r} with {data}") assert not self.store.read_only @@ -334,7 +334,7 @@ def set(self, key: str, data: DataObject) -> None: self.model[key] = data_buf @precondition(lambda self: len(self.model.keys()) > 0) - @rule(key=zarr_keys, data=st.data()) + @rule(key=zarr_keys(), data=st.data()) def get(self, key: str, data: DataObject) -> None: key = data.draw( st.sampled_from(sorted(self.model.keys())) @@ -344,7 +344,7 @@ def get(self, key: str, data: DataObject) -> None: # to bytes here necessary because data_buf set to model in set() assert self.model[key] == store_value - @rule(key=zarr_keys, data=st.data()) + @rule(key=zarr_keys(), data=st.data()) def get_invalid_zarr_keys(self, key: str, data: DataObject) -> None: note("(get_invalid)") assume(key not in self.model) @@ -408,7 +408,7 @@ def is_empty(self) -> None: # make sure they either both are or both aren't empty (same state) assert self.store.is_empty("") == (not self.model) - @rule(key=zarr_keys) + @rule(key=zarr_keys()) def exists(self, key: str) -> None: note("(exists)") diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 0e25e44592..96d664f5aa 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,10 +1,11 @@ +import math import sys from typing import Any, Literal import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np -from hypothesis import given, settings # noqa: F401 +from hypothesis import event, given, settings # noqa: F401 from hypothesis.strategies import SearchStrategy import zarr @@ -28,6 +29,16 @@ ) +@st.composite # type: ignore[misc] +def keys(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> Any: + return draw(st.lists(node_names, min_size=1, max_size=max_num_nodes).map("/".join)) + + +@st.composite # type: ignore[misc] +def paths(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> Any: + return draw(st.just("/") | keys(max_num_nodes=max_num_nodes)) + + def v3_dtypes() -> st.SearchStrategy[np.dtype]: return ( npst.boolean_dtypes() @@ -87,17 +98,19 @@ def clear_store(x: Store) -> Store: node_names = st.text(zarr_key_chars, min_size=1).filter( lambda t: t not in (".", "..") and not t.startswith("__") ) +short_node_names = st.text(zarr_key_chars, max_size=3, min_size=1).filter( + lambda t: t not in (".", "..") and not t.startswith("__") +) array_names = node_names attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) -keys = st.lists(node_names, min_size=1).map("/".join) -paths = st.just("/") | keys # st.builds will only call a new store constructor for different keyword arguments # i.e. stores.examples() will always return the same object per Store class. # So we map a clear to reset the store. stores = st.builds(MemoryStore, st.just({})).map(clear_store) compressors = st.sampled_from([None, "default"]) zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([3, 2]) -array_shapes = npst.array_shapes(max_dims=4, min_side=0) +# We de-prioritize arrays having dim sizes 0, 1, 2 +array_shapes = npst.array_shapes(max_dims=4, min_side=3) | npst.array_shapes(max_dims=4, min_side=0) @st.composite # type: ignore[misc] @@ -152,13 +165,15 @@ def numpy_arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, - zarr_formats: st.SearchStrategy[ZarrFormat] = zarr_formats, + dtype: np.dtype[Any] | None = None, + zarr_formats: st.SearchStrategy[ZarrFormat] | None = zarr_formats, ) -> Any: """ Generate numpy arrays that can be saved in the provided Zarr format. """ zarr_format = draw(zarr_formats) - dtype = draw(v3_dtypes() if zarr_format == 3 else v2_dtypes()) + if dtype is None: + dtype = draw(v3_dtypes() if zarr_format == 3 else v2_dtypes()) if np.issubdtype(dtype, np.str_): safe_unicode_strings = safe_unicode_for_dtype(dtype) return draw(npst.arrays(dtype=dtype, shape=shapes, elements=safe_unicode_strings)) @@ -174,11 +189,19 @@ def chunk_shapes(draw: st.DrawFn, *, shape: tuple[int, ...]) -> tuple[int, ...]: st.tuples(*[st.integers(min_value=0 if size == 0 else 1, max_value=size) for size in shape]) ) # 2. and now generate the chunks tuple - return tuple( + chunks = tuple( size // nchunks if nchunks > 0 else 0 for size, nchunks in zip(shape, numchunks, strict=True) ) + for c in chunks: + event("chunk size", c) + + if any((c != 0 and s % c != 0) for s, c in zip(shape, chunks, strict=True)): + event("smaller last chunk") + + return chunks + @st.composite # type: ignore[misc] def shard_shapes( @@ -211,7 +234,7 @@ def arrays( shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, compressors: st.SearchStrategy = compressors, stores: st.SearchStrategy[StoreLike] = stores, - paths: st.SearchStrategy[str | None] = paths, + paths: st.SearchStrategy[str | None] = paths(), # noqa: B008 array_names: st.SearchStrategy = array_names, arrays: st.SearchStrategy | None = None, attrs: st.SearchStrategy = attrs, @@ -267,23 +290,56 @@ def arrays( return a +@st.composite # type: ignore[misc] +def simple_arrays( + draw: st.DrawFn, + *, + shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, +) -> Any: + return draw( + arrays( + shapes=shapes, + paths=paths(max_num_nodes=2), + array_names=short_node_names, + attrs=st.none(), + compressors=st.sampled_from([None, "default"]), + ) + ) + + def is_negative_slice(idx: Any) -> bool: return isinstance(idx, slice) and idx.step is not None and idx.step < 0 +@st.composite # type: ignore[misc] +def end_slices(draw: st.DrawFn, *, shape: tuple[int]) -> Any: + """ + A strategy that slices ranges that include the last chunk. + This is intended to stress-test handling of a possibly smaller last chunk. + """ + slicers = [] + for size in shape: + start = draw(st.integers(min_value=size // 2, max_value=size - 1)) + length = draw(st.integers(min_value=0, max_value=size - start)) + slicers.append(slice(start, start + length)) + event("drawing end slice") + return tuple(slicers) + + @st.composite # type: ignore[misc] def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs: Any) -> Any: """Basic indices without unsupported negative slices.""" - return draw( - npst.basic_indices(shape=shape, **kwargs).filter( - lambda idxr: ( - not ( - is_negative_slice(idxr) - or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) - ) + strategy = npst.basic_indices(shape=shape, **kwargs).filter( + lambda idxr: ( + not ( + is_negative_slice(idxr) + or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) ) ) ) + if math.prod(shape) >= 3: + strategy = end_slices(shape=shape) | strategy + return draw(strategy) @st.composite # type: ignore[misc] diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 30d0d75f22..1bad861622 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -424,6 +424,18 @@ def test_orthogonal_indexing_fallback_on_getitem_2d( np.testing.assert_array_equal(z[index], expected_result) +@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") +def test_setitem_repeated_index(): + array = zarr.array(data=np.zeros((4,)), chunks=(1,)) + indexer = np.array([-1, -1, 0, 0]) + array.oindex[(indexer,)] = [0, 1, 2, 3] + np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) + + indexer = np.array([-1, 0, 0, -1]) + array.oindex[(indexer,)] = [0, 1, 2, 3] + np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) + + Index = list[int] | tuple[slice | int | list[int], ...] @@ -815,6 +827,25 @@ def test_set_orthogonal_selection_1d(store: StorePath) -> None: _test_set_orthogonal_selection(v, a, z, selection) +def test_set_item_1d_last_two_chunks(): + # regression test for GH2849 + g = zarr.open_group("foo.zarr", zarr_format=3, mode="w") + a = g.create_array("bar", shape=(10,), chunks=(3,), dtype=int) + data = np.array([7, 8, 9]) + a[slice(7, 10)] = data + np.testing.assert_array_equal(a[slice(7, 10)], data) + + z = zarr.open_group("foo.zarr", mode="w") + z.create_array("zoo", dtype=float, shape=()) + z["zoo"][...] = np.array(1) # why doesn't [:] work? + np.testing.assert_equal(z["zoo"][()], np.array(1)) + + z = zarr.open_group("foo.zarr", mode="w") + z.create_array("zoo", dtype=float, shape=()) + z["zoo"][...] = 1 # why doesn't [:] work? + np.testing.assert_equal(z["zoo"][()], np.array(1)) + + def _test_set_orthogonal_selection_2d( v: npt.NDArray[np.int_], a: npt.NDArray[np.int_], diff --git a/tests/test_properties.py b/tests/test_properties.py index 5643cf3853..68d8bb0a0e 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,3 +1,4 @@ +import numpy as np import pytest from numpy.testing import assert_array_equal @@ -18,6 +19,7 @@ basic_indices, numpy_arrays, orthogonal_indices, + simple_arrays, stores, zarr_formats, ) @@ -50,13 +52,13 @@ def test_array_creates_implicit_groups(array): @given(data=st.data()) def test_basic_indexing(data: st.DataObject) -> None: - zarray = data.draw(arrays()) + zarray = data.draw(simple_arrays()) nparray = zarray[:] indexer = data.draw(basic_indices(shape=nparray.shape)) actual = zarray[indexer] assert_array_equal(nparray[indexer], actual) - new_data = data.draw(npst.arrays(shape=st.just(actual.shape), dtype=nparray.dtype)) + new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) zarray[indexer] = new_data nparray[indexer] = new_data assert_array_equal(nparray, zarray[:]) @@ -65,7 +67,7 @@ def test_basic_indexing(data: st.DataObject) -> None: @given(data=st.data()) def test_oindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. - zarray = data.draw(arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) + zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) nparray = zarray[:] zindexer, npindexer = data.draw(orthogonal_indices(shape=nparray.shape)) @@ -73,7 +75,11 @@ def test_oindex(data: st.DataObject) -> None: assert_array_equal(nparray[npindexer], actual) assume(zarray.shards is None) # GH2834 - new_data = data.draw(npst.arrays(shape=st.just(actual.shape), dtype=nparray.dtype)) + for idxr in npindexer: + if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr).size: + # behaviour of setitem with repeated indices is not guaranteed in practice + assume(False) + new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[npindexer] = new_data zarray.oindex[zindexer] = new_data assert_array_equal(nparray, zarray[:]) @@ -82,7 +88,7 @@ def test_oindex(data: st.DataObject) -> None: @given(data=st.data()) def test_vindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. - zarray = data.draw(arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) + zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) nparray = zarray[:] indexer = data.draw( From 8d2fb47252a7ed5eb163ccfb0401e4e0486f72f6 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 23 Feb 2025 18:42:17 +0100 Subject: [PATCH 074/160] feat/batch creation (#2665) * sketch out batch creation routine * scratch state of easy batch creation * rename tupleize keys * tests and proper implementation for create_nodes and create_hierarchy * privatize * use Posixpath instead of Path in tests; avoid redundant cast * restore cast * pureposixpath instead of posixpath * group-level create_hierarchy * docstring * sketch out from_flat for groups * better concurrency for v2 * revert change to default concurrency * create root correctly * working _from_flat * working dict serialization for _ImplicitGroupMetadata * remove implicit group metadata, and add some key name normalization * add path normalization routines * use _join_paths for safer path concatenation * handle overwrite * rename _from_flat to _create_rooted_hierarchy, add sync version * add test for _create_rooted_hierarchy when the output should be an array, and for when the input is invalid * increase coverage, one way or another * remove replace kwarg for _set_return_key * shield lines from coverage * add some tests * lint * improve coverage with more tests * use store + path instead of StorePath for hierarchy api * docstrings * docstrings * release notes * refactor sync / async functions, and make tests more compact accordingly * keyerror -> filenotfounderror * keyerror -> filenotfounderror, fixup * add top-level exports * mildly refactor node input validation * simplify path normalization * refactor to separate sync and async routines * remove semaphore kwarg, and add test for concurrency limit sensitivity * wire up semaphore correctly, thanks to a test * export read_node * docstrings * docstrings * read_node -> get_node * Update src/zarr/api/synchronous.py Co-authored-by: Joe Hamman * update docstring * add function signature tests * update exception name * refactor: remove path kwarg, bring back ImplicitGroupMetadata * prune top-level synchronous API * more api pruning * put sync wrappers in sync_group module, move utils to utils * ensure we always have a root group * docs * fix group.create_hierarchy to properly prefix keys with the name of the group * docstrings * docstrings * docstring examples --------- Co-authored-by: Joe Hamman --- changes/2665.feature.rst | 1 + docs/quickstart.rst | 22 + docs/user-guide/groups.rst | 25 + src/zarr/__init__.py | 2 + src/zarr/api/asynchronous.py | 8 +- src/zarr/api/synchronous.py | 2 + src/zarr/core/group.py | 842 ++++++++++++++++++++++++++++++---- src/zarr/core/sync.py | 18 +- src/zarr/core/sync_group.py | 161 +++++++ src/zarr/storage/_utils.py | 46 +- tests/conftest.py | 227 ++++++++- tests/test_api.py | 21 +- tests/test_group.py | 557 +++++++++++++++++++++- tests/test_store/test_core.py | 47 +- 14 files changed, 1883 insertions(+), 96 deletions(-) create mode 100644 changes/2665.feature.rst create mode 100644 src/zarr/core/sync_group.py diff --git a/changes/2665.feature.rst b/changes/2665.feature.rst new file mode 100644 index 0000000000..40bec542ce --- /dev/null +++ b/changes/2665.feature.rst @@ -0,0 +1 @@ +Adds functions for concurrently creating multiple arrays and groups. \ No newline at end of file diff --git a/docs/quickstart.rst b/docs/quickstart.rst index d520554593..66bdae2a2e 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -119,6 +119,28 @@ Zarr allows you to create hierarchical groups, similar to directories:: This creates a group with two datasets: ``foo`` and ``bar``. +Batch Hierarchy Creation +~~~~~~~~~~~~~~~~~~~~~~~~ + +Zarr provides tools for creating a collection of arrays and groups with a single function call. +Suppose we want to copy existing groups and arrays into a new storage backend: + + >>> # Create nested groups and add arrays + >>> root = zarr.group("data/example-3.zarr", attributes={'name': 'root'}) + >>> foo = root.create_group(name="foo") + >>> bar = root.create_array( + ... name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" + ... ) + >>> nodes = {'': root.metadata} | {k: v.metadata for k,v in root.members()} + >>> print(nodes) + >>> from zarr.storage import MemoryStore + >>> new_nodes = dict(zarr.create_hierarchy(store=MemoryStore(), nodes=nodes)) + >>> new_root = new_nodes[''] + >>> assert new_root.attrs == root.attrs + +Note that :func:`zarr.create_hierarchy` will only initialize arrays and groups -- copying array data must +be done in a separate step. + Persistent Storage ------------------ diff --git a/docs/user-guide/groups.rst b/docs/user-guide/groups.rst index 1e72df3478..4268004f70 100644 --- a/docs/user-guide/groups.rst +++ b/docs/user-guide/groups.rst @@ -75,6 +75,31 @@ For more information on groups see the :class:`zarr.Group` API docs. .. _user-guide-diagnostics: +Batch Group Creation +-------------------- + +You can also create multiple groups concurrently with a single function call. :func:`zarr.create_hierarchy` takes +a :class:`zarr.storage.Store` instance and a dict of ``key : metadata`` pairs, parses that dict, and +writes metadata documents to storage: + + >>> from zarr import create_hierarchy + >>> from zarr.core.group import GroupMetadata + >>> from zarr.storage import LocalStore + >>> node_spec = {'a/b/c': GroupMetadata()} + >>> nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec)) + >>> print(sorted(nodes_created.items(), key=lambda kv: len(kv[0]))) + [('', ), ('a', ), ('a/b', ), ('a/b/c', )] + +Note that we only specified a single group named ``a/b/c``, but 4 groups were created. These additional groups +were created to ensure that the desired node ``a/b/c`` is connected to the root group ``''`` by a sequence +of intermediate groups. :func:`zarr.create_hierarchy` normalizes the ``nodes`` keyword argument to +ensure that the resulting hierarchy is complete, i.e. all groups or arrays are connected to the root +of the hierarchy via intermediate groups. + +Because :func:`zarr.create_hierarchy` concurrently creates metadata documents, it's more efficient +than repeated calls to :func:`create_group` or :func:`create_array`, provided you can statically define +the metadata for the groups and arrays you want to create. + Array and group diagnostics --------------------------- diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index bcbdaf7c19..4ffa4c9bbc 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -8,6 +8,7 @@ create, create_array, create_group, + create_hierarchy, empty, empty_like, full, @@ -50,6 +51,7 @@ "create", "create_array", "create_group", + "create_hierarchy", "empty", "empty_like", "full", diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3a3d03bb71..6059893920 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -23,7 +23,12 @@ _warn_write_empty_chunks_kwarg, parse_dtype, ) -from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata +from zarr.core.group import ( + AsyncGroup, + ConsolidatedMetadata, + GroupMetadata, + create_hierarchy, +) from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v2 import _default_compressor, _default_filters from zarr.errors import NodeTypeValidationError @@ -48,6 +53,7 @@ "copy_store", "create", "create_array", + "create_hierarchy", "empty", "empty_like", "full", diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index e1f92633cd..9424ae1fde 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -10,6 +10,7 @@ from zarr.core.array import Array, AsyncArray from zarr.core.group import Group from zarr.core.sync import sync +from zarr.core.sync_group import create_hierarchy if TYPE_CHECKING: from collections.abc import Iterable @@ -46,6 +47,7 @@ "copy_store", "create", "create_array", + "create_hierarchy", "empty", "empty_like", "full", diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 1f5d57c0ab..a7f8a6c022 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -6,7 +6,9 @@ import logging import warnings from collections import defaultdict +from collections.abc import Iterator, Mapping from dataclasses import asdict, dataclass, field, fields, replace +from itertools import accumulate from typing import TYPE_CHECKING, Literal, TypeVar, assert_never, cast, overload import numpy as np @@ -49,12 +51,19 @@ from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v3 import V3JsonEncoder from zarr.core.sync import SyncMixin, sync -from zarr.errors import MetadataValidationError +from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError from zarr.storage import StoreLike, StorePath from zarr.storage._common import ensure_no_existing_node, make_store_path +from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Generator, Iterable, Iterator + from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Coroutine, + Generator, + Iterable, + ) from typing import Any from zarr.core.array_spec import ArrayConfig, ArrayConfigLike @@ -407,6 +416,15 @@ def to_dict(self) -> dict[str, Any]: return result +@dataclass(frozen=True) +class ImplicitGroupMarker(GroupMetadata): + """ + Marker for an implicit group. Instances of this class are only used in the context of group + creation as a placeholder to represent groups that should only be created if they do not + already exist in storage + """ + + @dataclass(frozen=True) class AsyncGroup: """ @@ -416,6 +434,9 @@ class AsyncGroup: metadata: GroupMetadata store_path: StorePath + # TODO: make this correct and work + # TODO: ensure that this can be bound properly to subclass of AsyncGroup + @classmethod async def from_store( cls, @@ -662,55 +683,16 @@ async def getitem( """ store_path = self.store_path / key logger.debug("key=%s, store_path=%s", key, store_path) - metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata # Consolidated metadata lets us avoid some I/O operations so try that first. if self.metadata.consolidated_metadata is not None: return self._getitem_consolidated(store_path, key, prefix=self.name) - - # Note: - # in zarr-python v2, we first check if `key` references an Array, else if `key` references - # a group,using standalone `contains_array` and `contains_group` functions. These functions - # are reusable, but for v3 they would perform redundant I/O operations. - # Not clear how much of that strategy we want to keep here. - elif self.metadata.zarr_format == 3: - zarr_json_bytes = await (store_path / ZARR_JSON).get() - if zarr_json_bytes is None: - raise KeyError(key) - else: - zarr_json = json.loads(zarr_json_bytes.to_bytes()) - metadata = _build_metadata_v3(zarr_json) - return _build_node_v3(metadata, store_path) - - elif self.metadata.zarr_format == 2: - # Q: how do we like optimistically fetching .zgroup, .zarray, and .zattrs? - # This guarantees that we will always make at least one extra request to the store - zgroup_bytes, zarray_bytes, zattrs_bytes = await asyncio.gather( - (store_path / ZGROUP_JSON).get(), - (store_path / ZARRAY_JSON).get(), - (store_path / ZATTRS_JSON).get(), + try: + return await get_node( + store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format ) - - if zgroup_bytes is None and zarray_bytes is None: - raise KeyError(key) - - # unpack the zarray, if this is None then we must be opening a group - zarray = json.loads(zarray_bytes.to_bytes()) if zarray_bytes else None - zgroup = json.loads(zgroup_bytes.to_bytes()) if zgroup_bytes else None - # unpack the zattrs, this can be None if no attrs were written - zattrs = json.loads(zattrs_bytes.to_bytes()) if zattrs_bytes is not None else {} - - if zarray is not None: - metadata = _build_metadata_v2(zarray, zattrs) - return _build_node_v2(metadata=metadata, store_path=store_path) - else: - # this is just for mypy - if TYPE_CHECKING: - assert zgroup is not None - metadata = _build_metadata_v2(zgroup, zattrs) - return _build_node_v2(metadata=metadata, store_path=store_path) - else: - raise ValueError(f"unexpected zarr_format: {self.metadata.zarr_format}") + except FileNotFoundError as e: + raise KeyError(key) from e def _getitem_consolidated( self, store_path: StorePath, key: str, prefix: str @@ -1407,6 +1389,84 @@ async def _members( ): yield member + async def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] + ]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------- + tuple[str, AsyncArray | AsyncGroup]. + """ + # check that all the nodes have the same zarr_format as Self + prefix = self.path + nodes_parsed = {} + for key, value in nodes.items(): + if value.zarr_format != self.metadata.zarr_format: + msg = ( + "The zarr_format of the nodes must be the same as the parent group. " + f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" + f" has zarr_format {self.metadata.zarr_format}." + ) + raise ValueError(msg) + if normalize_path(key) == "": + msg = ( + "The input defines a root node, but a root node already exists, namely this Group instance." + "It is an error to use this method to create a root node. " + "Remove the root node from the input dict, or use a function like " + "create_rooted_hierarchy to create a rooted hierarchy." + ) + raise ValueError(msg) + else: + nodes_parsed[_join_paths([prefix, key])] = value + + async for key, node in create_hierarchy( + store=self.store, + nodes=nodes_parsed, + overwrite=overwrite, + ): + if prefix == "": + out_key = key + else: + out_key = key.removeprefix(prefix + "/") + yield out_key, node + async def keys(self) -> AsyncGenerator[str, None]: """Iterate over member names.""" async for key, _ in self.members(): @@ -2030,6 +2090,66 @@ def members(self, max_depth: int | None = 0) -> tuple[tuple[str, Array | Group], return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) + def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> Iterator[tuple[str, Group | Array]]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------- + tuple[str, Array | Group]. + + Examples + -------- + >>> import zarr + >>> from zarr.core.group import GroupMetadata + >>> root = zarr.create_group(store={}) + >>> for key, val in root.create_hierarchy({'a/b/c': GroupMetadata()}): + ... print(key, val) + ... + + + + """ + for key, node in self._sync_iter( + self._async_group.create_hierarchy(nodes, overwrite=overwrite) + ): + yield (key, _parse_async_node(node)) + def keys(self) -> Generator[str, None]: """Return an iterator over group member names. @@ -2774,11 +2894,361 @@ def array( ) +async def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] +]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncGroup | AsyncArray] + This function yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> from zarr.api.asynchronous import create_hierarchy + >>> from zarr.storage import MemoryStore + >>> from zarr.core.group import GroupMetadata + >>> import asyncio + >>> store = MemoryStore() + >>> nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + >>> async def run(): + ... print(dict([x async for x in create_hierarchy(store=store, nodes=nodes)])) + >>> asyncio.run(run()) + # {'a': , '': } + """ + # normalize the keys to be valid paths + nodes_normed_keys = _normalize_path_keys(nodes) + + # ensure that all nodes have the same zarr_format, and add implicit groups as needed + nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) + redundant_implicit_groups = [] + + # empty hierarchies should be a no-op + if len(nodes_parsed) > 0: + # figure out which zarr format we are using + zarr_format = next(iter(nodes_parsed.values())).zarr_format + + # check which implicit groups will require materialization + implicit_group_keys = tuple( + filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) + ) + # read potential group metadata for each implicit group + maybe_extant_group_coros = ( + _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys + ) + maybe_extant_groups = await asyncio.gather( + *maybe_extant_group_coros, return_exceptions=True + ) + + for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): + if isinstance(value, BaseException): + if isinstance(value, FileNotFoundError): + # this is fine -- there was no group there, so we will create one + pass + else: + raise value + else: + # a loop exists already at ``key``, so we can avoid creating anything there + redundant_implicit_groups.append(key) + + if overwrite: + # we will remove any nodes that collide with arrays and non-implicit groups defined in + # nodes + + # track the keys of nodes we need to delete + to_delete_keys = [] + to_delete_keys.extend( + [k for k, v in nodes_parsed.items() if k not in implicit_group_keys] + ) + await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) + else: + # This type is long. + coros: ( + Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] + | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] + ) + if zarr_format == 2: + coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) + elif zarr_format == 3: + coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) + else: # pragma: no cover + raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover + + extant_node_query = dict( + zip( + nodes_parsed.keys(), + await asyncio.gather(*coros, return_exceptions=True), + strict=False, + ) + ) + # iterate over the existing arrays / groups and figure out which of them conflict + # with the arrays / groups we want to create + for key, extant_node in extant_node_query.items(): + proposed_node = nodes_parsed[key] + if isinstance(extant_node, BaseException): + if isinstance(extant_node, FileNotFoundError): + # ignore FileNotFoundError, because they represent nodes we can safely create + pass + else: + # Any other exception is a real error + raise extant_node + else: + # this is a node that already exists, but a node with the same key was specified + # in nodes_parsed. + if isinstance(extant_node, GroupMetadata): + # a group already exists where we want to create a group + if isinstance(proposed_node, ImplicitGroupMarker): + # we have proposed an implicit group, which is OK -- we will just skip + # creating this particular metadata document + redundant_implicit_groups.append(key) + else: + # we have proposed an explicit group, which is an error, given that a + # group already exists. + raise ContainsGroupError(store, key) + elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): + # we are trying to overwrite an existing array. this is an error. + raise ContainsArrayError(store, key) + + nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} + + for k, v in nodes_parsed.items(): + if k not in redundant_implicit_groups: + if isinstance(v, ImplicitGroupMarker): + nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) + else: + nodes_explicit[k] = v + + async for key, node in create_nodes(store=store, nodes=nodes_explicit): + yield key, node + + +async def create_nodes( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] +]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + AsyncGroup | AsyncArray + The created nodes in the order they are created. + """ + + # Note: the only way to alter this value is via the config. If that's undesirable for some reason, + # then we should consider adding a keyword argument this this function + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + create_tasks: list[Coroutine[None, None, str]] = [] + + for key, value in nodes.items(): + # make the key absolute + create_tasks.extend(_persist_metadata(store, key, value, semaphore=semaphore)) + + created_object_keys = [] + + for coro in asyncio.as_completed(create_tasks): + created_key = await coro + # we need this to track which metadata documents were written so that we can yield a + # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. + created_object_keys.append(created_key) + + # get the node name from the object key + if len(created_key.split("/")) == 1: + # this is the root node + meta_out = nodes[""] + node_name = "" + else: + # turn "foo/" into "foo" + node_name = created_key[: created_key.rfind("/")] + meta_out = nodes[node_name] + if meta_out.zarr_format == 3: + yield node_name, _build_node(store=store, path=node_name, metadata=meta_out) + else: + # For zarr v2 + # we only want to yield when both the metadata and attributes are created + # so we track which keys have been created, and wait for both the meta key and + # the attrs key to be created before yielding back the AsyncArray / AsyncGroup + + attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys + + if isinstance(meta_out, GroupMetadata): + meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys + else: + meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys + + if meta_done and attrs_done: + yield node_name, _build_node(store=store, path=node_name, metadata=meta_out) + + continue + + +def _get_roots( + data: Iterable[str], +) -> tuple[str, ...]: + """ + Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of + path segments. + """ + if "" in data: + return ("",) + keys_split = sorted((key.split("/") for key in data), key=len) + groups: defaultdict[int, list[str]] = defaultdict(list) + for key_split in keys_split: + groups[len(key_split)].append("/".join(key_split)) + return tuple(groups[min(groups.keys())]) + + +def _parse_hierarchy_dict( + *, + data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: + """ + Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into + a dict of str: node pairs that models a valid, complete Zarr hierarchy. + + If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, + then return a dict with the exact same data as the input. + + Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make + the hierarchy complete. + + For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two + groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function + to that input will result in a return value of + {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups + were added. + + The input is also checked for the following conditions; an error is raised if any are violated: + + - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). + - All arrays and groups must have the same ``zarr_format`` value. + + This function ensures that the input is transformed into a specification of a complete and valid + Zarr hierarchy. + """ + + # ensure that all nodes have the same zarr format + data_purified = _ensure_consistent_zarr_format(data) + + # ensure that keys are normalized to zarr paths + data_normed_keys = _normalize_path_keys(data_purified) + + # insert an implicit root group if a root was not specified + # but not if an empty dict was provided, because any empty hierarchy has no nodes + if len(data_normed_keys) > 0 and "" not in data_normed_keys: + z_format = next(iter(data_normed_keys.values())).zarr_format + data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} + + out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} + + for k, v in data_normed_keys.items(): + key_split = k.split("/") + + # get every parent path + *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) + + for subpath in subpaths: + # If a component is not already in the output dict, add ImplicitGroupMetadata + if subpath not in out: + out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) + else: + if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): + msg = ( + f"The node at {subpath} contains other nodes, but it is not a Zarr group. " + "This is invalid. Only Zarr groups can contain other nodes." + ) + raise ValueError(msg) + return out + + +def _ensure_consistent_zarr_format( + data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: + """ + Ensure that all values of the input dict have the same zarr format. If any do not, + then a value error is raised. + """ + observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} + + for k, v in data.items(): + observed_zarr_formats[v.zarr_format].append(k) + + if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " + f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." + "Ensure that all nodes have the same Zarr format." + ) + raise ValueError(msg) + + return cast( + Mapping[str, GroupMetadata | ArrayV2Metadata] + | Mapping[str, GroupMetadata | ArrayV3Metadata], + data, + ) + + async def _getitem_semaphore( node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup: """ - Combine node.getitem with an optional semaphore. If the semaphore parameter is an + Wrap Group.getitem with an optional semaphore. + + If the semaphore parameter is an asyncio.Semaphore instance, then the getitem operation is performed inside an async context manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked without a context manager. @@ -2892,71 +3362,283 @@ async def _iter_members_deep( yield key, node -def _resolve_metadata_v2( - blobs: tuple[str | bytes | bytearray, str | bytes | bytearray], -) -> ArrayV2Metadata | GroupMetadata: - zarr_metadata = json.loads(blobs[0]) - attrs = json.loads(blobs[1]) - if "shape" in zarr_metadata: - return ArrayV2Metadata.from_dict(zarr_metadata | {"attrs": attrs}) +async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / zarr.json. If no such document is found, raise a + FileNotFoundError. + """ + zarr_json_bytes = await store.get( + _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() + ) + if zarr_json_bytes is None: + raise FileNotFoundError(path) + else: + zarr_json = json.loads(zarr_json_bytes.to_bytes()) + return _build_metadata_v3(zarr_json) + + +async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / (.zgroup | .zarray). If no such document is found, + raise a FileNotFoundError. + """ + # TODO: consider first fetching array metadata, and only fetching group metadata when we don't + # find an array + zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( + store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), + ) + + if zattrs_bytes is None: + zattrs = {} + else: + zattrs = json.loads(zattrs_bytes.to_bytes()) + + # TODO: decide how to handle finding both array and group metadata. The spec does not seem to + # consider this situation. A practical approach would be to ignore that combination, and only + # return the array metadata. + if zarray_bytes is not None: + zmeta = json.loads(zarray_bytes.to_bytes()) else: - return GroupMetadata.from_dict(zarr_metadata | {"attrs": attrs}) + if zgroup_bytes is None: + # neither .zarray or .zgroup were found results in KeyError + raise FileNotFoundError(path) + else: + zmeta = json.loads(zgroup_bytes.to_bytes()) + + return _build_metadata_v2(zmeta, zattrs) + + +async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v2(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v3(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta -def _build_metadata_v3(zarr_json: dict[str, Any]) -> ArrayV3Metadata | GroupMetadata: +async def _read_group_metadata( + store: Store, path: str, *, zarr_format: ZarrFormat +) -> GroupMetadata: + if zarr_format == 2: + return await _read_group_metadata_v2(store=store, path=path) + return await _read_group_metadata_v3(store=store, path=path) + + +def _build_metadata_v3(zarr_json: dict[str, JSON]) -> ArrayV3Metadata | GroupMetadata: """ - Take a dict and convert it into the correct metadata type. + Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. """ if "node_type" not in zarr_json: - raise KeyError("missing `node_type` key in metadata document.") + raise MetadataValidationError("node_type", "array or group", "nothing (the key is missing)") match zarr_json: case {"node_type": "array"}: return ArrayV3Metadata.from_dict(zarr_json) case {"node_type": "group"}: return GroupMetadata.from_dict(zarr_json) - case _: - raise ValueError("invalid value for `node_type` key in metadata document") + case _: # pragma: no cover + raise ValueError( + "invalid value for `node_type` key in metadata document" + ) # pragma: no cover def _build_metadata_v2( - zarr_json: dict[str, Any], attrs_json: dict[str, Any] + zarr_json: dict[str, object], attrs_json: dict[str, JSON] ) -> ArrayV2Metadata | GroupMetadata: """ - Take a dict and convert it into the correct metadata type. + Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. """ match zarr_json: case {"shape": _}: return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}) - case _: + case _: # pragma: no cover return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}) -def _build_node_v3( - metadata: ArrayV3Metadata | GroupMetadata, store_path: StorePath -) -> AsyncArray[ArrayV3Metadata] | AsyncGroup: +@overload +def _build_node( + *, store: Store, path: str, metadata: ArrayV2Metadata +) -> AsyncArray[ArrayV2Metadata]: ... + + +@overload +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata +) -> AsyncArray[ArrayV3Metadata]: ... + + +@overload +def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... + + +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: """ Take a metadata object and return a node (AsyncArray or AsyncGroup). """ + store_path = StorePath(store=store, path=path) match metadata: - case ArrayV3Metadata(): + case ArrayV2Metadata() | ArrayV3Metadata(): return AsyncArray(metadata, store_path=store_path) case GroupMetadata(): return AsyncGroup(metadata, store_path=store_path) - case _: - raise ValueError(f"Unexpected metadata type: {type(metadata)}") + case _: # pragma: no cover + raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover -def _build_node_v2( - metadata: ArrayV2Metadata | GroupMetadata, store_path: StorePath -) -> AsyncArray[ArrayV2Metadata] | AsyncGroup: +async def _get_node_v2(store: Store, path: str) -> AsyncArray[ArrayV2Metadata] | AsyncGroup: """ - Take a metadata object and return a node (AsyncArray or AsyncGroup). + Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup """ + metadata = await _read_metadata_v2(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) - match metadata: - case ArrayV2Metadata(): - return AsyncArray(metadata, store_path=store_path) - case GroupMetadata(): - return AsyncGroup(metadata, store_path=store_path) - case _: - raise ValueError(f"Unexpected metadata type: {type(metadata)}") + +async def _get_node_v3(store: Store, path: str) -> AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v3(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def get_node( + store: Store, path: str, zarr_format: ZarrFormat +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Get an AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + + match zarr_format: + case 2: + return await _get_node_v2(store=store, path=path) + case 3: + return await _get_node_v3(store=store, path=path) + case _: # pragma: no cover + raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover + + +async def _set_return_key( + *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None +) -> str: + """ + Write a value to storage at the given key. The key is returned. + Useful when saving values via routines that return results in execution order, + like asyncio.as_completed, because in this case we need to know which key was saved in order + to yield the right object to the caller. + + Parameters + ---------- + store : Store + The store to save the value to. + key : str + The key to save the value to. + value : Buffer + The value to save. + semaphore : asyncio.Semaphore | None + An optional semaphore to use to limit the number of concurrent writes. + """ + + if semaphore is not None: + async with semaphore: + await store.set(key, value) + else: + await store.set(key, value) + return key + + +def _persist_metadata( + store: Store, + path: str, + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata, + semaphore: asyncio.Semaphore | None = None, +) -> tuple[Coroutine[None, None, str], ...]: + """ + Prepare to save a metadata document to storage, returning a tuple of coroutines that must be awaited. + """ + + to_save = metadata.to_buffer_dict(default_buffer_prototype()) + return tuple( + _set_return_key(store=store, key=_join_paths([path, key]), value=value, semaphore=semaphore) + for key, value in to_save.items() + ) + + +async def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """ + Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. + This function ensures that its input contains a specification of a root node, + calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. + """ + roots = _get_roots(nodes.keys()) + if len(roots) != 1: + msg = ( + "The input does not specify a root node. " + "This function can only create hierarchies that contain a root node, which is " + "defined as a group that is ancestral to all the other arrays and " + "groups in the hierarchy, or a single array." + ) + raise ValueError(msg) + else: + root_key = roots[0] + + nodes_created = [ + x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) + ] + return dict(nodes_created)[root_key] diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 2bb5f24802..d9b4839e8e 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -6,14 +6,14 @@ import os import threading from concurrent.futures import ThreadPoolExecutor, wait -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from typing_extensions import ParamSpec from zarr.core.config import config if TYPE_CHECKING: - from collections.abc import AsyncIterator, Coroutine + from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from typing import Any logger = logging.getLogger(__name__) @@ -215,3 +215,17 @@ async def iter_to_list() -> list[T]: return [item async for item in async_iterator] return self._sync(iter_to_list()) + + +async def _with_semaphore( + func: Callable[[], Awaitable[T]], semaphore: asyncio.Semaphore | None = None +) -> T: + """ + Await the result of invoking the no-argument-callable ``func`` within the context manager + provided by a Semaphore, if one is provided. Otherwise, just await the result of invoking + ``func``. + """ + if semaphore is None: + return await func() + async with semaphore: + return await func() diff --git a/src/zarr/core/sync_group.py b/src/zarr/core/sync_group.py new file mode 100644 index 0000000000..39d8a17992 --- /dev/null +++ b/src/zarr/core/sync_group.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr.core.group import Group, GroupMetadata, _parse_async_node +from zarr.core.group import create_hierarchy as create_hierarchy_async +from zarr.core.group import create_nodes as create_nodes_async +from zarr.core.group import create_rooted_hierarchy as create_rooted_hierarchy_async +from zarr.core.group import get_node as get_node_async +from zarr.core.sync import _collect_aiterator, sync + +if TYPE_CHECKING: + from collections.abc import Iterator + + from zarr.abc.store import Store + from zarr.core.array import Array + from zarr.core.common import ZarrFormat + from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata + + +def create_nodes( + *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] +) -> Iterator[tuple[str, Group | Array]]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + Group | Array + The created nodes. + """ + coro = create_nodes_async(store=store, nodes=nodes) + + for key, value in sync(_collect_aiterator(coro)): + yield key, _parse_async_node(value) + + +def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> Iterator[tuple[str, Group | Array]]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, Group | Array] + This function yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> from zarr import create_hierarchy + >>> from zarr.storage import MemoryStore + >>> from zarr.core.group import GroupMetadata + + >>> store = MemoryStore() + >>> nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + >>> nodes_created = dict(create_hierarchy(store=store, nodes=nodes)) + >>> print(nodes) + # {'a': GroupMetadata(attributes={'name': 'leaf'}, zarr_format=3, consolidated_metadata=None, node_type='group')} + """ + coro = create_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite) + + for key, value in sync(_collect_aiterator(coro)): + yield key, _parse_async_node(value) + + +def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> Group | Array: + """ + Create a Zarr hierarchy with a root, and return the root node, which could be a ``Group`` + or ``Array`` instance. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + overwrite : bool + Whether to overwrite existing nodes. Default is ``False``. + + Returns + ------- + Group | Array + """ + async_node = sync(create_rooted_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite)) + return _parse_async_node(async_node) + + +def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> Array | Group: + """ + Get an Array or Group from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + Array | Group + """ + + return _parse_async_node(sync(get_node_async(store=store, path=path, zarr_format=zarr_format))) diff --git a/src/zarr/storage/_utils.py b/src/zarr/storage/_utils.py index 4fc3171eb8..eda4342f47 100644 --- a/src/zarr/storage/_utils.py +++ b/src/zarr/storage/_utils.py @@ -2,11 +2,13 @@ import re from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer @@ -66,3 +68,45 @@ def _normalize_byte_range_index(data: Buffer, byte_range: ByteRequest | None) -> else: raise ValueError(f"Unexpected byte_range, got {byte_range}.") return (start, stop) + + +def _join_paths(paths: Iterable[str]) -> str: + """ + Filter out instances of '' and join the remaining strings with '/'. + + Because the root node of a zarr hierarchy is represented by an empty string, + """ + return "/".join(filter(lambda v: v != "", paths)) + + +def _normalize_paths(paths: Iterable[str]) -> tuple[str, ...]: + """ + Normalize the input paths according to the normalization scheme used for zarr node paths. + If any two paths normalize to the same value, raise a ValueError. + """ + path_map: dict[str, str] = {} + for path in paths: + parsed = normalize_path(path) + if parsed in path_map: + msg = ( + f"After normalization, the value '{path}' collides with '{path_map[parsed]}'. " + f"Both '{path}' and '{path_map[parsed]}' normalize to the same value: '{parsed}'. " + f"You should use either '{path}' or '{path_map[parsed]}', but not both." + ) + raise ValueError(msg) + path_map[parsed] = path + return tuple(path_map.keys()) + + +T = TypeVar("T") + + +def _normalize_path_keys(data: Mapping[str, T]) -> dict[str, T]: + """ + Normalize the keys of the input dict according to the normalization scheme used for zarr node + paths. If any two keys in the input normalize to the same value, raise a ValueError. + Returns a dict where the keys are the elements of the input and the values are the + normalized form of each key. + """ + parsed_keys = _normalize_paths(data.keys()) + return dict(zip(parsed_keys, data.values(), strict=True)) diff --git a/tests/conftest.py b/tests/conftest.py index 9be675cb20..04034cb5b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,16 +11,30 @@ from zarr import AsyncGroup, config from zarr.abc.store import Store +from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation +from zarr.core.array import ( + _parse_chunk_encoding_v2, + _parse_chunk_encoding_v3, + _parse_chunk_key_encoding, +) +from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition +from zarr.core.common import JSON, parse_dtype, parse_shapelike +from zarr.core.config import config as zarr_config +from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import sync from zarr.storage import FsspecStore, LocalStore, MemoryStore, StorePath, ZipStore if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Generator, Iterable from typing import Any, Literal from _pytest.compat import LEGACY_PATH - from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat + from zarr.abc.codec import Codec + from zarr.core.array import CompressorsLike, FiltersLike, SerializerLike, ShardsLike + from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike + from zarr.core.common import ChunkCoords, MemoryOrder, ShapeLike, ZarrFormat async def parse_store( @@ -177,3 +191,212 @@ def pytest_collection_modifyitems(config: Any, items: Any) -> None: suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], verbosity=Verbosity.verbose, ) + +# TODO: uncomment these overrides when we can get mypy to accept them +""" +@overload +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"], + shards: None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: Literal[2], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: None, +) -> ArrayV2Metadata: ... + + +@overload +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"], + shards: ShardsLike | None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: None, + zarr_format: Literal[3], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV3Metadata: ... +""" + + +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + dimension_names: Iterable[str] | None = None, +) -> ArrayV2Metadata | ArrayV3Metadata: + """ + Create array metadata + """ + dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) + shape_parsed = parse_shapelike(shape) + chunk_key_encoding_parsed = _parse_chunk_key_encoding( + chunk_key_encoding, zarr_format=zarr_format + ) + + shard_shape_parsed, chunk_shape_parsed = _auto_partition( + array_shape=shape_parsed, shard_shape=shards, chunk_shape=chunks, dtype=dtype_parsed + ) + + if order is None: + order_parsed = zarr_config.get("array.order") + else: + order_parsed = order + chunks_out: tuple[int, ...] + + if zarr_format == 2: + filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( + compressor=compressors, filters=filters, dtype=np.dtype(dtype) + ) + return ArrayV2Metadata( + shape=shape_parsed, + dtype=np.dtype(dtype), + chunks=chunk_shape_parsed, + order=order_parsed, + dimension_separator=chunk_key_encoding_parsed.separator, + fill_value=fill_value, + compressor=compressor_parsed, + filters=filters_parsed, + attributes=attributes, + ) + elif zarr_format == 3: + array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( + compressors=compressors, + filters=filters, + serializer=serializer, + dtype=dtype_parsed, + ) + + sub_codecs: tuple[Codec, ...] = (*array_array, array_bytes, *bytes_bytes) + codecs_out: tuple[Codec, ...] + if shard_shape_parsed is not None: + index_location = None + if isinstance(shards, dict): + index_location = ShardingCodecIndexLocation(shards.get("index_location", None)) + if index_location is None: + index_location = ShardingCodecIndexLocation.end + sharding_codec = ShardingCodec( + chunk_shape=chunk_shape_parsed, + codecs=sub_codecs, + index_location=index_location, + ) + sharding_codec.validate( + shape=chunk_shape_parsed, + dtype=dtype_parsed, + chunk_grid=RegularChunkGrid(chunk_shape=shard_shape_parsed), + ) + codecs_out = (sharding_codec,) + chunks_out = shard_shape_parsed + else: + chunks_out = chunk_shape_parsed + codecs_out = sub_codecs + + return ArrayV3Metadata( + shape=shape_parsed, + data_type=dtype_parsed, + chunk_grid=RegularChunkGrid(chunk_shape=chunks_out), + chunk_key_encoding=chunk_key_encoding_parsed, + fill_value=fill_value, + codecs=codecs_out, + attributes=attributes, + dimension_names=dimension_names, + ) + + raise ValueError(f"Invalid Zarr format: {zarr_format}") + + +# TODO: uncomment these overrides when we can get mypy to accept them +""" +@overload +def meta_from_array( + array: np.ndarray[Any, Any], + chunks: ChunkCoords | Literal["auto"], + shards: None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: Literal[2], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV2Metadata: ... + + +@overload +def meta_from_array( + array: np.ndarray[Any, Any], + chunks: ChunkCoords | Literal["auto"], + shards: ShardsLike | None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: None, + zarr_format: Literal[3], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV3Metadata: ... + +""" + + +def meta_from_array( + array: np.ndarray[Any, Any], + *, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + dimension_names: Iterable[str] | None = None, +) -> ArrayV3Metadata | ArrayV2Metadata: + """ + Create array metadata from an array + """ + return create_array_metadata( + shape=array.shape, + dtype=array.dtype, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + ) diff --git a/tests/test_api.py b/tests/test_api.py index e9db33f6c5..3b565f8e60 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,4 +1,13 @@ -import pathlib +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pathlib + + from zarr.abc.store import Store + from zarr.core.common import JSON, MemoryOrder, ZarrFormat + import warnings from typing import Literal @@ -8,9 +17,9 @@ import zarr import zarr.api.asynchronous +import zarr.api.synchronous import zarr.core.group from zarr import Array, Group -from zarr.abc.store import Store from zarr.api.synchronous import ( create, create_array, @@ -23,7 +32,6 @@ save_array, save_group, ) -from zarr.core.common import JSON, MemoryOrder, ZarrFormat from zarr.errors import MetadataValidationError from zarr.storage import MemoryStore from zarr.storage._utils import normalize_path @@ -1124,6 +1132,13 @@ def test_open_array_with_mode_r_plus(store: Store) -> None: z2[:] = 3 +def test_api_exports() -> None: + """ + Test that the sync API and the async API export the same objects + """ + assert zarr.api.asynchronous.__all__ == zarr.api.synchronous.__all__ + + @gpu_test @pytest.mark.parametrize( "store", diff --git a/tests/test_group.py b/tests/test_group.py index 144054605e..521819ea0e 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1,8 +1,10 @@ from __future__ import annotations import contextlib +import inspect import operator import pickle +import re import time import warnings from typing import TYPE_CHECKING, Any, Literal @@ -17,18 +19,35 @@ import zarr.storage from zarr import Array, AsyncArray, AsyncGroup, Group from zarr.abc.store import Store +from zarr.core import sync_group from zarr.core._info import GroupInfo from zarr.core.buffer import default_buffer_prototype -from zarr.core.group import ConsolidatedMetadata, GroupMetadata -from zarr.core.sync import sync -from zarr.errors import ContainsArrayError, ContainsGroupError +from zarr.core.config import config as zarr_config +from zarr.core.group import ( + ConsolidatedMetadata, + GroupMetadata, + ImplicitGroupMarker, + _build_metadata_v3, + _get_roots, + _parse_hierarchy_dict, + create_hierarchy, + create_nodes, + create_rooted_hierarchy, + get_node, +) +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.core.sync import _collect_aiterator, sync +from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore from zarr.storage._common import make_store_path +from zarr.storage._utils import _join_paths, normalize_path from zarr.testing.store import LatencyStore -from .conftest import parse_store +from .conftest import meta_from_array, parse_store if TYPE_CHECKING: + from collections.abc import Callable + from _pytest.compat import LEGACY_PATH from zarr.core.common import JSON, ZarrFormat @@ -353,7 +372,7 @@ def test_group_getitem(store: Store, zarr_format: ZarrFormat, consolidated: bool ) with pytest.raises(KeyError): - # We've chosen to trust the consolidted metadata, which doesn't + # We've chosen to trust the consolidated metadata, which doesn't # contain this array group["subgroup/subarray"] @@ -1443,7 +1462,501 @@ def test_delitem_removes_children(store: Store, zarr_format: ZarrFormat) -> None @pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_group_members_performance(store: MemoryStore) -> None: +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_nodes( + impl: Literal["async", "sync"], store: Store, zarr_format: ZarrFormat +) -> None: + """ + Ensure that ``create_nodes`` can create a zarr hierarchy from a model of that + hierarchy in dict form. Note that this creates an incomplete Zarr hierarchy. + """ + node_spec = { + "group": GroupMetadata(attributes={"foo": 10}), + "group/array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "group/array_1": meta_from_array(np.arange(4), zarr_format=zarr_format), + "group/subgroup/array_0": meta_from_array(np.arange(4), zarr_format=zarr_format), + "group/subgroup/array_1": meta_from_array(np.arange(5), zarr_format=zarr_format), + } + if impl == "sync": + observed_nodes = dict(sync_group.create_nodes(store=store, nodes=node_spec)) + elif impl == "async": + observed_nodes = dict(await _collect_aiterator(create_nodes(store=store, nodes=node_spec))) + else: + raise ValueError(f"Invalid impl: {impl}") + + assert node_spec == {k: v.metadata for k, v in observed_nodes.items()} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_create_nodes_concurrency_limit(store: MemoryStore) -> None: + """ + Test that the execution time of create_nodes can be constrained by the async concurrency + configuration setting. + """ + set_latency = 0.02 + num_groups = 10 + groups = {str(idx): GroupMetadata() for idx in range(num_groups)} + + latency_store = LatencyStore(store, set_latency=set_latency) + + # check how long it takes to iterate over the groups + # if create_nodes is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + + with zarr_config.set({"async.concurrency": 1}): + start = time.time() + _ = tuple(sync_group.create_nodes(store=latency_store, nodes=groups)) + elapsed = time.time() - start + assert elapsed > num_groups * set_latency + + +@pytest.mark.parametrize( + ("a_func", "b_func"), + [ + (zarr.core.group.AsyncGroup.create_hierarchy, zarr.core.group.Group.create_hierarchy), + (zarr.core.group.create_hierarchy, zarr.core.sync_group.create_hierarchy), + (zarr.core.group.create_nodes, zarr.core.sync_group.create_nodes), + (zarr.core.group.create_rooted_hierarchy, zarr.core.sync_group.create_rooted_hierarchy), + (zarr.core.group.get_node, zarr.core.sync_group.get_node), + ], +) +def test_consistent_signatures( + a_func: Callable[[object], object], b_func: Callable[[object], object] +) -> None: + """ + Ensure that pairs of functions have consistent signatures + """ + base_sig = inspect.signature(a_func) + test_sig = inspect.signature(b_func) + assert test_sig.parameters == base_sig.parameters + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy( + impl: Literal["async", "sync"], store: Store, overwrite: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that ``create_hierarchy`` can create a complete Zarr hierarchy, even if the input describes + an incomplete one. + """ + + hierarchy_spec = { + "group": GroupMetadata(attributes={"path": "group"}, zarr_format=zarr_format), + "group/array_0": meta_from_array( + np.arange(3), attributes={"path": "group/array_0"}, zarr_format=zarr_format + ), + "group/subgroup/array_0": meta_from_array( + np.arange(4), attributes={"path": "group/subgroup/array_0"}, zarr_format=zarr_format + ), + } + pre_existing_nodes = { + "group/extra": GroupMetadata(zarr_format=zarr_format, attributes={"path": "group/extra"}), + "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "root"}), + } + # we expect create_hierarchy to insert a group that was missing from the hierarchy spec + expected_meta = hierarchy_spec | {"group/subgroup": GroupMetadata(zarr_format=zarr_format)} + + # initialize the group with some nodes + _ = dict(sync_group.create_nodes(store=store, nodes=pre_existing_nodes)) + + if impl == "sync": + created = dict( + sync_group.create_hierarchy(store=store, nodes=hierarchy_spec, overwrite=overwrite) + ) + elif impl == "async": + created = dict( + [ + a + async for a in create_hierarchy( + store=store, nodes=hierarchy_spec, overwrite=overwrite + ) + ] + ) + else: + raise ValueError(f"Invalid impl: {impl}") + if not overwrite: + extra_group = sync_group.get_node(store=store, path="group/extra", zarr_format=zarr_format) + assert extra_group.metadata.attributes == {"path": "group/extra"} + else: + with pytest.raises(FileNotFoundError): + await get_node(store=store, path="group/extra", zarr_format=zarr_format) + assert expected_meta == {k: v.metadata for k, v in created.items()} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("extant_node", ["array", "group"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_existing_nodes( + impl: Literal["async", "sync"], + store: Store, + extant_node: Literal["array", "group"], + zarr_format: ZarrFormat, +) -> None: + """ + Test that create_hierarchy with overwrite = False will not overwrite an existing array or group, + and raises an exception instead. + """ + extant_node_path = "node" + + if extant_node == "array": + extant_metadata = meta_from_array( + np.zeros(4), zarr_format=zarr_format, attributes={"extant": True} + ) + new_metadata = meta_from_array(np.zeros(4), zarr_format=zarr_format) + err_cls = ContainsArrayError + else: + extant_metadata = GroupMetadata(zarr_format=zarr_format, attributes={"extant": True}) + new_metadata = GroupMetadata(zarr_format=zarr_format) + err_cls = ContainsGroupError + + # write the extant metadata + tuple(sync_group.create_nodes(store=store, nodes={extant_node_path: extant_metadata})) + + msg = f"{extant_node} exists in store {store!r} at path {extant_node_path!r}." + # ensure that we cannot invoke create_hierarchy with overwrite=False here + if impl == "sync": + with pytest.raises(err_cls, match=re.escape(msg)): + tuple( + sync_group.create_hierarchy( + store=store, nodes={"node": new_metadata}, overwrite=False + ) + ) + elif impl == "async": + with pytest.raises(err_cls, match=re.escape(msg)): + tuple( + [ + x + async for x in create_hierarchy( + store=store, nodes={"node": new_metadata}, overwrite=False + ) + ] + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + # ensure that the extant metadata was not overwritten + assert ( + await get_node(store=store, path=extant_node_path, zarr_format=zarr_format) + ).metadata.attributes == {"extant": True} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("group_path", ["", "foo"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_group_create_hierarchy( + store: Store, + zarr_format: ZarrFormat, + overwrite: bool, + group_path: str, + impl: Literal["async", "sync"], +) -> None: + """ + Test that the Group.create_hierarchy method creates specified nodes and returns them in a dict. + Also test that off-target nodes are not deleted, and that the root group is not deleted + """ + root_attrs = {"root": True} + g = sync_group.create_rooted_hierarchy( + store=store, + nodes={group_path: GroupMetadata(zarr_format=zarr_format, attributes=root_attrs)}, + ) + node_spec = { + "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + "a/b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a/b"}), + "a/b/c": meta_from_array( + np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/b/c"} + ), + } + # This node should be kept if overwrite is True + extant_spec = {"b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "b"})} + if impl == "async": + extant_created = dict( + await _collect_aiterator(g._async_group.create_hierarchy(extant_spec, overwrite=False)) + ) + nodes_created = dict( + await _collect_aiterator( + g._async_group.create_hierarchy(node_spec, overwrite=overwrite) + ) + ) + elif impl == "sync": + extant_created = dict(g.create_hierarchy(extant_spec, overwrite=False)) + nodes_created = dict(g.create_hierarchy(node_spec, overwrite=overwrite)) + + all_members = dict(g.members(max_depth=None)) + for k, v in node_spec.items(): + assert all_members[k].metadata == v == nodes_created[k].metadata + + # if overwrite is True, the extant nodes should be erased + for k, v in extant_spec.items(): + if overwrite: + assert k in all_members + else: + assert all_members[k].metadata == v == extant_created[k].metadata + # ensure that we left the root group as-is + assert ( + sync_group.get_node(store=store, path=group_path, zarr_format=zarr_format).attrs.asdict() + == root_attrs + ) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_group_create_hierarchy_no_root( + store: Store, zarr_format: ZarrFormat, overwrite: bool +) -> None: + """ + Test that the Group.create_hierarchy method will error if the dict provided contains a root. + """ + g = Group.from_store(store, zarr_format=zarr_format) + tree = { + "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + } + with pytest.raises( + ValueError, match="It is an error to use this method to create a root node. " + ): + _ = dict(g.create_hierarchy(tree, overwrite=overwrite)) + + +class TestParseHierarchyDict: + """ + Tests for the function that parses dicts of str : Metadata pairs, ensuring that the output models a + valid Zarr hierarchy + """ + + @staticmethod + def test_normed_keys() -> None: + """ + Test that keys get normalized properly + """ + + nodes = { + "a": GroupMetadata(), + "/b": GroupMetadata(), + "": GroupMetadata(), + "/a//c////": GroupMetadata(), + } + observed = _parse_hierarchy_dict(data=nodes) + expected = {normalize_path(k): v for k, v in nodes.items()} + assert observed == expected + + @staticmethod + def test_empty() -> None: + """ + Test that an empty dict passes through + """ + assert _parse_hierarchy_dict(data={}) == {} + + @staticmethod + def test_implicit_groups() -> None: + """ + Test that implicit groups were added as needed. + """ + requested = {"a/b/c": GroupMetadata()} + expected = requested | { + "": ImplicitGroupMarker(), + "a": ImplicitGroupMarker(), + "a/b": ImplicitGroupMarker(), + } + observed = _parse_hierarchy_dict(data=requested) + assert observed == expected + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_create_hierarchy_invalid_mixed_zarr_format( + store: Store, zarr_format: ZarrFormat +) -> None: + """ + Test that ``Group.create_hierarchy`` will raise an error if the zarr_format of the nodes is + different from the parent group. + """ + other_format = 2 if zarr_format == 3 else 3 + g = Group.from_store(store, zarr_format=other_format) + tree = { + "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + "a/b": meta_from_array(np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/c"}), + } + + msg = "The zarr_format of the nodes must be the same as the parent group." + with pytest.raises(ValueError, match=msg): + _ = tuple(g.create_hierarchy(tree)) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("defect", ["array/array", "array/group"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_invalid_nested( + impl: Literal["async", "sync"], store: Store, defect: tuple[str, str], zarr_format: ZarrFormat +) -> None: + """ + Test that create_hierarchy will not create a Zarr array that contains a Zarr group + or Zarr array. + """ + if defect == "array/array": + hierarchy_spec = { + "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "array_0/subarray": meta_from_array(np.arange(4), zarr_format=zarr_format), + } + elif defect == "array/group": + hierarchy_spec = { + "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "array_0/subgroup": GroupMetadata(attributes={"foo": 10}, zarr_format=zarr_format), + } + + msg = "Only Zarr groups can contain other nodes." + if impl == "sync": + with pytest.raises(ValueError, match=msg): + tuple(sync_group.create_hierarchy(store=store, nodes=hierarchy_spec)) + elif impl == "async": + with pytest.raises(ValueError, match=msg): + await _collect_aiterator(create_hierarchy(store=store, nodes=hierarchy_spec)) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_invalid_mixed_format( + impl: Literal["async", "sync"], store: Store +) -> None: + """ + Test that create_hierarchy will not create a Zarr group that contains a both Zarr v2 and + Zarr v3 nodes. + """ + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + "The following keys map to Zarr v2 nodes: ['v2']. " + "The following keys map to Zarr v3 nodes: ['v3']." + "Ensure that all nodes have the same Zarr format." + ) + nodes = { + "v2": GroupMetadata(zarr_format=2), + "v3": GroupMetadata(zarr_format=3), + } + if impl == "sync": + with pytest.raises(ValueError, match=re.escape(msg)): + tuple( + sync_group.create_hierarchy( + store=store, + nodes=nodes, + ) + ) + elif impl == "async": + with pytest.raises(ValueError, match=re.escape(msg)): + await _collect_aiterator( + create_hierarchy( + store=store, + nodes=nodes, + ) + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + +@pytest.mark.parametrize("store", ["memory", "local"], indirect=True) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("root_key", ["", "root"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_group( + impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str +) -> None: + """ + Test that the _create_rooted_hierarchy can create a group. + """ + root_meta = {root_key: GroupMetadata(zarr_format=zarr_format, attributes={"path": root_key})} + group_names = ["a", "a/b"] + array_names = ["a/b/c", "a/b/d"] + + # just to ensure that we don't use the same name twice in tests + assert set(group_names) & set(array_names) == set() + + groups_expected_meta = { + _join_paths([root_key, node_name]): GroupMetadata( + zarr_format=zarr_format, attributes={"path": node_name} + ) + for node_name in group_names + } + + arrays_expected_meta = { + _join_paths([root_key, node_name]): meta_from_array(np.zeros(4), zarr_format=zarr_format) + for node_name in array_names + } + + nodes_create = root_meta | groups_expected_meta | arrays_expected_meta + if impl == "sync": + g = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create) + assert isinstance(g, Group) + members = g.members(max_depth=None) + elif impl == "async": + g = await create_rooted_hierarchy(store=store, nodes=nodes_create) + assert isinstance(g, AsyncGroup) + members = await _collect_aiterator(g.members(max_depth=None)) + else: + raise ValueError(f"Unknown implementation: {impl}") + + assert g.metadata.attributes == {"path": root_key} + + members_observed_meta = {k: v.metadata for k, v in members} + members_expected_meta_relative = { + k.removeprefix(root_key).lstrip("/"): v + for k, v in (groups_expected_meta | arrays_expected_meta).items() + } + assert members_observed_meta == members_expected_meta_relative + + +@pytest.mark.parametrize("store", ["memory", "local"], indirect=True) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("root_key", ["", "root"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_array( + impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str +) -> None: + """ + Test that _create_rooted_hierarchy can create an array. + """ + + root_meta = { + root_key: meta_from_array( + np.arange(3), zarr_format=zarr_format, attributes={"path": root_key} + ) + } + nodes_create = root_meta + + if impl == "sync": + a = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) + assert isinstance(a, Array) + elif impl == "async": + a = await create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) + assert isinstance(a, AsyncArray) + else: + raise ValueError(f"Invalid impl: {impl}") + assert a.metadata.attributes == {"path": root_key} + + +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_invalid(impl: Literal["async", "sync"]) -> None: + """ + Ensure _create_rooted_hierarchy will raise a ValueError if the input does not contain + a root node. + """ + zarr_format = 3 + nodes = { + "a": GroupMetadata(zarr_format=zarr_format), + "b": GroupMetadata(zarr_format=zarr_format), + } + msg = "The input does not specify a root node. " + if impl == "sync": + with pytest.raises(ValueError, match=msg): + sync_group.create_rooted_hierarchy(store=store, nodes=nodes) + elif impl == "async": + with pytest.raises(ValueError, match=msg): + await create_rooted_hierarchy(store=store, nodes=nodes) + else: + raise ValueError(f"Invalid impl: {impl}") + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_members_performance(store: Store) -> None: """ Test that the execution time of Group.members is less than the number of members times the latency for accessing each member. @@ -1505,3 +2018,35 @@ def test_group_members_concurrency_limit(store: MemoryStore) -> None: elapsed = time.time() - start assert elapsed > num_groups * get_latency + + +@pytest.mark.parametrize("option", ["array", "group", "invalid"]) +def test_build_metadata_v3(option: Literal["array", "group", "invalid"]) -> None: + """ + Test that _build_metadata_v3 returns the correct metadata for a v3 array or group + """ + match option: + case "array": + metadata_dict = meta_from_array(np.arange(10), zarr_format=3).to_dict() + assert _build_metadata_v3(metadata_dict) == ArrayV3Metadata.from_dict(metadata_dict) + case "group": + metadata_dict = GroupMetadata(attributes={"foo": 10}, zarr_format=3).to_dict() + assert _build_metadata_v3(metadata_dict) == GroupMetadata.from_dict(metadata_dict) + case "invalid": + metadata_dict = GroupMetadata(zarr_format=3).to_dict() + metadata_dict.pop("node_type") + # TODO: fix the error message + msg = "Invalid value for 'node_type'. Expected 'array or group'. Got 'nothing (the key is missing)'." + with pytest.raises(MetadataValidationError, match=re.escape(msg)): + _build_metadata_v3(metadata_dict) + + +@pytest.mark.parametrize("roots", [("",), ("a", "b")]) +def test_get_roots(roots: tuple[str, ...]): + root_nodes = {k: GroupMetadata(attributes={"name": k}) for k in roots} + child_nodes = { + _join_paths([k, "foo"]): GroupMetadata(attributes={"name": _join_paths([k, "foo"])}) + for k in roots + } + data = root_nodes | child_nodes + assert set(_get_roots(data)) == set(roots) diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 726da06a52..bce582a746 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -8,7 +8,7 @@ from zarr.core.common import AccessModeLiteral, ZarrFormat from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath from zarr.storage._common import contains_array, contains_group, make_store_path -from zarr.storage._utils import normalize_path +from zarr.storage._utils import _join_paths, _normalize_path_keys, _normalize_paths, normalize_path @pytest.mark.parametrize("path", ["foo", "foo/bar"]) @@ -174,3 +174,48 @@ def test_normalize_path_none(): def test_normalize_path_invalid(path: str): with pytest.raises(ValueError): normalize_path(path) + + +@pytest.mark.parametrize("paths", [("", "foo"), ("foo", "bar")]) +def test_join_paths(paths: tuple[str, str]) -> None: + """ + Test that _join_paths joins paths in a way that is robust to an empty string + """ + observed = _join_paths(paths) + if paths[0] == "": + assert observed == paths[1] + else: + assert observed == "/".join(paths) + + +class TestNormalizePaths: + @staticmethod + def test_valid() -> None: + """ + Test that path normalization works as expected + """ + paths = ["a", "b", "c", "d", "", "//a///b//"] + assert _normalize_paths(paths) == tuple([normalize_path(p) for p in paths]) + + @staticmethod + @pytest.mark.parametrize("paths", [("", "/"), ("///a", "a")]) + def test_invalid(paths: tuple[str, str]) -> None: + """ + Test that name collisions after normalization raise a ``ValueError`` + """ + msg = ( + f"After normalization, the value '{paths[1]}' collides with '{paths[0]}'. " + f"Both '{paths[1]}' and '{paths[0]}' normalize to the same value: '{normalize_path(paths[0])}'. " + f"You should use either '{paths[1]}' or '{paths[0]}', but not both." + ) + with pytest.raises(ValueError, match=msg): + _normalize_paths(paths) + + +def test_normalize_path_keys(): + """ + Test that ``_normalize_path_keys`` just applies the normalize_path function to each key of its + input + """ + data = {"a": 10, "//b": 10} + assert _normalize_path_keys(data) == {normalize_path(k): v for k, v in data.items()} From 64b9a370a94c8d8afcfd06f93ca25d77e668cdd6 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Tue, 25 Feb 2025 02:52:54 -0600 Subject: [PATCH 075/160] Avoid creating persistent files during tests (#2860) --- tests/test_indexing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 1bad861622..77363acff3 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -827,20 +827,20 @@ def test_set_orthogonal_selection_1d(store: StorePath) -> None: _test_set_orthogonal_selection(v, a, z, selection) -def test_set_item_1d_last_two_chunks(): +def test_set_item_1d_last_two_chunks(store: StorePath): # regression test for GH2849 - g = zarr.open_group("foo.zarr", zarr_format=3, mode="w") + g = zarr.open_group(store=store, zarr_format=3, mode="w") a = g.create_array("bar", shape=(10,), chunks=(3,), dtype=int) data = np.array([7, 8, 9]) a[slice(7, 10)] = data np.testing.assert_array_equal(a[slice(7, 10)], data) - z = zarr.open_group("foo.zarr", mode="w") + z = zarr.open_group(store=store, mode="w") z.create_array("zoo", dtype=float, shape=()) z["zoo"][...] = np.array(1) # why doesn't [:] work? np.testing.assert_equal(z["zoo"][()], np.array(1)) - z = zarr.open_group("foo.zarr", mode="w") + z = zarr.open_group(store=store, mode="w") z.create_array("zoo", dtype=float, shape=()) z["zoo"][...] = 1 # why doesn't [:] work? np.testing.assert_equal(z["zoo"][()], np.array(1)) From 7520870b9fea1fa42e2c441cb2da7438ef23e55f Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 27 Feb 2025 14:55:31 -0600 Subject: [PATCH 076/160] Prevent update_attributes from erasing all prior attributes (#2870) * Prevent update_attributes from erasing all prior attributes * Fix incorrect group attribute update tests * Add changelog entry --- changes/2870.bugfix.rst | 1 + src/zarr/core/array.py | 2 -- src/zarr/core/attributes.py | 1 + src/zarr/core/common.py | 4 ++-- src/zarr/core/group.py | 2 -- tests/test_attributes.py | 39 +++++++++++++++++++++++++++++++++++++ tests/test_group.py | 9 +++++++-- 7 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 changes/2870.bugfix.rst diff --git a/changes/2870.bugfix.rst b/changes/2870.bugfix.rst new file mode 100644 index 0000000000..7779835892 --- /dev/null +++ b/changes/2870.bugfix.rst @@ -0,0 +1 @@ +Prevent update_attributes calls from deleting old attributes \ No newline at end of file diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 9c2f8a7260..0e03cbcabb 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1609,8 +1609,6 @@ async def update_attributes(self, new_attributes: dict[str, JSON]) -> Self: - The updated attributes will be merged with existing attributes, and any conflicts will be overwritten by the new values. """ - # metadata.attributes is "frozen" so we simply clear and update the dict - self.metadata.attributes.clear() self.metadata.attributes.update(new_attributes) # Write new metadata diff --git a/src/zarr/core/attributes.py b/src/zarr/core/attributes.py index 7f9864d1b5..6aad39085d 100644 --- a/src/zarr/core/attributes.py +++ b/src/zarr/core/attributes.py @@ -50,6 +50,7 @@ def put(self, d: dict[str, JSON]) -> None: >>> attrs {'a': 3, 'c': 4} """ + self._obj.metadata.attributes.clear() self._obj = self._obj.update_attributes(d) def asdict(self) -> dict[str, JSON]: diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index ad3316b619..3308ca3247 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -4,7 +4,7 @@ import functools import operator import warnings -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from enum import Enum from itertools import starmap from typing import ( @@ -37,7 +37,7 @@ ChunkCoordsLike = Iterable[int] ZarrFormat = Literal[2, 3] NodeType = Literal["array", "group"] -JSON = str | int | float | Mapping[str, "JSON"] | tuple["JSON", ...] | None +JSON = str | int | float | Mapping[str, "JSON"] | Sequence["JSON"] | None MemoryOrder = Literal["C", "F"] AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index a7f8a6c022..8e7f7f3474 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1254,8 +1254,6 @@ async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: ------- self : AsyncGroup """ - # metadata.attributes is "frozen" so we simply clear and update the dict - self.metadata.attributes.clear() self.metadata.attributes.update(new_attributes) # Write new metadata diff --git a/tests/test_attributes.py b/tests/test_attributes.py index b26db5df89..16825c99e0 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -20,3 +20,42 @@ def test_asdict() -> None: ) result = attrs.asdict() assert result == {"a": 1, "b": 2} + + +def test_update_attributes_preserves_existing() -> None: + """ + Test that `update_attributes` only updates the specified attributes + and preserves existing ones. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + z.attrs["a"] = [] + z.attrs["b"] = 3 + assert dict(z.attrs) == {"a": [], "b": 3} + + z.update_attributes({"a": [3, 4], "c": 4}) + assert dict(z.attrs) == {"a": [3, 4], "b": 3, "c": 4} + + +def test_update_empty_attributes() -> None: + """ + Ensure updating when initial attributes are empty works. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + assert dict(z.attrs) == {} + z.update_attributes({"a": [3, 4], "c": 4}) + assert dict(z.attrs) == {"a": [3, 4], "c": 4} + + +def test_update_no_changes() -> None: + """ + Ensure updating when no new or modified attributes does not alter existing ones. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + z.attrs["a"] = [] + z.attrs["b"] = 3 + + z.update_attributes({}) + assert dict(z.attrs) == {"a": [], "b": 3} diff --git a/tests/test_group.py b/tests/test_group.py index 521819ea0e..12e116849a 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -604,7 +604,10 @@ def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: assert group.attrs == attrs new_attrs = {"bar": 100} new_group = group.update_attributes(new_attrs) - assert new_group.attrs == new_attrs + + updated_attrs = attrs.copy() + updated_attrs.update(new_attrs) + assert new_group.attrs == updated_attrs async def test_group_update_attributes_async(store: Store, zarr_format: ZarrFormat) -> None: @@ -1008,7 +1011,9 @@ async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrForma ) agroup_new_attributes = await agroup.update_attributes(attributes_new) - assert agroup_new_attributes.attrs == attributes_new + attributes_updated = attributes_old.copy() + attributes_updated.update(attributes_new) + assert agroup_new_attributes.attrs == attributes_updated @pytest.mark.parametrize("store", ["local"], indirect=["store"]) From cf371985b7d65fa2055663cb569c93922349229a Mon Sep 17 00:00:00 2001 From: AdityaHere Date: Fri, 28 Feb 2025 15:08:53 +0530 Subject: [PATCH 077/160] DOC: Fixed incorrect link in blosc.py (#2864) The "see more" under BloscCname class contains [old link](https://zarr.readthedocs.io/en/stable/tutorial.html#configuring-blosc) to refer more information about configuring Blosc but it is currently shifted to [new link](https://zarr.readthedocs.io/en/stable/user-guide/performance.html#configuring-blosc) --- src/zarr/codecs/blosc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index 54a23c9c57..2fcc041a6b 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -55,7 +55,7 @@ class BloscCname(Enum): zlib = "zlib" -# See https://zarr.readthedocs.io/en/stable/tutorial.html#configuring-blosc +# See https://zarr.readthedocs.io/en/stable/user-guide/performance.html#configuring-blosc numcodecs.blosc.use_threads = False From e9d31c4dcefce9f93e9d958e1a179535e2ff1d17 Mon Sep 17 00:00:00 2001 From: Asim <95623198+asimchoudhary@users.noreply.github.com> Date: Tue, 4 Mar 2025 09:17:28 +0530 Subject: [PATCH 078/160] Add other open modes in group creation test (#2840) * add other open modes in group creation test Signed-off-by: asim * minor fix Signed-off-by: asim * Update tests/test_api.py Add else block for write operations only Co-authored-by: Davis Bennett --------- Signed-off-by: asim Co-authored-by: Davis Bennett --- tests/test_api.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 3b565f8e60..94140ac784 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -209,7 +209,7 @@ def test_save(store: Store, n_args: int, n_kwargs: int) -> None: assert isinstance(array, Array) assert_array_equal(array[:], data) else: - save(store, *args, **kwargs) # type: ignore[arg-type] + save(store, *args, **kwargs) # type: ignore [arg-type] group = open(store) assert isinstance(group, Group) for array in group.array_values(): @@ -1095,11 +1095,17 @@ async def test_open_falls_back_to_open_group_async() -> None: assert group.attrs == {"key": "value"} -def test_open_mode_write_creates_group(tmp_path: pathlib.Path) -> None: +@pytest.mark.parametrize("mode", ["r", "r+", "w", "a"]) +def test_open_modes_creates_group(tmp_path: pathlib.Path, mode: str) -> None: # https://github.com/zarr-developers/zarr-python/issues/2490 - zarr_dir = tmp_path / "test.zarr" - group = zarr.open(zarr_dir, mode="w") - assert isinstance(group, Group) + zarr_dir = tmp_path / f"mode-{mode}-test.zarr" + if mode in ["r", "r+"]: + # Expect FileNotFoundError to be raised if 'r' or 'r+' mode + with pytest.raises(FileNotFoundError): + zarr.open(store=zarr_dir, mode=mode) + else: + group = zarr.open(store=zarr_dir, mode=mode) + assert isinstance(group, Group) async def test_metadata_validation_error() -> None: @@ -1107,13 +1113,13 @@ async def test_metadata_validation_error() -> None: MetadataValidationError, match="Invalid value for 'zarr_format'. Expected '2, 3, or None'. Got '3.0'.", ): - await zarr.api.asynchronous.open_group(zarr_format="3.0") # type: ignore[arg-type] + await zarr.api.asynchronous.open_group(zarr_format="3.0") # type: ignore [arg-type] with pytest.raises( MetadataValidationError, match="Invalid value for 'zarr_format'. Expected '2, 3, or None'. Got '3.0'.", ): - await zarr.api.asynchronous.open_array(shape=(1,), zarr_format="3.0") # type: ignore[arg-type] + await zarr.api.asynchronous.open_array(shape=(1,), zarr_format="3.0") # type: ignore [arg-type] @pytest.mark.parametrize( From fae8fb907d5d1afb7fb479d2d9a514ff321c5ed5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Mar 2025 08:15:19 +0000 Subject: [PATCH 079/160] chore: update pre-commit hooks (#2882) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.9.4 → v0.9.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.4...v0.9.9) - [github.com/pre-commit/mirrors-mypy: v1.14.1 → v1.15.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.14.1...v1.15.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 94d5342486..ba483d10e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ ci: default_stages: [pre-commit, pre-push] repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.4 + rev: v0.9.9 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -22,7 +22,7 @@ repos: - id: check-yaml - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.1 + rev: v1.15.0 hooks: - id: mypy files: src|tests From 73d5a6ca30bc2503d5092de1099206591156f880 Mon Sep 17 00:00:00 2001 From: OPMTerra <144823952+OPMTerra@users.noreply.github.com> Date: Tue, 4 Mar 2025 20:18:21 +0530 Subject: [PATCH 080/160] DOC: Add store import changes to v3 migration guide (fix #2733) (#2883) * DOC: Add store import changes to v3 migration guide (fix #2733) * Update docs/user-guide/v3_migration.rst Adjust heading level per review feedback Co-authored-by: David Stansby --------- Co-authored-by: David Stansby --- docs/user-guide/v3_migration.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/user-guide/v3_migration.rst b/docs/user-guide/v3_migration.rst index bda1ae64ed..678a3aafb7 100644 --- a/docs/user-guide/v3_migration.rst +++ b/docs/user-guide/v3_migration.rst @@ -124,6 +124,30 @@ The Store class The Store API has changed significant in Zarr-Python 3. The most notable changes to the Store API are: +Store Import Paths +^^^^^^^^^^^^^^^^^^ +Several store implementations have moved from the top-level module to ``zarr.storage``: + +.. code-block:: diff + :caption: Store import changes from v2 to v3 + + # Before (v2) + - from zarr import MemoryStore, DirectoryStore + + from zarr.storage import MemoryStore, LocalStore # LocalStore replaces DirectoryStore + +Common replacements: + ++-------------------------+------------------------------------+ +| v2 Import | v3 Import | ++=========================+====================================+ +| ``zarr.MemoryStore`` | ``zarr.storage.MemoryStore`` | ++-------------------------+------------------------------------+ +| ``zarr.DirectoryStore`` | ``zarr.storage.LocalStore`` | ++-------------------------+------------------------------------+ +| ``zarr.TempStore`` | Use ``tempfile.TemporaryDirectory``| +| | with ``LocalStore`` | ++-------------------------+------------------------------------+ + 1. Replaced the ``MutableMapping`` base class in favor of a custom abstract base class (:class:`zarr.abc.store.Store`). 2. Switched to an asynchronous interface for all store methods that result in IO. This From ee607923aa10b455f8c1f7cdcdbdee380c2af65d Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 4 Mar 2025 18:56:06 +0100 Subject: [PATCH 081/160] Normalize paths when creating StorePath (#2850) * add failing group name tests * normalize paths in storepath constructor * add asserts to hypothesis tests * test group creation too * code review * changelog --- changes/2850.fix.rst | 2 ++ src/zarr/storage/_common.py | 2 +- src/zarr/testing/strategies.py | 4 ++- tests/test_group.py | 65 ++++++++++++++++++++-------------- 4 files changed, 44 insertions(+), 29 deletions(-) create mode 100644 changes/2850.fix.rst diff --git a/changes/2850.fix.rst b/changes/2850.fix.rst new file mode 100644 index 0000000000..dbba120ce2 --- /dev/null +++ b/changes/2850.fix.rst @@ -0,0 +1,2 @@ +Fixed a bug where ``StorePath`` creation would not apply standard path normalization to the ``path`` parameter, +which led to the creation of arrays and groups with invalid keys. \ No newline at end of file diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 6ab539bb0a..d81369f142 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -41,7 +41,7 @@ class StorePath: def __init__(self, store: Store, path: str = "") -> None: self.store = store - self.path = path + self.path = normalize_path(path) @property def read_only(self) -> bool: diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 96d664f5aa..aa42329be7 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -19,6 +19,7 @@ from zarr.core.sync import sync from zarr.storage import MemoryStore, StoreLike from zarr.storage._common import _dereference_path +from zarr.storage._utils import normalize_path # Copied from Xarray _attr_keys = st.text(st.characters(), min_size=1) @@ -277,11 +278,12 @@ def arrays( if a.metadata.zarr_format == 3: assert a.fill_value is not None assert a.name is not None + assert a.path == normalize_path(array_path) + assert a.name == "/" + a.path assert isinstance(root[array_path], Array) assert nparray.shape == a.shape assert chunk_shape == a.chunks assert shard_shape == a.shards - assert array_path == a.path, (path, name, array_path, a.name, a.path) assert a.basename == name, (a.basename, name) assert dict(a.attrs) == expected_attrs diff --git a/tests/test_group.py b/tests/test_group.py index 12e116849a..1e4f31b5d6 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -130,24 +130,27 @@ async def test_create_creates_parents(store: Store, zarr_format: ZarrFormat) -> assert g.attrs == {} -def test_group_name_properties(store: Store, zarr_format: ZarrFormat) -> None: +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("root_name", ["", "/", "a", "/a"]) +@pytest.mark.parametrize("branch_name", ["foo", "/foo", "foo/bar", "/foo/bar"]) +def test_group_name_properties( + store: Store, zarr_format: ZarrFormat, root_name: str, branch_name: str +) -> None: """ - Test basic properties of groups + Test that the path, name, and basename attributes of a group and its subgroups are consistent """ - root = Group.from_store(store=store, zarr_format=zarr_format) - assert root.path == "" - assert root.name == "/" - assert root.basename == "" + root = Group.from_store(store=StorePath(store=store, path=root_name), zarr_format=zarr_format) + assert root.path == normalize_path(root_name) + assert root.name == "/" + root.path + assert root.basename == root.path - foo = root.create_group("foo") - assert foo.path == "foo" - assert foo.name == "/foo" - assert foo.basename == "foo" - - bar = root.create_group("foo/bar") - assert bar.path == "foo/bar" - assert bar.name == "/foo/bar" - assert bar.basename == "bar" + branch = root.create_group(branch_name) + if root.path == "": + assert branch.path == normalize_path(branch_name) + else: + assert branch.path == "/".join([root.path, normalize_path(branch_name)]) + assert branch.name == "/" + branch.path + assert branch.basename == branch_name.split("/")[-1] @pytest.mark.parametrize("consolidated_metadata", [True, False]) @@ -623,11 +626,13 @@ async def test_group_update_attributes_async(store: Store, zarr_format: ZarrForm @pytest.mark.parametrize("method", ["create_array", "array"]) +@pytest.mark.parametrize("name", ["a", "/a"]) def test_group_create_array( store: Store, zarr_format: ZarrFormat, overwrite: bool, method: Literal["create_array", "array"], + name: str, ) -> None: """ Test `Group.from_store` @@ -638,23 +643,26 @@ def test_group_create_array( data = np.arange(np.prod(shape)).reshape(shape).astype(dtype) if method == "create_array": - array = group.create_array(name="array", shape=shape, dtype=dtype) + array = group.create_array(name=name, shape=shape, dtype=dtype) array[:] = data elif method == "array": with pytest.warns(DeprecationWarning): - array = group.array(name="array", data=data, shape=shape, dtype=dtype) + array = group.array(name=name, data=data, shape=shape, dtype=dtype) else: raise AssertionError if not overwrite: if method == "create_array": with pytest.raises(ContainsArrayError): - a = group.create_array(name="array", shape=shape, dtype=dtype) + a = group.create_array(name=name, shape=shape, dtype=dtype) a[:] = data elif method == "array": with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): - a = group.array(name="array", shape=shape, dtype=dtype) + a = group.array(name=name, shape=shape, dtype=dtype) a[:] = data + + assert array.path == normalize_path(name) + assert array.name == "/" + array.path assert array.shape == shape assert array.dtype == np.dtype(dtype) assert np.array_equal(array[:], data) @@ -945,20 +953,23 @@ async def test_asyncgroup_delitem(store: Store, zarr_format: ZarrFormat) -> None raise AssertionError +@pytest.mark.parametrize("name", ["a", "/a"]) async def test_asyncgroup_create_group( store: Store, + name: str, zarr_format: ZarrFormat, ) -> None: agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) - sub_node_path = "sub_group" attributes = {"foo": 999} - subnode = await agroup.create_group(name=sub_node_path, attributes=attributes) - - assert isinstance(subnode, AsyncGroup) - assert subnode.attrs == attributes - assert subnode.store_path.path == sub_node_path - assert subnode.store_path.store == store - assert subnode.metadata.zarr_format == zarr_format + subgroup = await agroup.create_group(name=name, attributes=attributes) + + assert isinstance(subgroup, AsyncGroup) + assert subgroup.path == normalize_path(name) + assert subgroup.name == "/" + subgroup.path + assert subgroup.attrs == attributes + assert subgroup.store_path.path == subgroup.path + assert subgroup.store_path.store == store + assert subgroup.metadata.zarr_format == zarr_format async def test_asyncgroup_create_array( From 71a7768772ae9ed00f9507cb5e25da9b6fee66dc Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 6 Mar 2025 10:03:52 +0100 Subject: [PATCH 082/160] refer to zulip chat as developer chat (#2893) --- .github/ISSUE_TEMPLATE/config.yml | 2 +- README.md | 2 +- docs/index.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index edbd88eaf2..27239f5861 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,7 +5,7 @@ contact_links: about: A new major feature should be discussed in the Zarr specifications repository. - name: Discuss something on ZulipChat url: https://ossci.zulipchat.com/ - about: For questions like "How do I do X with Zarr?", you can move to our ZulipChat. + about: For questions like "How do I do X with Zarr?", consider posting your question to our developer chat. - name: Discuss something on GitHub Discussions url: https://github.com/zarr-developers/zarr-python/discussions about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions. diff --git a/README.md b/README.md index 5ee6748ada..7079b5afd5 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ - Zulip + Developer Chat diff --git a/docs/index.rst b/docs/index.rst index 6ab07b0693..83d427e290 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,7 +20,7 @@ Zarr-Python **Useful links**: `Source Repository `_ | `Issue Tracker `_ | -`Zulip Chat `_ | +`Developer Chat `_ | `Zarr specifications `_ Zarr-Python is a Python library for reading and writing Zarr groups and arrays. Highlights include: From 38a241712b243ebb12a8f969e499789700a4334d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 7 Mar 2025 09:53:15 -0800 Subject: [PATCH 083/160] 3.0.5 release prep (#2898) * fix release note file names * update release notes * add missing release notes section for 3.0.4 --- changes/2665.feature.rst | 1 - changes/2796.chore.rst | 1 - changes/2847.fix.rst | 1 - changes/2850.fix.rst | 2 -- changes/2851.bugfix.rst | 1 - changes/2870.bugfix.rst | 1 - docs/release-notes.rst | 36 ++++++++++++++++++++++++++++++++++++ 7 files changed, 36 insertions(+), 7 deletions(-) delete mode 100644 changes/2665.feature.rst delete mode 100644 changes/2796.chore.rst delete mode 100644 changes/2847.fix.rst delete mode 100644 changes/2850.fix.rst delete mode 100644 changes/2851.bugfix.rst delete mode 100644 changes/2870.bugfix.rst diff --git a/changes/2665.feature.rst b/changes/2665.feature.rst deleted file mode 100644 index 40bec542ce..0000000000 --- a/changes/2665.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Adds functions for concurrently creating multiple arrays and groups. \ No newline at end of file diff --git a/changes/2796.chore.rst b/changes/2796.chore.rst deleted file mode 100644 index 0ef1b8d31e..0000000000 --- a/changes/2796.chore.rst +++ /dev/null @@ -1 +0,0 @@ -The docs environment is now built with ``astroid`` pinned to a version less than 4. This allows the docs to build in CI. \ No newline at end of file diff --git a/changes/2847.fix.rst b/changes/2847.fix.rst deleted file mode 100644 index 148e191b98..0000000000 --- a/changes/2847.fix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug where ``ArrayV2Metadata`` could save ``filters`` as an empty array. \ No newline at end of file diff --git a/changes/2850.fix.rst b/changes/2850.fix.rst deleted file mode 100644 index dbba120ce2..0000000000 --- a/changes/2850.fix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a bug where ``StorePath`` creation would not apply standard path normalization to the ``path`` parameter, -which led to the creation of arrays and groups with invalid keys. \ No newline at end of file diff --git a/changes/2851.bugfix.rst b/changes/2851.bugfix.rst deleted file mode 100644 index 977f683847..0000000000 --- a/changes/2851.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug when setting values of a smaller last chunk. diff --git a/changes/2870.bugfix.rst b/changes/2870.bugfix.rst deleted file mode 100644 index 7779835892..0000000000 --- a/changes/2870.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent update_attributes calls from deleting old attributes \ No newline at end of file diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 93466a0992..21876770a2 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -3,6 +3,42 @@ Release notes .. towncrier release notes start +3.0.5 (2025-03-07) +------------------ + +Bugfixes +~~~~~~~~ + +- Fixed a bug where ``StorePath`` creation would not apply standard path normalization to the ``path`` parameter, + which led to the creation of arrays and groups with invalid keys. (:issue:`2850`) +- Prevent update_attributes calls from deleting old attributes (:issue:`2870`) + + +Misc +~~~~ + +- :issue:`2796` + +3.0.4 (2025-02-23) +------------------ + +Features +~~~~~~~~ + +- Adds functions for concurrently creating multiple arrays and groups. (:issue:`2665`) + +Bugfixes +~~~~~~~~ + +- Fixed a bug where ``ArrayV2Metadata`` could save ``filters`` as an empty array. (:issue:`2847`) +- Fix a bug when setting values of a smaller last chunk. (:issue:`2851`) + +Misc +~~~~ + +- :issue:`2828` + + 3.0.3 (2025-02-14) ------------------ From 5968fc79494fd3429e49592ada103f98fd1f66be Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Mon, 17 Mar 2025 15:59:10 -0600 Subject: [PATCH 084/160] fix: restore attrs del functionality (#2908) * fix: restore attrs del functionality * test: fully test * test: explicitly test both group and array - also reload * doc: add changelog entry * test: correct typing --- changes/2908.bugfix.rst | 1 + src/zarr/core/attributes.py | 2 +- tests/test_attributes.py | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 changes/2908.bugfix.rst diff --git a/changes/2908.bugfix.rst b/changes/2908.bugfix.rst new file mode 100644 index 0000000000..5b932b1b6b --- /dev/null +++ b/changes/2908.bugfix.rst @@ -0,0 +1 @@ +Restore functionality of `del z.attrs['key']` to actually delete the key. diff --git a/src/zarr/core/attributes.py b/src/zarr/core/attributes.py index 6aad39085d..e699c4f66d 100644 --- a/src/zarr/core/attributes.py +++ b/src/zarr/core/attributes.py @@ -28,7 +28,7 @@ def __setitem__(self, key: str, value: JSON) -> None: def __delitem__(self, key: str) -> None: new_attrs = dict(self._obj.metadata.attributes) del new_attrs[key] - self._obj = self._obj.update_attributes(new_attrs) + self.put(new_attrs) def __iter__(self) -> Iterator[str]: return iter(self._obj.metadata.attributes) diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 16825c99e0..127b2dbc36 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -1,3 +1,5 @@ +import pytest + import zarr.core import zarr.core.attributes import zarr.storage @@ -59,3 +61,24 @@ def test_update_no_changes() -> None: z.update_attributes({}) assert dict(z.attrs) == {"a": [], "b": 3} + + +@pytest.mark.parametrize("group", [True, False]) +def test_del_works(group: bool) -> None: + store = zarr.storage.MemoryStore() + z: zarr.Group | zarr.Array + if group: + z = zarr.create_group(store) + else: + z = zarr.create_array(store=store, shape=10, dtype=int) + assert dict(z.attrs) == {} + z.update_attributes({"a": [3, 4], "c": 4}) + del z.attrs["a"] + assert dict(z.attrs) == {"c": 4} + + z2: zarr.Group | zarr.Array + if group: + z2 = zarr.open_group(store) + else: + z2 = zarr.open_array(store) + assert dict(z2.attrs) == {"c": 4} From f95c453f3835da7f8097a1b5302ea1b7a3d6e57f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 20 Mar 2025 16:03:57 -0700 Subject: [PATCH 085/160] release: update release notes for 3.0.6 (#2923) --- changes/2908.bugfix.rst | 1 - docs/release-notes.rst | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) delete mode 100644 changes/2908.bugfix.rst diff --git a/changes/2908.bugfix.rst b/changes/2908.bugfix.rst deleted file mode 100644 index 5b932b1b6b..0000000000 --- a/changes/2908.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Restore functionality of `del z.attrs['key']` to actually delete the key. diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 21876770a2..9a6e680e4d 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -3,6 +3,15 @@ Release notes .. towncrier release notes start +3.0.6 (2025-03-20) +------------------ + +Bugfixes +~~~~~~~~ + +- Restore functionality of `del z.attrs['key']` to actually delete the key. (:issue:`2908`) + + 3.0.5 (2025-03-07) ------------------ From d3765339a55183c2f7a779bcfaf383bb058d4e3c Mon Sep 17 00:00:00 2001 From: Adam Dong Date: Sat, 22 Mar 2025 17:02:21 +0100 Subject: [PATCH 086/160] Modify the link in README.md from V2 to V3 docs (#2915) --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7079b5afd5..97f5617934 100644 --- a/README.md +++ b/README.md @@ -101,13 +101,13 @@ Zarr is a Python package providing an implementation of compressed, chunked, N-d ## Main Features -- [**Create**](https://zarr.readthedocs.io/en/stable/tutorial.html#creating-an-array) N-dimensional arrays with any NumPy `dtype`. -- [**Chunk arrays**](https://zarr.readthedocs.io/en/stable/tutorial.html#chunk-optimizations) along any dimension. -- [**Compress**](https://zarr.readthedocs.io/en/stable/tutorial.html#compressors) and/or filter chunks using any NumCodecs codec. -- [**Store arrays**](https://zarr.readthedocs.io/en/stable/tutorial.html#tutorial-storage) in memory, on disk, inside a zip file, on S3, etc... -- [**Read**](https://zarr.readthedocs.io/en/stable/tutorial.html#reading-and-writing-data) an array [**concurrently**](https://zarr.readthedocs.io/en/stable/tutorial.html#parallel-computing-and-synchronization) from multiple threads or processes. -- Write to an array concurrently from multiple threads or processes. -- Organize arrays into hierarchies via [**groups**](https://zarr.readthedocs.io/en/stable/tutorial.html#groups). +- [**Create**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#creating-an-array) N-dimensional arrays with any NumPy `dtype`. +- [**Chunk arrays**](https://zarr.readthedocs.io/en/stable/user-guide/performance.html#chunk-optimizations) along any dimension. +- [**Compress**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#compressors) and/or filter chunks using any NumCodecs codec. +- [**Store arrays**](https://zarr.readthedocs.io/en/stable/user-guide/storage.html) in memory, on disk, inside a zip file, on S3, etc... +- [**Read**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#reading-and-writing-data) an array [**concurrently**](https://zarr.readthedocs.io/en/stable/user-guide/performance.html#parallel-computing-and-synchronization) from multiple threads or processes. +- [**Write**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#reading-and-writing-data) to an array concurrently from multiple threads or processes. +- Organize arrays into hierarchies via [**groups**](https://zarr.readthedocs.io/en/stable/quickstart.html#hierarchical-groups). ## Where to get it From 8c24819809b649890cfbe5d27f7f29d77bfa0dd9 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sat, 22 Mar 2025 17:31:05 +0100 Subject: [PATCH 087/160] update version policy to use effver (#2910) * update version policy to use effver * style, fix missing verb * Update docs/developers/contributing.rst Co-authored-by: Joe Hamman * Update docs/developers/contributing.rst * Update docs/developers/contributing.rst Co-authored-by: Joe Hamman * Update docs/developers/contributing.rst Co-authored-by: David Stansby * Update docs/developers/contributing.rst Co-authored-by: David Stansby * Update docs/developers/contributing.rst Co-authored-by: David Stansby * add language about keeping changes smooth * Update docs/developers/contributing.rst * linewrap and style * chore: release notes --------- Co-authored-by: Joe Hamman Co-authored-by: David Stansby --- changes/2924.chore.rst | 2 + docs/developers/contributing.rst | 141 +++++++++++++++---------------- 2 files changed, 70 insertions(+), 73 deletions(-) create mode 100644 changes/2924.chore.rst diff --git a/changes/2924.chore.rst b/changes/2924.chore.rst new file mode 100644 index 0000000000..7bfbb2e1c7 --- /dev/null +++ b/changes/2924.chore.rst @@ -0,0 +1,2 @@ +Define a new versioning policy based on Effective Effort Versioning. This replaces the old +Semantic Versioning-based policy. \ No newline at end of file diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index de10fab2c6..fa65f71d48 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -261,8 +261,8 @@ Merging pull requests ~~~~~~~~~~~~~~~~~~~~~ Pull requests submitted by an external contributor should be reviewed and approved by at least -one core developers before being merged. Ideally, pull requests submitted by a core developer -should be reviewed and approved by at least one other core developers before being merged. +one core developer before being merged. Ideally, pull requests submitted by a core developer +should be reviewed and approved by at least one other core developer before being merged. Pull requests should not be merged until all CI checks have passed (GitHub Actions Codecov) against code that has had the latest main merged in. @@ -270,81 +270,74 @@ Codecov) against code that has had the latest main merged in. Compatibility and versioning policies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Because Zarr is a data storage library, there are two types of compatibility to -consider: API compatibility and data format compatibility. - -API compatibility -""""""""""""""""" - -All functions, classes and methods that are included in the API -documentation (files under ``docs/api/*.rst``) are considered as part of the Zarr **public API**, -except if they have been documented as an experimental feature, in which case they are part of -the **experimental API**. - -Any change to the public API that does **not** break existing third party -code importing Zarr, or cause third party code to behave in a different way, is a -**backwards-compatible API change**. For example, adding a new function, class or method is usually -a backwards-compatible change. However, removing a function, class or method; removing an argument -to a function or method; adding a required argument to a function or method; or changing the -behaviour of a function or method, are examples of **backwards-incompatible API changes**. - -If a release contains no changes to the public API (e.g., contains only bug fixes or -other maintenance work), then the micro version number should be incremented (e.g., -2.2.0 -> 2.2.1). If a release contains public API changes, but all changes are -backwards-compatible, then the minor version number should be incremented -(e.g., 2.2.1 -> 2.3.0). If a release contains any backwards-incompatible public API changes, -the major version number should be incremented (e.g., 2.3.0 -> 3.0.0). - -Backwards-incompatible changes to the experimental API can be included in a minor release, -although this should be minimised if possible. I.e., it would be preferable to save up -backwards-incompatible changes to the experimental API to be included in a major release, and to -stabilise those features at the same time (i.e., move from experimental to public API), rather than -frequently tinkering with the experimental API in minor releases. +Versioning +"""""""""" +Versions of this library are identified by a triplet of integers with the form +``..``, for example ``3.0.4``. A release of ``zarr-python`` is associated with a new +version identifier. That new identifier is generated by incrementing exactly one of the components of +the previous version identifier by 1. When incrementing the ``major`` component of the version identifier, +the ``minor`` and ``patch`` components is reset to 0. When incrementing the minor component, +the patch component is reset to 0. + +Releases are classified by the library changes contained in that release. This classification +determines which component of the version identifier is incremented on release. + +* ``major`` releases (for example, ``2.18.0`` -> ``3.0.0``) are for changes that will + require extensive adaptation efforts from many users and downstream projects. + For example, breaking changes to widely-used user-facing APIs should only be applied in a major release. + + + Users and downstream projects should carefully consider the impact of a major release before + adopting it. + In advance of a major release, developers should communicate the scope of the upcoming changes, + and help users prepare for them. + +* ``minor`` releases (or example, ``3.0.0`` -> ``3.1.0``) are for changes that do not require + significant effort from most users or downstream downstream projects to respond to. API changes + are possible in minor releases if the burden on users imposed by those changes is sufficiently small. + + For example, a recently released API may need fixes or refinements that are breaking, but low impact + due to the recency of the feature. Such API changes are permitted in a minor release. + + + Minor releases are safe for most users and downstream projects to adopt. + + +* ``patch`` releases (for example, ``3.1.0`` -> ``3.1.1``) are for changes that contain no breaking + or behaviour changes for downstream projects or users. Examples of changes suitable for a patch release are + bugfixes and documentation improvements. + + + Users should always feel safe upgrading to a the latest patch release. + +Note that this versioning scheme is not consistent with `Semantic Versioning `_. +Contrary to SemVer, the Zarr library may release breaking changes in ``minor`` releases, or even +``patch`` releases under exceptional circumstances. But we should strive to avoid doing so. + +A better model for our versioning scheme is `Intended Effort Versioning `_, +or "EffVer". The guiding principle off EffVer is to categorize releases based on the *expected effort +required to upgrade to that release*. + +Zarr developers should make changes as smooth as possible for users. This means making +backwards-compatible changes wherever possible. When a backwards-incompatible change is necessary, +users should be notified well in advance, e.g. via informative deprecation warnings. Data format compatibility -""""""""""""""""""""""""" - -The data format used by Zarr is defined by a specification document, which should be -platform-independent and contain sufficient detail to construct an interoperable -software library to read and/or write Zarr data using any programming language. The -latest version of the specification document is available on the -`Zarr specifications website `_. - -Here, **data format compatibility** means that all software libraries that implement a -particular version of the Zarr storage specification are interoperable, in the sense -that data written by any one library can be read by all others. It is obviously -desirable to maintain data format compatibility wherever possible. However, if a change -is needed to the storage specification, and that change would break data format -compatibility in any way, then the storage specification version number should be -incremented (e.g., 2 -> 3). - -The versioning of the Zarr software library is related to the versioning of the storage -specification as follows. A particular version of the Zarr library will -implement a particular version of the storage specification. For example, Zarr version -2.2.0 implements the Zarr storage specification version 2. If a release of the Zarr -library implements a different version of the storage specification, then the major -version number of the Zarr library should be incremented. E.g., if Zarr version 2.2.0 -implements the storage spec version 2, and the next release of the Zarr library -implements storage spec version 3, then the next library release should have version -number 3.0.0. Note however that the major version number of the Zarr library may not -always correspond to the spec version number. For example, Zarr versions 2.x, 3.x, and -4.x might all implement the same version of the storage spec and thus maintain data -format compatibility, although they will not maintain API compatibility. - -When to make a release -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Zarr library is an implementation of a file format standard defined externally -- +see the `Zarr specifications website `_ for the list of +Zarr file format specifications. -Ideally, any bug fixes that don't change the public API should be released as soon as -possible. It is fine for a micro release to contain only a single bug fix. -When to make a minor release is at the discretion of the core developers. There are no -hard-and-fast rules, e.g., it is fine to make a minor release to make a single new -feature available; equally, it is fine to make a minor release that includes a number of -changes. +If an existing Zarr format version changes, or a new version of the Zarr format is released, then +the Zarr library will generally require changes. It is very likely that a new Zarr format will +require extensive breaking changes to the Zarr library, and so support for a new Zarr format in the +Zarr library will almost certainly come in new ``major`` release. +When the Zarr library adds support for a new Zarr format, there may be a period of accelerated +changes as developers refine newly added APIs and deprecate old APIs. In such a transitional phase +breaking changes may be more frequent than usual. -Major releases obviously need to be given careful consideration, and should be done as -infrequently as possible, as they will break existing code and/or affect data -compatibility in some way. Release procedure ~~~~~~~~~~~~~~~~~ @@ -387,5 +380,7 @@ pre-releases will be available under Post-release """""""""""" -- Review and merge the pull request on the `conda-forge feedstock `_ that will be automatically generated. +- Review and merge the pull request on the + `conda-forge feedstock `_ that will be + automatically generated. - Create a new "Unreleased" section in the release notes From 9e8b50ae19cc63ad573f58569c3ef5826a5c60fc Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 24 Mar 2025 12:30:32 -0400 Subject: [PATCH 088/160] `obstore`-based Store implementation (#1661) Implementation of the Zarr Store API using obstore. --------- Co-authored-by: Davis Bennett Co-authored-by: Joe Hamman Co-authored-by: Deepak Cherian Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> Co-authored-by: Deepak Cherian Co-authored-by: Tom Augspurger --- .pre-commit-config.yaml | 13 +- changes/1661.feature.rst | 1 + docs/conf.py | 1 + docs/user-guide/storage.rst | 38 ++- pyproject.toml | 14 +- src/zarr/storage/__init__.py | 2 + src/zarr/storage/_obstore.py | 411 ++++++++++++++++++++++++++++++++ src/zarr/testing/store.py | 24 +- tests/test_store/test_object.py | 86 +++++++ 9 files changed, 567 insertions(+), 23 deletions(-) create mode 100644 changes/1661.feature.rst create mode 100644 src/zarr/storage/_obstore.py create mode 100644 tests/test_store/test_object.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba483d10e2..75ef0face8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,9 +8,9 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.9.9 hooks: - - id: ruff - args: ["--fix", "--show-fixes"] - - id: ruff-format + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format - repo: https://github.com/codespell-project/codespell rev: v2.4.1 hooks: @@ -19,8 +19,8 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - - id: check-yaml - - id: trailing-whitespace + - id: check-yaml + - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: @@ -31,9 +31,10 @@ repos: - packaging - donfig - numcodecs[crc32c] - - numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved + - numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved - typing_extensions - universal-pathlib + - obstore>=0.5.1 # Tests - pytest - repo: https://github.com/scientific-python/cookie diff --git a/changes/1661.feature.rst b/changes/1661.feature.rst new file mode 100644 index 0000000000..38d60b23c1 --- /dev/null +++ b/changes/1661.feature.rst @@ -0,0 +1 @@ +Add experimental ObjectStore storage class based on obstore. \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d69309d432..9bb1c48901 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -369,6 +369,7 @@ def setup(app: sphinx.application.Sphinx) -> None: "python": ("https://docs.python.org/3/", None), "numpy": ("https://numpy.org/doc/stable/", None), "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None), + "obstore": ("https://developmentseed.org/obstore/latest/", None), } diff --git a/docs/user-guide/storage.rst b/docs/user-guide/storage.rst index 46505271b4..4215cbaf20 100644 --- a/docs/user-guide/storage.rst +++ b/docs/user-guide/storage.rst @@ -47,7 +47,7 @@ Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers four built-in store: :class:`zarr.storage.LocalStore`, :class:`zarr.storage.FsspecStore`, -:class:`zarr.storage.ZipStore`, and :class:`zarr.storage.MemoryStore`. +:class:`zarr.storage.ZipStore`, :class:`zarr.storage.MemoryStore`, and :class:`zarr.storage.ObjectStore`. Local Store ~~~~~~~~~~~ @@ -99,6 +99,42 @@ Zarr data (metadata and chunks) to a dictionary.: >>> zarr.create_array(store=store, shape=(2,), dtype='float64') +Object Store +~~~~~~~~~~~~ + +:class:`zarr.storage.ObjectStore` stores the contents of the Zarr hierarchy using any ObjectStore +`storage implementation `_, including AWS S3 (:class:`obstore.store.S3Store`), Google Cloud Storage (:class:`obstore.store.GCSStore`), and Azure Blob Storage (:class:`obstore.store.AzureStore`). This store is backed by `obstore `_, which +builds on the production quality Rust library `object_store `_. + + + >>> from zarr.storage import ObjectStore + >>> from obstore.store import MemoryStore + >>> + >>> store = ObjectStore(MemoryStore()) + >>> zarr.create_array(store=store, shape=(2,), dtype='float64') + + +Here's an example of using ObjectStore for accessing remote data: + + >>> from zarr.storage import ObjectStore + >>> from obstore.store import S3Store + >>> + >>> s3_store = S3Store('noaa-nwm-retro-v2-zarr-pds', skip_signature=True, region="us-west-2") + >>> store = zarr.storage.ObjectStore(store=s3_store, read_only=True) + >>> group = zarr.open_group(store=store, mode='r') + >>> group.info + Name : + Type : Group + Zarr format : 2 + Read-only : True + Store type : ObjectStore + No. members : 12 + No. arrays : 12 + No. groups : 0 + +.. warning:: + The :class:`zarr.storage.ObjectStore` class is experimental. + .. _user-guide-custom-stores: Developing custom stores diff --git a/pyproject.toml b/pyproject.toml index 0137927039..d862cce0ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] # User extras remote = [ "fsspec>=2023.10.0", + "obstore>=0.5.1", ] gpu = [ "cupy-cuda12x", @@ -114,6 +115,12 @@ Discussions = "https://github.com/zarr-developers/zarr-python/discussions" Documentation = "https://zarr.readthedocs.io/" Homepage = "https://github.com/zarr-developers/zarr-python" +[dependency-groups] +dev = [ + "ipykernel>=6.29.5", + "pip>=25.0.1", +] + [tool.coverage.report] exclude_lines = [ "pragma: no cover", @@ -129,7 +136,9 @@ omit = [ [tool.hatch] version.source = "vcs" -build.hooks.vcs.version-file = "src/zarr/_version.py" + +[tool.hatch.build] +hooks.vcs.version-file = "src/zarr/_version.py" [tool.hatch.envs.test] dependencies = [ @@ -165,7 +174,7 @@ run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tes list-env = "pip list" [tool.hatch.envs.doctest] -features = ["test", "optional"] +features = ["test", "optional", "remote"] description = "Test environment for doctests" [tool.hatch.envs.doctest.scripts] @@ -211,6 +220,7 @@ dependencies = [ 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', 'typing_extensions @ git+https://github.com/python/typing_extensions', 'donfig @ git+https://github.com/pytroll/donfig', + 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', # test deps 'zarr[test]', ] diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index 649857f773..6721139375 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -8,6 +8,7 @@ from zarr.storage._local import LocalStore from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, MemoryStore +from zarr.storage._obstore import ObjectStore from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -17,6 +18,7 @@ "LocalStore", "LoggingStore", "MemoryStore", + "ObjectStore", "StoreLike", "StorePath", "WrapperStore", diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py new file mode 100644 index 0000000000..79afa08d15 --- /dev/null +++ b/src/zarr/storage/_obstore.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import asyncio +import contextlib +import pickle +from collections import defaultdict +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, TypedDict + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.buffer.core import BufferPrototype +from zarr.core.config import config + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Coroutine, Iterable + from typing import Any + + from obstore import ListResult, ListStream, ObjectMeta, OffsetRange, SuffixRange + from obstore.store import ObjectStore as _UpstreamObjectStore + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.common import BytesLike + +__all__ = ["ObjectStore"] + +_ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, +) + + +class ObjectStore(Store): + """A Zarr store that uses obstore for fast read/write from AWS, GCP, Azure. + + Parameters + ---------- + store : obstore.store.ObjectStore + An obstore store instance that is set up with the proper credentials. + read_only : bool + Whether to open the store in read-only mode. + + Warnings + -------- + ObjectStore is experimental and subject to API changes without notice. Please + raise an issue with any comments/concerns about the store. + """ + + store: _UpstreamObjectStore + """The underlying obstore instance.""" + + def __eq__(self, value: object) -> bool: + if not isinstance(value, ObjectStore): + return False + + if not self.read_only == value.read_only: + return False + + return self.store == value.store + + def __init__(self, store: _UpstreamObjectStore, *, read_only: bool = False) -> None: + if not store.__class__.__module__.startswith("obstore"): + raise TypeError(f"expected ObjectStore class, got {store!r}") + super().__init__(read_only=read_only) + self.store = store + + def __str__(self) -> str: + return f"object_store://{self.store}" + + def __repr__(self) -> str: + return f"{type(self).__name__}({self})" + + def __getstate__(self) -> dict[Any, Any]: + state = self.__dict__.copy() + state["store"] = pickle.dumps(self.store) + return state + + def __setstate__(self, state: dict[Any, Any]) -> None: + state["store"] = pickle.loads(state["store"]) + self.__dict__.update(state) + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: + # docstring inherited + import obstore as obs + + try: + if byte_range is None: + resp = await obs.get_async(self.store, key) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + elif isinstance(byte_range, RangeByteRequest): + bytes = await obs.get_range_async( + self.store, key, start=byte_range.start, end=byte_range.end + ) + return prototype.buffer.from_bytes(bytes) # type: ignore[arg-type] + elif isinstance(byte_range, OffsetByteRequest): + resp = await obs.get_async( + self.store, key, options={"range": {"offset": byte_range.offset}} + ) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + elif isinstance(byte_range, SuffixByteRequest): + resp = await obs.get_async( + self.store, key, options={"range": {"suffix": byte_range.suffix}} + ) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}") + except _ALLOWED_EXCEPTIONS: + return None + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + return await _get_partial_values(self.store, prototype=prototype, key_ranges=key_ranges) + + async def exists(self, key: str) -> bool: + # docstring inherited + import obstore as obs + + try: + await obs.head_async(self.store, key) + except FileNotFoundError: + return False + else: + return True + + @property + def supports_writes(self) -> bool: + # docstring inherited + return True + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + + buf = value.to_bytes() + await obs.put_async(self.store, key, buf) + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + buf = value.to_bytes() + with contextlib.suppress(obs.exceptions.AlreadyExistsError): + await obs.put_async(self.store, key, buf, mode="create") + + @property + def supports_deletes(self) -> bool: + # docstring inherited + return True + + async def delete(self, key: str) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + await obs.delete_async(self.store, key) + + @property + def supports_partial_writes(self) -> bool: + # docstring inherited + return False + + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, BytesLike]] + ) -> None: + # docstring inherited + raise NotImplementedError + + @property + def supports_listing(self) -> bool: + # docstring inherited + return True + + def list(self) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + objects: ListStream[list[ObjectMeta]] = obs.list(self.store) + return _transform_list(objects) + + def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + objects: ListStream[list[ObjectMeta]] = obs.list(self.store, prefix=prefix) + return _transform_list(objects) + + def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + coroutine = obs.list_with_delimiter_async(self.store, prefix=prefix) + return _transform_list_dir(coroutine, prefix) + + +async def _transform_list( + list_stream: ListStream[list[ObjectMeta]], +) -> AsyncGenerator[str, None]: + """ + Transform the result of list into an async generator of paths. + """ + async for batch in list_stream: + for item in batch: + yield item["path"] + + +async def _transform_list_dir( + list_result_coroutine: Coroutine[Any, Any, ListResult[list[ObjectMeta]]], prefix: str +) -> AsyncGenerator[str, None]: + """ + Transform the result of list_with_delimiter into an async generator of paths. + """ + list_result = await list_result_coroutine + + # We assume that the underlying object-store implementation correctly handles the + # prefix, so we don't double-check that the returned results actually start with the + # given prefix. + prefixes = [obj.lstrip(prefix).lstrip("/") for obj in list_result["common_prefixes"]] + objects = [obj["path"].removeprefix(prefix).lstrip("/") for obj in list_result["objects"]] + for item in prefixes + objects: + yield item + + +class _BoundedRequest(TypedDict): + """Range request with a known start and end byte. + + These requests can be multiplexed natively on the Rust side with + `obstore.get_ranges_async`. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + start: int + """Start byte offset.""" + + end: int + """End byte offset.""" + + +class _OtherRequest(TypedDict): + """Offset or suffix range requests. + + These requests cannot be concurrent on the Rust side, and each need their own call + to `obstore.get_async`, passing in the `range` parameter. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + path: str + """The path to request from.""" + + range: OffsetRange | SuffixRange | None + """The range request type.""" + + +class _Response(TypedDict): + """A response buffer associated with the original index that it should be restored to.""" + + original_request_index: int + """The positional index in the original key_ranges input""" + + buffer: Buffer + """The buffer returned from obstore's range request.""" + + +async def _make_bounded_requests( + store: _UpstreamObjectStore, + path: str, + requests: list[_BoundedRequest], + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make all bounded requests for a specific file. + + `obstore.get_ranges_async` allows for making concurrent requests for multiple ranges + within a single file, and will e.g. merge concurrent requests. This only uses one + single Python coroutine. + """ + import obstore as obs + + starts = [r["start"] for r in requests] + ends = [r["end"] for r in requests] + async with semaphore: + responses = await obs.get_ranges_async(store, path=path, starts=starts, ends=ends) + + buffer_responses: list[_Response] = [] + for request, response in zip(requests, responses, strict=True): + buffer_responses.append( + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(response), # type: ignore[arg-type] + } + ) + + return buffer_responses + + +async def _make_other_request( + store: _UpstreamObjectStore, + request: _OtherRequest, + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make suffix or offset requests. + + We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all + futures can be gathered together. + """ + import obstore as obs + + async with semaphore: + if request["range"] is None: + resp = await obs.get_async(store, request["path"]) + else: + resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) + buffer = await resp.bytes_async() + + return [ + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] + } + ] + + +async def _get_partial_values( + store: _UpstreamObjectStore, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], +) -> list[Buffer | None]: + """Make multiple range requests. + + ObjectStore has a `get_ranges` method that will additionally merge nearby ranges, + but it's _per_ file. So we need to split these key_ranges into **per-file** key + ranges, and then reassemble the results in the original order. + + We separate into different requests: + + - One call to `obstore.get_ranges_async` **per target file** + - One call to `obstore.get_async` for each other request. + """ + key_ranges = list(key_ranges) + per_file_bounded_requests: dict[str, list[_BoundedRequest]] = defaultdict(list) + other_requests: list[_OtherRequest] = [] + + for idx, (path, byte_range) in enumerate(key_ranges): + if byte_range is None: + other_requests.append( + { + "original_request_index": idx, + "path": path, + "range": None, + } + ) + elif isinstance(byte_range, RangeByteRequest): + per_file_bounded_requests[path].append( + {"original_request_index": idx, "start": byte_range.start, "end": byte_range.end} + ) + elif isinstance(byte_range, OffsetByteRequest): + other_requests.append( + { + "original_request_index": idx, + "path": path, + "range": {"offset": byte_range.offset}, + } + ) + elif isinstance(byte_range, SuffixByteRequest): + other_requests.append( + { + "original_request_index": idx, + "path": path, + "range": {"suffix": byte_range.suffix}, + } + ) + else: + raise ValueError(f"Unsupported range input: {byte_range}") + + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + + futs: list[Coroutine[Any, Any, list[_Response]]] = [] + for path, bounded_ranges in per_file_bounded_requests.items(): + futs.append( + _make_bounded_requests(store, path, bounded_ranges, prototype, semaphore=semaphore) + ) + + for request in other_requests: + futs.append(_make_other_request(store, request, prototype, semaphore=semaphore)) # noqa: PERF401 + + buffers: list[Buffer | None] = [None] * len(key_ranges) + + for responses in await asyncio.gather(*futs): + for resp in responses: + buffers[resp["original_request_index"]] = resp["buffer"] + + return buffers diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 112f6261e9..867df2121f 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -42,7 +42,7 @@ class StoreTests(Generic[S, B]): async def set(self, store: S, key: str, value: Buffer) -> None: """ Insert a value into a storage backend, with a specific key. - This should not not use any store methods. Bypassing the store methods allows them to be + This should not use any store methods. Bypassing the store methods allows them to be tested. """ ... @@ -51,7 +51,7 @@ async def set(self, store: S, key: str, value: Buffer) -> None: async def get(self, store: S, key: str) -> Buffer: """ Retrieve a value from a storage backend, by key. - This should not not use any store methods. Bypassing the store methods allows them to be + This should not use any store methods. Bypassing the store methods allows them to be tested. """ ... @@ -150,9 +150,15 @@ async def test_read_only_store_raises(self, open_kwargs: dict[str, Any]) -> None await store.delete("foo") @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) - @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) @pytest.mark.parametrize( - "byte_range", [None, RangeByteRequest(1, 4), OffsetByteRequest(1), SuffixByteRequest(1)] + ("data", "byte_range"), + [ + (b"\x01\x02\x03\x04", None), + (b"\x01\x02\x03\x04", RangeByteRequest(1, 4)), + (b"\x01\x02\x03\x04", OffsetByteRequest(1)), + (b"\x01\x02\x03\x04", SuffixByteRequest(1)), + (b"", None), + ], ) async def test_get(self, store: S, key: str, data: bytes, byte_range: ByteRequest) -> None: """ @@ -273,16 +279,6 @@ async def test_set_many(self, store: S) -> None: for k, v in store_dict.items(): assert (await self.get(store, k)).to_bytes() == v.to_bytes() - async def test_set_invalid_buffer(self, store: S) -> None: - """ - Ensure that set raises a Type or Value Error for invalid buffer arguments. - """ - with pytest.raises( - (ValueError, TypeError), - match=r"\S+\.set\(\): `value` must be a Buffer instance. Got an instance of instead.", - ): - await store.set("c/0", 0) # type: ignore[arg-type] - @pytest.mark.parametrize( "key_ranges", [ diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py new file mode 100644 index 0000000000..943564abc8 --- /dev/null +++ b/tests/test_store/test_object.py @@ -0,0 +1,86 @@ +# ruff: noqa: E402 +from typing import Any + +import pytest + +obstore = pytest.importorskip("obstore") +import pytest +from hypothesis.stateful import ( + run_state_machine_as_test, +) +from obstore.store import LocalStore, MemoryStore + +from zarr.core.buffer import Buffer, cpu +from zarr.storage import ObjectStore +from zarr.testing.stateful import ZarrHierarchyStateMachine +from zarr.testing.store import StoreTests + + +class TestObjectStore(StoreTests[ObjectStore, cpu.Buffer]): + store_cls = ObjectStore + buffer_cls = cpu.Buffer + + @pytest.fixture + def store_kwargs(self, tmpdir) -> dict[str, Any]: + store = LocalStore(prefix=tmpdir) + return {"store": store, "read_only": False} + + @pytest.fixture + def store(self, store_kwargs: dict[str, str | bool]) -> ObjectStore: + return self.store_cls(**store_kwargs) + + async def get(self, store: ObjectStore, key: str) -> Buffer: + assert isinstance(store.store, LocalStore) + new_local_store = LocalStore(prefix=store.store.prefix) + return self.buffer_cls.from_bytes(obstore.get(new_local_store, key).bytes()) + + async def set(self, store: ObjectStore, key: str, value: Buffer) -> None: + assert isinstance(store.store, LocalStore) + new_local_store = LocalStore(prefix=store.store.prefix) + obstore.put(new_local_store, key, value.to_bytes()) + + def test_store_repr(self, store: ObjectStore) -> None: + from fnmatch import fnmatch + + pattern = "ObjectStore(object_store://LocalStore(*))" + assert fnmatch(f"{store!r}", pattern) + + def test_store_supports_writes(self, store: ObjectStore) -> None: + assert store.supports_writes + + async def test_store_supports_partial_writes(self, store: ObjectStore) -> None: + assert not store.supports_partial_writes + with pytest.raises(NotImplementedError): + await store.set_partial_values([("foo", 0, b"\x01\x02\x03\x04")]) + + def test_store_supports_listing(self, store: ObjectStore) -> None: + assert store.supports_listing + + def test_store_equal(self, store: ObjectStore) -> None: + """Test store equality""" + # Test equality against a different instance type + assert store != 0 + # Test equality against a different store type + new_memory_store = ObjectStore(MemoryStore()) + assert store != new_memory_store + # Test equality against a read only store + new_local_store = ObjectStore(LocalStore(prefix=store.store.prefix), read_only=True) + assert store != new_local_store + # Test two memory stores cannot be equal + second_memory_store = ObjectStore(MemoryStore()) + assert new_memory_store != second_memory_store + + def test_store_init_raises(self) -> None: + """Test __init__ raises appropriate error for improper store type""" + with pytest.raises(TypeError): + ObjectStore("path/to/store") + + +@pytest.mark.slow_hypothesis +def test_zarr_hierarchy(): + sync_store = ObjectStore(MemoryStore()) + + def mk_test_instance_sync() -> ZarrHierarchyStateMachine: + return ZarrHierarchyStateMachine(sync_store) + + run_state_machine_as_test(mk_test_instance_sync) From 54b3d44b36e1876eab93239bc0de43c3d70e1e92 Mon Sep 17 00:00:00 2001 From: Tom White Date: Thu, 3 Apr 2025 17:51:34 +0100 Subject: [PATCH 089/160] Avoid memory copy in local store write (#2944) * Avoid memory copy in local store write --- changes/2944.misc.rst | 1 + src/zarr/storage/_local.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/2944.misc.rst diff --git a/changes/2944.misc.rst b/changes/2944.misc.rst new file mode 100644 index 0000000000..48356a1fef --- /dev/null +++ b/changes/2944.misc.rst @@ -0,0 +1 @@ +Avoid an unnecessary memory copy when writing Zarr to a local file diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index b20a601bed..bd5bfc1da2 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -51,15 +51,17 @@ def _put( if start is not None: with path.open("r+b") as f: f.seek(start) - f.write(value.as_numpy_array().tobytes()) + # write takes any object supporting the buffer protocol + f.write(value.as_numpy_array()) # type: ignore[arg-type] return None else: - view = memoryview(value.as_numpy_array().tobytes()) + view = memoryview(value.as_numpy_array()) # type: ignore[arg-type] if exclusive: mode = "xb" else: mode = "wb" with path.open(mode=mode) as f: + # write takes any object supporting the buffer protocol return f.write(view) From 3b6565bd4377fc534a790a28e3436081e38c62b5 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Fri, 4 Apr 2025 10:05:27 -0500 Subject: [PATCH 090/160] Fix fill_value serialization issues (#2802) * Fix fill_value serialization of NaN * Round trip serialization for array metadata v2/v3 * Unify metadata v2 fill value parsing * Test structured fill_value parsing * Remove redundancies, fix integral handling * Reorganize structured fill parsing * Bump up hypothesis deadline * Remove hypothesis deadline * Update tests/test_v2.py Co-authored-by: David Stansby --------- Co-authored-by: Deepak Cherian Co-authored-by: David Stansby --- changes/2802.fix.rst | 1 + src/zarr/core/group.py | 2 +- src/zarr/core/metadata/v2.py | 112 +++++++++++++++------ src/zarr/testing/strategies.py | 2 +- tests/test_properties.py | 175 ++++++++++++++++++++++++++++++--- tests/test_v2.py | 84 ++++++++++++++++ 6 files changed, 330 insertions(+), 46 deletions(-) create mode 100644 changes/2802.fix.rst diff --git a/changes/2802.fix.rst b/changes/2802.fix.rst new file mode 100644 index 0000000000..471ddf66f4 --- /dev/null +++ b/changes/2802.fix.rst @@ -0,0 +1 @@ +Fix `fill_value` serialization for `NaN` in `ArrayV2Metadata` and add property-based testing of round-trip serialization \ No newline at end of file diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 8e7f7f3474..5500bdd4a5 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -3456,7 +3456,7 @@ def _build_metadata_v3(zarr_json: dict[str, JSON]) -> ArrayV3Metadata | GroupMet def _build_metadata_v2( - zarr_json: dict[str, object], attrs_json: dict[str, JSON] + zarr_json: dict[str, JSON], attrs_json: dict[str, JSON] ) -> ArrayV2Metadata | GroupMetadata: """ Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 823944e067..11f14b37aa 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -2,17 +2,17 @@ import base64 import warnings -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from enum import Enum from functools import cached_property -from typing import TYPE_CHECKING, TypedDict, cast +from typing import TYPE_CHECKING, Any, TypedDict, cast import numcodecs.abc from zarr.abc.metadata import Metadata if TYPE_CHECKING: - from typing import Any, Literal, Self + from typing import Literal, Self import numpy.typing as npt @@ -20,6 +20,7 @@ from zarr.core.common import ChunkCoords import json +import numbers from dataclasses import dataclass, field, fields, replace import numcodecs @@ -146,41 +147,39 @@ def _json_convert( raise TypeError zarray_dict = self.to_dict() + zarray_dict["fill_value"] = _serialize_fill_value(self.fill_value, self.dtype) zattrs_dict = zarray_dict.pop("attributes", {}) json_indent = config.get("json_indent") return { ZARRAY_JSON: prototype.buffer.from_bytes( - json.dumps(zarray_dict, default=_json_convert, indent=json_indent).encode() + json.dumps( + zarray_dict, default=_json_convert, indent=json_indent, allow_nan=False + ).encode() ), ZATTRS_JSON: prototype.buffer.from_bytes( - json.dumps(zattrs_dict, indent=json_indent).encode() + json.dumps(zattrs_dict, indent=json_indent, allow_nan=False).encode() ), } @classmethod def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: - # make a copy to protect the original from modification + # Make a copy to protect the original from modification. _data = data.copy() - # check that the zarr_format attribute is correct + # Check that the zarr_format attribute is correct. _ = parse_zarr_format(_data.pop("zarr_format")) - dtype = parse_dtype(_data["dtype"]) - if dtype.kind in "SV": - fill_value_encoded = _data.get("fill_value") - if fill_value_encoded is not None: - fill_value = base64.standard_b64decode(fill_value_encoded) - _data["fill_value"] = fill_value - - # zarr v2 allowed arbitrary keys here. - # We don't want the ArrayV2Metadata constructor to fail just because someone put an - # extra key in the metadata. + # zarr v2 allowed arbitrary keys in the metadata. + # Filter the keys to only those expected by the constructor. expected = {x.name for x in fields(cls)} - # https://github.com/zarr-developers/zarr-python/issues/2269 - # handle the renames expected |= {"dtype", "chunks"} # check if `filters` is an empty sequence; if so use None instead and raise a warning - if _data["filters"] is not None and len(_data["filters"]) == 0: + filters = _data.get("filters") + if ( + isinstance(filters, Sequence) + and not isinstance(filters, (str, bytes)) + and len(filters) == 0 + ): msg = ( "Found an empty list of filters in the array metadata document. " "This is contrary to the Zarr V2 specification, and will cause an error in the future. " @@ -196,13 +195,6 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: def to_dict(self) -> dict[str, JSON]: zarray_dict = super().to_dict() - if self.dtype.kind in "SV" and self.fill_value is not None: - # There's a relationship between self.dtype and self.fill_value - # that mypy isn't aware of. The fact that we have S or V dtype here - # means we should have a bytes-type fill_value. - fill_value = base64.standard_b64encode(cast(bytes, self.fill_value)).decode("ascii") - zarray_dict["fill_value"] = fill_value - _ = zarray_dict.pop("dtype") dtype_json: JSON # In the case of zarr v2, the simplest i.e., '|VXX' dtype is represented as a string @@ -300,7 +292,26 @@ def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: return data -def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: +def _parse_structured_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: + """Handle structured dtype/fill value pairs""" + print("FILL VALUE", fill_value, "DT", dtype) + try: + if isinstance(fill_value, list): + return np.array([tuple(fill_value)], dtype=dtype)[0] + elif isinstance(fill_value, tuple): + return np.array([fill_value], dtype=dtype)[0] + elif isinstance(fill_value, bytes): + return np.frombuffer(fill_value, dtype=dtype)[0] + elif isinstance(fill_value, str): + decoded = base64.standard_b64decode(fill_value) + return np.frombuffer(decoded, dtype=dtype)[0] + else: + return np.array(fill_value, dtype=dtype)[()] + except Exception as e: + raise ValueError(f"Fill_value {fill_value} is not valid for dtype {dtype}.") from e + + +def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: """ Parse a potential fill value into a value that is compatible with the provided dtype. @@ -317,13 +328,16 @@ def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: """ if fill_value is None or dtype.hasobject: - # no fill value pass + elif dtype.fields is not None: + # the dtype is structured (has multiple fields), so the fill_value might be a + # compound value (e.g., a tuple or dict) that needs field-wise processing. + # We use parse_structured_fill_value to correctly convert each component. + fill_value = _parse_structured_fill_value(fill_value, dtype) elif not isinstance(fill_value, np.void) and fill_value == 0: # this should be compatible across numpy versions for any array type, including # structured arrays fill_value = np.zeros((), dtype=dtype)[()] - elif dtype.kind == "U": # special case unicode because of encoding issues on Windows if passed through numpy # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713 @@ -332,6 +346,11 @@ def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: raise ValueError( f"fill_value {fill_value!r} is not valid for dtype {dtype}; must be a unicode string" ) + elif dtype.kind in "SV" and isinstance(fill_value, str): + fill_value = base64.standard_b64decode(fill_value) + elif dtype.kind == "c" and isinstance(fill_value, list) and len(fill_value) == 2: + complex_val = complex(float(fill_value[0]), float(fill_value[1])) + fill_value = np.array(complex_val, dtype=dtype)[()] else: try: if isinstance(fill_value, bytes) and dtype.kind == "V": @@ -347,6 +366,39 @@ def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: return fill_value +def _serialize_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> JSON: + serialized: JSON + + if fill_value is None: + serialized = None + elif dtype.kind in "SV": + # There's a relationship between dtype and fill_value + # that mypy isn't aware of. The fact that we have S or V dtype here + # means we should have a bytes-type fill_value. + serialized = base64.standard_b64encode(cast(bytes, fill_value)).decode("ascii") + elif isinstance(fill_value, np.datetime64): + serialized = np.datetime_as_string(fill_value) + elif isinstance(fill_value, numbers.Integral): + serialized = int(fill_value) + elif isinstance(fill_value, numbers.Real): + float_fv = float(fill_value) + if np.isnan(float_fv): + serialized = "NaN" + elif np.isinf(float_fv): + serialized = "Infinity" if float_fv > 0 else "-Infinity" + else: + serialized = float_fv + elif isinstance(fill_value, numbers.Complex): + serialized = [ + _serialize_fill_value(fill_value.real, dtype), + _serialize_fill_value(fill_value.imag, dtype), + ] + else: + serialized = fill_value + + return serialized + + def _default_fill_value(dtype: np.dtype[Any]) -> Any: """ Get the default fill value for a type. diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index aa42329be7..f2dc38483a 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -5,7 +5,7 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np -from hypothesis import event, given, settings # noqa: F401 +from hypothesis import event from hypothesis.strategies import SearchStrategy import zarr diff --git a/tests/test_properties.py b/tests/test_properties.py index 68d8bb0a0e..d48dfe2fef 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,3 +1,8 @@ +import dataclasses +import json +import numbers +from typing import Any + import numpy as np import pytest from numpy.testing import assert_array_equal @@ -8,9 +13,10 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st -from hypothesis import assume, given +from hypothesis import assume, given, settings from zarr.abc.store import Store +from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync from zarr.testing.strategies import ( @@ -25,8 +31,53 @@ ) +def deep_equal(a: Any, b: Any) -> bool: + """Deep equality check with handling of special cases for array metadata classes""" + if isinstance(a, (complex, np.complexfloating)) and isinstance( + b, (complex, np.complexfloating) + ): + a_real, a_imag = float(a.real), float(a.imag) + b_real, b_imag = float(b.real), float(b.imag) + if np.isnan(a_real) and np.isnan(b_real): + real_eq = True + else: + real_eq = a_real == b_real + if np.isnan(a_imag) and np.isnan(b_imag): + imag_eq = True + else: + imag_eq = a_imag == b_imag + return real_eq and imag_eq + + if isinstance(a, (float, np.floating)) and isinstance(b, (float, np.floating)): + if np.isnan(a) and np.isnan(b): + return True + return a == b + + if isinstance(a, np.datetime64) and isinstance(b, np.datetime64): + if np.isnat(a) and np.isnat(b): + return True + return a == b + + if isinstance(a, np.ndarray) and isinstance(b, np.ndarray): + if a.shape != b.shape: + return False + return all(deep_equal(x, y) for x, y in zip(a.flat, b.flat, strict=False)) + + if isinstance(a, dict) and isinstance(b, dict): + if set(a.keys()) != set(b.keys()): + return False + return all(deep_equal(a[k], b[k]) for k in a) + + if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): + if len(a) != len(b): + return False + return all(deep_equal(x, y) for x, y in zip(a, b, strict=False)) + + return a == b + + @given(data=st.data(), zarr_format=zarr_formats) -def test_roundtrip(data: st.DataObject, zarr_format: int) -> None: +def test_array_roundtrip(data: st.DataObject, zarr_format: int) -> None: nparray = data.draw(numpy_arrays(zarr_formats=st.just(zarr_format))) zarray = data.draw(arrays(arrays=st.just(nparray), zarr_formats=st.just(zarr_format))) assert_array_equal(nparray, zarray[:]) @@ -50,6 +101,8 @@ def test_array_creates_implicit_groups(array): ) +# this decorator removes timeout; not ideal but it should avoid intermittent CI failures +@settings(deadline=None) @given(data=st.data()) def test_basic_indexing(data: st.DataObject) -> None: zarray = data.draw(simple_arrays()) @@ -109,9 +162,17 @@ def test_vindex(data: st.DataObject) -> None: @given(store=stores, meta=array_metadata()) # type: ignore[misc] -async def test_roundtrip_array_metadata( +async def test_roundtrip_array_metadata_from_store( store: Store, meta: ArrayV2Metadata | ArrayV3Metadata ) -> None: + """ + Verify that the I/O for metadata in a store are lossless. + + This test serializes an ArrayV2Metadata or ArrayV3Metadata object to a dict + of buffers via `to_buffer_dict`, writes each buffer to a store under keys + prefixed with "0/", and then reads them back. The test asserts that each + retrieved buffer exactly matches the original buffer. + """ asdict = meta.to_buffer_dict(prototype=default_buffer_prototype()) for key, expected in asdict.items(): await store.set(f"0/{key}", expected) @@ -119,18 +180,39 @@ async def test_roundtrip_array_metadata( assert actual == expected -@given(store=stores, meta=array_metadata()) # type: ignore[misc] -def test_array_metadata_meets_spec(store: Store, meta: ArrayV2Metadata | ArrayV3Metadata) -> None: - # TODO: fill this out - asdict = meta.to_dict() - if isinstance(meta, ArrayV2Metadata): - assert asdict["filters"] != () - assert asdict["filters"] is None or isinstance(asdict["filters"], tuple) - assert asdict["zarr_format"] == 2 - elif isinstance(meta, ArrayV3Metadata): - assert asdict["zarr_format"] == 3 +@given(data=st.data(), zarr_format=zarr_formats) +def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: int) -> None: + """ + Verify that JSON serialization and deserialization of metadata is lossless. + + For Zarr v2: + - The metadata is split into two JSON documents (one for array data and one + for attributes). The test merges the attributes back before deserialization. + For Zarr v3: + - All metadata is stored in a single JSON document. No manual merger is necessary. + + The test then converts both the original and round-tripped metadata objects + into dictionaries using `dataclasses.asdict` and uses a deep equality check + to verify that the roundtrip has preserved all fields (including special + cases like NaN, Infinity, complex numbers, and datetime values). + """ + metadata = data.draw(array_metadata(zarr_formats=st.just(zarr_format))) + buffer_dict = metadata.to_buffer_dict(prototype=default_buffer_prototype()) + + if zarr_format == 2: + zarray_dict = json.loads(buffer_dict[ZARRAY_JSON].to_bytes().decode()) + zattrs_dict = json.loads(buffer_dict[ZATTRS_JSON].to_bytes().decode()) + # zattrs and zarray are separate in v2, we have to add attributes back prior to `from_dict` + zarray_dict["attributes"] = zattrs_dict + metadata_roundtripped = ArrayV2Metadata.from_dict(zarray_dict) else: - raise NotImplementedError + zarray_dict = json.loads(buffer_dict[ZARR_JSON].to_bytes().decode()) + metadata_roundtripped = ArrayV3Metadata.from_dict(zarray_dict) + + orig = dataclasses.asdict(metadata) + rt = dataclasses.asdict(metadata_roundtripped) + + assert deep_equal(orig, rt), f"Roundtrip mismatch:\nOriginal: {orig}\nRoundtripped: {rt}" # @st.composite @@ -155,3 +237,68 @@ def test_array_metadata_meets_spec(store: Store, meta: ArrayV2Metadata | ArrayV3 # nparray = data.draw(np_arrays) # zarray = data.draw(arrays(arrays=st.just(nparray))) # assert_array_equal(nparray, zarray[:]) + + +def serialized_float_is_valid(serialized: numbers.Real | str) -> bool: + """ + Validate that the serialized representation of a float conforms to the spec. + + The specification requires that a serialized float must be either: + - A JSON number, or + - One of the strings "NaN", "Infinity", or "-Infinity". + + Args: + serialized: The value produced by JSON serialization for a floating point number. + + Returns: + bool: True if the serialized value is valid according to the spec, False otherwise. + """ + if isinstance(serialized, numbers.Real): + return True + return serialized in ("NaN", "Infinity", "-Infinity") + + +@given(meta=array_metadata()) # type: ignore[misc] +def test_array_metadata_meets_spec(meta: ArrayV2Metadata | ArrayV3Metadata) -> None: + """ + Validate that the array metadata produced by the library conforms to the relevant spec (V2 vs V3). + + For ArrayV2Metadata: + - Ensures that 'zarr_format' is 2. + - Verifies that 'filters' is either None or a tuple (and not an empty tuple). + For ArrayV3Metadata: + - Ensures that 'zarr_format' is 3. + + For both versions: + - If the dtype is a floating point of some kind, verifies of fill values: + * NaN is serialized as the string "NaN" + * Positive Infinity is serialized as the string "Infinity" + * Negative Infinity is serialized as the string "-Infinity" + * Other fill values are preserved as-is. + - If the dtype is a complex number of some kind, verifies that each component of the fill + value (real and imaginary) satisfies the serialization rules for floating point numbers. + - If the dtype is a datetime of some kind, verifies that `NaT` values are serialized as "NaT". + + Note: + This test validates spec-compliance for array metadata serialization. + It is a work-in-progress and should be expanded as further edge cases are identified. + """ + asdict_dict = meta.to_dict() + + # version-specific validations + if isinstance(meta, ArrayV2Metadata): + assert asdict_dict["filters"] != () + assert asdict_dict["filters"] is None or isinstance(asdict_dict["filters"], tuple) + assert asdict_dict["zarr_format"] == 2 + else: + assert asdict_dict["zarr_format"] == 3 + + # version-agnostic validations + if meta.dtype.kind == "f": + assert serialized_float_is_valid(asdict_dict["fill_value"]) + elif meta.dtype.kind == "c": + # fill_value should be a two-element array [real, imag]. + assert serialized_float_is_valid(asdict_dict["fill_value"].real) + assert serialized_float_is_valid(asdict_dict["fill_value"].imag) + elif meta.dtype.kind == "M" and np.isnat(meta.fill_value): + assert asdict_dict["fill_value"] == "NaT" diff --git a/tests/test_v2.py b/tests/test_v2.py index 0a4487cfcc..3a36bc01fd 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -15,6 +15,7 @@ from zarr import config from zarr.abc.store import Store from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.metadata.v2 import _parse_structured_fill_value from zarr.core.sync import sync from zarr.storage import MemoryStore, StorePath @@ -315,6 +316,89 @@ def test_structured_dtype_roundtrip(fill_value, tmp_path) -> None: assert (a == za[:]).all() +@pytest.mark.parametrize( + ( + "fill_value", + "dtype", + "expected_result", + ), + [ + ( + ("Alice", 30), + np.dtype([("name", "U10"), ("age", "i4")]), + np.array([("Alice", 30)], dtype=[("name", "U10"), ("age", "i4")])[0], + ), + ( + ["Bob", 25], + np.dtype([("name", "U10"), ("age", "i4")]), + np.array([("Bob", 25)], dtype=[("name", "U10"), ("age", "i4")])[0], + ), + ( + b"\x01\x00\x00\x00\x02\x00\x00\x00", + np.dtype([("x", "i4"), ("y", "i4")]), + np.array([(1, 2)], dtype=[("x", "i4"), ("y", "i4")])[0], + ), + ( + "BQAAAA==", + np.dtype([("val", "i4")]), + np.array([(5,)], dtype=[("val", "i4")])[0], + ), + ( + {"x": 1, "y": 2}, + np.dtype([("location", "O")]), + np.array([({"x": 1, "y": 2},)], dtype=[("location", "O")])[0], + ), + ( + {"x": 1, "y": 2, "z": 3}, + np.dtype([("location", "O")]), + np.array([({"x": 1, "y": 2, "z": 3},)], dtype=[("location", "O")])[0], + ), + ], + ids=[ + "tuple_input", + "list_input", + "bytes_input", + "string_input", + "dictionary_input", + "dictionary_input_extra_fields", + ], +) +def test_parse_structured_fill_value_valid( + fill_value: Any, dtype: np.dtype[Any], expected_result: Any +) -> None: + result = _parse_structured_fill_value(fill_value, dtype) + assert result.dtype == expected_result.dtype + assert result == expected_result + if isinstance(expected_result, np.void): + for name in expected_result.dtype.names or []: + assert result[name] == expected_result[name] + + +@pytest.mark.parametrize( + ( + "fill_value", + "dtype", + ), + [ + (("Alice", 30), np.dtype([("name", "U10"), ("age", "i4"), ("city", "U20")])), + (b"\x01\x00\x00\x00", np.dtype([("x", "i4"), ("y", "i4")])), + ("this_is_not_base64", np.dtype([("val", "i4")])), + ("hello", np.dtype([("age", "i4")])), + ({"x": 1, "y": 2}, np.dtype([("location", "i4")])), + ], + ids=[ + "tuple_list_wrong_length", + "bytes_wrong_length", + "invalid_base64", + "wrong_data_type", + "wrong_dictionary", + ], +) +def test_parse_structured_fill_value_invalid(fill_value: Any, dtype: np.dtype[Any]) -> None: + with pytest.raises(ValueError): + _parse_structured_fill_value(fill_value, dtype) + + @pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"]) def test_other_dtype_roundtrip(fill_value, tmp_path) -> None: a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7") From 50abf3d40a4b32270dd5e4a4580cf6c043e0648e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 9 Apr 2025 15:11:07 +0200 Subject: [PATCH 091/160] Clean up warning filters in tests (#2714) * Clean up warning filters in tests * Ignore some more warnings * Ignore some more warnings * Ignore boto3 deprecation warning * Ignore unclosed client warning * Filter another fsspec warning * Ignore zip warning * Filter warning in test_stateful * pre-commit fixes * Filter warning in test_wrapper * Filter warning in memorystore * Close pool in multiprocessing test * pre-commit fixes * Filter two more warnings * Filter another warning * Filter more warnings * Add changelog * Ignore unclosed client sessions * Put back dtype filter * Make client session filter better --- changes/2714.misc.rst | 1 + pyproject.toml | 18 ++++++++++++------ tests/test_array.py | 5 ++--- tests/test_store/test_fsspec.py | 12 ++++++++++++ tests/test_store/test_memory.py | 7 +++++++ tests/test_store/test_stateful.py | 6 +++++- tests/test_store/test_wrapper.py | 9 +++++++++ tests/test_store/test_zip.py | 10 ++++++++++ tests/test_sync.py | 1 + 9 files changed, 59 insertions(+), 10 deletions(-) create mode 100644 changes/2714.misc.rst diff --git a/changes/2714.misc.rst b/changes/2714.misc.rst new file mode 100644 index 0000000000..9ab55089d2 --- /dev/null +++ b/changes/2714.misc.rst @@ -0,0 +1 @@ +Make warning filters in the tests more specific, so warnings emitted by tests added in the future are more likely to be caught instead of ignored. diff --git a/pyproject.toml b/pyproject.toml index d862cce0ac..1ee5678f82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -397,12 +397,18 @@ addopts = [ "--durations=10", "-ra", "--strict-config", "--strict-markers", ] filterwarnings = [ - "error:::zarr.*", - "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", - "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", - "ignore:Creating a zarr.buffer.gpu.*:UserWarning", - "ignore:Duplicate name:UserWarning", # from ZipFile - "ignore:.*is currently not part in the Zarr format 3 specification.*:UserWarning", + "error", + # TODO: explicitly filter or catch the warnings below where we expect them to be emitted in the tests + "ignore:Consolidated metadata is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__.*:UserWarning", + "ignore:Automatic shard shape inference is experimental and may change without notice.*:UserWarning", + "ignore:The codec .* is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:The dtype .* is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:Use zarr.create_array instead.:DeprecationWarning", + "ignore:Duplicate name.*:UserWarning", + "ignore:The `compressor` argument is deprecated. Use `compressors` instead.:UserWarning", + "ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.:UserWarning", + "ignore:Unclosed client session None: store._is_open = True +# TODO: work out where warning is coming from and fix +@pytest.mark.filterwarnings( + "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" +) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) async def test_wrapped_set(store: Store, capsys: pytest.CaptureFixture[str]) -> None: # define a class that prints when it sets @@ -89,6 +97,7 @@ async def set(self, key: str, value: Buffer) -> None: assert await store_wrapped.get(key, buffer_prototype) == value +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) async def test_wrapped_get(store: Store, capsys: pytest.CaptureFixture[str]) -> None: # define a class that prints when it sets diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index 839656108b..0237258ab1 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -19,6 +19,14 @@ from typing import Any +# TODO: work out where this is coming from and fix +pytestmark = [ + pytest.mark.filterwarnings( + "ignore:coroutine method 'aclose' of 'ZipStore.list' was never awaited:RuntimeWarning" + ) +] + + class TestZipStore(StoreTests[ZipStore, cpu.Buffer]): store_cls = ZipStore buffer_cls = cpu.Buffer @@ -66,6 +74,8 @@ def test_store_supports_partial_writes(self, store: ZipStore) -> None: def test_store_supports_listing(self, store: ZipStore) -> None: assert store.supports_listing + # TODO: fix this warning + @pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") def test_api_integration(self, store: ZipStore) -> None: root = zarr.open_group(store=store, mode="a") diff --git a/tests/test_sync.py b/tests/test_sync.py index e0002fc5a7..13b475f8da 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -90,6 +90,7 @@ def test_sync_raises_if_loop_is_closed() -> None: foo.assert_not_awaited() +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_calling_sync_from_within_a_running_loop( sync_loop: asyncio.AbstractEventLoop | None, From 06f7796b001a1bec94d54e408f35b8b8b9401764 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:45:41 +0200 Subject: [PATCH 092/160] Return scalar when accessing zero dimensional array (#2718) * Return scalar when accessing zero dimensional array * returning npt.ArrayLike instead of NDArrayLike because of scalar return values * returning npt.ArrayLike instead of NDArrayLike because of scalar return values * fix mypy in tests * fix mypy in tests * fix mypy in tests * improve test_scalar_array * fix typo * add ScalarWrapper * use ScalarWrapper as NDArrayLike * Revert "fix mypy in tests" * Revert "fix mypy in tests" This reverts commit 75d6cdf2669702dc75fe857c867e0e21944937f5. * Revert "fix mypy in tests" This reverts commit 34bf2606e402ee14b7d6f41372ac6611b437f47b. * format * Revert "returning npt.ArrayLike instead of NDArrayLike because of scalar return values" This reverts commit 1a290c73d92bc4d171b31e3a0057009a35f86e0c. * Revert "returning npt.ArrayLike instead of NDArrayLike because of scalar return values" This reverts commit 3348439d * fix mypy for ScalarWrapper * add missing import NDArrayLike * ignore unavoidable mypy error * format * fix __array__ * extend tests * format * fix typing in test_scalar_array * add dtype to ScalarWrapper * correct dtype type * fix test_basic_indexing * fix test_basic_indexing * fix test_basic_indexing for dtype=datetime64[Y] * increase codecov * fix typing * document changes * move test_scalar_wrapper to test_buffer.py * remove ScalarWrapper usage * create NDArrayOrScalarLike * fix NDArrayOrScalarLike * fix mypy * fix mypy * fix mypy * fix mypy in asynchronous.py * fix mypy in test_api.py * fix mypy in test_api.py and synchronous.py * fix mypy in test_api.py and test_array.py * fix mypy in test_array.py * fix mypy in test_array.py * fix mypy in test_array.py * fix mypy in test_array.py * fix mypy in test_array.py, test_api.py, test_buffer.py, test_sharding.py * add bytes, str and datetime to ScalarType * only support numpy datetime64 in ScalarType * remove ScalarWrapper and update changes * undo wrong code changes * rename ``NDArrayOrScalarLike`` to ``NDArrayLikeOrScalar`` * rename ``NDArrayOrScalarLike`` to ``NDArrayLikeOrScalar`` * fix mypy in test_array.py * fix mypy in test_array.py * handle datetype scalars for different units * fix mypy * fix mypy * format --- changes/2718.bugfix.rst | 3 ++ src/zarr/api/asynchronous.py | 3 +- src/zarr/api/synchronous.py | 4 +- src/zarr/core/array.py | 48 +++++++++-------- src/zarr/core/buffer/__init__.py | 2 + src/zarr/core/buffer/core.py | 19 +++++++ src/zarr/core/indexing.py | 10 ++-- tests/test_api.py | 21 ++++++-- tests/test_array.py | 83 +++++++++++++++++++----------- tests/test_buffer.py | 8 ++- tests/test_codecs/test_codecs.py | 2 +- tests/test_codecs/test_sharding.py | 6 ++- 12 files changed, 142 insertions(+), 67 deletions(-) create mode 100644 changes/2718.bugfix.rst diff --git a/changes/2718.bugfix.rst b/changes/2718.bugfix.rst new file mode 100644 index 0000000000..48ddf8b5a8 --- /dev/null +++ b/changes/2718.bugfix.rst @@ -0,0 +1,3 @@ +0-dimensional arrays are now returning a scalar. Therefore, the return type of ``__getitem__`` changed +to NDArrayLikeOrScalar. This change is to make the behavior of 0-dimensional arrays consistent with +``numpy`` scalars. \ No newline at end of file diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 6059893920..d067160f92 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -38,6 +38,7 @@ from collections.abc import Iterable from zarr.abc.codec import Codec + from zarr.core.buffer import NDArrayLikeOrScalar from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.storage import StoreLike @@ -238,7 +239,7 @@ async def load( path: str | None = None, zarr_format: ZarrFormat | None = None, zarr_version: ZarrFormat | None = None, -) -> NDArrayLike | dict[str, NDArrayLike]: +) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: """Load data from an array or group into memory. Parameters diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 9424ae1fde..51cb0ed4f6 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -27,7 +27,7 @@ ShardsLike, ) from zarr.core.array_spec import ArrayConfigLike - from zarr.core.buffer import NDArrayLike + from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike from zarr.core.common import ( JSON, @@ -121,7 +121,7 @@ def load( path: str | None = None, zarr_format: ZarrFormat | None = None, zarr_version: ZarrFormat | None = None, -) -> NDArrayLike | dict[str, NDArrayLike]: +) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: """Load data from an array or group into memory. Parameters diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 0e03cbcabb..98c8adeea6 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -35,6 +35,7 @@ from zarr.core.buffer import ( BufferPrototype, NDArrayLike, + NDArrayLikeOrScalar, NDBuffer, default_buffer_prototype, ) @@ -1256,7 +1257,7 @@ async def _get_selection( prototype: BufferPrototype, out: NDBuffer | None = None, fields: Fields | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: # check fields are sensible out_dtype = check_fields(fields, self.dtype) @@ -1298,6 +1299,8 @@ async def _get_selection( out_buffer, drop_axes=indexer.drop_axes, ) + if isinstance(indexer, BasicIndexer) and indexer.shape == (): + return out_buffer.as_scalar() return out_buffer.as_ndarray_like() async def getitem( @@ -1305,7 +1308,7 @@ async def getitem( selection: BasicSelection, *, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """ Asynchronous function that retrieves a subset of the array's data based on the provided selection. @@ -1318,7 +1321,7 @@ async def getitem( Returns ------- - NDArrayLike + NDArrayLikeOrScalar The retrieved subset of the array's data. Examples @@ -2268,14 +2271,15 @@ def __array__( msg = "`copy=False` is not supported. This method always creates a copy." raise ValueError(msg) - arr_np = self[...] + arr = self[...] + arr_np: NDArrayLike = np.array(arr, dtype=dtype) if dtype is not None: arr_np = arr_np.astype(dtype) return arr_np - def __getitem__(self, selection: Selection) -> NDArrayLike: + def __getitem__(self, selection: Selection) -> NDArrayLikeOrScalar: """Retrieve data for an item or region of the array. Parameters @@ -2286,8 +2290,8 @@ def __getitem__(self, selection: Selection) -> NDArrayLike: Returns ------- - NDArrayLike - An array-like containing the data for the requested region. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested region. Examples -------- @@ -2533,7 +2537,7 @@ def get_basic_selection( out: NDBuffer | None = None, prototype: BufferPrototype | None = None, fields: Fields | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve data for an item or region of the array. Parameters @@ -2551,8 +2555,8 @@ def get_basic_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested region. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested region. Examples -------- @@ -2753,7 +2757,7 @@ def get_orthogonal_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve data by making a selection for each dimension of the array. For example, if an array has 2 dimensions, allows selecting specific rows and/or columns. The selection for each dimension can be either an integer (indexing a @@ -2775,8 +2779,8 @@ def get_orthogonal_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested selection. Examples -------- @@ -2989,7 +2993,7 @@ def get_mask_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing a Boolean array of the same shape as the array against which the selection is being made, where True values indicate a selected item. @@ -3009,8 +3013,8 @@ def get_mask_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested selection. Examples -------- @@ -3151,7 +3155,7 @@ def get_coordinate_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing the indices (coordinates) for each selected item. @@ -3169,8 +3173,8 @@ def get_coordinate_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested coordinate selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested coordinate selection. Examples -------- @@ -3339,7 +3343,7 @@ def get_block_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing the indices (coordinates) for each selected item. @@ -3357,8 +3361,8 @@ def get_block_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested block selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested block selection. Examples -------- diff --git a/src/zarr/core/buffer/__init__.py b/src/zarr/core/buffer/__init__.py index ccb41e291c..ebec61a372 100644 --- a/src/zarr/core/buffer/__init__.py +++ b/src/zarr/core/buffer/__init__.py @@ -3,6 +3,7 @@ Buffer, BufferPrototype, NDArrayLike, + NDArrayLikeOrScalar, NDBuffer, default_buffer_prototype, ) @@ -13,6 +14,7 @@ "Buffer", "BufferPrototype", "NDArrayLike", + "NDArrayLikeOrScalar", "NDBuffer", "default_buffer_prototype", "numpy_buffer_prototype", diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index ccab103e0f..70d408d32b 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -105,6 +105,10 @@ def __eq__(self, other: object) -> Self: # type: ignore[explicit-override, over """ +ScalarType = int | float | complex | bytes | str | bool | np.generic +NDArrayLikeOrScalar = ScalarType | NDArrayLike + + def check_item_key_is_1d_contiguous(key: Any) -> None: """Raises error if `key` isn't a 1d contiguous slice""" if not isinstance(key, slice): @@ -419,6 +423,21 @@ def as_numpy_array(self) -> npt.NDArray[Any]: """ ... + def as_scalar(self) -> ScalarType: + """Returns the buffer as a scalar value""" + if self._data.size != 1: + raise ValueError("Buffer does not contain a single scalar value") + item = self.as_numpy_array().item() + scalar: ScalarType + + if np.issubdtype(self.dtype, np.datetime64): + unit: str = np.datetime_data(self.dtype)[0] # Extract the unit (e.g., 'Y', 'D', etc.) + scalar = np.datetime64(item, unit) + else: + scalar = self.dtype.type(item) # Regular conversion for non-datetime types + + return scalar + @property def dtype(self) -> np.dtype[Any]: return self._data.dtype diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index c197f6f397..998fe156a1 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from zarr.core.array import Array - from zarr.core.buffer import NDArrayLike + from zarr.core.buffer import NDArrayLikeOrScalar from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import ChunkCoords @@ -937,7 +937,7 @@ class OIndex: array: Array # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool - def __getitem__(self, selection: OrthogonalSelection | Array) -> NDArrayLike: + def __getitem__(self, selection: OrthogonalSelection | Array) -> NDArrayLikeOrScalar: from zarr.core.array import Array # if input is a Zarr array, we materialize it now. @@ -1046,7 +1046,7 @@ def __iter__(self) -> Iterator[ChunkProjection]: class BlockIndex: array: Array - def __getitem__(self, selection: BasicSelection) -> NDArrayLike: + def __getitem__(self, selection: BasicSelection) -> NDArrayLikeOrScalar: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) @@ -1236,7 +1236,9 @@ class VIndex: array: Array # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool - def __getitem__(self, selection: CoordinateSelection | MaskSelection | Array) -> NDArrayLike: + def __getitem__( + self, selection: CoordinateSelection | MaskSelection | Array + ) -> NDArrayLikeOrScalar: from zarr.core.array import Array # if input is a Zarr array, we materialize it now. diff --git a/tests/test_api.py b/tests/test_api.py index 94140ac784..f03fd53f7a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -32,6 +32,7 @@ save_array, save_group, ) +from zarr.core.buffer import NDArrayLike from zarr.errors import MetadataValidationError from zarr.storage import MemoryStore from zarr.storage._utils import normalize_path @@ -244,7 +245,9 @@ def test_open_with_mode_r(tmp_path: pathlib.Path) -> None: z2 = zarr.open(store=tmp_path, mode="r") assert isinstance(z2, Array) assert z2.fill_value == 1 - assert (z2[:] == 1).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() with pytest.raises(ValueError): z2[:] = 3 @@ -256,7 +259,9 @@ def test_open_with_mode_r_plus(tmp_path: pathlib.Path) -> None: zarr.ones(store=tmp_path, shape=(3, 3)) z2 = zarr.open(store=tmp_path, mode="r+") assert isinstance(z2, Array) - assert (z2[:] == 1).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() z2[:] = 3 @@ -272,7 +277,9 @@ async def test_open_with_mode_a(tmp_path: pathlib.Path) -> None: arr[...] = 1 z2 = zarr.open(store=tmp_path, mode="a") assert isinstance(z2, Array) - assert (z2[:] == 1).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() z2[:] = 3 @@ -284,7 +291,9 @@ def test_open_with_mode_w(tmp_path: pathlib.Path) -> None: arr[...] = 3 z2 = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) assert isinstance(z2, Array) - assert not (z2[:] == 3).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert not (result == 3).all() z2[:] = 3 @@ -1134,7 +1143,9 @@ def test_open_array_with_mode_r_plus(store: Store) -> None: zarr.ones(store=store, shape=(3, 3)) z2 = zarr.open_array(store=store, mode="r+") assert isinstance(z2, Array) - assert (z2[:] == 1).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() z2[:] = 3 diff --git a/tests/test_array.py b/tests/test_array.py index 3751d5409e..2c28892b68 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -37,7 +37,7 @@ chunks_initialized, create_array, ) -from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype from zarr.core.buffer.cpu import NDBuffer from zarr.core.chunk_grids import _auto_partition from zarr.core.common import JSON, MemoryOrder, ZarrFormat @@ -654,35 +654,43 @@ def test_resize_1d(store: MemoryStore, zarr_format: ZarrFormat) -> None: ) a = np.arange(105, dtype="i4") z[:] = a + result = z[:] + assert isinstance(result, NDArrayLike) assert (105,) == z.shape - assert (105,) == z[:].shape + assert (105,) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10,) == z.chunks - np.testing.assert_array_equal(a, z[:]) + np.testing.assert_array_equal(a, result) z.resize(205) + result = z[:] + assert isinstance(result, NDArrayLike) assert (205,) == z.shape - assert (205,) == z[:].shape + assert (205,) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10,) == z.chunks np.testing.assert_array_equal(a, z[:105]) np.testing.assert_array_equal(np.zeros(100, dtype="i4"), z[105:]) z.resize(55) + result = z[:] + assert isinstance(result, NDArrayLike) assert (55,) == z.shape - assert (55,) == z[:].shape + assert (55,) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10,) == z.chunks - np.testing.assert_array_equal(a[:55], z[:]) + np.testing.assert_array_equal(a[:55], result) # via shape setter new_shape = (105,) z.shape = new_shape + result = z[:] + assert isinstance(result, NDArrayLike) assert new_shape == z.shape - assert new_shape == z[:].shape + assert new_shape == result.shape @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -697,44 +705,54 @@ def test_resize_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: ) a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) z[:] = a + result = z[:] + assert isinstance(result, NDArrayLike) assert (105, 105) == z.shape - assert (105, 105) == z[:].shape + assert (105, 105) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks - np.testing.assert_array_equal(a, z[:]) + np.testing.assert_array_equal(a, result) z.resize((205, 205)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (205, 205) == z.shape - assert (205, 205) == z[:].shape + assert (205, 205) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a, z[:105, :105]) np.testing.assert_array_equal(np.zeros((100, 205), dtype="i4"), z[105:, :]) np.testing.assert_array_equal(np.zeros((205, 100), dtype="i4"), z[:, 105:]) z.resize((55, 55)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (55, 55) == z.shape - assert (55, 55) == z[:].shape + assert (55, 55) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks - np.testing.assert_array_equal(a[:55, :55], z[:]) + np.testing.assert_array_equal(a[:55, :55], result) z.resize((55, 1)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (55, 1) == z.shape - assert (55, 1) == z[:].shape + assert (55, 1) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks - np.testing.assert_array_equal(a[:55, :1], z[:]) + np.testing.assert_array_equal(a[:55, :1], result) z.resize((1, 55)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (1, 55) == z.shape - assert (1, 55) == z[:].shape + assert (1, 55) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a[:1, :10], z[:, :10]) np.testing.assert_array_equal(np.zeros((1, 55 - 10), dtype="i4"), z[:, 10:55]) @@ -742,8 +760,10 @@ def test_resize_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: # via shape setter new_shape = (105, 105) z.shape = new_shape + result = z[:] + assert isinstance(result, NDArrayLike) assert new_shape == z.shape - assert new_shape == z[:].shape + assert new_shape == result.shape @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -1319,11 +1339,14 @@ async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) -> ) -async def test_scalar_array() -> None: - arr = zarr.array(1.5) - assert arr[...] == 1.5 - assert arr[()] == 1.5 +@pytest.mark.parametrize("value", [1, 1.4, "a", b"a", np.array(1)]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +def test_scalar_array(value: Any, zarr_format: ZarrFormat) -> None: + arr = zarr.array(value, zarr_format=zarr_format) + assert arr[...] == value assert arr.shape == () + assert arr.ndim == 0 + assert isinstance(arr[()], NDArrayLikeOrScalar) async def test_orthogonal_set_total_slice() -> None: @@ -1434,4 +1457,6 @@ async def test_sharding_coordinate_selection() -> None: shards=(2, 4, 4), ) arr[:] = np.arange(2 * 3 * 4).reshape((2, 3, 4)) - assert (arr[1, [0, 1]] == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all() # type: ignore[index] + result = arr[1, [0, 1]] # type: ignore[index] + assert isinstance(result, NDArrayLike) + assert (result == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all() diff --git a/tests/test_buffer.py b/tests/test_buffer.py index baef0b8109..33ac0266eb 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -30,6 +30,8 @@ cp = None +import zarr.api.asynchronous + if TYPE_CHECKING: import types @@ -64,7 +66,7 @@ async def test_async_array_prototype() -> None: got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=my_prototype) # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol # The test passes, so it clearly does. - assert isinstance(got, TestNDArrayLike) # type: ignore[unreachable] + assert isinstance(got, TestNDArrayLike) assert np.array_equal(expect, got) # type: ignore[unreachable] @@ -117,7 +119,7 @@ async def test_codecs_use_of_prototype() -> None: got = await a.getitem(selection=(slice(0, 10), slice(0, 10)), prototype=my_prototype) # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol # The test passes, so it clearly does. - assert isinstance(got, TestNDArrayLike) # type: ignore[unreachable] + assert isinstance(got, TestNDArrayLike) assert np.array_equal(expect, got) # type: ignore[unreachable] @@ -151,3 +153,5 @@ def test_numpy_buffer_prototype() -> None: ndbuffer = cpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) assert isinstance(buffer.as_array_like(), np.ndarray) assert isinstance(ndbuffer.as_ndarray_like(), np.ndarray) + with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): + ndbuffer.as_scalar() diff --git a/tests/test_codecs/test_codecs.py b/tests/test_codecs/test_codecs.py index e36a332440..b8122b4ac2 100644 --- a/tests/test_codecs/test_codecs.py +++ b/tests/test_codecs/test_codecs.py @@ -23,7 +23,7 @@ if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.buffer.core import NDArrayLike + from zarr.core.buffer import NDArrayLike from zarr.core.common import MemoryOrder diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 2ba57d7a39..403fd80e81 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -16,7 +16,7 @@ ShardingCodecIndexLocation, TransposeCodec, ) -from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer import NDArrayLike, default_buffer_prototype from zarr.storage import StorePath from ..conftest import ArrayRequest @@ -66,6 +66,7 @@ def test_sharding( assert np.all(arr[empty_region] == arr.metadata.fill_value) read_data = arr[write_region] + assert isinstance(read_data, NDArrayLike) assert data.shape == read_data.shape assert np.array_equal(data, read_data) @@ -130,6 +131,7 @@ def test_sharding_partial( assert np.all(read_data == 0) read_data = a[10:, 10:, 10:] + assert isinstance(read_data, NDArrayLike) assert data.shape == read_data.shape assert np.array_equal(data, read_data) @@ -275,6 +277,7 @@ def test_nested_sharding( a[:, :, :] = data read_data = a[0 : data.shape[0], 0 : data.shape[1], 0 : data.shape[2]] + assert isinstance(read_data, NDArrayLike) assert data.shape == read_data.shape assert np.array_equal(data, read_data) @@ -322,6 +325,7 @@ def test_nested_sharding_create_array( a[:, :, :] = data read_data = a[0 : data.shape[0], 0 : data.shape[1], 0 : data.shape[2]] + assert isinstance(read_data, NDArrayLike) assert data.shape == read_data.shape assert np.array_equal(data, read_data) From 018f61d93207112f68eba06ea8c2560a489767f6 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:49:55 +0200 Subject: [PATCH 093/160] `zarr.array` from from an existing `zarr.Array` (#2622) * add creation from other zarr * remove duplicated tests * improve test * test_iter_grid for non-squares * concurrent streaming for equal chunk sizes * fix merge * fix mypy * fix mypy * fix test_iter_grid * extract to zarr.from_array * fix mypy * fix mypy * format * fix test_creation_from_other_zarr_format * distinguish between keep and auto for from_array arguments * partition concurrency along new_array chunks * fix mypy * improve test_creation_from_other_zarr_format * add typing in test * Update src/zarr/core/array.py Co-authored-by: Norman Rzepka * add from_array with npt.ArrayLike * add write_data argument * improve tests * improve docstrings and add examples * fix mypy and readthedocs * fix mypy and readthedocs * fix mypy and readthedocs * fix mypy and readthedocs * fix readthedocs ERROR: Unexpected indentation * add release notes * format docstring examples * add write_data attr to synchronous.create_array * `create_array` calls `from_array` calls `init_array` * document changes * fix serializer from_array v2 to v3 * fix mypy * improve codecov * fix mypy * from_array: copy zarr format on default * in ``from_array`` make all arguments except ``store`` keyword-only, to match ``create_array`` * in ``from_array`` default shards="keep" * redundant ``ChunkKeyEncoding | ChunkKeyEncodingLike`` * fix argument order in calls of `from_array` * fix numpydoc-validation * add docstring to store2 pytest fixture * extract `_parse_keep_array_attr` from `from_array` * extract `_parse_keep_array_attr` from `from_array` * correct _parse_keep_array_attr * fix merge * fix merge --------- Co-authored-by: Norman Rzepka --- changes/2622.feature.rst | 1 + docs/release-notes.rst | 2 + src/zarr/__init__.py | 2 + src/zarr/api/asynchronous.py | 10 +- src/zarr/api/synchronous.py | 227 ++++++++++++++++++- src/zarr/core/array.py | 422 ++++++++++++++++++++++++++++++++--- src/zarr/core/group.py | 8 +- tests/conftest.py | 8 + tests/test_array.py | 120 ++++++++++ tests/test_indexing.py | 12 +- 10 files changed, 772 insertions(+), 40 deletions(-) create mode 100644 changes/2622.feature.rst diff --git a/changes/2622.feature.rst b/changes/2622.feature.rst new file mode 100644 index 0000000000..f5c7cbe192 --- /dev/null +++ b/changes/2622.feature.rst @@ -0,0 +1 @@ +Add ``zarr.from_array`` using concurrent streaming of source data \ No newline at end of file diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 9a6e680e4d..c585e4f0d3 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -145,6 +145,8 @@ Other 3.0.1 (Jan. 17, 2025) --------------------- +* Implement ``zarr.from_array`` using concurrent streaming (:issue:`2622`). + Bug fixes ~~~~~~~~~ * Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`). diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index 4ffa4c9bbc..31796601b3 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -11,6 +11,7 @@ create_hierarchy, empty, empty_like, + from_array, full, full_like, group, @@ -54,6 +55,7 @@ "create_hierarchy", "empty", "empty_like", + "from_array", "full", "full_like", "group", diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index d067160f92..285d777258 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -9,7 +9,7 @@ import numpy.typing as npt from typing_extensions import deprecated -from zarr.core.array import Array, AsyncArray, create_array, get_array_metadata +from zarr.core.array import Array, AsyncArray, create_array, from_array, get_array_metadata from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams from zarr.core.buffer import NDArrayLike from zarr.core.common import ( @@ -57,6 +57,7 @@ "create_hierarchy", "empty", "empty_like", + "from_array", "full", "full_like", "group", @@ -534,7 +535,7 @@ async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = async def array( - data: npt.ArrayLike, **kwargs: Any + data: npt.ArrayLike | Array, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Create an array filled with `data`. @@ -551,13 +552,16 @@ async def array( The new array. """ + if isinstance(data, Array): + return await from_array(data=data, **kwargs) + # ensure data is array-like if not hasattr(data, "shape") or not hasattr(data, "dtype"): data = np.asanyarray(data) # setup dtype kw_dtype = kwargs.get("dtype") - if kw_dtype is None: + if kw_dtype is None and hasattr(data, "dtype"): kwargs["dtype"] = data.dtype else: kwargs["dtype"] = kw_dtype diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 51cb0ed4f6..4c577936cd 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -50,6 +50,7 @@ "create_hierarchy", "empty", "empty_like", + "from_array", "full", "full_like", "group", @@ -359,7 +360,7 @@ def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> An # TODO: add type annotations for kwargs -def array(data: npt.ArrayLike, **kwargs: Any) -> Array: +def array(data: npt.ArrayLike | Array, **kwargs: Any) -> Array: """Create an array filled with `data`. Parameters @@ -759,11 +760,12 @@ def create_array( order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, + write_data: bool = True, ) -> Array: """Create an array. @@ -857,6 +859,11 @@ def create_array( Whether to overwrite an array with the same name in the store, if one exists. config : ArrayConfigLike, optional Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the ``data`` parameter + then ``write_data`` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If ``write_data`` is ``False``, then the + array will be left empty. Returns ------- @@ -866,7 +873,7 @@ def create_array( Examples -------- >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') + >>> store = zarr.storage.MemoryStore() >>> arr = await zarr.create_array( >>> store=store, >>> shape=(100,100), @@ -897,6 +904,220 @@ def create_array( storage_options=storage_options, overwrite=overwrite, config=config, + write_data=write_data, + ) + ) + ) + + +def from_array( + store: str | StoreLike, + *, + data: Array | npt.ArrayLike, + write_data: bool = True, + name: str | None = None, + chunks: Literal["auto", "keep"] | ChunkCoords = "keep", + shards: ShardsLike | None | Literal["keep"] = "keep", + filters: FiltersLike | Literal["keep"] = "keep", + compressors: CompressorsLike | Literal["keep"] = "keep", + serializer: SerializerLike | Literal["keep"] = "keep", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: Iterable[str] | None = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, +) -> Array: + """Create an array from an existing array or array-like. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file for the new array. + data : Array | array-like + The array to copy. + write_data : bool, default True + Whether to copy the data from the input array to the new array. + If ``write_data`` is ``False``, the new array will be created with the same metadata as the + input array, but without any data. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + chunks : ChunkCoords or "auto" or "keep", optional + Chunk shape of the array. + Following values are supported: + + - "auto": Automatically determine the chunk shape based on the array's shape and dtype. + - "keep": Retain the chunk shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the chunk shape. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". + shards : ChunkCoords, optional + Shard shape of the array. + Following values are supported: + + - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. + - "keep": Retain the shard shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the shard shape. + - None: No sharding. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise None. + filters : Iterable[Codec] or "auto" or "keep", optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + + Following values are supported: + + - Iterable[Codec]: List of filters to apply to the array. + - "auto": Automatically determine the filters based on the array's dtype. + - "keep": Retain the filters of the data array if it is a zarr Array. + + If no ``filters`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + compressors : Iterable[Codec] or "auto" or "keep", optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + + Following values are supported: + + - Iterable[Codec]: List of compressors to apply to the array. + - "auto": Automatically determine the compressors based on the array's dtype. + - "keep": Retain the compressors of the input array if it is a zarr Array. + + If no ``compressors`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + + Following values are supported: + + - dict[str, JSON]: A dict representation of an ``ArrayBytesCodec``. + - ArrayBytesCodec: An instance of ``ArrayBytesCodec``. + - "auto": a default serializer will be used. These defaults can be changed by modifying the value of + ``array.v3_default_serializer`` in :mod:`zarr.core.config`. + - "keep": Retain the serializer of the input array if it is a zarr Array. + + fill_value : Any, optional + Fill value for the array. + If not specified, defaults to the fill value of the data array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If not specified, defaults to the memory order of the data array. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + If not specified, defaults to the zarr format of the data array. + attributes : dict, optional + Attributes for the array. + If not specified, defaults to the attributes of the data array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + If not specified and the data array has the same zarr format as the target array, + the chunk key encoding of the data array is used. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + If not specified, defaults to the dimension names of the data array. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + Array + The array. + + Examples + -------- + Create an array from an existing Array:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> store2 = zarr.storage.LocalStore('example.zarr') + >>> arr = zarr.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='int32', + >>> fill_value=0) + >>> arr2 = zarr.from_array(store2, data=arr) + + + Create an array from an existing NumPy array:: + + >>> import numpy as np + >>> arr3 = zarr.from_array( + zarr.storage.MemoryStore(), + >>> data=np.arange(10000, dtype='i4').reshape(100, 100), + >>> ) + + + Create an array from any array-like object:: + + >>> arr4 = zarr.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=[[1, 2], [3, 4]], + >>> ) + + >>> arr4[...] + array([[1, 2],[3, 4]]) + + Create an array from an existing Array without copying the data:: + + >>> arr5 = zarr.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=arr4, + >>> write_data=False, + >>> ) + + >>> arr5[...] + array([[0, 0],[0, 0]]) + """ + return Array( + sync( + zarr.core.array.from_array( + store, + data=data, + write_data=write_data, + name=name, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, ) ) ) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 98c8adeea6..f2c88c508b 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -25,6 +25,7 @@ import numpy.typing as npt from typing_extensions import deprecated +import zarr from zarr._compat import _deprecate_positional_args from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.abc.store import Store, set_or_delete @@ -889,7 +890,7 @@ async def open( Examples -------- >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') + >>> store = zarr.storage.MemoryStore() >>> async_arr = await AsyncArray.open(store) # doctest: +ELLIPSIS """ @@ -1327,7 +1328,7 @@ async def getitem( Examples -------- >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') + >>> store = zarr.storage.MemoryStore() >>> async_arr = await zarr.api.asynchronous.create_array( ... store=store, ... shape=(100,100), @@ -3798,6 +3799,269 @@ class ShardsConfigParam(TypedDict): ShardsLike: TypeAlias = ChunkCoords | ShardsConfigParam | Literal["auto"] +async def from_array( + store: str | StoreLike, + *, + data: Array | npt.ArrayLike, + write_data: bool = True, + name: str | None = None, + chunks: Literal["auto", "keep"] | ChunkCoords = "keep", + shards: ShardsLike | None | Literal["keep"] = "keep", + filters: FiltersLike | Literal["keep"] = "keep", + compressors: CompressorsLike | Literal["keep"] = "keep", + serializer: SerializerLike | Literal["keep"] = "keep", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: Iterable[str] | None = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfig | ArrayConfigLike | None = None, +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array from an existing array or array-like. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file for the new array. + data : Array | array-like + The array to copy. + write_data : bool, default True + Whether to copy the data from the input array to the new array. + If ``write_data`` is ``False``, the new array will be created with the same metadata as the + input array, but without any data. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + chunks : ChunkCoords or "auto" or "keep", optional + Chunk shape of the array. + Following values are supported: + + - "auto": Automatically determine the chunk shape based on the array's shape and dtype. + - "keep": Retain the chunk shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the chunk shape. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". + shards : ChunkCoords, optional + Shard shape of the array. + Following values are supported: + + - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. + - "keep": Retain the shard shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the shard shape. + - None: No sharding. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise None. + filters : Iterable[Codec] or "auto" or "keep", optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + + Following values are supported: + + - Iterable[Codec]: List of filters to apply to the array. + - "auto": Automatically determine the filters based on the array's dtype. + - "keep": Retain the filters of the data array if it is a zarr Array. + + If no ``filters`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + compressors : Iterable[Codec] or "auto" or "keep", optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + + Following values are supported: + + - Iterable[Codec]: List of compressors to apply to the array. + - "auto": Automatically determine the compressors based on the array's dtype. + - "keep": Retain the compressors of the input array if it is a zarr Array. + + If no ``compressors`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + + Following values are supported: + + - dict[str, JSON]: A dict representation of an ``ArrayBytesCodec``. + - ArrayBytesCodec: An instance of ``ArrayBytesCodec``. + - "auto": a default serializer will be used. These defaults can be changed by modifying the value of + ``array.v3_default_serializer`` in :mod:`zarr.core.config`. + - "keep": Retain the serializer of the input array if it is a zarr Array. + + fill_value : Any, optional + Fill value for the array. + If not specified, defaults to the fill value of the data array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If not specified, defaults to the memory order of the data array. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + If not specified, defaults to the zarr format of the data array. + attributes : dict, optional + Attributes for the array. + If not specified, defaults to the attributes of the data array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + If not specified and the data array has the same zarr format as the target array, + the chunk key encoding of the data array is used. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + If not specified, defaults to the dimension names of the data array. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + AsyncArray + The array. + + Examples + -------- + Create an array from an existing Array:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> store2 = zarr.storage.LocalStore('example.zarr') + >>> arr = zarr.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='int32', + >>> fill_value=0) + >>> arr2 = await zarr.api.asynchronous.from_array(store2, data=arr) + + + Create an array from an existing NumPy array:: + + >>> arr3 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=np.arange(10000, dtype='i4').reshape(100, 100), + >>> ) + + + Create an array from any array-like object:: + + >>> arr4 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=[[1, 2], [3, 4]], + >>> ) + + >>> await arr4.getitem(...) + array([[1, 2],[3, 4]]) + + Create an array from an existing Array without copying the data:: + + >>> arr5 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=Array(arr4), + >>> write_data=False, + >>> ) + + >>> await arr5.getitem(...) + array([[0, 0],[0, 0]]) + """ + mode: Literal["a"] = "a" + config_parsed = parse_array_config(config) + store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + + ( + chunks, + shards, + filters, + compressors, + serializer, + fill_value, + order, + zarr_format, + chunk_key_encoding, + dimension_names, + ) = _parse_keep_array_attr( + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + ) + if not hasattr(data, "dtype") or not hasattr(data, "shape"): + data = np.array(data) + + result = await init_array( + store_path=store_path, + shape=data.shape, + dtype=data.dtype, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + overwrite=overwrite, + config=config_parsed, + ) + + if write_data: + if isinstance(data, Array): + + async def _copy_array_region(chunk_coords: ChunkCoords | slice, _data: Array) -> None: + arr = await _data._async_array.getitem(chunk_coords) + await result.setitem(chunk_coords, arr) + + # Stream data from the source array to the new array + await concurrent_map( + [(region, data) for region in result._iter_chunk_regions()], + _copy_array_region, + zarr.core.config.config.get("async.concurrency"), + ) + else: + + async def _copy_arraylike_region(chunk_coords: slice, _data: NDArrayLike) -> None: + await result.setitem(chunk_coords, _data[chunk_coords]) + + # Stream data from the source array to the new array + await concurrent_map( + [(region, data) for region in result._iter_chunk_regions()], + _copy_arraylike_region, + zarr.core.config.config.get("async.concurrency"), + ) + return result + + async def init_array( *, store_path: StorePath, @@ -4141,38 +4405,138 @@ async def create_array( >>> fill_value=0) """ - mode: Literal["a"] = "a" - store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) - data_parsed, shape_parsed, dtype_parsed = _parse_data_params( data=data, shape=shape, dtype=dtype ) - result = await init_array( - store_path=store_path, - shape=shape_parsed, - dtype=dtype_parsed, - chunks=chunks, - shards=shards, - filters=filters, - compressors=compressors, - serializer=serializer, - fill_value=fill_value, - order=order, - zarr_format=zarr_format, - attributes=attributes, - chunk_key_encoding=chunk_key_encoding, - dimension_names=dimension_names, - overwrite=overwrite, - config=config, - ) + if data_parsed is not None: + return await from_array( + store, + data=data_parsed, + write_data=write_data, + name=name, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + ) + else: + mode: Literal["a"] = "a" - if write_data is True and data_parsed is not None: - await result._set_selection( - BasicIndexer(..., shape=result.shape, chunk_grid=result.metadata.chunk_grid), - data_parsed, - prototype=default_buffer_prototype(), + store_path = await make_store_path( + store, path=name, mode=mode, storage_options=storage_options ) - return result + return await init_array( + store_path=store_path, + shape=shape_parsed, + dtype=dtype_parsed, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + overwrite=overwrite, + config=config, + ) + + +def _parse_keep_array_attr( + data: Array | npt.ArrayLike, + chunks: Literal["auto", "keep"] | ChunkCoords, + shards: ShardsLike | None | Literal["keep"], + filters: FiltersLike | Literal["keep"], + compressors: CompressorsLike | Literal["keep"], + serializer: SerializerLike | Literal["keep"], + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: ZarrFormat | None, + chunk_key_encoding: ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> tuple[ + ChunkCoords | Literal["auto"], + ShardsLike | None, + FiltersLike, + CompressorsLike, + SerializerLike, + Any | None, + MemoryOrder | None, + ZarrFormat, + ChunkKeyEncodingLike | None, + Iterable[str] | None, +]: + if isinstance(data, Array): + if chunks == "keep": + chunks = data.chunks + if shards == "keep": + shards = data.shards + if zarr_format is None: + zarr_format = data.metadata.zarr_format + if filters == "keep": + if zarr_format == data.metadata.zarr_format: + filters = data.filters or None + else: + filters = "auto" + if compressors == "keep": + if zarr_format == data.metadata.zarr_format: + compressors = data.compressors or None + else: + compressors = "auto" + if serializer == "keep": + if zarr_format == 3 and data.metadata.zarr_format == 3: + serializer = cast(SerializerLike, data.serializer) + else: + serializer = "auto" + if fill_value is None: + fill_value = data.fill_value + if order is None: + order = data.order + if chunk_key_encoding is None and zarr_format == data.metadata.zarr_format: + if isinstance(data.metadata, ArrayV2Metadata): + chunk_key_encoding = {"name": "v2", "separator": data.metadata.dimension_separator} + elif isinstance(data.metadata, ArrayV3Metadata): + chunk_key_encoding = data.metadata.chunk_key_encoding + if dimension_names is None and data.metadata.zarr_format == 3: + dimension_names = data.metadata.dimension_names + else: + if chunks == "keep": + chunks = "auto" + if shards == "keep": + shards = None + if zarr_format is None: + zarr_format = 3 + if filters == "keep": + filters = "auto" + if compressors == "keep": + compressors = "auto" + if serializer == "keep": + serializer = "auto" + return ( + chunks, + shards, + filters, + compressors, + serializer, + fill_value, + order, + zarr_format, + chunk_key_encoding, + dimension_names, + ) def _parse_chunk_key_encoding( diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 5500bdd4a5..da2aa5f754 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -68,7 +68,7 @@ from zarr.core.array_spec import ArrayConfig, ArrayConfigLike from zarr.core.buffer import Buffer, BufferPrototype - from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike + from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike from zarr.core.common import MemoryOrder logger = logging.getLogger("zarr.group") @@ -998,7 +998,7 @@ async def create_array( fill_value: Any | None = 0, order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, @@ -2369,7 +2369,7 @@ def create_array( fill_value: Any | None = 0, order: MemoryOrder | None = "C", attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, @@ -2763,7 +2763,7 @@ def array( fill_value: Any | None = 0, order: MemoryOrder | None = "C", attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: Iterable[str] | None = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, diff --git a/tests/conftest.py b/tests/conftest.py index 04034cb5b8..74a140c5c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,6 +89,14 @@ async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: return await parse_store(param, str(tmpdir)) +@pytest.fixture +async def store2(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: + """Fixture to create a second store for testing copy operations between stores""" + param = request.param + store2_path = tmpdir.mkdir("store2") + return await parse_store(param, str(store2_path)) + + @pytest.fixture(params=["local", "memory", "zip"]) def sync_store(request: pytest.FixtureRequest, tmp_path: LEGACY_PATH) -> Store: result = sync(parse_store(request.param, str(tmp_path))) diff --git a/tests/test_array.py b/tests/test_array.py index 2c28892b68..5c3c556dfb 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -11,6 +11,7 @@ import numcodecs import numpy as np +import numpy.typing as npt import pytest from packaging.version import Version @@ -1349,6 +1350,125 @@ def test_scalar_array(value: Any, zarr_format: ZarrFormat) -> None: assert isinstance(arr[()], NDArrayLikeOrScalar) +@pytest.mark.parametrize("store", ["local"], indirect=True) +@pytest.mark.parametrize("store2", ["local"], indirect=["store2"]) +@pytest.mark.parametrize("src_format", [2, 3]) +@pytest.mark.parametrize("new_format", [2, 3, None]) +async def test_creation_from_other_zarr_format( + store: Store, + store2: Store, + src_format: ZarrFormat, + new_format: ZarrFormat | None, +) -> None: + if src_format == 2: + src = zarr.create( + (50, 50), chunks=(10, 10), store=store, zarr_format=src_format, dimension_separator="/" + ) + else: + src = zarr.create( + (50, 50), + chunks=(10, 10), + store=store, + zarr_format=src_format, + chunk_key_encoding=("default", "."), + ) + + src[:] = np.arange(50 * 50).reshape((50, 50)) + result = zarr.from_array( + store=store2, + data=src, + zarr_format=new_format, + ) + np.testing.assert_array_equal(result[:], src[:]) + assert result.fill_value == src.fill_value + assert result.dtype == src.dtype + assert result.chunks == src.chunks + expected_format = src_format if new_format is None else new_format + assert result.metadata.zarr_format == expected_format + if src_format == new_format: + assert result.metadata == src.metadata + + result2 = zarr.array( + data=src, + store=store2, + overwrite=True, + zarr_format=new_format, + ) + np.testing.assert_array_equal(result2[:], src[:]) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) +@pytest.mark.parametrize("store2", ["local", "memory", "zip"], indirect=["store2"]) +@pytest.mark.parametrize("src_chunks", [(40, 10), (11, 50)]) +@pytest.mark.parametrize("new_chunks", [(40, 10), (11, 50)]) +async def test_from_array( + store: Store, + store2: Store, + src_chunks: tuple[int, int], + new_chunks: tuple[int, int], + zarr_format: ZarrFormat, +) -> None: + src_fill_value = 2 + src_dtype = np.dtype("uint8") + src_attributes = None + + src = zarr.create( + (100, 10), + chunks=src_chunks, + dtype=src_dtype, + store=store, + fill_value=src_fill_value, + attributes=src_attributes, + ) + src[:] = np.arange(1000).reshape((100, 10)) + + new_fill_value = 3 + new_attributes: dict[str, JSON] = {"foo": "bar"} + + result = zarr.from_array( + data=src, + store=store2, + chunks=new_chunks, + fill_value=new_fill_value, + attributes=new_attributes, + ) + + np.testing.assert_array_equal(result[:], src[:]) + assert result.fill_value == new_fill_value + assert result.dtype == src_dtype + assert result.attrs == new_attributes + assert result.chunks == new_chunks + + +@pytest.mark.parametrize("store", ["local"], indirect=True) +@pytest.mark.parametrize("chunks", ["keep", "auto"]) +@pytest.mark.parametrize("write_data", [True, False]) +@pytest.mark.parametrize( + "src", + [ + np.arange(1000).reshape(10, 10, 10), + zarr.ones((10, 10, 10)), + 5, + [1, 2, 3], + [[1, 2, 3], [4, 5, 6]], + ], +) # add other npt.ArrayLike? +async def test_from_array_arraylike( + store: Store, + chunks: Literal["auto", "keep"] | tuple[int, int], + write_data: bool, + src: Array | npt.ArrayLike, +) -> None: + fill_value = 42 + result = zarr.from_array( + store, data=src, chunks=chunks, write_data=write_data, fill_value=fill_value + ) + if write_data: + np.testing.assert_array_equal(result[...], np.array(src)) + else: + np.testing.assert_array_equal(result[...], np.full_like(src, fill_value)) + + async def test_orthogonal_set_total_slice() -> None: """Ensure that a whole chunk overwrite does not read chunks""" store = MemoryStore() diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 77363acff3..b1707c88a3 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -1902,7 +1902,7 @@ def test_iter_grid( """ Test that iter_grid works as expected for 1, 2, and 3 dimensions. """ - grid_shape = (5,) * ndim + grid_shape = (10, 5, 7)[:ndim] if origin_0d is not None: origin_kwarg = origin_0d * ndim @@ -1984,3 +1984,13 @@ def test_vectorized_indexing_incompatible_shape(store) -> None: ) with pytest.raises(ValueError, match="Attempting to set"): arr[np.array([1, 2]), np.array([1, 2])] = np.array([[-1, -2], [-3, -4]]) + + +def test_iter_chunk_regions(): + chunks = (2, 3) + a = zarr.create((10, 10), chunks=chunks) + a[:] = 1 + for region in a._iter_chunk_regions(): + assert_array_equal(a[region], np.ones_like(a[region])) + a[region] = 0 + assert_array_equal(a[region], np.zeros_like(a[region])) From 5f49d24eb7ff1a463d954bdb5b3d14330b86cda5 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 16 Apr 2025 10:25:57 -0400 Subject: [PATCH 094/160] Fix hatch matrix setup for minimal and optional dependencies (#2872) * Run minimal tests without fsspec, requests, aiohttp * Retain existing test env names * Use importorskip * Specify which matrix config to upload codecov on * Remove redundant gpu env * Add obstore to min_deps definition * Fix optional dependency set * Add remote_tests set to doctest --- .github/workflows/test.yml | 1 + pyproject.toml | 33 ++++++++++++++------------------- tests/test_store/test_core.py | 2 ++ 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d82a95662..4160fa3506 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,6 +64,7 @@ jobs: run: | hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage - name: Upload coverage + if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 1ee5678f82..0b351c3b27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,18 +73,19 @@ gpu = [ test = [ "coverage", "pytest", + "pytest-asyncio", "pytest-cov", + "pytest-accept", + "rich", + "mypy", + "hypothesis", +] +remote_tests = [ 'zarr[remote]', "botocore", "s3fs", "moto[s3,server]", - "pytest-asyncio", - "pytest-accept", "requests", - "rich", - "mypy", - "hypothesis", - "universal-pathlib", ] optional = ["rich", "universal-pathlib"] docs = [ @@ -143,28 +144,21 @@ hooks.vcs.version-file = "src/zarr/_version.py" [tool.hatch.envs.test] dependencies = [ "numpy~={matrix:numpy}", - "universal_pathlib", ] features = ["test"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] numpy = ["1.25", "2.1"] -version = ["minimal"] - -[[tool.hatch.envs.test.matrix]] -python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] -features = ["optional"] +deps = ["minimal", "optional"] -[[tool.hatch.envs.test.matrix]] -python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] -features = ["gpu"] +[tool.hatch.envs.test.overrides] +matrix.deps.dependencies = [ + {value = "zarr[remote, remote_tests, test, optional]", if = ["optional"]} +] [tool.hatch.envs.test.scripts] run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" -run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" run = "run-coverage --no-cov" run-pytest = "run" @@ -174,7 +168,7 @@ run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tes list-env = "pip list" [tool.hatch.envs.doctest] -features = ["test", "optional", "remote"] +features = ["test", "optional", "remote", "remote_tests"] description = "Test environment for doctests" [tool.hatch.envs.doctest.scripts] @@ -255,6 +249,7 @@ dependencies = [ 'universal_pathlib==0.0.22', 'typing_extensions==4.9.*', 'donfig==0.8.*', + 'obstore==0.5.*', # test deps 'zarr[test]', ] diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index bce582a746..87d0e6e40d 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -121,6 +121,8 @@ async def test_make_store_path_invalid() -> None: async def test_make_store_path_fsspec(monkeypatch) -> None: pytest.importorskip("fsspec") + pytest.importorskip("requests") + pytest.importorskip("aiohttp") store_path = await make_store_path("http://foo.com/bar") assert isinstance(store_path.store, FsspecStore) From f9b5a3bf47e981c58a66222a15bd0df9413b8662 Mon Sep 17 00:00:00 2001 From: Christine Smit Date: Thu, 17 Apr 2025 19:01:37 -0400 Subject: [PATCH 095/160] =?UTF-8?q?updated=20migration=20documention=20to?= =?UTF-8?q?=20say=20that=20'.'=20syntax=20is=20no=20longer=20all=E2=80=A6?= =?UTF-8?q?=20(#2997)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updated migration documention to say that '.' syntax is no longer allowed for variables in groups * fixed spelling error --- changes/2991.doc.rst | 1 + docs/user-guide/v3_migration.rst | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changes/2991.doc.rst diff --git a/changes/2991.doc.rst b/changes/2991.doc.rst new file mode 100644 index 0000000000..828cfcdb2f --- /dev/null +++ b/changes/2991.doc.rst @@ -0,0 +1 @@ +Updated the 3.0 migration guide to include the removal of "." syntax for getting group members. diff --git a/docs/user-guide/v3_migration.rst b/docs/user-guide/v3_migration.rst index 678a3aafb7..a6258534e4 100644 --- a/docs/user-guide/v3_migration.rst +++ b/docs/user-guide/v3_migration.rst @@ -117,6 +117,8 @@ The Group class - Use :func:`zarr.Group.create_array` in place of :func:`zarr.Group.create_dataset` - Use :func:`zarr.Group.require_array` in place of :func:`zarr.Group.require_dataset` +3. Disallow "." syntax for getting group members. To get a member of a group named ``foo``, + use ``group["foo"]`` in place of ``group.foo``. The Store class ~~~~~~~~~~~~~~~ From 84d3284664b0328ce650f679dbdfc51a7e2fad0b Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Fri, 18 Apr 2025 10:51:21 -0400 Subject: [PATCH 096/160] Fix nan encoding in consolidated metadata (#2996) * Fix nan encoding in consolidated metadata Co-authored-by: Davis Bennett --- changes/2996.bugfix.rst | 4 +++ src/zarr/core/group.py | 8 +++--- tests/test_metadata/test_consolidated.py | 34 ++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 changes/2996.bugfix.rst diff --git a/changes/2996.bugfix.rst b/changes/2996.bugfix.rst new file mode 100644 index 0000000000..977dc79d0b --- /dev/null +++ b/changes/2996.bugfix.rst @@ -0,0 +1,4 @@ +Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be +consistent with the behavior of `ArrayMetadata`. + + diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index da2aa5f754..925252ccf0 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -49,7 +49,7 @@ ) from zarr.core.config import config from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata -from zarr.core.metadata.v3 import V3JsonEncoder +from zarr.core.metadata.v3 import V3JsonEncoder, _replace_special_floats from zarr.core.sync import SyncMixin, sync from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError from zarr.storage import StoreLike, StorePath @@ -334,7 +334,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: if self.zarr_format == 3: return { ZARR_JSON: prototype.buffer.from_bytes( - json.dumps(self.to_dict(), cls=V3JsonEncoder).encode() + json.dumps(_replace_special_floats(self.to_dict()), cls=V3JsonEncoder).encode() ) } else: @@ -355,10 +355,10 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: assert isinstance(consolidated_metadata, dict) for k, v in consolidated_metadata.items(): attrs = v.pop("attributes", None) - d[f"{k}/{ZATTRS_JSON}"] = attrs + d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats(attrs) if "shape" in v: # it's an array - d[f"{k}/{ZARRAY_JSON}"] = v + d[f"{k}/{ZARRAY_JSON}"] = _replace_special_floats(v) else: d[f"{k}/{ZGROUP_JSON}"] = { "zarr_format": self.zarr_format, diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index c1ff2e130a..a179982e94 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -573,3 +573,37 @@ async def test_use_consolidated_false( assert len([x async for x in good.members()]) == 2 assert good.metadata.consolidated_metadata assert sorted(good.metadata.consolidated_metadata.metadata) == ["a", "b"] + + +@pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf]) +async def test_consolidated_metadata_encodes_special_chars( + memory_store: Store, zarr_format: ZarrFormat, fill_value: float +): + root = await group(store=memory_store, zarr_format=zarr_format) + _child = await root.create_group("child", attributes={"test": fill_value}) + _time = await root.create_array("time", shape=(12,), dtype=np.float64, fill_value=fill_value) + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + root = await group(store=memory_store, zarr_format=zarr_format) + root_buffer = root.metadata.to_buffer_dict(default_buffer_prototype()) + + if zarr_format == 2: + root_metadata = json.loads(root_buffer[".zmetadata"].to_bytes().decode("utf-8"))["metadata"] + elif zarr_format == 3: + root_metadata = json.loads(root_buffer["zarr.json"].to_bytes().decode("utf-8"))[ + "consolidated_metadata" + ]["metadata"] + + if np.isnan(fill_value): + expected_fill_value = "NaN" + elif np.isneginf(fill_value): + expected_fill_value = "-Infinity" + elif np.isinf(fill_value): + expected_fill_value = "Infinity" + + if zarr_format == 2: + assert root_metadata["child/.zattrs"]["test"] == expected_fill_value + assert root_metadata["time/.zarray"]["fill_value"] == expected_fill_value + elif zarr_format == 3: + assert root_metadata["child"]["attributes"]["test"] == expected_fill_value + assert root_metadata["time"]["fill_value"] == expected_fill_value From bb55f0c58320a6d27be3a0ba918feee398a53db4 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 19 Apr 2025 10:05:28 -0400 Subject: [PATCH 097/160] Use unsigned bytes to back Buffer (#2738) This makes compressors consistent with v2, and buffers consistents with `bytes` types. Fixes #2735 Co-authored-by: Davis Bennett Co-authored-by: David Stansby Co-authored-by: jakirkham --- src/zarr/codecs/bytes.py | 2 +- src/zarr/codecs/crc32c_.py | 2 +- src/zarr/core/buffer/core.py | 4 ++-- src/zarr/core/buffer/cpu.py | 8 ++++---- src/zarr/core/buffer/gpu.py | 10 +++++----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 78c7b22fbc..750707d36a 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -114,7 +114,7 @@ async def _encode_single( nd_array = chunk_array.as_ndarray_like() # Flatten the nd-array (only copy if needed) and reinterpret as bytes - nd_array = nd_array.ravel().view(dtype="b") + nd_array = nd_array.ravel().view(dtype="B") return chunk_spec.prototype.buffer.from_array_like(nd_array) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index 3a6624ad25..ab8a57eba7 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -57,7 +57,7 @@ async def _encode_single( # Calculate the checksum and "cast" it to a numpy array checksum = np.array([crc32c(cast(typing_extensions.Buffer, data))], dtype=np.uint32) # Append the checksum (as bytes) to the data - return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("b"))) + return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B"))) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length + 4 diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 70d408d32b..1318f868a0 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -143,7 +143,7 @@ class Buffer(ABC): def __init__(self, array_like: ArrayLike) -> None: if array_like.ndim != 1: raise ValueError("array_like: only 1-dim allowed") - if array_like.dtype != np.dtype("b"): + if array_like.dtype != np.dtype("B"): raise ValueError("array_like: only byte dtype allowed") self._data = array_like @@ -306,7 +306,7 @@ class NDBuffer: Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer - is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 225adb6f5c..8464518818 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -49,7 +49,7 @@ def __init__(self, array_like: ArrayLike) -> None: @classmethod def create_zero_length(cls) -> Self: - return cls(np.array([], dtype="b")) + return cls(np.array([], dtype="B")) @classmethod def from_buffer(cls, buffer: core.Buffer) -> Self: @@ -92,7 +92,7 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self: ------- New buffer representing `bytes_like` """ - return cls.from_array_like(np.frombuffer(bytes_like, dtype="b")) + return cls.from_array_like(np.frombuffer(bytes_like, dtype="B")) def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). @@ -111,7 +111,7 @@ def __add__(self, other: core.Buffer) -> Self: """Concatenate two buffers""" other_array = other.as_array_like() - assert other_array.dtype == np.dtype("b") + assert other_array.dtype == np.dtype("B") return self.__class__( np.concatenate((np.asanyarray(self._data), np.asanyarray(other_array))) ) @@ -131,7 +131,7 @@ class NDBuffer(core.NDBuffer): Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer - is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index aac6792cff..77d2731c71 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -59,7 +59,7 @@ def __init__(self, array_like: ArrayLike) -> None: if array_like.ndim != 1: raise ValueError("array_like: only 1-dim allowed") - if array_like.dtype != np.dtype("b"): + if array_like.dtype != np.dtype("B"): raise ValueError("array_like: only byte dtype allowed") if not hasattr(array_like, "__cuda_array_interface__"): @@ -84,7 +84,7 @@ def create_zero_length(cls) -> Self: ------- New empty 0-length buffer """ - return cls(cp.array([], dtype="b")) + return cls(cp.array([], dtype="B")) @classmethod def from_buffer(cls, buffer: core.Buffer) -> Self: @@ -100,14 +100,14 @@ def from_buffer(cls, buffer: core.Buffer) -> Self: @classmethod def from_bytes(cls, bytes_like: BytesLike) -> Self: - return cls.from_array_like(cp.frombuffer(bytes_like, dtype="b")) + return cls.from_array_like(cp.frombuffer(bytes_like, dtype="B")) def as_numpy_array(self) -> npt.NDArray[Any]: return cast(npt.NDArray[Any], cp.asnumpy(self._data)) def __add__(self, other: core.Buffer) -> Self: other_array = other.as_array_like() - assert other_array.dtype == np.dtype("b") + assert other_array.dtype == np.dtype("B") gpu_other = Buffer(other_array) gpu_other_array = gpu_other.as_array_like() return self.__class__( @@ -129,7 +129,7 @@ class NDBuffer(core.NDBuffer): Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer - is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. From a0761ac5f2ce36f630a8c63a0505d5eb6d998311 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 21 Apr 2025 07:47:03 -0400 Subject: [PATCH 098/160] fix: Special-case suffix requests in obstore backend to support Azure (#2994) * Special-case suffix requests in obstore backend Co-authored-by: Davis Bennett --- src/zarr/storage/_obstore.py | 92 +++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index 79afa08d15..4381acb2ae 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -106,10 +106,25 @@ async def get( ) return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] elif isinstance(byte_range, SuffixByteRequest): - resp = await obs.get_async( - self.store, key, options={"range": {"suffix": byte_range.suffix}} - ) - return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + # some object stores (Azure) don't support suffix requests. In this + # case, our workaround is to first get the length of the object and then + # manually request the byte range at the end. + try: + resp = await obs.get_async( + self.store, key, options={"range": {"suffix": byte_range.suffix}} + ) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + except obs.exceptions.NotSupportedError: + head_resp = await obs.head_async(self.store, key) + file_size = head_resp["size"] + suffix_len = byte_range.suffix + buffer = await obs.get_range_async( + self.store, + key, + start=file_size - suffix_len, + length=suffix_len, + ) + return prototype.buffer.from_bytes(buffer) # type: ignore[arg-type] else: raise ValueError(f"Unexpected byte_range, got {byte_range}") except _ALLOWED_EXCEPTIONS: @@ -265,10 +280,29 @@ class _OtherRequest(TypedDict): path: str """The path to request from.""" - range: OffsetRange | SuffixRange | None + range: OffsetRange | None + # Note: suffix requests are handled separately because some object stores (Azure) + # don't support them """The range request type.""" +class _SuffixRequest(TypedDict): + """Offset or suffix range requests. + + These requests cannot be concurrent on the Rust side, and each need their own call + to `obstore.get_async`, passing in the `range` parameter. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + path: str + """The path to request from.""" + + range: SuffixRange + """The suffix range.""" + + class _Response(TypedDict): """A response buffer associated with the original index that it should be restored to.""" @@ -317,7 +351,7 @@ async def _make_other_request( prototype: BufferPrototype, semaphore: asyncio.Semaphore, ) -> list[_Response]: - """Make suffix or offset requests. + """Make offset or full-file requests. We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all futures can be gathered together. @@ -339,6 +373,46 @@ async def _make_other_request( ] +async def _make_suffix_request( + store: _UpstreamObjectStore, + request: _SuffixRequest, + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make suffix requests. + + This is separated out from `_make_other_request` because some object stores (Azure) + don't support suffix requests. In this case, our workaround is to first get the + length of the object and then manually request the byte range at the end. + + We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all + futures can be gathered together. + """ + import obstore as obs + + async with semaphore: + try: + resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) + buffer = await resp.bytes_async() + except obs.exceptions.NotSupportedError: + head_resp = await obs.head_async(store, request["path"]) + file_size = head_resp["size"] + suffix_len = request["range"]["suffix"] + buffer = await obs.get_range_async( + store, + request["path"], + start=file_size - suffix_len, + length=suffix_len, + ) + + return [ + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] + } + ] + + async def _get_partial_values( store: _UpstreamObjectStore, prototype: BufferPrototype, @@ -358,6 +432,7 @@ async def _get_partial_values( key_ranges = list(key_ranges) per_file_bounded_requests: dict[str, list[_BoundedRequest]] = defaultdict(list) other_requests: list[_OtherRequest] = [] + suffix_requests: list[_SuffixRequest] = [] for idx, (path, byte_range) in enumerate(key_ranges): if byte_range is None: @@ -381,7 +456,7 @@ async def _get_partial_values( } ) elif isinstance(byte_range, SuffixByteRequest): - other_requests.append( + suffix_requests.append( { "original_request_index": idx, "path": path, @@ -402,6 +477,9 @@ async def _get_partial_values( for request in other_requests: futs.append(_make_other_request(store, request, prototype, semaphore=semaphore)) # noqa: PERF401 + for suffix_request in suffix_requests: + futs.append(_make_suffix_request(store, suffix_request, prototype, semaphore=semaphore)) # noqa: PERF401 + buffers: list[Buffer | None] = [None] * len(key_ranges) for responses in await asyncio.gather(*futs): From 45186538b0525a5c3bf5a4420ae0f27adf37c050 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 21 Apr 2025 18:18:54 +0100 Subject: [PATCH 099/160] More consistent store docstrings (#2976) Co-authored-by: Davis Bennett --- src/zarr/storage/_fsspec.py | 2 +- src/zarr/storage/_local.py | 2 +- src/zarr/storage/_logging.py | 2 +- src/zarr/storage/_memory.py | 8 +++++--- src/zarr/storage/_obstore.py | 3 ++- src/zarr/storage/_wrapper.py | 3 ++- src/zarr/storage/_zip.py | 2 +- 7 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index a4730a93d9..40f1b2fbc0 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -32,7 +32,7 @@ class FsspecStore(Store): """ - A remote Store based on FSSpec + Store for remote data based on FSSpec. Parameters ---------- diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index bd5bfc1da2..85d244f17b 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -67,7 +67,7 @@ def _put( class LocalStore(Store): """ - Local file system store. + Store for the local file system. Parameters ---------- diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py index 5f1a97acd9..a2164a418f 100644 --- a/src/zarr/storage/_logging.py +++ b/src/zarr/storage/_logging.py @@ -24,7 +24,7 @@ class LoggingStore(WrapperStore[T_Store]): """ - Store wrapper that logs all calls to the wrapped store. + Store that logs all calls to another wrapped store. Parameters ---------- diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index b37fc8d5c9..ea25f82a3b 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -19,7 +19,7 @@ class MemoryStore(Store): """ - In-memory store. + Store for local memory. Parameters ---------- @@ -173,8 +173,10 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: class GpuMemoryStore(MemoryStore): - """A GPU only memory store that stores every chunk in GPU memory irrespective - of the original location. + """ + Store for GPU memory. + + Stores every chunk in GPU memory irrespective of the original location. The dictionary of buffers to initialize this memory store with *must* be GPU Buffers. diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index 4381acb2ae..8c2469747d 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -37,7 +37,8 @@ class ObjectStore(Store): - """A Zarr store that uses obstore for fast read/write from AWS, GCP, Azure. + """ + Store that uses obstore for fast read/write from AWS, GCP, Azure. Parameters ---------- diff --git a/src/zarr/storage/_wrapper.py b/src/zarr/storage/_wrapper.py index 349048e495..f21d378191 100644 --- a/src/zarr/storage/_wrapper.py +++ b/src/zarr/storage/_wrapper.py @@ -18,7 +18,8 @@ class WrapperStore(Store, Generic[T_Store]): """ - A store class that wraps an existing ``Store`` instance. + Store that wraps an existing Store. + By default all of the store methods are delegated to the wrapped store instance, which is accessible via the ``._store`` attribute of this class. diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index bbfe6c67aa..f9eb8d8808 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -24,7 +24,7 @@ class ZipStore(Store): """ - Storage class using a ZIP file. + Store using a ZIP file. Parameters ---------- From cf879eb78cfaebe85c5cc42fd2bd0daab898b023 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 22 Apr 2025 23:18:26 +0100 Subject: [PATCH 100/160] Improve array and group docstringspc (#2975) --- src/zarr/core/array.py | 4 +++- src/zarr/core/group.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index f2c88c508b..62efe44e4c 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1711,7 +1711,9 @@ def _info( # TODO: Array can be a frozen data class again once property setters (e.g. shape) are removed @dataclass(frozen=False) class Array: - """Instantiate an array from an initialized store.""" + """ + A Zarr array. + """ _async_array: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 925252ccf0..3f8dad1740 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1744,6 +1744,10 @@ async def move(self, source: str, dest: str) -> None: @dataclass(frozen=True) class Group(SyncMixin): + """ + A Zarr group. + """ + _async_group: AsyncGroup @classmethod From 630897b7ba55a0c6d0aa376b86399a55ef8088b5 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 23 Apr 2025 16:51:30 +0200 Subject: [PATCH 101/160] 3.0.7 release notes (#3008) * create 3.0.7 release notes * describe misc changes * add pr link for 2997 --- changes/1661.feature.rst | 1 - changes/2622.feature.rst | 1 - changes/2714.misc.rst | 1 - changes/2718.bugfix.rst | 3 --- changes/2802.fix.rst | 1 - changes/2924.chore.rst | 2 -- changes/2944.misc.rst | 1 - changes/2991.doc.rst | 1 - changes/2996.bugfix.rst | 4 ---- docs/release-notes.rst | 36 ++++++++++++++++++++++++++++++++++++ 10 files changed, 36 insertions(+), 15 deletions(-) delete mode 100644 changes/1661.feature.rst delete mode 100644 changes/2622.feature.rst delete mode 100644 changes/2714.misc.rst delete mode 100644 changes/2718.bugfix.rst delete mode 100644 changes/2802.fix.rst delete mode 100644 changes/2924.chore.rst delete mode 100644 changes/2944.misc.rst delete mode 100644 changes/2991.doc.rst delete mode 100644 changes/2996.bugfix.rst diff --git a/changes/1661.feature.rst b/changes/1661.feature.rst deleted file mode 100644 index 38d60b23c1..0000000000 --- a/changes/1661.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add experimental ObjectStore storage class based on obstore. \ No newline at end of file diff --git a/changes/2622.feature.rst b/changes/2622.feature.rst deleted file mode 100644 index f5c7cbe192..0000000000 --- a/changes/2622.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``zarr.from_array`` using concurrent streaming of source data \ No newline at end of file diff --git a/changes/2714.misc.rst b/changes/2714.misc.rst deleted file mode 100644 index 9ab55089d2..0000000000 --- a/changes/2714.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Make warning filters in the tests more specific, so warnings emitted by tests added in the future are more likely to be caught instead of ignored. diff --git a/changes/2718.bugfix.rst b/changes/2718.bugfix.rst deleted file mode 100644 index 48ddf8b5a8..0000000000 --- a/changes/2718.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -0-dimensional arrays are now returning a scalar. Therefore, the return type of ``__getitem__`` changed -to NDArrayLikeOrScalar. This change is to make the behavior of 0-dimensional arrays consistent with -``numpy`` scalars. \ No newline at end of file diff --git a/changes/2802.fix.rst b/changes/2802.fix.rst deleted file mode 100644 index 471ddf66f4..0000000000 --- a/changes/2802.fix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix `fill_value` serialization for `NaN` in `ArrayV2Metadata` and add property-based testing of round-trip serialization \ No newline at end of file diff --git a/changes/2924.chore.rst b/changes/2924.chore.rst deleted file mode 100644 index 7bfbb2e1c7..0000000000 --- a/changes/2924.chore.rst +++ /dev/null @@ -1,2 +0,0 @@ -Define a new versioning policy based on Effective Effort Versioning. This replaces the old -Semantic Versioning-based policy. \ No newline at end of file diff --git a/changes/2944.misc.rst b/changes/2944.misc.rst deleted file mode 100644 index 48356a1fef..0000000000 --- a/changes/2944.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid an unnecessary memory copy when writing Zarr to a local file diff --git a/changes/2991.doc.rst b/changes/2991.doc.rst deleted file mode 100644 index 828cfcdb2f..0000000000 --- a/changes/2991.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Updated the 3.0 migration guide to include the removal of "." syntax for getting group members. diff --git a/changes/2996.bugfix.rst b/changes/2996.bugfix.rst deleted file mode 100644 index 977dc79d0b..0000000000 --- a/changes/2996.bugfix.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be -consistent with the behavior of `ArrayMetadata`. - - diff --git a/docs/release-notes.rst b/docs/release-notes.rst index c585e4f0d3..341a32c364 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -3,6 +3,42 @@ Release notes .. towncrier release notes start +3.0.7 (2025-04-22) +------------------ + +Features +~~~~~~~~ + +- Add experimental ObjectStore storage class based on obstore. (:issue:`1661`) +- Add ``zarr.from_array`` using concurrent streaming of source data (:issue:`2622`) + + +Bugfixes +~~~~~~~~ + +- 0-dimensional arrays are now returning a scalar. Therefore, the return type of ``__getitem__`` changed + to NDArrayLikeOrScalar. This change is to make the behavior of 0-dimensional arrays consistent with + ``numpy`` scalars. (:issue:`2718`) +- Fix `fill_value` serialization for `NaN` in `ArrayV2Metadata` and add property-based testing of round-trip serialization (:issue:`2802`) +- Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be + consistent with the behavior of `ArrayMetadata`. (:issue:`2996`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Updated the 3.0 migration guide to include the removal of "." syntax for getting group members. (:issue:`2991`, :issue:`2997`) + + +Misc +~~~~ +- Define a new versioning policy based on Effective Effort Versioning. This replaces the old Semantic + Versioning-based policy. (:issue:`2924`, :issue:`2910`) +- Make warning filters in the tests more specific, so warnings emitted by tests added in the future + are more likely to be caught instead of ignored. (:issue:`2714`) +- Avoid an unnecessary memory copy when writing Zarr to a local file (:issue:`2944`) + + 3.0.6 (2025-03-20) ------------------ From 5f4aeb457072a503d92e6c63c6b66f920cb91611 Mon Sep 17 00:00:00 2001 From: Altay Sansal Date: Thu, 24 Apr 2025 11:26:31 -0500 Subject: [PATCH 102/160] remove debug print statement (#3007) --- src/zarr/core/metadata/v2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 11f14b37aa..d19193963f 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -294,7 +294,6 @@ def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: def _parse_structured_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: """Handle structured dtype/fill value pairs""" - print("FILL VALUE", fill_value, "DT", dtype) try: if isinstance(fill_value, list): return np.array([tuple(fill_value)], dtype=dtype)[0] From 0351c4e524e60b849dbaca2c00fabee20e882f46 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 25 Apr 2025 08:27:45 -0600 Subject: [PATCH 103/160] hypothesis: Don't generate node name: 'zarr.json' (#3020) --- src/zarr/testing/strategies.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index f2dc38483a..663d46034d 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -96,11 +96,15 @@ def clear_store(x: Store) -> Store: zarr_key_chars = st.sampled_from( ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ) -node_names = st.text(zarr_key_chars, min_size=1).filter( - lambda t: t not in (".", "..") and not t.startswith("__") +node_names = ( + st.text(zarr_key_chars, min_size=1) + .filter(lambda t: t not in (".", "..") and not t.startswith("__")) + .filter(lambda name: name.lower() != "zarr.json") ) -short_node_names = st.text(zarr_key_chars, max_size=3, min_size=1).filter( - lambda t: t not in (".", "..") and not t.startswith("__") +short_node_names = ( + st.text(zarr_key_chars, max_size=3, min_size=1) + .filter(lambda t: t not in (".", "..") and not t.startswith("__")) + .filter(lambda name: name.lower() != "zarr.json") ) array_names = node_names attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) From 0c7677890d497918ae11d7a633c5fac93781f211 Mon Sep 17 00:00:00 2001 From: Ilan Gold Date: Wed, 30 Apr 2025 18:10:38 +0200 Subject: [PATCH 104/160] (fix): structured dtype fill value consolidated metadata (#3015) * (fix): structured dtype consolidated metadata fill value * (chore): relnote * (chore): test * (fix): more robust testing --------- Co-authored-by: Davis Bennett --- changes/2998.bugfix.md | 1 + src/zarr/core/group.py | 9 ++++++++- tests/test_metadata/test_v2.py | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 changes/2998.bugfix.md diff --git a/changes/2998.bugfix.md b/changes/2998.bugfix.md new file mode 100644 index 0000000000..7b94223122 --- /dev/null +++ b/changes/2998.bugfix.md @@ -0,0 +1 @@ +Fix structured `dtype` fill value serialization for consolidated metadata \ No newline at end of file diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 3f8dad1740..3f4f15b9e9 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import base64 import itertools import json import logging @@ -358,7 +359,13 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats(attrs) if "shape" in v: # it's an array - d[f"{k}/{ZARRAY_JSON}"] = _replace_special_floats(v) + if isinstance(v.get("fill_value", None), np.void): + v["fill_value"] = base64.standard_b64encode( + cast(bytes, v["fill_value"]) + ).decode("ascii") + else: + v = _replace_special_floats(v) + d[f"{k}/{ZARRAY_JSON}"] = v else: d[f"{k}/{ZGROUP_JSON}"] = { "zarr_format": self.zarr_format, diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 4600a977d4..08b9cb2507 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -316,3 +316,28 @@ def test_zstd_checksum() -> None: arr.metadata.to_buffer_dict(default_buffer_prototype())[".zarray"].to_bytes() ) assert "checksum" not in metadata["compressor"] + + +@pytest.mark.parametrize( + "fill_value", [None, np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))] +) +def test_structured_dtype_fill_value_serialization(tmp_path, fill_value): + group_path = tmp_path / "test.zarr" + root_group = zarr.open_group(group_path, mode="w", zarr_format=2) + dtype = np.dtype([("foo", "i4"), ("bar", "i4")]) + root_group.create_array( + name="structured_dtype", + shape=(100, 100), + chunks=(100, 100), + dtype=dtype, + fill_value=fill_value, + ) + + zarr.consolidate_metadata(root_group.store, zarr_format=2) + root_group = zarr.open_group(group_path, mode="r") + assert ( + root_group.metadata.consolidated_metadata.to_dict()["metadata"]["structured_dtype"][ + "fill_value" + ] + == fill_value + ) From 36a1bac850c50fbaa1708f8986edd5766f5fd67b Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 30 Apr 2025 17:55:27 +0100 Subject: [PATCH 105/160] Fix specifying memory order in v2 arrays (#2951) * Fix specifying memory order in v2 arrays * Re-work test_order * Fix getting array order in v3 * Fix order of arrays for v2 * Fix order with V3 arrays * Fix mypy * Remove errant print() * Fix order with v3 arrays * Fix v2 test * Add numpy order parametrization * Add changelog entry --- changes/2950.bufgix.rst | 1 + src/zarr/api/asynchronous.py | 17 +++++++-------- src/zarr/core/array.py | 15 ++++++++++--- tests/test_api.py | 7 +++--- tests/test_array.py | 42 ++++++++++++++++++++---------------- tests/test_v2.py | 41 ++++++++++++++++++----------------- 6 files changed, 68 insertions(+), 55 deletions(-) create mode 100644 changes/2950.bufgix.rst diff --git a/changes/2950.bufgix.rst b/changes/2950.bufgix.rst new file mode 100644 index 0000000000..67cd61f377 --- /dev/null +++ b/changes/2950.bufgix.rst @@ -0,0 +1 @@ +Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 285d777258..9b8b43a517 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1040,15 +1040,13 @@ async def create( ) warnings.warn(UserWarning(msg), stacklevel=1) config_dict["write_empty_chunks"] = write_empty_chunks - if order is not None: - if config is not None: - msg = ( - "Both order and config keyword arguments are set. " - "This is redundant. When both are set, order will be ignored and " - "config will be used." - ) - warnings.warn(UserWarning(msg), stacklevel=1) - config_dict["order"] = order + if order is not None and config is not None: + msg = ( + "Both order and config keyword arguments are set. " + "This is redundant. When both are set, order will be ignored and " + "config will be used." + ) + warnings.warn(UserWarning(msg), stacklevel=1) config_parsed = ArrayConfig.from_dict(config_dict) @@ -1062,6 +1060,7 @@ async def create( overwrite=overwrite, filters=filters, dimension_separator=dimension_separator, + order=order, zarr_format=zarr_format, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 62efe44e4c..b0e8b03cd7 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -609,6 +609,7 @@ async def _create( if order is not None: _warn_order_kwarg() + config_parsed = replace(config_parsed, order=order) result = await cls._create_v3( store_path, @@ -1044,7 +1045,10 @@ def order(self) -> MemoryOrder: bool Memory order of the array """ - return self._config.order + if self.metadata.zarr_format == 2: + return self.metadata.order + else: + return self._config.order @property def attrs(self) -> dict[str, JSON]: @@ -1276,14 +1280,14 @@ async def _get_selection( out_buffer = prototype.nd_buffer.create( shape=indexer.shape, dtype=out_dtype, - order=self._config.order, + order=self.order, fill_value=self.metadata.fill_value, ) if product(indexer.shape) > 0: # need to use the order from the metadata for v2 _config = self._config if self.metadata.zarr_format == 2: - _config = replace(_config, order=self.metadata.order) + _config = replace(_config, order=self.order) # reading chunks and decoding them await self.codec_pipeline.read( @@ -4256,6 +4260,11 @@ async def init_array( chunks_out = chunk_shape_parsed codecs_out = sub_codecs + if config is None: + config = {} + if order is not None and isinstance(config, dict): + config["order"] = config.get("order", order) + meta = AsyncArray._create_metadata_v3( shape=shape_parsed, dtype=dtype_parsed, diff --git a/tests/test_api.py b/tests/test_api.py index f03fd53f7a..9f03a1067a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -326,13 +326,12 @@ def test_array_order(zarr_format: ZarrFormat) -> None: def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None: with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) - expected = order or zarr.config.get("array.order") - assert arr.order == expected + assert arr.order == order vals = np.asarray(arr) - if expected == "C": + if order == "C": assert vals.flags.c_contiguous - elif expected == "F": + elif order == "F": assert vals.flags.f_contiguous else: raise AssertionError diff --git a/tests/test_array.py b/tests/test_array.py index 5c3c556dfb..4be9bbde43 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1262,9 +1262,11 @@ async def test_data_ignored_params(store: Store) -> None: await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) @staticmethod - @pytest.mark.parametrize("order_config", ["C", "F", None]) + @pytest.mark.parametrize("order", ["C", "F", None]) + @pytest.mark.parametrize("with_config", [True, False]) def test_order( - order_config: MemoryOrder | None, + order: MemoryOrder | None, + with_config: bool, zarr_format: ZarrFormat, store: MemoryStore, ) -> None: @@ -1272,29 +1274,31 @@ def test_order( Test that the arrays generated by array indexing have a memory order defined by the config order value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly. """ - config: ArrayConfigLike = {} - if order_config is None: + config: ArrayConfigLike | None = {} + if order is None: config = {} expected = zarr.config.get("array.order") else: - config = {"order": order_config} - expected = order_config + config = {"order": order} + expected = order + + if not with_config: + # Test without passing config parameter + config = None + + arr = zarr.create_array( + store=store, + shape=(2, 2), + zarr_format=zarr_format, + dtype="i4", + order=order, + config=config, + ) + assert arr.order == expected if zarr_format == 2: - arr = zarr.create_array( - store=store, - shape=(2, 2), - zarr_format=zarr_format, - dtype="i4", - order=expected, - config=config, - ) - # guard for type checking assert arr.metadata.zarr_format == 2 assert arr.metadata.order == expected - else: - arr = zarr.create_array( - store=store, shape=(2, 2), zarr_format=zarr_format, dtype="i4", config=config - ) + vals = np.asarray(arr) if expected == "C": assert vals.flags.c_contiguous diff --git a/tests/test_v2.py b/tests/test_v2.py index 3a36bc01fd..8f0e1b2d29 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -195,12 +195,13 @@ def test_create_array_defaults(store: Store): ) -@pytest.mark.parametrize("array_order", ["C", "F"]) -@pytest.mark.parametrize("data_order", ["C", "F"]) -@pytest.mark.parametrize("memory_order", ["C", "F"]) -def test_v2_non_contiguous( - array_order: Literal["C", "F"], data_order: Literal["C", "F"], memory_order: Literal["C", "F"] -) -> None: +@pytest.mark.parametrize("numpy_order", ["C", "F"]) +@pytest.mark.parametrize("zarr_order", ["C", "F"]) +def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal["C", "F"]) -> None: + """ + Make sure zarr v2 arrays save data using the memory order given to the zarr array, + not the memory order of the original numpy array. + """ store = MemoryStore() arr = zarr.create_array( store, @@ -212,12 +213,11 @@ def test_v2_non_contiguous( filters=None, compressors=None, overwrite=True, - order=array_order, - config={"order": memory_order}, + order=zarr_order, ) - # Non-contiguous write - a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=data_order) + # Non-contiguous write, using numpy memory order + a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=numpy_order) arr[6:9, 3:6] = a[6:9, 3:6] # The slice on the RHS is important np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6]) @@ -225,13 +225,15 @@ def test_v2_non_contiguous( a[6:9, 3:6], np.frombuffer( sync(store.get("2.1", default_buffer_prototype())).to_bytes(), dtype="float64" - ).reshape((3, 3), order=array_order), + ).reshape((3, 3), order=zarr_order), ) - if memory_order == "F": + # After writing and reading from zarr array, order should be same as zarr order + if zarr_order == "F": assert (arr[6:9, 3:6]).flags.f_contiguous else: assert (arr[6:9, 3:6]).flags.c_contiguous + # Contiguous write store = MemoryStore() arr = zarr.create_array( store, @@ -243,18 +245,17 @@ def test_v2_non_contiguous( compressors=None, filters=None, overwrite=True, - order=array_order, - config={"order": memory_order}, + order=zarr_order, ) - # Contiguous write - a = np.arange(9).reshape((3, 3), order=data_order) - if data_order == "F": - assert a.flags.f_contiguous - else: - assert a.flags.c_contiguous + a = np.arange(9).reshape((3, 3), order=numpy_order) arr[6:9, 3:6] = a np.testing.assert_array_equal(arr[6:9, 3:6], a) + # After writing and reading from zarr array, order should be same as zarr order + if zarr_order == "F": + assert (arr[6:9, 3:6]).flags.f_contiguous + else: + assert (arr[6:9, 3:6]).flags.c_contiguous def test_default_compressor_deprecation_warning(): From 0b97e784788c7b4386fd295b4574bb5794dc0e37 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Fri, 2 May 2025 16:08:45 +0200 Subject: [PATCH 106/160] simplify NDBuffer.as_scalar (#3027) * index with an empty tuple to get scalar * changelog * add as_scalar test --- changes/3027.misc.rst | 1 + src/zarr/core/buffer/core.py | 11 +---------- tests/test_buffer.py | 6 ++++++ 3 files changed, 8 insertions(+), 10 deletions(-) create mode 100644 changes/3027.misc.rst diff --git a/changes/3027.misc.rst b/changes/3027.misc.rst new file mode 100644 index 0000000000..ffbfe9b808 --- /dev/null +++ b/changes/3027.misc.rst @@ -0,0 +1 @@ +Simplified scalar indexing of size-1 arrays. \ No newline at end of file diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 1318f868a0..cfcd7e6633 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -427,16 +427,7 @@ def as_scalar(self) -> ScalarType: """Returns the buffer as a scalar value""" if self._data.size != 1: raise ValueError("Buffer does not contain a single scalar value") - item = self.as_numpy_array().item() - scalar: ScalarType - - if np.issubdtype(self.dtype, np.datetime64): - unit: str = np.datetime_data(self.dtype)[0] # Extract the unit (e.g., 'Y', 'D', etc.) - scalar = np.datetime64(item, unit) - else: - scalar = self.dtype.type(item) # Regular conversion for non-datetime types - - return scalar + return cast(ScalarType, self.as_numpy_array()[()]) @property def dtype(self) -> np.dtype[Any]: diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 33ac0266eb..73b3a16677 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -155,3 +155,9 @@ def test_numpy_buffer_prototype() -> None: assert isinstance(ndbuffer.as_ndarray_like(), np.ndarray) with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): ndbuffer.as_scalar() + + +# TODO: the same test for other buffer classes +def test_cpu_buffer_as_scalar() -> None: + buf = cpu.buffer_prototype.nd_buffer.create(shape=(), dtype="int64") + assert buf.as_scalar() == buf.as_ndarray_like()[()] # type: ignore[index] From 213863ba0548647cba1ab1d53f8eaa3631f4e0d0 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Sat, 3 May 2025 23:15:26 +0200 Subject: [PATCH 107/160] Use a dictionary comprehension instead (#3029) --- tests/test_group.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_group.py b/tests/test_group.py index 1e4f31b5d6..b4dace2568 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1583,14 +1583,12 @@ async def test_create_hierarchy( sync_group.create_hierarchy(store=store, nodes=hierarchy_spec, overwrite=overwrite) ) elif impl == "async": - created = dict( - [ - a - async for a in create_hierarchy( - store=store, nodes=hierarchy_spec, overwrite=overwrite - ) - ] - ) + created = { + k: v + async for k, v in create_hierarchy( + store=store, nodes=hierarchy_spec, overwrite=overwrite + ) + } else: raise ValueError(f"Invalid impl: {impl}") if not overwrite: From c51150cba4666952978b563279503a9119bd1bc2 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 7 May 2025 19:11:19 +0100 Subject: [PATCH 108/160] Pin minimum s3fs (#3041) --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0b351c3b27..09615b6b22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ test = [ remote_tests = [ 'zarr[remote]', "botocore", - "s3fs", + "s3fs>=2023.10.0", "moto[s3,server]", "requests", ] @@ -104,7 +104,7 @@ docs = [ # Optional dependencies to run examples 'numcodecs[msgpack]', 'rich', - 's3fs', + 's3fs>=2023.10.0', 'astroid<4' ] From 693324c19d00dd013aa614643c6513ea20740950 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 7 May 2025 19:37:47 +0100 Subject: [PATCH 109/160] Fix some mypy errors (#3044) * Clean mypy config * Fix typing errors in entrypoint test package * Fix test_transpose typing errors * Fix typing errors in test_config --------- Co-authored-by: Davis Bennett --- pyproject.toml | 14 +++++++------- tests/package_with_entrypoint/__init__.py | 23 ++++++++++++----------- tests/test_codecs/test_transpose.py | 3 ++- tests/test_config.py | 14 +++++++------- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 09615b6b22..9244a9ec0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -348,27 +348,27 @@ python_version = "3.11" ignore_missing_imports = true namespace_packages = false - strict = true warn_unreachable = true - enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] + [[tool.mypy.overrides]] module = [ - "zarr.v2.*", + "tests.package_with_entrypoint.*", + "tests.test_codecs.test_transpose", + "tests.test_config" ] -ignore_errors = true +strict = false +# TODO: Move the next modules up to the strict = false section +# and fix the errors [[tool.mypy.overrides]] module = [ "zarr.testing.stateful", # lots of hypothesis decorator errors - "tests.package_with_entrypoint.*", "tests.test_codecs.test_codecs", - "tests.test_codecs.test_transpose", "tests.test_metadata.*", "tests.test_store.*", - "tests.test_config", "tests.test_group", "tests.test_indexing", "tests.test_properties", diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py index b818adf8ea..cfbd4f23a9 100644 --- a/tests/package_with_entrypoint/__init__.py +++ b/tests/package_with_entrypoint/__init__.py @@ -1,13 +1,14 @@ from collections.abc import Iterable +from typing import Any -from numpy import ndarray +import numpy as np +import numpy.typing as npt import zarr.core.buffer -from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecOutput, CodecPipeline +from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline from zarr.codecs import BytesCodec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer -from zarr.core.common import BytesLike class TestEntrypointCodec(ArrayBytesCodec): @@ -16,14 +17,14 @@ class TestEntrypointCodec(ArrayBytesCodec): async def encode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], - ) -> Iterable[CodecOutput | None]: - pass + ) -> Iterable[Buffer | None]: + return [None] async def decode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], - ) -> ndarray: - pass + ) -> npt.NDArray[Any]: + return np.array(1) def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: return input_byte_length @@ -35,13 +36,13 @@ def __init__(self, batch_size: int = 1) -> None: async def encode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] - ) -> BytesLike: - pass + ) -> Iterable[Buffer | None]: + return [None] async def decode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] - ) -> ndarray: - pass + ) -> Iterable[NDBuffer | None]: + return np.array(1) class TestEntrypointBuffer(Buffer): diff --git a/tests/test_codecs/test_transpose.py b/tests/test_codecs/test_transpose.py index 18ea8e65d0..06ec668ad3 100644 --- a/tests/test_codecs/test_transpose.py +++ b/tests/test_codecs/test_transpose.py @@ -48,6 +48,7 @@ async def test_transpose( read_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, read_data) + assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] @@ -90,5 +91,5 @@ def test_transpose_invalid( dtype=data.dtype, fill_value=0, chunk_key_encoding={"name": "v2", "separator": "."}, - filters=[TransposeCodec(order=order)], + filters=[TransposeCodec(order=order)], # type: ignore[arg-type] ) diff --git a/tests/test_config.py b/tests/test_config.py index 1a2453d646..2cbf172752 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -10,7 +10,7 @@ import zarr import zarr.api from zarr import zeros -from zarr.abc.codec import CodecInput, CodecOutput, CodecPipeline +from zarr.abc.codec import CodecPipeline from zarr.abc.store import ByteSetter, Store from zarr.codecs import ( BloscCodec, @@ -21,6 +21,7 @@ ) from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer +from zarr.core.buffer.core import Buffer from zarr.core.codec_pipeline import BatchedCodecPipeline from zarr.core.config import BadConfigError, config from zarr.core.indexing import SelectorTuple @@ -144,7 +145,7 @@ def test_config_codec_pipeline_class(store: Store) -> None: class MockCodecPipeline(BatchedCodecPipeline): async def write( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -174,7 +175,7 @@ async def write( class MockEnvCodecPipeline(CodecPipeline): pass - register_pipeline(MockEnvCodecPipeline) + register_pipeline(MockEnvCodecPipeline) # type: ignore[type-abstract] with mock.patch.dict( os.environ, {"ZARR_CODEC_PIPELINE__PATH": fully_qualified_name(MockEnvCodecPipeline)} @@ -191,10 +192,9 @@ def test_config_codec_implementation(store: Store) -> None: _mock = Mock() class MockBloscCodec(BloscCodec): - async def _encode_single( - self, chunk_data: CodecInput, chunk_spec: ArraySpec - ) -> CodecOutput | None: + async def _encode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer | None: _mock.call() + return None register_codec("blosc", MockBloscCodec) with config.set({"codecs.blosc": fully_qualified_name(MockBloscCodec)}): @@ -245,7 +245,7 @@ def test_config_buffer_implementation() -> None: # has default value assert fully_qualified_name(get_buffer_class()) == config.defaults[0]["buffer"] - arr = zeros(shape=(100), store=StoreExpectingTestBuffer()) + arr = zeros(shape=(100,), store=StoreExpectingTestBuffer()) # AssertionError of StoreExpectingTestBuffer when not using my buffer with pytest.raises(AssertionError): From 260974832d3d2771ce8cf5b1ae67b95ab028c28d Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 7 May 2025 20:37:38 -0400 Subject: [PATCH 110/160] Fix and test sharding with GPU buffers (#2978) --- changes/2978.bugfix.rst | 1 + src/zarr/codecs/sharding.py | 2 +- src/zarr/testing/utils.py | 2 +- tests/test_buffer.py | 38 +++++++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 changes/2978.bugfix.rst diff --git a/changes/2978.bugfix.rst b/changes/2978.bugfix.rst new file mode 100644 index 0000000000..fe9f3d3f64 --- /dev/null +++ b/changes/2978.bugfix.rst @@ -0,0 +1 @@ +Fixed sharding with GPU buffers. diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 42b1313fac..bee36b3160 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -683,7 +683,7 @@ def _get_index_chunk_spec(self, chunks_per_shard: ChunkCoords) -> ArraySpec: config=ArrayConfig( order="C", write_empty_chunks=False ), # Note: this is hard-coded for simplicity -- it is not surfaced into user code, - prototype=numpy_buffer_prototype(), + prototype=default_buffer_prototype(), ) def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec: diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 0a93b93fdb..28d6774286 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -44,7 +44,7 @@ def has_cupy() -> bool: # Decorator for GPU tests def gpu_test(func: T_Callable) -> T_Callable: return cast( - T_Callable, + "T_Callable", pytest.mark.gpu( pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")( func diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 73b3a16677..11ff7cd96c 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -148,6 +148,34 @@ async def test_codecs_use_of_gpu_prototype() -> None: assert cp.array_equal(expect, got) +@gpu_test +@pytest.mark.asyncio +async def test_sharding_use_of_gpu_prototype() -> None: + with zarr.config.enable_gpu(): + expect = cp.zeros((10, 10), dtype="uint16", order="F") + + a = await zarr.api.asynchronous.create_array( + StorePath(MemoryStore()) / "test_codecs_use_of_gpu_prototype", + shape=expect.shape, + chunks=(5, 5), + shards=(10, 10), + dtype=expect.dtype, + fill_value=0, + ) + expect[:] = cp.arange(100).reshape(10, 10) + + await a.setitem( + selection=(slice(0, 10), slice(0, 10)), + value=expect[:], + prototype=gpu.buffer_prototype, + ) + got = await a.getitem( + selection=(slice(0, 10), slice(0, 10)), prototype=gpu.buffer_prototype + ) + assert isinstance(got, cp.ndarray) + assert cp.array_equal(expect, got) + + def test_numpy_buffer_prototype() -> None: buffer = cpu.buffer_prototype.buffer.create_zero_length() ndbuffer = cpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) @@ -157,6 +185,16 @@ def test_numpy_buffer_prototype() -> None: ndbuffer.as_scalar() +@gpu_test +def test_gpu_buffer_prototype() -> None: + buffer = gpu.buffer_prototype.buffer.create_zero_length() + ndbuffer = gpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=cp.dtype("int64")) + assert isinstance(buffer.as_array_like(), cp.ndarray) + assert isinstance(ndbuffer.as_ndarray_like(), cp.ndarray) + with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): + ndbuffer.as_scalar() + + # TODO: the same test for other buffer classes def test_cpu_buffer_as_scalar() -> None: buf = cpu.buffer_prototype.nd_buffer.create(shape=(), dtype="int64") From 0465c2b37d937c905c6adb6b6734422fb4f99866 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 8 May 2025 15:15:44 +0100 Subject: [PATCH 111/160] Fix typing errors in testing.stateful (#3045) * Fix typing errors in testing.stateful * Add a dimensionnames type * Add bugfix entry * Fix properties test --- .pre-commit-config.yaml | 1 + changes/3045.bugfix.rst | 1 + pyproject.toml | 2 +- src/zarr/api/asynchronous.py | 3 +- src/zarr/api/synchronous.py | 7 +-- src/zarr/core/array.py | 35 +++++++------- src/zarr/core/common.py | 1 + src/zarr/core/group.py | 7 +-- src/zarr/core/metadata/v3.py | 5 +- src/zarr/testing/stateful.py | 4 +- src/zarr/testing/strategies.py | 86 +++++++++++++++++++++------------- tests/conftest.py | 8 ++-- 12 files changed, 94 insertions(+), 66 deletions(-) create mode 100644 changes/3045.bugfix.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75ef0face8..474d109c80 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,7 @@ repos: - obstore>=0.5.1 # Tests - pytest + - hypothesis - repo: https://github.com/scientific-python/cookie rev: 2025.01.22 hooks: diff --git a/changes/3045.bugfix.rst b/changes/3045.bugfix.rst new file mode 100644 index 0000000000..a3886717a7 --- /dev/null +++ b/changes/3045.bugfix.rst @@ -0,0 +1 @@ +Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. diff --git a/pyproject.toml b/pyproject.toml index 9244a9ec0b..1c534f7927 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -356,6 +356,7 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] [[tool.mypy.overrides]] module = [ "tests.package_with_entrypoint.*", + "zarr.testing.stateful", "tests.test_codecs.test_transpose", "tests.test_config" ] @@ -365,7 +366,6 @@ strict = false # and fix the errors [[tool.mypy.overrides]] module = [ - "zarr.testing.stateful", # lots of hypothesis decorator errors "tests.test_codecs.test_codecs", "tests.test_metadata.*", "tests.test_store.*", diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 9b8b43a517..ac143f6dea 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -16,6 +16,7 @@ JSON, AccessModeLiteral, ChunkCoords, + DimensionNames, MemoryOrder, ZarrFormat, _default_zarr_format, @@ -865,7 +866,7 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, **kwargs: Any, diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 4c577936cd..5662f5c247 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -33,6 +33,7 @@ JSON, AccessModeLiteral, ChunkCoords, + DimensionNames, MemoryOrder, ShapeLike, ZarrFormat, @@ -626,7 +627,7 @@ def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, **kwargs: Any, @@ -761,7 +762,7 @@ def create_array( zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, @@ -926,7 +927,7 @@ def from_array( zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index b0e8b03cd7..c6217a3d93 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -54,6 +54,7 @@ ZARRAY_JSON, ZATTRS_JSON, ChunkCoords, + DimensionNames, MemoryOrder, ShapeLike, ZarrFormat, @@ -330,7 +331,7 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, @@ -358,7 +359,7 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, @@ -386,7 +387,7 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, @@ -421,7 +422,7 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, @@ -473,7 +474,7 @@ async def create( These defaults can be changed by modifying the value of ``array.v3_default_filters``, ``array.v3_default_serializer`` and ``array.v3_default_compressors`` in :mod:`zarr.core.config`. - dimension_names : Iterable[str], optional + dimension_names : Iterable[str | None], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. chunks : ShapeLike, optional @@ -562,7 +563,7 @@ async def _create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, @@ -672,7 +673,7 @@ def _create_metadata_v3( fill_value: Any | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, attributes: dict[str, JSON] | None = None, ) -> ArrayV3Metadata: """ @@ -723,7 +724,7 @@ async def _create_v3( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, attributes: dict[str, JSON] | None = None, overwrite: bool = False, ) -> AsyncArray[ArrayV3Metadata]: @@ -1743,7 +1744,7 @@ def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ChunkCoords | None = None, dimension_separator: Literal[".", "/"] | None = None, @@ -1788,7 +1789,7 @@ def create( These defaults can be changed by modifying the value of ``array.v3_default_filters``, ``array.v3_default_serializer`` and ``array.v3_default_compressors`` in :mod:`zarr.core.config`. - dimension_names : Iterable[str], optional + dimension_names : Iterable[str | None], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. chunks : ChunkCoords, optional @@ -1872,7 +1873,7 @@ def _create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ChunkCoords | None = None, dimension_separator: Literal[".", "/"] | None = None, @@ -3821,7 +3822,7 @@ async def from_array( zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, @@ -3929,7 +3930,7 @@ async def from_array( For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. If not specified and the data array has the same zarr format as the target array, the chunk key encoding of the data array is used. - dimension_names : Iterable[str], optional + dimension_names : Iterable[str | None], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. If not specified, defaults to the dimension names of the data array. @@ -4083,7 +4084,7 @@ async def init_array( zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, overwrite: bool = False, config: ArrayConfigLike | None, ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: @@ -4298,7 +4299,7 @@ async def create_array( zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, @@ -4477,7 +4478,7 @@ def _parse_keep_array_attr( order: MemoryOrder | None, zarr_format: ZarrFormat | None, chunk_key_encoding: ChunkKeyEncodingLike | None, - dimension_names: Iterable[str] | None, + dimension_names: DimensionNames, ) -> tuple[ ChunkCoords | Literal["auto"], ShardsLike | None, @@ -4488,7 +4489,7 @@ def _parse_keep_array_attr( MemoryOrder | None, ZarrFormat, ChunkKeyEncodingLike | None, - Iterable[str] | None, + DimensionNames, ]: if isinstance(data, Array): if chunks == "keep": diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 3308ca3247..a670834206 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -40,6 +40,7 @@ JSON = str | int | float | Mapping[str, "JSON"] | Sequence["JSON"] | None MemoryOrder = Literal["C", "F"] AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] +DimensionNames = Iterable[str | None] | None def product(tup: ChunkCoords) -> int: diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 3f4f15b9e9..5c470e29ca 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -43,6 +43,7 @@ ZGROUP_JSON, ZMETADATA_V2_JSON, ChunkCoords, + DimensionNames, NodeType, ShapeLike, ZarrFormat, @@ -1006,7 +1007,7 @@ async def create_array( order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, @@ -2381,7 +2382,7 @@ def create_array( order: MemoryOrder | None = "C", attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, @@ -2775,7 +2776,7 @@ def array( order: MemoryOrder | None = "C", attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 9154762648..63f6515e44 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -32,6 +32,7 @@ JSON, ZARR_JSON, ChunkCoords, + DimensionNames, parse_named_configuration, parse_shapelike, ) @@ -242,7 +243,7 @@ class ArrayV3Metadata(Metadata): fill_value: Any codecs: tuple[Codec, ...] attributes: dict[str, Any] = field(default_factory=dict) - dimension_names: tuple[str, ...] | None = None + dimension_names: tuple[str | None, ...] | None = None zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["array"] = field(default="array", init=False) storage_transformers: tuple[dict[str, JSON], ...] @@ -257,7 +258,7 @@ def __init__( fill_value: Any, codecs: Iterable[Codec | dict[str, JSON]], attributes: dict[str, JSON] | None, - dimension_names: Iterable[str] | None, + dimension_names: DimensionNames, storage_transformers: Iterable[dict[str, JSON]] | None = None, ) -> None: """ diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py index ede83201ae..acc5f63f19 100644 --- a/src/zarr/testing/stateful.py +++ b/src/zarr/testing/stateful.py @@ -326,8 +326,8 @@ def init_store(self) -> None: self.store.clear() @rule(key=zarr_keys(), data=st.binary(min_size=0, max_size=MAX_BINARY_SIZE)) - def set(self, key: str, data: DataObject) -> None: - note(f"(set) Setting {key!r} with {data}") + def set(self, key: str, data: bytes) -> None: + note(f"(set) Setting {key!r} with {data!r}") assert not self.store.read_only data_buf = cpu.Buffer.from_bytes(data) self.store.set(key, data_buf) diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 663d46034d..3b10592ec0 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,10 +1,12 @@ import math import sys +from collections.abc import Callable, Mapping from typing import Any, Literal import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np +import numpy.typing as npt from hypothesis import event from hypothesis.strategies import SearchStrategy @@ -14,7 +16,7 @@ from zarr.core.array import Array from zarr.core.chunk_grids import RegularChunkGrid from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding -from zarr.core.common import ZarrFormat +from zarr.core.common import JSON, ZarrFormat from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync from zarr.storage import MemoryStore, StoreLike @@ -30,17 +32,17 @@ ) -@st.composite # type: ignore[misc] -def keys(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> Any: +@st.composite +def keys(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: return draw(st.lists(node_names, min_size=1, max_size=max_num_nodes).map("/".join)) -@st.composite # type: ignore[misc] -def paths(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> Any: +@st.composite +def paths(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: return draw(st.just("/") | keys(max_num_nodes=max_num_nodes)) -def v3_dtypes() -> st.SearchStrategy[np.dtype]: +def v3_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return ( npst.boolean_dtypes() | npst.integer_dtypes(endianness="=") @@ -54,7 +56,7 @@ def v3_dtypes() -> st.SearchStrategy[np.dtype]: ) -def v2_dtypes() -> st.SearchStrategy[np.dtype]: +def v2_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return ( npst.boolean_dtypes() | npst.integer_dtypes(endianness="=") @@ -107,7 +109,9 @@ def clear_store(x: Store) -> Store: .filter(lambda name: name.lower() != "zarr.json") ) array_names = node_names -attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) +attrs: st.SearchStrategy[Mapping[str, JSON] | None] = st.none() | st.dictionaries( + _attr_keys, _attr_values +) # st.builds will only call a new store constructor for different keyword arguments # i.e. stores.examples() will always return the same object per Store class. # So we map a clear to reset the store. @@ -118,19 +122,19 @@ def clear_store(x: Store) -> Store: array_shapes = npst.array_shapes(max_dims=4, min_side=3) | npst.array_shapes(max_dims=4, min_side=0) -@st.composite # type: ignore[misc] +@st.composite def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[None | str] | None: simple_text = st.text(zarr_key_chars, min_size=0) - return draw(st.none() | st.lists(st.none() | simple_text, min_size=ndim, max_size=ndim)) # type: ignore[no-any-return] + return draw(st.none() | st.lists(st.none() | simple_text, min_size=ndim, max_size=ndim)) # type: ignore[arg-type] -@st.composite # type: ignore[misc] +@st.composite def array_metadata( draw: st.DrawFn, *, - array_shapes: st.SearchStrategy[tuple[int, ...]] = npst.array_shapes, + array_shapes: Callable[..., st.SearchStrategy[tuple[int, ...]]] = npst.array_shapes, zarr_formats: st.SearchStrategy[Literal[2, 3]] = zarr_formats, - attributes: st.SearchStrategy[dict[str, Any]] = attrs, + attributes: SearchStrategy[Mapping[str, JSON] | None] = attrs, ) -> ArrayV2Metadata | ArrayV3Metadata: zarr_format = draw(zarr_formats) # separator = draw(st.sampled_from(['/', '\\'])) @@ -146,7 +150,7 @@ def array_metadata( dtype=dtype, fill_value=fill_value, order=draw(st.sampled_from(["C", "F"])), - attributes=draw(attributes), + attributes=draw(attributes), # type: ignore[arg-type] dimension_separator=draw(st.sampled_from([".", "/"])), filters=None, compressor=None, @@ -157,7 +161,7 @@ def array_metadata( data_type=dtype, chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), fill_value=fill_value, - attributes=draw(attributes), + attributes=draw(attributes), # type: ignore[arg-type] dimension_names=draw(dimension_names(ndim=ndim)), chunk_key_encoding=DefaultChunkKeyEncoding(separator="/"), # FIXME codecs=[BytesCodec()], @@ -165,14 +169,14 @@ def array_metadata( ) -@st.composite # type: ignore[misc] +@st.composite def numpy_arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, dtype: np.dtype[Any] | None = None, - zarr_formats: st.SearchStrategy[ZarrFormat] | None = zarr_formats, -) -> Any: + zarr_formats: st.SearchStrategy[ZarrFormat] = zarr_formats, +) -> npt.NDArray[Any]: """ Generate numpy arrays that can be saved in the provided Zarr format. """ @@ -186,7 +190,7 @@ def numpy_arrays( return draw(npst.arrays(dtype=dtype, shape=shapes)) -@st.composite # type: ignore[misc] +@st.composite def chunk_shapes(draw: st.DrawFn, *, shape: tuple[int, ...]) -> tuple[int, ...]: # We want this strategy to shrink towards arrays with smaller number of chunks # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks @@ -208,7 +212,7 @@ def chunk_shapes(draw: st.DrawFn, *, shape: tuple[int, ...]) -> tuple[int, ...]: return chunks -@st.composite # type: ignore[misc] +@st.composite def shard_shapes( draw: st.DrawFn, *, shape: tuple[int, ...], chunk_shape: tuple[int, ...] ) -> tuple[int, ...]: @@ -220,9 +224,11 @@ def shard_shapes( return tuple(m * c for m, c in zip(multiples, chunk_shape, strict=True)) -@st.composite # type: ignore[misc] +@st.composite def np_array_and_chunks( - draw: st.DrawFn, *, arrays: st.SearchStrategy[np.ndarray] = numpy_arrays + draw: st.DrawFn, + *, + arrays: st.SearchStrategy[npt.NDArray[Any]] = numpy_arrays(), # noqa: B008 ) -> tuple[np.ndarray, tuple[int, ...]]: # type: ignore[type-arg] """A hypothesis strategy to generate small sized random arrays. @@ -232,14 +238,14 @@ def np_array_and_chunks( return (array, draw(chunk_shapes(shape=array.shape))) -@st.composite # type: ignore[misc] +@st.composite def arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, compressors: st.SearchStrategy = compressors, stores: st.SearchStrategy[StoreLike] = stores, - paths: st.SearchStrategy[str | None] = paths(), # noqa: B008 + paths: st.SearchStrategy[str] = paths(), # noqa: B008 array_names: st.SearchStrategy = array_names, arrays: st.SearchStrategy | None = None, attrs: st.SearchStrategy = attrs, @@ -296,7 +302,7 @@ def arrays( return a -@st.composite # type: ignore[misc] +@st.composite def simple_arrays( draw: st.DrawFn, *, @@ -317,7 +323,7 @@ def is_negative_slice(idx: Any) -> bool: return isinstance(idx, slice) and idx.step is not None and idx.step < 0 -@st.composite # type: ignore[misc] +@st.composite def end_slices(draw: st.DrawFn, *, shape: tuple[int]) -> Any: """ A strategy that slices ranges that include the last chunk. @@ -332,14 +338,28 @@ def end_slices(draw: st.DrawFn, *, shape: tuple[int]) -> Any: return tuple(slicers) -@st.composite # type: ignore[misc] -def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs: Any) -> Any: +@st.composite +def basic_indices( + draw: st.DrawFn, + *, + shape: tuple[int], + min_dims: int = 0, + max_dims: int | None = None, + allow_newaxis: bool = False, + allow_ellipsis: bool = True, +) -> Any: """Basic indices without unsupported negative slices.""" - strategy = npst.basic_indices(shape=shape, **kwargs).filter( + strategy = npst.basic_indices( + shape=shape, + min_dims=min_dims, + max_dims=max_dims, + allow_newaxis=allow_newaxis, + allow_ellipsis=allow_ellipsis, + ).filter( lambda idxr: ( not ( is_negative_slice(idxr) - or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) + or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) # type: ignore[redundant-expr] ) ) ) @@ -348,7 +368,7 @@ def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs: Any) -> Any: return draw(strategy) -@st.composite # type: ignore[misc] +@st.composite def orthogonal_indices( draw: st.DrawFn, *, shape: tuple[int] ) -> tuple[tuple[np.ndarray[Any, Any], ...], tuple[np.ndarray[Any, Any], ...]]: @@ -386,8 +406,8 @@ def orthogonal_indices( def key_ranges( - keys: SearchStrategy = node_names, max_size: int = sys.maxsize -) -> SearchStrategy[list[int]]: + keys: SearchStrategy[str] = node_names, max_size: int = sys.maxsize +) -> SearchStrategy[list[tuple[str, RangeByteRequest]]]: """ Function to generate key_ranges strategy for get_partial_values() returns list strategy w/ form:: diff --git a/tests/conftest.py b/tests/conftest.py index 74a140c5c7..948d3cd055 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ _parse_chunk_key_encoding, ) from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition -from zarr.core.common import JSON, parse_dtype, parse_shapelike +from zarr.core.common import JSON, DimensionNames, parse_dtype, parse_shapelike from zarr.core.config import config as zarr_config from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ArrayV3Metadata @@ -26,7 +26,7 @@ from zarr.storage import FsspecStore, LocalStore, MemoryStore, StorePath, ZipStore if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Generator from typing import Any, Literal from _pytest.compat import LEGACY_PATH @@ -255,7 +255,7 @@ def create_array_metadata( zarr_format: ZarrFormat, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, ) -> ArrayV2Metadata | ArrayV3Metadata: """ Create array metadata @@ -388,7 +388,7 @@ def meta_from_array( zarr_format: ZarrFormat = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, ) -> ArrayV3Metadata | ArrayV2Metadata: """ Create array metadata from an array From 5ff3fbe5fe1488310301e9d2ae56a9880d1ddfb2 Mon Sep 17 00:00:00 2001 From: Ilan Gold Date: Thu, 8 May 2025 16:45:30 +0200 Subject: [PATCH 112/160] (fix): use `typesize` on `Blosc` codec (#2962) * (fix): use `typesize` on `Blosc` codec * (chore): relnote * (fix): intersphinx * (fix): look at that compression ratio! * (fix): add test * (fix): min version * (fix): parenthesis? * (fix): try assertion error * (fix): windows size * (fix): add bytes print * (fix): aghh windows latest is correct, error for non latest * (fix): conditions for sizes * (fix): try clearer data * (fix): awesome! * (fix): pre-commit --------- Co-authored-by: David Stansby --- changes/2962.fix.rst | 1 + docs/user-guide/arrays.rst | 4 ++-- src/zarr/codecs/blosc.py | 4 ++++ tests/test_codecs/test_blosc.py | 19 +++++++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 changes/2962.fix.rst diff --git a/changes/2962.fix.rst b/changes/2962.fix.rst new file mode 100644 index 0000000000..83d24b72ce --- /dev/null +++ b/changes/2962.fix.rst @@ -0,0 +1 @@ +Internally use `typesize` constructor parameter for :class:`numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. \ No newline at end of file diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst index a62b2ea0fa..e6d1bcdc54 100644 --- a/docs/user-guide/arrays.rst +++ b/docs/user-guide/arrays.rst @@ -209,8 +209,8 @@ prints additional diagnostics, e.g.:: Serializer : BytesCodec(endian=) Compressors : (BloscCodec(typesize=4, cname=, clevel=3, shuffle=, blocksize=0),) No. bytes : 400000000 (381.5M) - No. bytes stored : 9696520 - Storage ratio : 41.3 + No. bytes stored : 3558573 + Storage ratio : 112.4 Chunks Initialized : 100 .. note:: diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index 2fcc041a6b..9a999e10d7 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -8,6 +8,7 @@ import numcodecs from numcodecs.blosc import Blosc +from packaging.version import Version from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer.cpu import as_numpy_array_wrapper @@ -163,6 +164,9 @@ def _blosc_codec(self) -> Blosc: "shuffle": map_shuffle_str_to_int[self.shuffle], "blocksize": self.blocksize, } + # See https://github.com/zarr-developers/numcodecs/pull/713 + if Version(numcodecs.__version__) >= Version("0.16.0"): + config_dict["typesize"] = self.typesize return Blosc.from_config(config_dict) async def _decode_single( diff --git a/tests/test_codecs/test_blosc.py b/tests/test_codecs/test_blosc.py index c1c5c92329..6e6e9df383 100644 --- a/tests/test_codecs/test_blosc.py +++ b/tests/test_codecs/test_blosc.py @@ -1,7 +1,9 @@ import json +import numcodecs import numpy as np import pytest +from packaging.version import Version import zarr from zarr.abc.store import Store @@ -54,3 +56,20 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None: assert blosc_configuration_json["shuffle"] == "bitshuffle" else: assert blosc_configuration_json["shuffle"] == "shuffle" + + +async def test_typesize() -> None: + a = np.arange(1000000, dtype=np.uint64) + codecs = [zarr.codecs.BytesCodec(), zarr.codecs.BloscCodec()] + z = zarr.array(a, chunks=(10000), codecs=codecs) + data = await z.store.get("c/0", prototype=default_buffer_prototype()) + assert data is not None + bytes = data.to_bytes() + size = len(bytes) + msg = f"Blosc size mismatch. First 10 bytes: {bytes[:20]!r} and last 10 bytes: {bytes[-20:]!r}" + if Version(numcodecs.__version__) >= Version("0.16.0"): + expected_size = 402 + assert size == expected_size, msg + else: + expected_size = 10216 + assert size == expected_size, msg From 520bc1f6c2af511832c3bab6da1eb5e2e2c38900 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 12 May 2025 16:32:54 +0200 Subject: [PATCH 113/160] fix/unbreak chunks initialized (#2862) * unbreak chunks initialized * Update src/zarr/storage/_utils.py Co-authored-by: Tom Augspurger * update docstring * make relativize_paths kw-only, and add tests --------- Co-authored-by: Tom Augspurger --- changes/2862.bugfix.rst | 1 + docs/user-guide/groups.rst | 2 +- src/zarr/core/array.py | 8 +++++- src/zarr/storage/_utils.py | 53 ++++++++++++++++++++++++++++++++++- tests/test_array.py | 5 ++-- tests/test_store/test_core.py | 32 ++++++++++++++++++++- 6 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 changes/2862.bugfix.rst diff --git a/changes/2862.bugfix.rst b/changes/2862.bugfix.rst new file mode 100644 index 0000000000..bbe6f0746e --- /dev/null +++ b/changes/2862.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that prevented the number of initialized chunks being counted properly. \ No newline at end of file diff --git a/docs/user-guide/groups.rst b/docs/user-guide/groups.rst index 4268004f70..d5a0a7ccee 100644 --- a/docs/user-guide/groups.rst +++ b/docs/user-guide/groups.rst @@ -140,7 +140,7 @@ property. E.g.:: No. bytes : 8000000 (7.6M) No. bytes stored : 1614 Storage ratio : 4956.6 - Chunks Initialized : 0 + Chunks Initialized : 10 >>> baz.info Type : Array Zarr format : 3 diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index c6217a3d93..9852bf8d5f 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -117,6 +117,7 @@ get_pipeline_class, ) from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path +from zarr.storage._utils import _relativize_path if TYPE_CHECKING: from collections.abc import Iterator, Sequence @@ -3737,7 +3738,12 @@ async def chunks_initialized( store_contents = [ x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path) ] - return tuple(chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents) + store_contents_relative = [ + _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents + ] + return tuple( + chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents_relative + ) def _build_parents( diff --git a/src/zarr/storage/_utils.py b/src/zarr/storage/_utils.py index eda4342f47..145790278c 100644 --- a/src/zarr/storage/_utils.py +++ b/src/zarr/storage/_utils.py @@ -74,11 +74,62 @@ def _join_paths(paths: Iterable[str]) -> str: """ Filter out instances of '' and join the remaining strings with '/'. - Because the root node of a zarr hierarchy is represented by an empty string, + Parameters + ---------- + paths : Iterable[str] + + Returns + ------- + str + + Examples + -------- + >>> _join_paths(["", "a", "b"]) + 'a/b' + >>> _join_paths(["a", "b", "c"]) + 'a/b/c' """ return "/".join(filter(lambda v: v != "", paths)) +def _relativize_path(*, path: str, prefix: str) -> str: + """ + Make a "/"-delimited path relative to some prefix. If the prefix is '', then the path is + returned as-is. Otherwise, the prefix is removed from the path as well as the separator + string "/". + + If ``prefix`` is not the empty string and ``path`` does not start with ``prefix`` + followed by a "/" character, then an error is raised. + + This function assumes that the prefix does not end with "/". + + Parameters + ---------- + path : str + The path to make relative to the prefix. + prefix : str + The prefix to make the path relative to. + + Returns + ------- + str + + Examples + -------- + >>> _relativize_path(path="", prefix="a/b") + 'a/b' + >>> _relativize_path(path="a/b", prefix="a/b/c") + 'c' + """ + if prefix == "": + return path + else: + _prefix = prefix + "/" + if not path.startswith(_prefix): + raise ValueError(f"The first component of {path} does not start with {prefix}.") + return path.removeprefix(f"{prefix}/") + + def _normalize_paths(paths: Iterable[str]) -> tuple[str, ...]: """ Normalize the input paths according to the normalization scheme used for zarr node paths. diff --git a/tests/test_array.py b/tests/test_array.py index 4be9bbde43..989fe30592 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -387,12 +387,13 @@ async def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]] assert observed == expected -async def test_chunks_initialized() -> None: +@pytest.mark.parametrize("path", ["", "foo"]) +async def test_chunks_initialized(path: str) -> None: """ Test that chunks_initialized accurately returns the keys of stored chunks. """ store = MemoryStore() - arr = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="i4") + arr = zarr.create_array(store, name=path, shape=(100,), chunks=(10,), dtype="i4") chunks_accumulated = tuple( accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_chunk_keys())) diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 87d0e6e40d..1ac410954b 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -8,7 +8,13 @@ from zarr.core.common import AccessModeLiteral, ZarrFormat from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath from zarr.storage._common import contains_array, contains_group, make_store_path -from zarr.storage._utils import _join_paths, _normalize_path_keys, _normalize_paths, normalize_path +from zarr.storage._utils import ( + _join_paths, + _normalize_path_keys, + _normalize_paths, + _relativize_path, + normalize_path, +) @pytest.mark.parametrize("path", ["foo", "foo/bar"]) @@ -221,3 +227,27 @@ def test_normalize_path_keys(): """ data = {"a": 10, "//b": 10} assert _normalize_path_keys(data) == {normalize_path(k): v for k, v in data.items()} + + +@pytest.mark.parametrize( + ("path", "prefix", "expected"), + [ + ("a", "", "a"), + ("a/b/c", "a/b", "c"), + ("a/b/c", "a", "b/c"), + ], +) +def test_relativize_path_valid(path: str, prefix: str, expected: str) -> None: + """ + Test the normal behavior of the _relativize_path function. Prefixes should be removed from the + path argument. + """ + assert _relativize_path(path=path, prefix=prefix) == expected + + +def test_relativize_path_invalid() -> None: + path = "a/b/c" + prefix = "b" + msg = f"The first component of {path} does not start with {prefix}." + with pytest.raises(ValueError, match=msg): + _relativize_path(path="a/b/c", prefix="b") From 7584b96f22b5d517cf61f13b415961d7be99b428 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 13 May 2025 11:48:46 +0100 Subject: [PATCH 114/160] Fix typing in a bunch of store tests (#3052) * Ignore explicit test store files * Fix test_zip * Fix test_local * Fix test_fsspec * Fix typing in test_memory * Remove walrus Co-authored-by: Davis Bennett --------- Co-authored-by: Davis Bennett --- .pre-commit-config.yaml | 1 + pyproject.toml | 12 ++++++++-- src/zarr/testing/store.py | 2 +- src/zarr/testing/utils.py | 9 ++++---- tests/test_store/test_fsspec.py | 38 +++++++++++++++++++----------- tests/test_store/test_local.py | 10 ++++---- tests/test_store/test_memory.py | 41 ++++++++++++++++++--------------- tests/test_store/test_zip.py | 19 +++++++++------ 8 files changed, 80 insertions(+), 52 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 474d109c80..80743a5dec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,6 +38,7 @@ repos: # Tests - pytest - hypothesis + - s3fs - repo: https://github.com/scientific-python/cookie rev: 2025.01.22 hooks: diff --git a/pyproject.toml b/pyproject.toml index 1c534f7927..033c9dc114 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -358,7 +358,11 @@ module = [ "tests.package_with_entrypoint.*", "zarr.testing.stateful", "tests.test_codecs.test_transpose", - "tests.test_config" + "tests.test_config", + "tests.test_store.test_zip", + "tests.test_store.test_local", + "tests.test_store.test_fsspec", + "tests.test_store.test_memory", ] strict = false @@ -368,7 +372,11 @@ strict = false module = [ "tests.test_codecs.test_codecs", "tests.test_metadata.*", - "tests.test_store.*", + "tests.test_store.test_core", + "tests.test_store.test_logging", + "tests.test_store.test_object", + "tests.test_store.test_stateful", + "tests.test_store.test_wrapper", "tests.test_group", "tests.test_indexing", "tests.test_properties", diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 867df2121f..0e73599791 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -58,7 +58,7 @@ async def get(self, store: S, key: str) -> Buffer: @abstractmethod @pytest.fixture - def store_kwargs(self) -> dict[str, Any]: + def store_kwargs(self, *args: Any, **kwargs: Any) -> dict[str, Any]: """Kwargs for instantiating a store""" ... diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 28d6774286..7cf57ab9d6 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,7 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Coroutine -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, TypeVar, cast import pytest @@ -38,13 +37,13 @@ def has_cupy() -> bool: return False -T_Callable = TypeVar("T_Callable", bound=Callable[..., Coroutine[Any, Any, None] | None]) +T = TypeVar("T") # Decorator for GPU tests -def gpu_test(func: T_Callable) -> T_Callable: +def gpu_test(func: T) -> T: return cast( - "T_Callable", + "T", pytest.mark.gpu( pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")( func diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index 08cf2f286d..c10471809c 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -3,7 +3,7 @@ import json import os import re -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest from packaging.version import parse as parse_version @@ -17,8 +17,13 @@ if TYPE_CHECKING: from collections.abc import Generator + from pathlib import Path import botocore.client + import s3fs + + from zarr.core.common import JSON + # Warning filter due to https://github.com/boto/boto3/issues/3889 pytestmark = [ @@ -109,10 +114,13 @@ async def test_basic() -> None: data = b"hello" await store.set("foo", cpu.Buffer.from_bytes(data)) assert await store.exists("foo") - assert (await store.get("foo", prototype=default_buffer_prototype())).to_bytes() == data + buf = await store.get("foo", prototype=default_buffer_prototype()) + assert buf is not None + assert buf.to_bytes() == data out = await store.get_partial_values( prototype=default_buffer_prototype(), key_ranges=[("foo", OffsetByteRequest(1))] ) + assert out[0] is not None assert out[0].to_bytes() == data[1:] @@ -121,7 +129,7 @@ class TestFsspecStoreS3(StoreTests[FsspecStore, cpu.Buffer]): buffer_cls = cpu.Buffer @pytest.fixture - def store_kwargs(self, request) -> dict[str, str | bool]: + def store_kwargs(self) -> dict[str, str | bool]: try: from fsspec import url_to_fs except ImportError: @@ -133,7 +141,7 @@ def store_kwargs(self, request) -> dict[str, str | bool]: return {"fs": fs, "path": path} @pytest.fixture - def store(self, store_kwargs: dict[str, str | bool]) -> FsspecStore: + async def store(self, store_kwargs: dict[str, Any]) -> FsspecStore: return self.store_cls(**store_kwargs) async def get(self, store: FsspecStore, key: str) -> Buffer: @@ -168,7 +176,11 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: "anon": False, } - meta = {"attributes": {"key": "value"}, "zarr_format": 3, "node_type": "group"} + meta: dict[str, JSON] = { + "attributes": {"key": "value"}, + "zarr_format": 3, + "node_type": "group", + } await store.set( "zarr.json", @@ -179,7 +191,7 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value"} - meta["attributes"]["key"] = "value-2" + meta["attributes"]["key"] = "value-2" # type: ignore[index] await store.set( "directory-2/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), @@ -189,7 +201,7 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value-2"} - meta["attributes"]["key"] = "value-3" + meta["attributes"]["key"] = "value-3" # type: ignore[index] await store.set( "directory-3/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), @@ -216,7 +228,7 @@ def test_from_upath(self) -> None: assert result.fs.asynchronous assert result.path == f"{test_bucket_name}/foo/bar" - def test_init_raises_if_path_has_scheme(self, store_kwargs) -> None: + def test_init_raises_if_path_has_scheme(self, store_kwargs: dict[str, Any]) -> None: # regression test for https://github.com/zarr-developers/zarr-python/issues/2342 store_kwargs["path"] = "s3://" + store_kwargs["path"] with pytest.raises( @@ -237,7 +249,7 @@ def test_init_warns_if_fs_asynchronous_is_false(self) -> None: with pytest.warns(UserWarning, match=r".* was not created with `asynchronous=True`.*"): self.store_cls(**store_kwargs) - async def test_empty_nonexistent_path(self, store_kwargs) -> None: + async def test_empty_nonexistent_path(self, store_kwargs: dict[str, Any]) -> None: # regression test for https://github.com/zarr-developers/zarr-python/pull/2343 store_kwargs["path"] += "/abc" store = await self.store_cls.open(**store_kwargs) @@ -256,7 +268,7 @@ async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) -def test_wrap_sync_filesystem(): +def test_wrap_sync_filesystem() -> None: """The local fs is not async so we should expect it to be wrapped automatically""" from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper @@ -270,7 +282,7 @@ def test_wrap_sync_filesystem(): parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) -def test_no_wrap_async_filesystem(): +def test_no_wrap_async_filesystem() -> None: """An async fs should not be wrapped automatically; fsspec's https filesystem is such an fs""" from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper @@ -284,12 +296,12 @@ def test_no_wrap_async_filesystem(): parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) -async def test_delete_dir_wrapped_filesystem(tmpdir) -> None: +async def test_delete_dir_wrapped_filesystem(tmp_path: Path) -> None: from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from fsspec.implementations.local import LocalFileSystem wrapped_fs = AsyncFileSystemWrapper(LocalFileSystem(auto_mkdir=True)) - store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmpdir}/test/path") + store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmp_path}/test/path") assert isinstance(store.fs, AsyncFileSystemWrapper) assert store.fs.asynchronous diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py index d9d941c6f0..8699a85082 100644 --- a/tests/test_store/test_local.py +++ b/tests/test_store/test_local.py @@ -28,7 +28,7 @@ async def set(self, store: LocalStore, key: str, value: Buffer) -> None: (store.root / key).write_bytes(value.to_bytes()) @pytest.fixture - def store_kwargs(self, tmpdir) -> dict[str, str]: + def store_kwargs(self, tmpdir: str) -> dict[str, str]: return {"root": str(tmpdir)} def test_store_repr(self, store: LocalStore) -> None: @@ -48,14 +48,14 @@ async def test_empty_with_empty_subdir(self, store: LocalStore) -> None: (store.root / "foo/bar").mkdir(parents=True) assert await store.is_empty("") - def test_creates_new_directory(self, tmp_path: pathlib.Path): + def test_creates_new_directory(self, tmp_path: pathlib.Path) -> None: target = tmp_path.joinpath("a", "b", "c") assert not target.exists() store = self.store_cls(root=target) zarr.group(store=store) - def test_invalid_root_raises(self): + def test_invalid_root_raises(self) -> None: """ Test that a TypeError is raised when a non-str/Path type is used for the `root` argument """ @@ -63,9 +63,9 @@ def test_invalid_root_raises(self): TypeError, match=r"'root' must be a string or Path instance. Got an instance of instead.", ): - LocalStore(root=0) + LocalStore(root=0) # type: ignore[arg-type] - async def test_get_with_prototype_default(self, store: LocalStore): + async def test_get_with_prototype_default(self, store: LocalStore) -> None: """ Ensure that data can be read via ``store.get`` if the prototype keyword argument is unspecified, i.e. set to ``None``. """ diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py index e520c7d054..a090f56951 100644 --- a/tests/test_store/test_memory.py +++ b/tests/test_store/test_memory.py @@ -1,12 +1,15 @@ from __future__ import annotations import re -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np +import numpy.typing as npt import pytest import zarr +import zarr.core +import zarr.core.array from zarr.core.buffer import Buffer, cpu, gpu from zarr.storage import GpuMemoryStore, MemoryStore from zarr.testing.store import StoreTests @@ -31,16 +34,16 @@ async def get(self, store: MemoryStore, key: str) -> Buffer: return store._store_dict[key] @pytest.fixture(params=[None, True]) - def store_kwargs( - self, request: pytest.FixtureRequest - ) -> dict[str, str | dict[str, Buffer] | None]: - kwargs = {"store_dict": None} + def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: + kwargs: dict[str, Any] if request.param is True: - kwargs["store_dict"] = {} + kwargs = {"store_dict": {}} + else: + kwargs = {"store_dict": None} return kwargs @pytest.fixture - def store(self, store_kwargs: str | dict[str, Buffer] | None) -> MemoryStore: + async def store(self, store_kwargs: dict[str, Any]) -> MemoryStore: return self.store_cls(**store_kwargs) def test_store_repr(self, store: MemoryStore) -> None: @@ -55,13 +58,13 @@ def test_store_supports_listing(self, store: MemoryStore) -> None: def test_store_supports_partial_writes(self, store: MemoryStore) -> None: assert store.supports_partial_writes - def test_list_prefix(self, store: MemoryStore) -> None: + async def test_list_prefix(self, store: MemoryStore) -> None: assert True @pytest.mark.parametrize("dtype", ["uint8", "float32", "int64"]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_deterministic_size( - self, store: MemoryStore, dtype, zarr_format: ZarrFormat + self, store: MemoryStore, dtype: npt.DTypeLike, zarr_format: ZarrFormat ) -> None: a = zarr.empty( store=store, @@ -85,23 +88,23 @@ class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): store_cls = GpuMemoryStore buffer_cls = gpu.Buffer - async def set(self, store: GpuMemoryStore, key: str, value: Buffer) -> None: + async def set(self, store: GpuMemoryStore, key: str, value: gpu.Buffer) -> None: # type: ignore[override] store._store_dict[key] = value async def get(self, store: MemoryStore, key: str) -> Buffer: return store._store_dict[key] @pytest.fixture(params=[None, True]) - def store_kwargs( - self, request: pytest.FixtureRequest - ) -> dict[str, str | dict[str, Buffer] | None]: - kwargs = {"store_dict": None} + def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: + kwargs: dict[str, Any] if request.param is True: - kwargs["store_dict"] = {} + kwargs = {"store_dict": {}} + else: + kwargs = {"store_dict": None} return kwargs @pytest.fixture - def store(self, store_kwargs: str | dict[str, gpu.Buffer] | None) -> GpuMemoryStore: + async def store(self, store_kwargs: dict[str, Any]) -> GpuMemoryStore: return self.store_cls(**store_kwargs) def test_store_repr(self, store: GpuMemoryStore) -> None: @@ -116,15 +119,15 @@ def test_store_supports_listing(self, store: GpuMemoryStore) -> None: def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: assert store.supports_partial_writes - def test_list_prefix(self, store: GpuMemoryStore) -> None: + async def test_list_prefix(self, store: GpuMemoryStore) -> None: assert True def test_dict_reference(self, store: GpuMemoryStore) -> None: - store_dict = {} + store_dict: dict[str, Any] = {} result = GpuMemoryStore(store_dict=store_dict) assert result._store_dict is store_dict - def test_from_dict(self): + def test_from_dict(self) -> None: d = { "a": gpu.Buffer.from_bytes(b"aaaa"), "b": cpu.Buffer.from_bytes(b"bbbb"), diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index 0237258ab1..fa99ca61bd 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -11,6 +11,7 @@ import zarr from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.core.group import Group from zarr.storage import ZipStore from zarr.testing.store import StoreTests @@ -32,7 +33,7 @@ class TestZipStore(StoreTests[ZipStore, cpu.Buffer]): buffer_cls = cpu.Buffer @pytest.fixture - def store_kwargs(self, request) -> dict[str, str | bool]: + def store_kwargs(self) -> dict[str, str | bool]: fd, temp_path = tempfile.mkstemp() os.close(fd) os.unlink(temp_path) @@ -40,12 +41,14 @@ def store_kwargs(self, request) -> dict[str, str | bool]: return {"path": temp_path, "mode": "w", "read_only": False} async def get(self, store: ZipStore, key: str) -> Buffer: - return store._get(key, prototype=default_buffer_prototype()) + buf = store._get(key, prototype=default_buffer_prototype()) + assert buf is not None + return buf async def set(self, store: ZipStore, key: str, value: Buffer) -> None: return store._set(key, value) - def test_store_read_only(self, store: ZipStore, store_kwargs: dict[str, Any]) -> None: + def test_store_read_only(self, store: ZipStore) -> None: assert not store.read_only async def test_read_only_store_raises(self, store_kwargs: dict[str, Any]) -> None: @@ -109,7 +112,7 @@ def test_api_integration(self, store: ZipStore) -> None: async def test_store_open_read_only( self, store_kwargs: dict[str, Any], read_only: bool ) -> None: - if read_only == "r": + if read_only: # create an empty zipfile with zipfile.ZipFile(store_kwargs["path"], mode="w"): pass @@ -129,9 +132,11 @@ def test_externally_zipped_store(self, tmp_path: Path) -> None: zarr_path = tmp_path / "foo.zarr" root = zarr.open_group(store=zarr_path, mode="w") root.require_group("foo") - root["foo"]["bar"] = np.array([1]) - shutil.make_archive(zarr_path, "zip", zarr_path) + assert isinstance(foo := root["foo"], Group) # noqa: RUF018 + foo["bar"] = np.array([1]) + shutil.make_archive(str(zarr_path), "zip", zarr_path) zip_path = tmp_path / "foo.zarr.zip" zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") assert list(zipped.keys()) == list(root.keys()) - assert list(zipped["foo"].keys()) == list(root["foo"].keys()) + assert isinstance(group := zipped["foo"], Group) + assert list(group.keys()) == list(group.keys()) From 629b4e5094d566cdc96aa187d0faccee9b28861f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 14 May 2025 09:35:21 +0100 Subject: [PATCH 115/160] Allow no compressor for v2 arrays (#3039) * Allow no compressor for v2 arrays * Use typing aliases for compressors * Test v2 array w/ v3 codec errors * Add changelog entry * Update type comment * fix test names Co-authored-by: Davis Bennett --------- Co-authored-by: Davis Bennett --- changes/3039.bugfix.rst | 5 +++++ src/zarr/api/asynchronous.py | 13 +++++++++--- src/zarr/api/synchronous.py | 4 ++-- src/zarr/core/array.py | 38 +++++++++++++++++++++++++----------- src/zarr/core/metadata/v2.py | 10 +++++++--- tests/test_api.py | 19 ++++++++++++++++++ 6 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 changes/3039.bugfix.rst diff --git a/changes/3039.bugfix.rst b/changes/3039.bugfix.rst new file mode 100644 index 0000000000..be2b424cf5 --- /dev/null +++ b/changes/3039.bugfix.rst @@ -0,0 +1,5 @@ +It is now possible to specify no compressor when creating a zarr format 2 array. +This can be done by passing ``compressor=None`` to the various array creation routines. + +The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. +To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index ac143f6dea..59261cca8a 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -9,7 +9,14 @@ import numpy.typing as npt from typing_extensions import deprecated -from zarr.core.array import Array, AsyncArray, create_array, from_array, get_array_metadata +from zarr.core.array import ( + Array, + AsyncArray, + CompressorLike, + create_array, + from_array, + get_array_metadata, +) from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams from zarr.core.buffer import NDArrayLike from zarr.core.common import ( @@ -838,7 +845,7 @@ async def create( *, # Note: this is a change from v2 chunks: ChunkCoords | int | None = None, # TODO: v2 allowed chunks=True dtype: npt.DTypeLike | None = None, - compressor: dict[str, JSON] | None = None, # TODO: default and type change + compressor: CompressorLike = "auto", fill_value: Any | None = 0, # TODO: need type order: MemoryOrder | None = None, store: str | StoreLike | None = None, @@ -991,7 +998,7 @@ async def create( dtype = parse_dtype(dtype, zarr_format) if not filters: filters = _default_filters(dtype) - if not compressor: + if compressor == "auto": compressor = _default_compressor(dtype) elif zarr_format == 3 and chunk_shape is None: # type: ignore[redundant-expr] if chunks is not None: diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 5662f5c247..24ab937db5 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -7,7 +7,7 @@ import zarr.api.asynchronous as async_api import zarr.core.array from zarr._compat import _deprecate_positional_args -from zarr.core.array import Array, AsyncArray +from zarr.core.array import Array, AsyncArray, CompressorLike from zarr.core.group import Group from zarr.core.sync import sync from zarr.core.sync_group import create_hierarchy @@ -599,7 +599,7 @@ def create( *, # Note: this is a change from v2 chunks: ChunkCoords | int | bool | None = None, dtype: npt.DTypeLike | None = None, - compressor: dict[str, JSON] | None = None, # TODO: default and type change + compressor: CompressorLike = "auto", fill_value: Any | None = 0, # TODO: need type order: MemoryOrder | None = None, store: str | StoreLike | None = None, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 9852bf8d5f..cf4c36cc22 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -102,6 +102,7 @@ T_ArrayMetadata, ) from zarr.core.metadata.v2 import ( + CompressorLikev2, _default_compressor, _default_filters, parse_compressor, @@ -303,7 +304,7 @@ async def create( dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLikev2 | Literal["auto"] = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, @@ -394,7 +395,7 @@ async def create( dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, @@ -429,7 +430,7 @@ async def create( dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, @@ -570,7 +571,7 @@ async def _create( dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, @@ -604,7 +605,7 @@ async def _create( raise ValueError( "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead." ) - if compressor is not None: + if compressor != "auto": raise ValueError( "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead." ) @@ -768,7 +769,7 @@ def _create_metadata_v2( dimension_separator: Literal[".", "/"] | None = None, fill_value: float | None = None, filters: Iterable[dict[str, JSON] | numcodecs.abc.Codec] | None = None, - compressor: dict[str, JSON] | numcodecs.abc.Codec | None = None, + compressor: CompressorLikev2 = None, attributes: dict[str, JSON] | None = None, ) -> ArrayV2Metadata: if dimension_separator is None: @@ -809,7 +810,7 @@ async def _create_v2( dimension_separator: Literal[".", "/"] | None = None, fill_value: float | None = None, filters: Iterable[dict[str, JSON] | numcodecs.abc.Codec] | None = None, - compressor: dict[str, JSON] | numcodecs.abc.Codec | None = None, + compressor: CompressorLike = "auto", attributes: dict[str, JSON] | None = None, overwrite: bool = False, ) -> AsyncArray[ArrayV2Metadata]: @@ -821,6 +822,17 @@ async def _create_v2( else: await ensure_no_existing_node(store_path, zarr_format=2) + compressor_parsed: CompressorLikev2 + if compressor == "auto": + compressor_parsed = _default_compressor(dtype) + elif isinstance(compressor, BytesBytesCodec): + raise ValueError( + "Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. " + "Use a numcodecs codec directly instead." + ) + else: + compressor_parsed = compressor + metadata = cls._create_metadata_v2( shape=shape, dtype=dtype, @@ -829,7 +841,7 @@ async def _create_v2( dimension_separator=dimension_separator, fill_value=fill_value, filters=filters, - compressor=compressor, + compressor=compressor_parsed, attributes=attributes, ) @@ -1751,7 +1763,7 @@ def create( dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, config: ArrayConfigLike | None = None, @@ -1880,7 +1892,7 @@ def _create( dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, config: ArrayConfigLike | None = None, @@ -3792,7 +3804,11 @@ def _get_default_codecs( | Literal["auto"] | None ) -CompressorLike: TypeAlias = dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec | None +# Union of acceptable types for users to pass in for both v2 and v3 compressors +CompressorLike: TypeAlias = ( + dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec | Literal["auto"] | None +) + CompressorsLike: TypeAlias = ( Iterable[dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec] | dict[str, JSON] diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index d19193963f..029a3e09a7 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -5,7 +5,7 @@ from collections.abc import Iterable, Sequence from enum import Enum from functools import cached_property -from typing import TYPE_CHECKING, Any, TypedDict, cast +from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict, cast import numcodecs.abc @@ -43,6 +43,10 @@ class ArrayV2MetadataDict(TypedDict): attributes: dict[str, JSON] +# Union of acceptable types for v2 compressors +CompressorLikev2: TypeAlias = dict[str, JSON] | numcodecs.abc.Codec | None + + @dataclass(frozen=True, kw_only=True) class ArrayV2Metadata(Metadata): shape: ChunkCoords @@ -52,7 +56,7 @@ class ArrayV2Metadata(Metadata): order: MemoryOrder = "C" filters: tuple[numcodecs.abc.Codec, ...] | None = None dimension_separator: Literal[".", "/"] = "." - compressor: numcodecs.abc.Codec | None = None + compressor: CompressorLikev2 attributes: dict[str, JSON] = field(default_factory=dict) zarr_format: Literal[2] = field(init=False, default=2) @@ -65,7 +69,7 @@ def __init__( fill_value: Any, order: MemoryOrder, dimension_separator: Literal[".", "/"] = ".", - compressor: numcodecs.abc.Codec | dict[str, JSON] | None = None, + compressor: CompressorLikev2 = None, filters: Iterable[numcodecs.abc.Codec | dict[str, JSON]] | None = None, attributes: dict[str, JSON] | None = None, ) -> None: diff --git a/tests/test_api.py b/tests/test_api.py index 9f03a1067a..d1912f7238 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING +import zarr.codecs + if TYPE_CHECKING: import pathlib @@ -1190,3 +1192,20 @@ def test_gpu_basic(store: Store, zarr_format: ZarrFormat | None) -> None: # assert_array_equal doesn't check the type assert isinstance(result, type(src)) cp.testing.assert_array_equal(result, src[:10, :10]) + + +def test_v2_without_compressor() -> None: + # Make sure it's possible to set no compressor for v2 arrays + arr = zarr.create(store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=None) + assert arr.compressors == () + + +def test_v2_with_v3_compressor() -> None: + # Check trying to create a v2 array with a v3 compressor fails + with pytest.raises( + ValueError, + match="Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. Use a numcodecs codec directly instead.", + ): + zarr.create( + store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=zarr.codecs.BloscCodec() + ) From d615783f1588da8f555476e006622833743f15dc Mon Sep 17 00:00:00 2001 From: Tom White Date: Wed, 14 May 2025 13:10:37 +0100 Subject: [PATCH 116/160] Avoid memory copy in obstore write (#2972) * Avoid memory copy in obstore write * Add as_bytes_like method to Buffer * Add changelog entry * No need to take unsigned bytes view following #2738 * Change method name to `as_buffer_like` --------- Co-authored-by: jakirkham Co-authored-by: Davis Bennett --- changes/2972.misc.rst | 1 + src/zarr/core/buffer/core.py | 13 +++++++++++++ src/zarr/storage/_local.py | 4 ++-- src/zarr/storage/_obstore.py | 4 ++-- 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 changes/2972.misc.rst diff --git a/changes/2972.misc.rst b/changes/2972.misc.rst new file mode 100644 index 0000000000..f0258c1d05 --- /dev/null +++ b/changes/2972.misc.rst @@ -0,0 +1 @@ +Avoid an unnecessary memory copy when writing Zarr with obstore diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index cfcd7e6633..94cd91f026 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -255,6 +255,19 @@ def as_numpy_array(self) -> npt.NDArray[Any]: """ ... + def as_buffer_like(self) -> BytesLike: + """Returns the buffer as an object that implements the Python buffer protocol. + + Notes + ----- + Might have to copy data, since the implementation uses `.as_numpy_array()`. + + Returns + ------- + An object that implements the Python buffer protocol + """ + return memoryview(self.as_numpy_array()) # type: ignore[arg-type] + def to_bytes(self) -> bytes: """Returns the buffer as `bytes` (host memory). diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index 85d244f17b..f2af75f43e 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -52,10 +52,10 @@ def _put( with path.open("r+b") as f: f.seek(start) # write takes any object supporting the buffer protocol - f.write(value.as_numpy_array()) # type: ignore[arg-type] + f.write(value.as_buffer_like()) return None else: - view = memoryview(value.as_numpy_array()) # type: ignore[arg-type] + view = value.as_buffer_like() if exclusive: mode = "xb" else: diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index 8c2469747d..738754a8b9 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -161,7 +161,7 @@ async def set(self, key: str, value: Buffer) -> None: self._check_writable() - buf = value.to_bytes() + buf = value.as_buffer_like() await obs.put_async(self.store, key, buf) async def set_if_not_exists(self, key: str, value: Buffer) -> None: @@ -169,7 +169,7 @@ async def set_if_not_exists(self, key: str, value: Buffer) -> None: import obstore as obs self._check_writable() - buf = value.to_bytes() + buf = value.as_buffer_like() with contextlib.suppress(obs.exceptions.AlreadyExistsError): await obs.put_async(self.store, key, buf, mode="create") From aa3341573b8397ce655058ec80335931131badbf Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 14 May 2025 20:26:43 +0200 Subject: [PATCH 117/160] Replace redundant list comprehension with generator (#3040) * Replace redundant list comprehension with generator * Partially revert There is no such thing as a "tuple comprehension": https://stackoverflow.com/questions/52285419/aggregating-an-async-generator-to-a-tuple#52285420 Fixes CI error: FAILED tests/test_group.py::test_create_hierarchy_existing_nodes[zarr2-async-array-memory] - TypeError: 'async_generator' object is not iterable --- tests/test_store/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 1ac410954b..4b1858afb5 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -203,7 +203,7 @@ def test_valid() -> None: Test that path normalization works as expected """ paths = ["a", "b", "c", "d", "", "//a///b//"] - assert _normalize_paths(paths) == tuple([normalize_path(p) for p in paths]) + assert _normalize_paths(paths) == tuple(normalize_path(p) for p in paths) @staticmethod @pytest.mark.parametrize("paths", [("", "/"), ("///a", "a")]) From 882641519d8b79c32aa803b99e9cc70eccf6e066 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Thu, 15 May 2025 11:58:41 +0200 Subject: [PATCH 118/160] Additional testing for `AsyncArray`, `Array` (#3049) * remove duplicate metadata parsing * add test cases * add test cases * tests for different zarr_formats in test_storage_transformers * tests for different zarr_formats in test_storage_transformers * ignore mypy arg-type error for deprecation test * fix typing in tests * test_chunk_key_encoding * test_invalid_v2_arguments * test_array_repr * type annotation for parse_array_metadata * test_v2_and_v3_exist_at_same_path * remove duplicate check for dimension_separator in v3 * tests for invalid arguments in creation * format * revert typing * document changes --- changes/3049.misc.rst | 1 + src/zarr/api/asynchronous.py | 5 - src/zarr/core/array.py | 23 ++--- tests/test_api.py | 65 +++++++++---- tests/test_array.py | 178 +++++++++++++++++++++++++++++++---- 5 files changed, 213 insertions(+), 59 deletions(-) create mode 100644 changes/3049.misc.rst diff --git a/changes/3049.misc.rst b/changes/3049.misc.rst new file mode 100644 index 0000000000..79ecd6ed95 --- /dev/null +++ b/changes/3049.misc.rst @@ -0,0 +1 @@ +Added tests for ``AsyncArray``, ``Array`` and removed duplicate argument parsing. \ No newline at end of file diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 59261cca8a..cdedd5b033 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1019,11 +1019,6 @@ async def create( warnings.warn("object_codec is not yet implemented", RuntimeWarning, stacklevel=2) if read_only is not None: warnings.warn("read_only is not yet implemented", RuntimeWarning, stacklevel=2) - if dimension_separator is not None and zarr_format == 3: - raise ValueError( - "dimension_separator is not supported for zarr format 3, use chunk_key_encoding instead" - ) - if order is not None: _warn_order_kwarg() if write_empty_chunks is not None: diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index cf4c36cc22..78b5e92ed6 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -140,7 +140,8 @@ def parse_array_metadata(data: Any) -> ArrayMetadata: if isinstance(data, ArrayMetadata): return data elif isinstance(data, dict): - if data["zarr_format"] == 3: + zarr_format = data.get("zarr_format") + if zarr_format == 3: meta_out = ArrayV3Metadata.from_dict(data) if len(meta_out.storage_transformers) > 0: msg = ( @@ -149,9 +150,11 @@ def parse_array_metadata(data: Any) -> ArrayMetadata: ) raise ValueError(msg) return meta_out - elif data["zarr_format"] == 2: + elif zarr_format == 2: return ArrayV2Metadata.from_dict(data) - raise TypeError + else: + raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3") + raise TypeError # pragma: no cover def create_codec_pipeline(metadata: ArrayMetadata) -> CodecPipeline: @@ -160,8 +163,7 @@ def create_codec_pipeline(metadata: ArrayMetadata) -> CodecPipeline: elif isinstance(metadata, ArrayV2Metadata): v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor) return get_pipeline_class().from_codecs([v2_codec]) - else: - raise TypeError + raise TypeError # pragma: no cover async def get_array_metadata( @@ -268,17 +270,6 @@ def __init__( store_path: StorePath, config: ArrayConfigLike | None = None, ) -> None: - if isinstance(metadata, dict): - zarr_format = metadata["zarr_format"] - # TODO: remove this when we extensively type the dict representation of metadata - _metadata = cast(dict[str, JSON], metadata) - if zarr_format == 2: - metadata = ArrayV2Metadata.from_dict(_metadata) - elif zarr_format == 3: - metadata = ArrayV3Metadata.from_dict(_metadata) - else: - raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3") - metadata_parsed = parse_array_metadata(metadata) config_parsed = parse_array_config(config) diff --git a/tests/test_api.py b/tests/test_api.py index d1912f7238..6904f91fe7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import TYPE_CHECKING import zarr.codecs @@ -72,13 +73,19 @@ def test_create(memory_store: Store) -> None: # TODO: parametrize over everything this function takes @pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_create_array(store: Store) -> None: +def test_create_array(store: Store, zarr_format: ZarrFormat) -> None: attrs: dict[str, JSON] = {"foo": 100} # explicit type annotation to avoid mypy error shape = (10, 10) path = "foo" data_val = 1 array_w = create_array( - store, name=path, shape=shape, attributes=attrs, chunks=shape, dtype="uint8" + store, + name=path, + shape=shape, + attributes=attrs, + chunks=shape, + dtype="uint8", + zarr_format=zarr_format, ) array_w[:] = data_val assert array_w.shape == shape @@ -87,18 +94,27 @@ def test_create_array(store: Store) -> None: @pytest.mark.parametrize("write_empty_chunks", [True, False]) -def test_write_empty_chunks_warns(write_empty_chunks: bool) -> None: +def test_write_empty_chunks_warns(write_empty_chunks: bool, zarr_format: ZarrFormat) -> None: """ Test that using the `write_empty_chunks` kwarg on array access will raise a warning. """ match = "The `write_empty_chunks` keyword argument .*" with pytest.warns(RuntimeWarning, match=match): _ = zarr.array( - data=np.arange(10), shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks + data=np.arange(10), + shape=(10,), + dtype="uint8", + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, ) with pytest.warns(RuntimeWarning, match=match): - _ = zarr.create(shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks) + _ = zarr.create( + shape=(10,), + dtype="uint8", + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + ) @pytest.mark.parametrize("path", ["foo", "/", "/foo", "///foo/bar"]) @@ -115,18 +131,18 @@ def test_open_normalized_path( assert node.path == normalize_path(path) -async def test_open_array(memory_store: MemoryStore) -> None: +async def test_open_array(memory_store: MemoryStore, zarr_format: ZarrFormat) -> None: store = memory_store # open array, create if doesn't exist - z = open(store=store, shape=100) + z = open(store=store, shape=100, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (100,) # open array, overwrite # store._store_dict = {} store = MemoryStore() - z = open(store=store, shape=200) + z = open(store=store, shape=200, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (200,) @@ -140,7 +156,16 @@ async def test_open_array(memory_store: MemoryStore) -> None: # path not found with pytest.raises(FileNotFoundError): - open(store="doesnotexist", mode="r") + open(store="doesnotexist", mode="r", zarr_format=zarr_format) + + +@pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True) +def test_v2_and_v3_exist_at_same_path(store: Store) -> None: + zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=3) + zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=2) + msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store}. Zarr v3 will be used." + with pytest.warns(UserWarning, match=re.escape(msg)): + zarr.open(store=store, mode="r") @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -163,9 +188,9 @@ async def test_open_group(memory_store: MemoryStore) -> None: assert "foo" in g # open group, overwrite - # g = open_group(store=store) - # assert isinstance(g, Group) - # assert "foo" not in g + g = open_group(store=store, mode="w") + assert isinstance(g, Group) + assert "foo" not in g # open group, read-only store_cls = type(store) @@ -308,7 +333,6 @@ def test_open_with_mode_w_minus(tmp_path: pathlib.Path) -> None: zarr.open(store=tmp_path, mode="w-") -@pytest.mark.parametrize("zarr_format", [2, 3]) def test_array_order(zarr_format: ZarrFormat) -> None: arr = zarr.ones(shape=(2, 2), order=None, zarr_format=zarr_format) expected = zarr.config.get("array.order") @@ -324,7 +348,6 @@ def test_array_order(zarr_format: ZarrFormat) -> None: @pytest.mark.parametrize("order", ["C", "F"]) -@pytest.mark.parametrize("zarr_format", [2, 3]) def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None: with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) @@ -1095,13 +1118,16 @@ def test_open_falls_back_to_open_group() -> None: assert group.attrs == {"key": "value"} -async def test_open_falls_back_to_open_group_async() -> None: +async def test_open_falls_back_to_open_group_async(zarr_format: ZarrFormat) -> None: # https://github.com/zarr-developers/zarr-python/issues/2309 store = MemoryStore() - await zarr.api.asynchronous.open_group(store, attributes={"key": "value"}) + await zarr.api.asynchronous.open_group( + store, attributes={"key": "value"}, zarr_format=zarr_format + ) group = await zarr.api.asynchronous.open(store=store) assert isinstance(group, zarr.core.group.AsyncGroup) + assert group.metadata.zarr_format == zarr_format assert group.attrs == {"key": "value"} @@ -1137,13 +1163,14 @@ async def test_metadata_validation_error() -> None: ["local", "memory", "zip"], indirect=True, ) -def test_open_array_with_mode_r_plus(store: Store) -> None: +def test_open_array_with_mode_r_plus(store: Store, zarr_format: ZarrFormat) -> None: # 'r+' means read/write (must exist) with pytest.raises(FileNotFoundError): - zarr.open_array(store=store, mode="r+") - zarr.ones(store=store, shape=(3, 3)) + zarr.open_array(store=store, mode="r+", zarr_format=zarr_format) + zarr.ones(store=store, shape=(3, 3), zarr_format=zarr_format) z2 = zarr.open_array(store=store, mode="r+") assert isinstance(z2, Array) + assert z2.metadata.zarr_format == zarr_format result = z2[:] assert isinstance(result, NDArrayLike) assert (result == 1).all() diff --git a/tests/test_array.py b/tests/test_array.py index 989fe30592..eb19f0e7f3 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -41,6 +41,7 @@ from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype from zarr.core.buffer.cpu import NDBuffer from zarr.core.chunk_grids import _auto_partition +from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams from zarr.core.common import JSON, MemoryOrder, ZarrFormat from zarr.core.group import AsyncGroup from zarr.core.indexing import BasicIndexer, ceildiv @@ -51,7 +52,7 @@ if TYPE_CHECKING: from zarr.core.array_spec import ArrayConfigLike - from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.core.metadata.v2 import ArrayV2Metadata @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @@ -227,10 +228,13 @@ def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str assert arr.fill_value.dtype == arr.dtype -def test_create_positional_args_deprecated() -> None: - store = MemoryStore() - with pytest.warns(FutureWarning, match="Pass"): - zarr.Array.create(store, (2, 2), dtype="f8") +async def test_create_deprecated() -> None: + with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning, match=re.escape("Pass shape=(2, 2) as keyword args")): + await zarr.AsyncArray.create(MemoryStore(), (2, 2), dtype="f8") # type: ignore[call-overload] + with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning, match=re.escape("Pass shape=(2, 2) as keyword args")): + zarr.Array.create(MemoryStore(), (2, 2), dtype="f8") def test_selection_positional_args_deprecated() -> None: @@ -321,24 +325,47 @@ def test_serializable_sync_array(store: LocalStore, zarr_format: ZarrFormat) -> @pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_storage_transformers(store: MemoryStore) -> None: +@pytest.mark.parametrize("zarr_format", [2, 3, "invalid"]) +def test_storage_transformers(store: MemoryStore, zarr_format: ZarrFormat | str) -> None: """ Test that providing an actual storage transformer produces a warning and otherwise passes through """ - metadata_dict: dict[str, JSON] = { - "zarr_format": 3, - "node_type": "array", - "shape": (10,), - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, - "data_type": "uint8", - "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "/"}}, - "codecs": (BytesCodec().to_dict(),), - "fill_value": 0, - "storage_transformers": ({"test": "should_raise"}), - } - match = "Arrays with storage transformers are not supported in zarr-python at this time." - with pytest.raises(ValueError, match=match): + metadata_dict: dict[str, JSON] + if zarr_format == 3: + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": "uint8", + "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "/"}}, + "codecs": (BytesCodec().to_dict(),), + "fill_value": 0, + "storage_transformers": ({"test": "should_raise"}), + } + else: + metadata_dict = { + "zarr_format": zarr_format, + "shape": (10,), + "chunks": (1,), + "dtype": "uint8", + "dimension_separator": ".", + "codecs": (BytesCodec().to_dict(),), + "fill_value": 0, + "order": "C", + "storage_transformers": ({"test": "should_raise"}), + } + if zarr_format == 3: + match = "Arrays with storage transformers are not supported in zarr-python at this time." + with pytest.raises(ValueError, match=match): + Array.from_dict(StorePath(store), data=metadata_dict) + elif zarr_format == 2: + # no warning Array.from_dict(StorePath(store), data=metadata_dict) + else: + match = f"Invalid zarr_format: {zarr_format}. Expected 2 or 3" + with pytest.raises(ValueError, match=match): + Array.from_dict(StorePath(store), data=metadata_dict) @pytest.mark.parametrize("test_cls", [Array, AsyncArray[Any]]) @@ -1106,6 +1133,111 @@ async def test_v3_chunk_encoding( assert arr.filters == filters_expected assert arr.compressors == compressors_expected + @staticmethod + @pytest.mark.parametrize("name", ["v2", "default", "invalid"]) + @pytest.mark.parametrize("separator", [".", "/"]) + async def test_chunk_key_encoding( + name: str, separator: Literal[".", "/"], zarr_format: ZarrFormat, store: MemoryStore + ) -> None: + chunk_key_encoding = ChunkKeyEncodingParams(name=name, separator=separator) # type: ignore[typeddict-item] + error_msg = "" + if name == "invalid": + error_msg = "Unknown chunk key encoding." + if zarr_format == 2 and name == "default": + error_msg = "Invalid chunk key encoding. For Zarr format 2 arrays, the `name` field of the chunk key encoding must be 'v2'." + if error_msg: + with pytest.raises(ValueError, match=re.escape(error_msg)): + arr = await create_array( + store=store, + dtype="uint8", + shape=(10,), + chunks=(1,), + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + ) + else: + arr = await create_array( + store=store, + dtype="uint8", + shape=(10,), + chunks=(1,), + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + ) + if isinstance(arr.metadata, ArrayV2Metadata): + assert arr.metadata.dimension_separator == separator + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ({"serializer": "bytes"}, "Zarr format 2 arrays do not support `serializer`."), + ({"dimension_names": ["test"]}, "Zarr format 2 arrays do not support dimension names."), + ], + ) + async def test_create_array_invalid_v2_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + with pytest.raises(ValueError, match=re.escape(error_msg)): + await zarr.api.asynchronous.create_array( + store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs + ) + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ( + {"dimension_names": ["test"]}, + "dimension_names cannot be used for arrays with zarr_format 2.", + ), + ( + {"chunk_key_encoding": {"name": "default", "separator": "/"}}, + "chunk_key_encoding cannot be used for arrays with zarr_format 2. Use dimension_separator instead.", + ), + ( + {"codecs": "bytes"}, + "codecs cannot be used for arrays with zarr_format 2. Use filters and compressor instead.", + ), + ], + ) + async def test_create_invalid_v2_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + with pytest.raises(ValueError, match=re.escape(error_msg)): + await zarr.api.asynchronous.create( + store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs + ) + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ( + {"chunk_shape": (1,), "chunks": (2,)}, + "Only one of chunk_shape or chunks can be provided.", + ), + ( + {"dimension_separator": "/"}, + "dimension_separator cannot be used for arrays with zarr_format 3. Use chunk_key_encoding instead.", + ), + ( + {"filters": []}, + "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead", + ), + ( + {"compressor": "blosc"}, + "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead", + ), + ], + ) + async def test_invalid_v3_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + kwargs.setdefault("chunks", (1,)) + with pytest.raises(ValueError, match=re.escape(error_msg)): + zarr.create(store=store, dtype="uint8", shape=(10,), zarr_format=3, **kwargs) + @staticmethod @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) @pytest.mark.parametrize( @@ -1585,3 +1717,11 @@ async def test_sharding_coordinate_selection() -> None: result = arr[1, [0, 1]] # type: ignore[index] assert isinstance(result, NDArrayLike) assert (result == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all() + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_array_repr(store: Store) -> None: + shape = (2, 3, 4) + dtype = "uint8" + arr = zarr.create_array(store, shape=shape, dtype=dtype) + assert str(arr) == f"" From ee1d70f9b036e3827dd6287a294e73d86bfed86a Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 16 May 2025 10:24:06 +0100 Subject: [PATCH 119/160] Update pre-commit hooks (#3058) * Update pre-commit hooks * Update type checking code --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80743a5dec..fd50366a1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ ci: default_stages: [pre-commit, pre-push] repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.9 + rev: v0.11.9 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -40,7 +40,7 @@ repos: - hypothesis - s3fs - repo: https://github.com/scientific-python/cookie - rev: 2025.01.22 + rev: 2025.05.02 hooks: - id: sp-repo-review - repo: https://github.com/pre-commit/pygrep-hooks diff --git a/pyproject.toml b/pyproject.toml index 033c9dc114..f1c290e1b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -310,7 +310,7 @@ extend-select = [ "RUF", "SIM", # flake8-simplify "SLOT", # flake8-slots - "TCH", # flake8-type-checking + "TC", # flake8-type-checking "TRY", # tryceratops "UP", # pyupgrade "W", # pycodestyle warnings @@ -338,6 +338,7 @@ ignore = [ "Q003", "COM812", "COM819", + "TC006", ] [tool.ruff.lint.extend-per-file-ignores] From d57fbf78a83689d7886676a2f52a3d7a52462942 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 16 May 2025 17:35:16 +0100 Subject: [PATCH 120/160] Don't compress data in hypothesis store testing (#3063) * Don't compress data in hypothesis testing * Add comment --- src/zarr/testing/stateful.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py index acc5f63f19..f4f7b33318 100644 --- a/src/zarr/testing/stateful.py +++ b/src/zarr/testing/stateful.py @@ -17,6 +17,7 @@ import zarr from zarr import Array from zarr.abc.store import Store +from zarr.codecs.bytes import BytesCodec from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype from zarr.core.sync import SyncMixin from zarr.storage import LocalStore, MemoryStore @@ -108,7 +109,15 @@ def add_array( assume(self.can_add(path)) note(f"Adding array: path='{path}' shape={array.shape} chunks={chunks}") for store in [self.store, self.model]: - zarr.array(array, chunks=chunks, path=path, store=store, fill_value=fill_value) + zarr.array( + array, + chunks=chunks, + path=path, + store=store, + fill_value=fill_value, + # Chose bytes codec to avoid wasting time compressing the data being written + codecs=[BytesCodec()], + ) self.all_arrays.add(path) # @precondition(lambda self: bool(self.all_groups)) From a941224ab51dfb8ca98f9a53afb91f3b6d2c6955 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 16 May 2025 13:23:27 -0400 Subject: [PATCH 121/160] feat: add `print_debug_info` function (#2913) * feat: add function * doc: add docstring * doc: add change log for print_debug_info * Update .github/ISSUE_TEMPLATE/bug_report.yml * feat: expand debug printout + better test * feat: debug - also print out rich * test: print debug test for upstream * feat: better formatting for print_debug_info * restore original issue template --------- Co-authored-by: Davis Bennett Co-authored-by: David Stansby --- changes/2913.feature.rst | 1 + src/zarr/__init__.py | 49 ++++++++++++++++++++++++++++++++++++++++ tests/test_zarr.py | 18 +++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 changes/2913.feature.rst diff --git a/changes/2913.feature.rst b/changes/2913.feature.rst new file mode 100644 index 0000000000..e0bfcba791 --- /dev/null +++ b/changes/2913.feature.rst @@ -0,0 +1 @@ +Added a `print_debug_info` function for bug reports. diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index 31796601b3..0d58ecf8e8 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -37,6 +37,54 @@ # in case setuptools scm screw up and find version to be 0.0.0 assert not __version__.startswith("0.0.0") + +def print_debug_info() -> None: + """ + Print version info for use in bug reports. + """ + import platform + from importlib.metadata import version + + def print_packages(packages: list[str]) -> None: + not_installed = [] + for package in packages: + try: + print(f"{package}: {version(package)}") + except ModuleNotFoundError: + not_installed.append(package) + if not_installed: + print("\n**Not Installed:**") + for package in not_installed: + print(package) + + required = [ + "packaging", + "numpy", + "numcodecs", + "typing_extensions", + "donfig", + ] + optional = [ + "botocore", + "cupy-cuda12x", + "fsspec", + "numcodecs", + "s3fs", + "gcsfs", + "universal-pathlib", + "rich", + "obstore", + ] + + print(f"platform: {platform.platform()}") + print(f"python: {platform.python_version()}") + print(f"zarr: {__version__}\n") + print("**Required dependencies:**") + print_packages(required) + print("\n**Optional dependencies:**") + print_packages(optional) + + __all__ = [ "Array", "AsyncArray", @@ -67,6 +115,7 @@ "open_consolidated", "open_group", "open_like", + "print_debug_info", "save", "save_array", "save_group", diff --git a/tests/test_zarr.py b/tests/test_zarr.py index 2aa62e4231..f49873132e 100644 --- a/tests/test_zarr.py +++ b/tests/test_zarr.py @@ -1,3 +1,5 @@ +import pytest + import zarr @@ -9,3 +11,19 @@ def test_exports() -> None: for export in __all__: getattr(zarr, export) + + +def test_print_debug_info(capsys: pytest.CaptureFixture[str]) -> None: + """ + Ensure that print_debug_info does not raise an error + """ + from importlib.metadata import version + + from zarr import __version__, print_debug_info + + print_debug_info() + captured = capsys.readouterr() + # test that at least some of what we expect is + # printed out + assert f"zarr: {__version__}" in captured.out + assert f"numpy: {version('numpy')}" in captured.out From dd161df93ec908b5f3ecad03b0f18916885c1f90 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 19 May 2025 11:27:02 +0100 Subject: [PATCH 122/160] Fix overwrite modes (#3062) * Fix overwrite modes * Add many more tests that data doesn't disappear * Add bugfix entry. * Fix function name --- changes/3062.bugfix.rst | 1 + src/zarr/api/asynchronous.py | 3 +- src/zarr/api/synchronous.py | 1 + tests/test_api.py | 81 ++++++++++++++++++++++++++++++++---- 4 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 changes/3062.bugfix.rst diff --git a/changes/3062.bugfix.rst b/changes/3062.bugfix.rst new file mode 100644 index 0000000000..9e1e52ddb7 --- /dev/null +++ b/changes/3062.bugfix.rst @@ -0,0 +1 @@ +Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index cdedd5b033..4f3c9c3f8f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -88,7 +88,7 @@ _READ_MODES: tuple[AccessModeLiteral, ...] = ("r", "r+", "a") _CREATE_MODES: tuple[AccessModeLiteral, ...] = ("a", "w", "w-") -_OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("a", "r+", "w") +_OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("w",) def _infer_overwrite(mode: AccessModeLiteral) -> bool: @@ -817,7 +817,6 @@ async def open_group( warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) - if attributes is None: attributes = {} diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 24ab937db5..d4b652ad6e 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -858,6 +858,7 @@ def create_array( Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. + If `True`, all existing paths in the store will be deleted. config : ArrayConfigLike, optional Runtime configuration for the array. write_data : bool diff --git a/tests/test_api.py b/tests/test_api.py index 6904f91fe7..ae112756c5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING import zarr.codecs +import zarr.storage if TYPE_CHECKING: import pathlib @@ -11,6 +12,7 @@ from zarr.abc.store import Store from zarr.core.common import JSON, MemoryOrder, ZarrFormat +import contextlib import warnings from typing import Literal @@ -27,9 +29,9 @@ create, create_array, create_group, + from_array, group, load, - open, open_group, save, save_array, @@ -41,6 +43,10 @@ from zarr.storage._utils import normalize_path from zarr.testing.utils import gpu_test +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + def test_create(memory_store: Store) -> None: store = memory_store @@ -135,28 +141,28 @@ async def test_open_array(memory_store: MemoryStore, zarr_format: ZarrFormat) -> store = memory_store # open array, create if doesn't exist - z = open(store=store, shape=100, zarr_format=zarr_format) + z = zarr.api.synchronous.open(store=store, shape=100, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (100,) # open array, overwrite # store._store_dict = {} store = MemoryStore() - z = open(store=store, shape=200, zarr_format=zarr_format) + z = zarr.api.synchronous.open(store=store, shape=200, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (200,) # open array, read-only store_cls = type(store) ro_store = await store_cls.open(store_dict=store._store_dict, read_only=True) - z = open(store=ro_store, mode="r") + z = zarr.api.synchronous.open(store=ro_store, mode="r") assert isinstance(z, Array) assert z.shape == (200,) assert z.read_only # path not found with pytest.raises(FileNotFoundError): - open(store="doesnotexist", mode="r", zarr_format=zarr_format) + zarr.api.synchronous.open(store="doesnotexist", mode="r", zarr_format=zarr_format) @pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True) @@ -233,12 +239,12 @@ def test_save(store: Store, n_args: int, n_kwargs: int) -> None: save(store) elif n_args == 1 and n_kwargs == 0: save(store, *args) - array = open(store) + array = zarr.api.synchronous.open(store) assert isinstance(array, Array) assert_array_equal(array[:], data) else: save(store, *args, **kwargs) # type: ignore [arg-type] - group = open(store) + group = zarr.api.synchronous.open(store) assert isinstance(group, Group) for array in group.array_values(): assert_array_equal(array[:], data) @@ -1077,7 +1083,7 @@ def test_tree() -> None: def test_open_positional_args_deprecated() -> None: store = MemoryStore() with pytest.warns(FutureWarning, match="pass"): - open(store, "w", shape=(1,)) + zarr.api.synchronous.open(store, "w", shape=(1,)) def test_save_array_positional_args_deprecated() -> None: @@ -1236,3 +1242,62 @@ def test_v2_with_v3_compressor() -> None: zarr.create( store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=zarr.codecs.BloscCodec() ) + + +def add_empty_file(path: Path) -> Path: + fpath = path / "a.txt" + fpath.touch() + return fpath + + +@pytest.mark.parametrize("create_function", [create_array, from_array]) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_no_overwrite_array(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + create_function(store=store, data=np.ones(shape=(1,)), overwrite=overwrite) + if overwrite: + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +@pytest.mark.parametrize("create_function", [create_group, group]) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_no_overwrite_group(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + create_function(store=store, overwrite=overwrite) + if overwrite: + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +@pytest.mark.parametrize("open_func", [zarr.open, open_group]) +@pytest.mark.parametrize("mode", ["r", "r+", "a", "w", "w-"]) +def test_no_overwrite_open(tmp_path: Path, open_func: Callable, mode: str) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + with contextlib.suppress(FileExistsError, FileNotFoundError): + open_func(store=store, mode=mode) + if mode == "w": + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +def test_no_overwrite_load(tmp_path: Path) -> None: + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + with contextlib.suppress(NotImplementedError): + zarr.load(store) + assert existing_fpath.exists() From 57107260291fc9c6f32c95346c321b8a28a6b6e8 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 19 May 2025 15:13:07 +0100 Subject: [PATCH 123/160] Changelog for 3.0.8 (#3071) * Changelog for 3.0.8 * Add warning to top of release notes * fix warning --- changes/2862.bugfix.rst | 1 - changes/2913.feature.rst | 1 - changes/2972.misc.rst | 1 - changes/2978.bugfix.rst | 1 - changes/2998.bugfix.md | 1 - changes/3027.misc.rst | 1 - changes/3039.bugfix.rst | 5 ----- changes/3045.bugfix.rst | 1 - changes/3049.misc.rst | 1 - changes/3062.bugfix.rst | 1 - docs/release-notes.rst | 36 ++++++++++++++++++++++++++++++++++++ 11 files changed, 36 insertions(+), 14 deletions(-) delete mode 100644 changes/2862.bugfix.rst delete mode 100644 changes/2913.feature.rst delete mode 100644 changes/2972.misc.rst delete mode 100644 changes/2978.bugfix.rst delete mode 100644 changes/2998.bugfix.md delete mode 100644 changes/3027.misc.rst delete mode 100644 changes/3039.bugfix.rst delete mode 100644 changes/3045.bugfix.rst delete mode 100644 changes/3049.misc.rst delete mode 100644 changes/3062.bugfix.rst diff --git a/changes/2862.bugfix.rst b/changes/2862.bugfix.rst deleted file mode 100644 index bbe6f0746e..0000000000 --- a/changes/2862.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug that prevented the number of initialized chunks being counted properly. \ No newline at end of file diff --git a/changes/2913.feature.rst b/changes/2913.feature.rst deleted file mode 100644 index e0bfcba791..0000000000 --- a/changes/2913.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added a `print_debug_info` function for bug reports. diff --git a/changes/2972.misc.rst b/changes/2972.misc.rst deleted file mode 100644 index f0258c1d05..0000000000 --- a/changes/2972.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid an unnecessary memory copy when writing Zarr with obstore diff --git a/changes/2978.bugfix.rst b/changes/2978.bugfix.rst deleted file mode 100644 index fe9f3d3f64..0000000000 --- a/changes/2978.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed sharding with GPU buffers. diff --git a/changes/2998.bugfix.md b/changes/2998.bugfix.md deleted file mode 100644 index 7b94223122..0000000000 --- a/changes/2998.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fix structured `dtype` fill value serialization for consolidated metadata \ No newline at end of file diff --git a/changes/3027.misc.rst b/changes/3027.misc.rst deleted file mode 100644 index ffbfe9b808..0000000000 --- a/changes/3027.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Simplified scalar indexing of size-1 arrays. \ No newline at end of file diff --git a/changes/3039.bugfix.rst b/changes/3039.bugfix.rst deleted file mode 100644 index be2b424cf5..0000000000 --- a/changes/3039.bugfix.rst +++ /dev/null @@ -1,5 +0,0 @@ -It is now possible to specify no compressor when creating a zarr format 2 array. -This can be done by passing ``compressor=None`` to the various array creation routines. - -The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. -To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. diff --git a/changes/3045.bugfix.rst b/changes/3045.bugfix.rst deleted file mode 100644 index a3886717a7..0000000000 --- a/changes/3045.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. diff --git a/changes/3049.misc.rst b/changes/3049.misc.rst deleted file mode 100644 index 79ecd6ed95..0000000000 --- a/changes/3049.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Added tests for ``AsyncArray``, ``Array`` and removed duplicate argument parsing. \ No newline at end of file diff --git a/changes/3062.bugfix.rst b/changes/3062.bugfix.rst deleted file mode 100644 index 9e1e52ddb7..0000000000 --- a/changes/3062.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 341a32c364..f8b00f83e7 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -3,6 +3,42 @@ Release notes .. towncrier release notes start +3.0.8 (2025-05-19) +------------------ + +.. warning:: + + In versions 3.0.0 to 3.0.7 opening arrays or groups with ``mode='a'`` (the default for many builtin functions) + would cause any existing paths in the store to be deleted. This is fixed in 3.0.8, and + we recommend all users upgrade to avoid this bug that could cause unintentional data loss. + +Features +~~~~~~~~ + +- Added a `print_debug_info` function for bug reports. (:issue:`2913`) + + +Bugfixes +~~~~~~~~ + +- Fix a bug that prevented the number of initialized chunks being counted properly. (:issue:`2862`) +- Fixed sharding with GPU buffers. (:issue:`2978`) +- Fix structured `dtype` fill value serialization for consolidated metadata (:issue:`2998`) +- It is now possible to specify no compressor when creating a zarr format 2 array. + This can be done by passing ``compressor=None`` to the various array creation routines. + + The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. + To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. (:issue:`3039`) +- Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. (:issue:`3045`) +- Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. (:issue:`3062`) + + +Misc +~~~~ + +- :issue:`2972`, :issue:`3027`, :issue:`3049` + + 3.0.7 (2025-04-22) ------------------ From 80a09d710531fda387b9521a5e613fc7cd7706cb Mon Sep 17 00:00:00 2001 From: Ilan Gold Date: Tue, 20 May 2025 14:31:49 +0200 Subject: [PATCH 124/160] (chore): release notes cleanup (#3074) * (chore): release notes cleanup * (chore): add double space --- changes/2950.bufgix.rst | 1 - changes/2962.fix.rst | 1 - docs/release-notes.rst | 2 ++ 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 changes/2950.bufgix.rst delete mode 100644 changes/2962.fix.rst diff --git a/changes/2950.bufgix.rst b/changes/2950.bufgix.rst deleted file mode 100644 index 67cd61f377..0000000000 --- a/changes/2950.bufgix.rst +++ /dev/null @@ -1 +0,0 @@ -Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed. diff --git a/changes/2962.fix.rst b/changes/2962.fix.rst deleted file mode 100644 index 83d24b72ce..0000000000 --- a/changes/2962.fix.rst +++ /dev/null @@ -1 +0,0 @@ -Internally use `typesize` constructor parameter for :class:`numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. \ No newline at end of file diff --git a/docs/release-notes.rst b/docs/release-notes.rst index f8b00f83e7..a89046dd6d 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -31,6 +31,8 @@ Bugfixes To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. (:issue:`3039`) - Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. (:issue:`3045`) - Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. (:issue:`3062`) +- Internally use `typesize` constructor parameter for :class:`numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. (:issue:`2962`) +- Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed. (:issue:`2950`) Misc From 18c4a9eba1f6a94d434f97aebf282a476bd2cda2 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Wed, 21 May 2025 11:01:45 +0200 Subject: [PATCH 125/160] Implement Store.move (#3021) * move for LocalStore * move for ZipStore * remove redundant check * open and close zipstore * fix zipstore.move * fix localstore.move for ndim>1 * format * remove abstract Store .move * document changes * shutil.move entire folder * fix test for windows --- changes/3021.feature.rst | 1 + src/zarr/storage/_local.py | 12 ++++++++++ src/zarr/storage/_zip.py | 13 ++++++++++ tests/test_store/test_local.py | 43 ++++++++++++++++++++++++++++++---- tests/test_store/test_zip.py | 15 ++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 changes/3021.feature.rst diff --git a/changes/3021.feature.rst b/changes/3021.feature.rst new file mode 100644 index 0000000000..8805797ce3 --- /dev/null +++ b/changes/3021.feature.rst @@ -0,0 +1 @@ +Implemented ``move`` for ``LocalStore`` and ``ZipStore``. This allows users to move the store to a different root path. \ No newline at end of file diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index f2af75f43e..15b043b1dc 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -253,5 +253,17 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: except (FileNotFoundError, NotADirectoryError): pass + async def move(self, dest_root: Path | str) -> None: + """ + Move the store to another path. The old root directory is deleted. + """ + if isinstance(dest_root, str): + dest_root = Path(dest_root) + os.makedirs(dest_root.parent, exist_ok=True) + if os.path.exists(dest_root): + raise FileExistsError(f"Destination root {dest_root} already exists.") + shutil.move(self.root, dest_root) + self.root = dest_root + async def getsize(self, key: str) -> int: return os.path.getsize(self.root / key) diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index f9eb8d8808..5d147deded 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shutil import threading import time import zipfile @@ -288,3 +289,15 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: if k not in seen: seen.add(k) yield k + + async def move(self, path: Path | str) -> None: + """ + Move the store to another path. + """ + if isinstance(path, str): + path = Path(path) + self.close() + os.makedirs(path.parent, exist_ok=True) + shutil.move(self.path, path) + self.path = path + await self._open() diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py index 8699a85082..7974d0d633 100644 --- a/tests/test_store/test_local.py +++ b/tests/test_store/test_local.py @@ -1,18 +1,18 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import pathlib +import re +import numpy as np import pytest import zarr +from zarr import create_array from zarr.core.buffer import Buffer, cpu from zarr.storage import LocalStore from zarr.testing.store import StoreTests from zarr.testing.utils import assert_bytes_equal -if TYPE_CHECKING: - import pathlib - class TestLocalStore(StoreTests[LocalStore, cpu.Buffer]): store_cls = LocalStore @@ -74,3 +74,38 @@ async def test_get_with_prototype_default(self, store: LocalStore) -> None: await self.set(store, key, data_buf) observed = await store.get(key, prototype=None) assert_bytes_equal(observed, data_buf) + + @pytest.mark.parametrize("ndim", [0, 1, 3]) + @pytest.mark.parametrize( + "destination", ["destination", "foo/bar/destintion", pathlib.Path("foo/bar/destintion")] + ) + async def test_move( + self, tmp_path: pathlib.Path, ndim: int, destination: pathlib.Path | str + ) -> None: + origin = tmp_path / "origin" + if isinstance(destination, str): + destination = str(tmp_path / destination) + else: + destination = tmp_path / destination + + print(type(destination)) + store = await LocalStore.open(root=origin) + shape = (4,) * ndim + chunks = (2,) * ndim + data = np.arange(4**ndim) + if ndim > 0: + data = data.reshape(*shape) + array = create_array(store, data=data, chunks=chunks or "auto") + + await store.move(destination) + + assert store.root == pathlib.Path(destination) + assert pathlib.Path(destination).exists() + assert not origin.exists() + assert np.array_equal(array[...], data) + + store2 = await LocalStore.open(root=origin) + with pytest.raises( + FileExistsError, match=re.escape(f"Destination root {destination} already exists") + ): + await store2.move(destination) diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index fa99ca61bd..24b25ed315 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -10,6 +10,7 @@ import pytest import zarr +from zarr import create_array from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.group import Group from zarr.storage import ZipStore @@ -140,3 +141,17 @@ def test_externally_zipped_store(self, tmp_path: Path) -> None: assert list(zipped.keys()) == list(root.keys()) assert isinstance(group := zipped["foo"], Group) assert list(group.keys()) == list(group.keys()) + + async def test_move(self, tmp_path: Path) -> None: + origin = tmp_path / "origin.zip" + destination = tmp_path / "some_folder" / "destination.zip" + + store = await ZipStore.open(path=origin, mode="a") + array = create_array(store, data=np.arange(10)) + + await store.move(str(destination)) + + assert store.path == destination + assert destination.exists() + assert not origin.exists() + assert np.array_equal(array[...], np.arange(10)) From 481550a2b98d1bf51d731653208c26b5bcfce454 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 12:44:18 +0100 Subject: [PATCH 126/160] Error on invalid store mode (#3068) * Error on invalid store mode * Fix tests * Add release note * Fix test --- changes/3068.bugfix.rst | 1 + src/zarr/storage/_common.py | 11 +++++++---- tests/test_api.py | 4 ++-- tests/test_array.py | 4 ++-- tests/test_store/test_core.py | 8 ++++++++ 5 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 changes/3068.bugfix.rst diff --git a/changes/3068.bugfix.rst b/changes/3068.bugfix.rst new file mode 100644 index 0000000000..9ada322c13 --- /dev/null +++ b/changes/3068.bugfix.rst @@ -0,0 +1 @@ +Trying to open an array with ``mode='r'`` when the store is not read-only now raises an error. diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index d81369f142..b2fefe96d7 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -2,7 +2,7 @@ import json from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, Self from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, default_buffer_prototype @@ -48,9 +48,7 @@ def read_only(self) -> bool: return self.store.read_only @classmethod - async def open( - cls, store: Store, path: str, mode: AccessModeLiteral | None = None - ) -> StorePath: + async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = None) -> Self: """ Open StorePath based on the provided mode. @@ -67,6 +65,9 @@ async def open( ------ FileExistsError If the mode is 'w-' and the store path already exists. + ValueError + If the mode is not "r" and the store is read-only, or + if the mode is "r" and the store is not read-only. """ await store._ensure_open() @@ -78,6 +79,8 @@ async def open( if store.read_only and mode != "r": raise ValueError(f"Store is read-only but mode is '{mode}'") + if not store.read_only and mode == "r": + raise ValueError(f"Store is not read-only but mode is '{mode}'") match mode: case "w-": diff --git a/tests/test_api.py b/tests/test_api.py index ae112756c5..640478e9c1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -171,7 +171,7 @@ def test_v2_and_v3_exist_at_same_path(store: Store) -> None: zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=2) msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store}. Zarr v3 will be used." with pytest.warns(UserWarning, match=re.escape(msg)): - zarr.open(store=store, mode="r") + zarr.open(store=store) @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -1285,7 +1285,7 @@ def test_no_overwrite_open(tmp_path: Path, open_func: Callable, mode: str) -> No existing_fpath = add_empty_file(tmp_path) assert existing_fpath.exists() - with contextlib.suppress(FileExistsError, FileNotFoundError): + with contextlib.suppress(FileExistsError, FileNotFoundError, ValueError): open_func(store=store, mode=mode) if mode == "w": assert not existing_fpath.exists() diff --git a/tests/test_array.py b/tests/test_array.py index eb19f0e7f3..32f7887007 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1473,7 +1473,7 @@ async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) -> for parent_path in parents: # this will raise if these groups were not created _ = await zarr.api.asynchronous.open_group( - store=store, path=parent_path, mode="r", zarr_format=zarr_format + store=store, path=parent_path, zarr_format=zarr_format ) @@ -1661,7 +1661,7 @@ def test_roundtrip_numcodecs() -> None: BYTES_CODEC = {"name": "bytes", "configuration": {"endian": "little"}} # Read in the array again and check compressor config - root = zarr.open_group(store, mode="r") + root = zarr.open_group(store) metadata = root["test"].metadata.to_dict() expected = (*filters, BYTES_CODEC, *compressors) assert metadata["codecs"] == expected diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 4b1858afb5..6a94cc0ac2 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -4,6 +4,7 @@ import pytest from _pytest.compat import LEGACY_PATH +import zarr from zarr import Group from zarr.core.common import AccessModeLiteral, ZarrFormat from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath @@ -251,3 +252,10 @@ def test_relativize_path_invalid() -> None: msg = f"The first component of {path} does not start with {prefix}." with pytest.raises(ValueError, match=msg): _relativize_path(path="a/b/c", prefix="b") + + +def test_invalid_open_mode() -> None: + store = MemoryStore() + zarr.create((100,), store=store, zarr_format=2, path="a") + with pytest.raises(ValueError, match="Store is not read-only but mode is 'r'"): + zarr.open_array(store=store, path="a", zarr_format=2, mode="r") From f6742368229b8817a50335c28bb19a10e7a778e9 Mon Sep 17 00:00:00 2001 From: Ilan Gold Date: Fri, 23 May 2025 10:57:22 +0200 Subject: [PATCH 127/160] (feat): use `np.zeros` for buffer creation with `fill_value=0` (#3082) * (feat): use `np.zeros` for buffer creation when possible * (fix): add comment and check * (chore): relnote --- changes/3082.feature.rst | 1 + src/zarr/core/buffer/cpu.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changes/3082.feature.rst diff --git a/changes/3082.feature.rst b/changes/3082.feature.rst new file mode 100644 index 0000000000..e990d1f3a0 --- /dev/null +++ b/changes/3082.feature.rst @@ -0,0 +1 @@ +Use :py:func:`numpy.zeros` instead of :py:func:`np.full` for a performance speedup when creating a `zarr.core.buffer.NDBuffer` with `fill_value=0`. \ No newline at end of file diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 8464518818..3022bafb6f 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -154,7 +154,8 @@ def create( order: Literal["C", "F"] = "C", fill_value: Any | None = None, ) -> Self: - if fill_value is None: + # np.zeros is much faster than np.full, and therefore using it when possible is better. + if fill_value is None or (isinstance(fill_value, int) and fill_value == 0): return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order)) else: return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order)) From 2361cd76d1a57a666a313b02be1298ce4df3b6d5 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 25 May 2025 11:24:31 +0200 Subject: [PATCH 128/160] add fill_value output to info (#3081) * add fill_value output to info * changelog * fix tests * fix example in docs --- changes/3081.feature.rst | 1 + docs/user-guide/arrays.rst | 4 ++++ docs/user-guide/groups.rst | 2 ++ docs/user-guide/performance.rst | 3 +++ src/zarr/core/_info.py | 4 +++- src/zarr/core/array.py | 1 + tests/test_array.py | 6 ++++++ tests/test_info.py | 4 ++++ 8 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 changes/3081.feature.rst diff --git a/changes/3081.feature.rst b/changes/3081.feature.rst new file mode 100644 index 0000000000..8cf83ea7c2 --- /dev/null +++ b/changes/3081.feature.rst @@ -0,0 +1 @@ +Adds ``fill_value`` to the list of attributes displayed in the output of the ``AsyncArray.info()`` method. \ No newline at end of file diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst index e6d1bcdc54..5bd6b1500f 100644 --- a/docs/user-guide/arrays.rst +++ b/docs/user-guide/arrays.rst @@ -183,6 +183,7 @@ which can be used to print useful diagnostics, e.g.:: Type : Array Zarr format : 3 Data type : DataType.int32 + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C @@ -200,6 +201,7 @@ prints additional diagnostics, e.g.:: Type : Array Zarr format : 3 Data type : DataType.int32 + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C @@ -287,6 +289,7 @@ Here is an example using a delta filter with the Blosc compressor:: Type : Array Zarr format : 3 Data type : DataType.int32 + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C @@ -601,6 +604,7 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za Type : Array Zarr format : 3 Data type : DataType.uint8 + Fill value : 0 Shape : (10000, 10000) Shard shape : (1000, 1000) Chunk shape : (100, 100) diff --git a/docs/user-guide/groups.rst b/docs/user-guide/groups.rst index d5a0a7ccee..99234bad4e 100644 --- a/docs/user-guide/groups.rst +++ b/docs/user-guide/groups.rst @@ -129,6 +129,7 @@ property. E.g.:: Type : Array Zarr format : 3 Data type : DataType.int64 + Fill value : 0 Shape : (1000000,) Chunk shape : (100000,) Order : C @@ -145,6 +146,7 @@ property. E.g.:: Type : Array Zarr format : 3 Data type : DataType.float32 + Fill value : 0.0 Shape : (1000, 1000) Chunk shape : (100, 100) Order : C diff --git a/docs/user-guide/performance.rst b/docs/user-guide/performance.rst index 42d830780f..88329f11b8 100644 --- a/docs/user-guide/performance.rst +++ b/docs/user-guide/performance.rst @@ -92,6 +92,7 @@ To use sharding, you need to specify the ``shards`` parameter when creating the Type : Array Zarr format : 3 Data type : DataType.uint8 + Fill value : 0 Shape : (10000, 10000, 1000) Shard shape : (1000, 1000, 1000) Chunk shape : (100, 100, 100) @@ -122,6 +123,7 @@ ratios, depending on the correlation structure within the data. E.g.:: Type : Array Zarr format : 3 Data type : DataType.int32 + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C @@ -141,6 +143,7 @@ ratios, depending on the correlation structure within the data. E.g.:: Type : Array Zarr format : 3 Data type : DataType.int32 + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : F diff --git a/src/zarr/core/_info.py b/src/zarr/core/_info.py index 845552c8be..ee953d4591 100644 --- a/src/zarr/core/_info.py +++ b/src/zarr/core/_info.py @@ -67,7 +67,7 @@ def byte_info(size: int) -> str: return f"{size} ({human_readable_size(size)})" -@dataclasses.dataclass(kw_only=True) +@dataclasses.dataclass(kw_only=True, frozen=True, slots=True) class ArrayInfo: """ Visual summary for an Array. @@ -79,6 +79,7 @@ class ArrayInfo: _type: Literal["Array"] = "Array" _zarr_format: ZarrFormat _data_type: np.dtype[Any] | DataType + _fill_value: object _shape: tuple[int, ...] _shard_shape: tuple[int, ...] | None = None _chunk_shape: tuple[int, ...] | None = None @@ -97,6 +98,7 @@ def __repr__(self) -> str: Type : {_type} Zarr format : {_zarr_format} Data type : {_data_type} + Fill value : {_fill_value} Shape : {_shape}""") if self._shard_shape is not None: diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 78b5e92ed6..00ac7c2138 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1702,6 +1702,7 @@ def _info( return ArrayInfo( _zarr_format=self.metadata.zarr_format, _data_type=_data_type, + _fill_value=self.metadata.fill_value, _shape=self.shape, _order=self.order, _shard_shape=self.shards, diff --git a/tests/test_array.py b/tests/test_array.py index 32f7887007..a6bcd17c4b 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -522,6 +522,7 @@ def test_info_v2(self, chunks: tuple[int, int], shards: tuple[int, int] | None) expected = ArrayInfo( _zarr_format=2, _data_type=np.dtype("float64"), + _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=None, @@ -539,6 +540,7 @@ def test_info_v3(self, chunks: tuple[int, int], shards: tuple[int, int] | None) expected = ArrayInfo( _zarr_format=3, _data_type=DataType.parse("float64"), + _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, @@ -564,6 +566,7 @@ def test_info_complete(self, chunks: tuple[int, int], shards: tuple[int, int] | expected = ArrayInfo( _zarr_format=3, _data_type=DataType.parse("float64"), + _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, @@ -599,6 +602,7 @@ async def test_info_v2_async( expected = ArrayInfo( _zarr_format=2, _data_type=np.dtype("float64"), + _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=(2, 2), _shard_shape=None, @@ -624,6 +628,7 @@ async def test_info_v3_async( expected = ArrayInfo( _zarr_format=3, _data_type=DataType.parse("float64"), + _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, @@ -651,6 +656,7 @@ async def test_info_complete_async( expected = ArrayInfo( _zarr_format=3, _data_type=DataType.parse("float64"), + _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, diff --git a/tests/test_info.py b/tests/test_info.py index db0fd0ef76..04e6339092 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -54,6 +54,7 @@ def test_array_info(zarr_format: ZarrFormat) -> None: info = ArrayInfo( _zarr_format=zarr_format, _data_type=np.dtype("int32"), + _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), _order="C", @@ -66,6 +67,7 @@ def test_array_info(zarr_format: ZarrFormat) -> None: Type : Array Zarr format : {zarr_format} Data type : int32 + Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) Order : C @@ -92,6 +94,7 @@ def test_array_info_complete( info = ArrayInfo( _zarr_format=zarr_format, _data_type=np.dtype("int32"), + _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), _order="C", @@ -107,6 +110,7 @@ def test_array_info_complete( Type : Array Zarr format : {zarr_format} Data type : int32 + Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) Order : C From 26811ebd93f6e56c9bf6fa848a1066bf0dd48851 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Sun, 25 May 2025 05:45:07 -0400 Subject: [PATCH 129/160] issues: add pep-723 to issue template (#3087) * issues: add pep-723 to issue template * use zarr main branch + make copyable * corret example link + not uv specific --- .github/ISSUE_TEMPLATE/bug_report.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 705cd31cb5..84bb89d82a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -57,7 +57,22 @@ body: id: reproduce attributes: label: Steps to reproduce - description: Minimal, reproducible code sample, a copy-pastable example if possible. + description: Minimal, reproducible code sample. Must list dependencies in [inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/#example). When put in a file named `issue.py` calling `uv run issue.py` should show the issue. + value: | + ```python + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "zarr@git+https://github.com/zarr-developers/zarr-python.git@main", + # ] + # /// + # + # This script automatically imports the development branch of zarr to check for issues + + import zarr + # your reproducer code + # zarr.print_debug_info() + ``` validations: required: true - type: textarea From 70af54a6a4c40173df51f848c75ea8674dbf9504 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sun, 25 May 2025 15:16:25 +0100 Subject: [PATCH 130/160] Fix typing in test_codecs (#3095) --- pyproject.toml | 2 +- tests/test_codecs/test_codecs.py | 33 +++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f1c290e1b1..0da6b1acdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -364,6 +364,7 @@ module = [ "tests.test_store.test_local", "tests.test_store.test_fsspec", "tests.test_store.test_memory", + "tests.test_codecs.test_codecs", ] strict = false @@ -371,7 +372,6 @@ strict = false # and fix the errors [[tool.mypy.overrides]] module = [ - "tests.test_codecs.test_codecs", "tests.test_metadata.*", "tests.test_store.test_core", "tests.test_store.test_logging", diff --git a/tests/test_codecs/test_codecs.py b/tests/test_codecs/test_codecs.py index b8122b4ac2..468f395254 100644 --- a/tests/test_codecs/test_codecs.py +++ b/tests/test_codecs/test_codecs.py @@ -2,7 +2,7 @@ import json from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np import pytest @@ -18,32 +18,33 @@ TransposeCodec, ) from zarr.core.buffer import default_buffer_prototype -from zarr.core.indexing import Selection, morton_order_iter +from zarr.core.indexing import BasicSelection, morton_order_iter +from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.storage import StorePath if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.buffer import NDArrayLike - from zarr.core.common import MemoryOrder + from zarr.core.buffer.core import NDArrayLikeOrScalar + from zarr.core.common import ChunkCoords, MemoryOrder @dataclass(frozen=True) class _AsyncArrayProxy: - array: AsyncArray + array: AsyncArray[Any] - def __getitem__(self, selection: Selection) -> _AsyncArraySelectionProxy: + def __getitem__(self, selection: BasicSelection) -> _AsyncArraySelectionProxy: return _AsyncArraySelectionProxy(self.array, selection) @dataclass(frozen=True) class _AsyncArraySelectionProxy: - array: AsyncArray - selection: Selection + array: AsyncArray[Any] + selection: BasicSelection - async def get(self) -> NDArrayLike: + async def get(self) -> NDArrayLikeOrScalar: return await self.array.getitem(self.selection) - async def set(self, value: np.ndarray) -> None: + async def set(self, value: np.ndarray[Any, Any]) -> None: return await self.array.setitem(self.selection, value) @@ -101,6 +102,7 @@ async def test_order( read_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, read_data) + assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] @@ -142,6 +144,7 @@ def test_order_implicit( read_data = a[:, :] assert np.array_equal(data, read_data) + assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] @@ -209,7 +212,7 @@ def test_morton() -> None: [3, 2, 1, 6, 4, 5, 2], ], ) -def test_morton2(shape) -> None: +def test_morton2(shape: ChunkCoords) -> None: order = list(morton_order_iter(shape)) for i, x in enumerate(order): assert x not in order[:i] # no duplicates @@ -263,7 +266,10 @@ async def test_dimension_names(store: Store) -> None: dimension_names=("x", "y"), ) - assert (await zarr.api.asynchronous.open_array(store=spath)).metadata.dimension_names == ( + assert isinstance( + meta := (await zarr.api.asynchronous.open_array(store=spath)).metadata, ArrayV3Metadata + ) + assert meta.dimension_names == ( "x", "y", ) @@ -277,7 +283,8 @@ async def test_dimension_names(store: Store) -> None: fill_value=0, ) - assert (await AsyncArray.open(spath2)).metadata.dimension_names is None + assert isinstance(meta := (await AsyncArray.open(spath2)).metadata, ArrayV3Metadata) + assert meta.dimension_names is None zarr_json_buffer = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype()) assert zarr_json_buffer is not None assert "dimension_names" not in json.loads(zarr_json_buffer.to_bytes()) From 0dd797f8a7eeab05982a1e4115990200d97d9d98 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Sun, 25 May 2025 23:29:18 +0200 Subject: [PATCH 131/160] =?UTF-8?q?ruff=20rules:=20`TCH`=20=E2=86=92=20`TC?= =?UTF-8?q?`=20(#3032)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * The ruff ruleset is TC, not TCH * Apply ruff/flake8-type-checking rule TC006 TC006 Add quotes to type expression in `typing.cast()` * Apply ruff/flake8-type-checking rule TC003 TC003 Move standard library import into a type-checking block --------- Co-authored-by: David Stansby --- pyproject.toml | 2 +- src/zarr/api/asynchronous.py | 2 +- src/zarr/codecs/crc32c_.py | 6 +++-- src/zarr/codecs/sharding.py | 2 +- src/zarr/codecs/transpose.py | 2 +- src/zarr/core/array.py | 24 ++++++++++---------- src/zarr/core/array_spec.py | 2 +- src/zarr/core/buffer/core.py | 12 +++++----- src/zarr/core/buffer/gpu.py | 4 ++-- src/zarr/core/chunk_key_encodings.py | 4 ++-- src/zarr/core/common.py | 4 ++-- src/zarr/core/config.py | 2 +- src/zarr/core/group.py | 12 +++++----- src/zarr/core/indexing.py | 34 ++++++++++++++-------------- src/zarr/core/metadata/v2.py | 6 ++--- src/zarr/core/metadata/v3.py | 6 ++--- src/zarr/core/strings.py | 4 ++-- src/zarr/testing/utils.py | 2 +- 18 files changed, 66 insertions(+), 64 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0da6b1acdd..a43e51abd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -291,8 +291,8 @@ extend-exclude = [ extend-select = [ "ANN", # flake8-annotations "B", # flake8-bugbear - "EXE", # flake8-executable "C4", # flake8-comprehensions + "EXE", # flake8-executable "FA", # flake8-future-annotations "FLY", # flynt "FURB", # refurb diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 4f3c9c3f8f..5b9c0bee3d 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -329,7 +329,7 @@ async def open( try: metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we fix typing for array metadata dicts - _metadata_dict = cast(ArrayMetadataDict, metadata_dict) + _metadata_dict = cast("ArrayMetadataDict", metadata_dict) # for v2, the above would already have raised an exception if not an array zarr_format = _metadata_dict["zarr_format"] is_v3_array = zarr_format == 3 and _metadata_dict.get("node_type") == "array" diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index ab8a57eba7..6da673ceac 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -40,7 +40,9 @@ async def _decode_single( inner_bytes = data[:-4] # Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved - computed_checksum = np.uint32(crc32c(cast(typing_extensions.Buffer, inner_bytes))).tobytes() + computed_checksum = np.uint32( + crc32c(cast("typing_extensions.Buffer", inner_bytes)) + ).tobytes() stored_checksum = bytes(crc32_bytes) if computed_checksum != stored_checksum: raise ValueError( @@ -55,7 +57,7 @@ async def _encode_single( ) -> Buffer | None: data = chunk_bytes.as_numpy_array() # Calculate the checksum and "cast" it to a numpy array - checksum = np.array([crc32c(cast(typing_extensions.Buffer, data))], dtype=np.uint32) + checksum = np.array([crc32c(cast("typing_extensions.Buffer", data))], dtype=np.uint32) # Append the checksum (as bytes) to the data return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B"))) diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index bee36b3160..4638d973cb 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -115,7 +115,7 @@ class _ShardIndex(NamedTuple): def chunks_per_shard(self) -> ChunkCoords: result = tuple(self.offsets_and_lengths.shape[0:-1]) # The cast is required until https://github.com/numpy/numpy/pull/27211 is merged - return cast(ChunkCoords, result) + return cast("ChunkCoords", result) def _localize_chunk(self, chunk_coords: ChunkCoords) -> ChunkCoords: return tuple( diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index 1aa1eb40e2..85e4526b8b 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -23,7 +23,7 @@ def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]: raise TypeError(f"Expected an iterable. Got {data} instead.") if not all(isinstance(a, int) for a in data): raise TypeError(f"Expected an iterable of integers. Got {data} instead.") - return tuple(cast(Iterable[int], data)) + return tuple(cast("Iterable[int]", data)) @dataclass(frozen=True) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 00ac7c2138..62af644f7d 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -903,7 +903,7 @@ async def open( store_path = await make_store_path(store) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints - _metadata_dict = cast(ArrayV3MetadataDict, metadata_dict) + _metadata_dict = cast("ArrayV3MetadataDict", metadata_dict) return cls(store_path=store_path, metadata=_metadata_dict) @property @@ -1399,7 +1399,7 @@ async def _set_selection( if isinstance(array_like, np._typing._SupportsArrayFunc): # TODO: need to handle array types that don't support __array_function__ # like PyTorch and JAX - array_like_ = cast(np._typing._SupportsArrayFunc, array_like) + array_like_ = cast("np._typing._SupportsArrayFunc", array_like) value = np.asanyarray(value, dtype=self.metadata.dtype, like=array_like_) else: if not hasattr(value, "shape"): @@ -1413,7 +1413,7 @@ async def _set_selection( value = value.astype(dtype=self.metadata.dtype, order="A") else: value = np.array(value, dtype=self.metadata.dtype, order="A") - value = cast(NDArrayLike, value) + value = cast("NDArrayLike", value) # We accept any ndarray like object from the user and convert it # to a NDBuffer (or subclass). From this point onwards, we only pass # Buffer and NDBuffer between components. @@ -2437,11 +2437,11 @@ def __getitem__(self, selection: Selection) -> NDArrayLikeOrScalar: """ fields, pure_selection = pop_fields(selection) if is_pure_fancy_indexing(pure_selection, self.ndim): - return self.vindex[cast(CoordinateSelection | MaskSelection, selection)] + return self.vindex[cast("CoordinateSelection | MaskSelection", selection)] elif is_pure_orthogonal_indexing(pure_selection, self.ndim): return self.get_orthogonal_selection(pure_selection, fields=fields) else: - return self.get_basic_selection(cast(BasicSelection, pure_selection), fields=fields) + return self.get_basic_selection(cast("BasicSelection", pure_selection), fields=fields) def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None: """Modify data for an item or region of the array. @@ -2536,11 +2536,11 @@ def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None: """ fields, pure_selection = pop_fields(selection) if is_pure_fancy_indexing(pure_selection, self.ndim): - self.vindex[cast(CoordinateSelection | MaskSelection, selection)] = value + self.vindex[cast("CoordinateSelection | MaskSelection", selection)] = value elif is_pure_orthogonal_indexing(pure_selection, self.ndim): self.set_orthogonal_selection(pure_selection, value, fields=fields) else: - self.set_basic_selection(cast(BasicSelection, pure_selection), value, fields=fields) + self.set_basic_selection(cast("BasicSelection", pure_selection), value, fields=fields) @_deprecate_positional_args def get_basic_selection( @@ -3658,7 +3658,7 @@ def update_attributes(self, new_attributes: dict[str, JSON]) -> Array: # TODO: remove this cast when type inference improves new_array = sync(self._async_array.update_attributes(new_attributes)) # TODO: remove this cast when type inference improves - _new_array = cast(AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], new_array) + _new_array = cast("AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]", new_array) return type(self)(_new_array) def __repr__(self) -> str: @@ -4253,7 +4253,7 @@ async def init_array( serializer=serializer, dtype=dtype_parsed, ) - sub_codecs = cast(tuple[Codec, ...], (*array_array, array_bytes, *bytes_bytes)) + sub_codecs = cast("tuple[Codec, ...]", (*array_array, array_bytes, *bytes_bytes)) codecs_out: tuple[Codec, ...] if shard_shape_parsed is not None: index_location = None @@ -4524,7 +4524,7 @@ def _parse_keep_array_attr( compressors = "auto" if serializer == "keep": if zarr_format == 3 and data.metadata.zarr_format == 3: - serializer = cast(SerializerLike, data.serializer) + serializer = cast("SerializerLike", data.serializer) else: serializer = "auto" if fill_value is None: @@ -4702,7 +4702,7 @@ def _parse_chunk_encoding_v3( if isinstance(filters, dict | Codec): maybe_array_array = (filters,) else: - maybe_array_array = cast(Iterable[Codec | dict[str, JSON]], filters) + maybe_array_array = cast("Iterable[Codec | dict[str, JSON]]", filters) out_array_array = tuple(_parse_array_array_codec(c) for c in maybe_array_array) if serializer == "auto": @@ -4719,7 +4719,7 @@ def _parse_chunk_encoding_v3( if isinstance(compressors, dict | Codec): maybe_bytes_bytes = (compressors,) else: - maybe_bytes_bytes = cast(Iterable[Codec | dict[str, JSON]], compressors) + maybe_bytes_bytes = cast("Iterable[Codec | dict[str, JSON]]", compressors) out_bytes_bytes = tuple(_parse_bytes_bytes_codec(c) for c in maybe_bytes_bytes) diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py index 59d3cc6b40..6cd27b30eb 100644 --- a/src/zarr/core/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -64,7 +64,7 @@ def from_dict(cls, data: ArrayConfigParams) -> Self: """ kwargs_out: ArrayConfigParams = {} for f in fields(ArrayConfig): - field_name = cast(Literal["order", "write_empty_chunks"], f.name) + field_name = cast("Literal['order', 'write_empty_chunks']", f.name) if field_name not in data: kwargs_out[field_name] = zarr_config.get(f"array.{field_name}") else: diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 94cd91f026..d0a2d992d2 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -159,7 +159,7 @@ def create_zero_length(cls) -> Self: if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( - cast(ArrayLike, None) + cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod @@ -207,7 +207,7 @@ def from_buffer(cls, buffer: Buffer) -> Self: if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( - cast(ArrayLike, None) + cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod @@ -227,7 +227,7 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self: if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( - cast(ArrayLike, None) + cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker def as_array_like(self) -> ArrayLike: @@ -371,7 +371,7 @@ def create( "Cannot call abstract method on the abstract class 'NDBuffer'" ) return cls( - cast(NDArrayLike, None) + cast("NDArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod @@ -408,7 +408,7 @@ def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: "Cannot call abstract method on the abstract class 'NDBuffer'" ) return cls( - cast(NDArrayLike, None) + cast("NDArrayLike", None) ) # This line will never be reached, but it satisfies the type checker def as_ndarray_like(self) -> NDArrayLike: @@ -440,7 +440,7 @@ def as_scalar(self) -> ScalarType: """Returns the buffer as a scalar value""" if self._data.size != 1: raise ValueError("Buffer does not contain a single scalar value") - return cast(ScalarType, self.as_numpy_array()[()]) + return cast("ScalarType", self.as_numpy_array()[()]) @property def dtype(self) -> np.dtype[Any]: diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index 77d2731c71..88746c5fac 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -103,7 +103,7 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self: return cls.from_array_like(cp.frombuffer(bytes_like, dtype="B")) def as_numpy_array(self) -> npt.NDArray[Any]: - return cast(npt.NDArray[Any], cp.asnumpy(self._data)) + return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) def __add__(self, other: core.Buffer) -> Self: other_array = other.as_array_like() @@ -204,7 +204,7 @@ def as_numpy_array(self) -> npt.NDArray[Any]: ------- NumPy array of this buffer (might be a data copy) """ - return cast(npt.NDArray[Any], cp.asnumpy(self._data)) + return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) def __getitem__(self, key: Any) -> Self: return self.__class__(self._data.__getitem__(key)) diff --git a/src/zarr/core/chunk_key_encodings.py b/src/zarr/core/chunk_key_encodings.py index 103472c3b4..91dfc90365 100644 --- a/src/zarr/core/chunk_key_encodings.py +++ b/src/zarr/core/chunk_key_encodings.py @@ -20,7 +20,7 @@ def parse_separator(data: JSON) -> SeparatorLiteral: if data not in (".", "/"): raise ValueError(f"Expected an '.' or '/' separator. Got {data} instead.") - return cast(SeparatorLiteral, data) + return cast("SeparatorLiteral", data) class ChunkKeyEncodingParams(TypedDict): @@ -48,7 +48,7 @@ def from_dict(cls, data: dict[str, JSON] | ChunkKeyEncodingLike) -> ChunkKeyEnco data = {"name": data["name"], "configuration": {"separator": data["separator"]}} # TODO: remove this cast when we are statically typing the JSON metadata completely. - data = cast(dict[str, JSON], data) + data = cast("dict[str, JSON]", data) # configuration is optional for chunk key encodings name_parsed, config_parsed = parse_named_configuration(data, require_configuration=False) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index a670834206..be37dc5109 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -158,7 +158,7 @@ def parse_fill_value(data: Any) -> Any: def parse_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): - return cast(Literal["C", "F"], data) + return cast("Literal['C', 'F']", data) raise ValueError(f"Expected one of ('C', 'F'), got {data} instead.") @@ -202,4 +202,4 @@ def _warn_order_kwarg() -> None: def _default_zarr_format() -> ZarrFormat: """Return the default zarr_version""" - return cast(ZarrFormat, int(zarr_config.get("default_zarr_format", 3))) + return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index c565cb0708..2a10943d80 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -134,6 +134,6 @@ def enable_gpu(self) -> ConfigSet: def parse_indexing_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): - return cast(Literal["C", "F"], data) + return cast("Literal['C', 'F']", data) msg = f"Expected one of ('C', 'F'), got {data} instead." raise ValueError(msg) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 5c470e29ca..86cc6a3c6b 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -7,7 +7,6 @@ import logging import warnings from collections import defaultdict -from collections.abc import Iterator, Mapping from dataclasses import asdict, dataclass, field, fields, replace from itertools import accumulate from typing import TYPE_CHECKING, Literal, TypeVar, assert_never, cast, overload @@ -65,6 +64,8 @@ Coroutine, Generator, Iterable, + Iterator, + Mapping, ) from typing import Any @@ -81,7 +82,7 @@ def parse_zarr_format(data: Any) -> ZarrFormat: """Parse the zarr_format field from metadata.""" if data in (2, 3): - return cast(ZarrFormat, data) + return cast("ZarrFormat", data) msg = f"Invalid zarr_format. Expected one of 2 or 3. Got {data}." raise ValueError(msg) @@ -89,7 +90,7 @@ def parse_zarr_format(data: Any) -> ZarrFormat: def parse_node_type(data: Any) -> NodeType: """Parse the node_type field from metadata.""" if data in ("array", "group"): - return cast(Literal["array", "group"], data) + return cast("Literal['array', 'group']", data) raise MetadataValidationError("node_type", "array or group", data) @@ -362,7 +363,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: # it's an array if isinstance(v.get("fill_value", None), np.void): v["fill_value"] = base64.standard_b64encode( - cast(bytes, v["fill_value"]) + cast("bytes", v["fill_value"]) ).decode("ascii") else: v = _replace_special_floats(v) @@ -3246,8 +3247,7 @@ def _ensure_consistent_zarr_format( raise ValueError(msg) return cast( - Mapping[str, GroupMetadata | ArrayV2Metadata] - | Mapping[str, GroupMetadata | ArrayV3Metadata], + "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", data, ) diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 998fe156a1..c11889f7f4 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -466,7 +466,7 @@ def replace_ellipsis(selection: Any, shape: ChunkCoords) -> SelectionNormalized: # check selection not too long check_selection_length(selection, shape) - return cast(SelectionNormalized, selection) + return cast("SelectionNormalized", selection) def replace_lists(selection: SelectionNormalized) -> SelectionNormalized: @@ -481,7 +481,7 @@ def replace_lists(selection: SelectionNormalized) -> SelectionNormalized: def ensure_tuple(v: Any) -> SelectionNormalized: if not isinstance(v, tuple): v = (v,) - return cast(SelectionNormalized, v) + return cast("SelectionNormalized", v) class ChunkProjection(NamedTuple): @@ -818,7 +818,7 @@ def ix_(selection: Any, shape: ChunkCoords) -> npt.NDArray[np.intp]: # now get numpy to convert to a coordinate selection selection = np.ix_(*selection) - return cast(npt.NDArray[np.intp], selection) + return cast("npt.NDArray[np.intp]", selection) def oindex(a: npt.NDArray[Any], selection: Selection) -> npt.NDArray[Any]: @@ -948,7 +948,7 @@ def __getitem__(self, selection: OrthogonalSelection | Array) -> NDArrayLikeOrSc new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.get_orthogonal_selection( - cast(OrthogonalSelection, new_selection), fields=fields + cast("OrthogonalSelection", new_selection), fields=fields ) def __setitem__(self, selection: OrthogonalSelection, value: npt.ArrayLike) -> None: @@ -956,7 +956,7 @@ def __setitem__(self, selection: OrthogonalSelection, value: npt.ArrayLike) -> N new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.set_orthogonal_selection( - cast(OrthogonalSelection, new_selection), value, fields=fields + cast("OrthogonalSelection", new_selection), value, fields=fields ) @@ -1050,14 +1050,14 @@ def __getitem__(self, selection: BasicSelection) -> NDArrayLikeOrScalar: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) - return self.array.get_block_selection(cast(BasicSelection, new_selection), fields=fields) + return self.array.get_block_selection(cast("BasicSelection", new_selection), fields=fields) def __setitem__(self, selection: BasicSelection, value: npt.ArrayLike) -> None: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.set_block_selection( - cast(BasicSelection, new_selection), value, fields=fields + cast("BasicSelection", new_selection), value, fields=fields ) @@ -1105,12 +1105,12 @@ def __init__( nchunks = reduce(operator.mul, cdata_shape, 1) # some initial normalization - selection_normalized = cast(CoordinateSelectionNormalized, ensure_tuple(selection)) + selection_normalized = cast("CoordinateSelectionNormalized", ensure_tuple(selection)) selection_normalized = tuple( np.asarray([i]) if is_integer(i) else i for i in selection_normalized ) selection_normalized = cast( - CoordinateSelectionNormalized, replace_lists(selection_normalized) + "CoordinateSelectionNormalized", replace_lists(selection_normalized) ) # validation @@ -1214,8 +1214,8 @@ def __iter__(self) -> Iterator[ChunkProjection]: class MaskIndexer(CoordinateIndexer): def __init__(self, selection: MaskSelection, shape: ChunkCoords, chunk_grid: ChunkGrid) -> None: # some initial normalization - selection_normalized = cast(tuple[MaskSelection], ensure_tuple(selection)) - selection_normalized = cast(tuple[MaskSelection], replace_lists(selection_normalized)) + selection_normalized = cast("tuple[MaskSelection]", ensure_tuple(selection)) + selection_normalized = cast("tuple[MaskSelection]", replace_lists(selection_normalized)) # validation if not is_mask_selection(selection_normalized, shape): @@ -1311,14 +1311,14 @@ def pop_fields(selection: SelectionWithFields) -> tuple[Fields | None, Selection elif not isinstance(selection, tuple): # single selection item, no fields # leave selection as-is - return None, cast(Selection, selection) + return None, cast("Selection", selection) else: # multiple items, split fields from selection items fields: Fields = [f for f in selection if isinstance(f, str)] fields = fields[0] if len(fields) == 1 else fields selection_tuple = tuple(s for s in selection if not isinstance(s, str)) selection = cast( - Selection, selection_tuple[0] if len(selection_tuple) == 1 else selection_tuple + "Selection", selection_tuple[0] if len(selection_tuple) == 1 else selection_tuple ) return fields, selection @@ -1380,12 +1380,12 @@ def get_indexer( new_selection = ensure_tuple(selection) new_selection = replace_lists(new_selection) if is_coordinate_selection(new_selection, shape): - return CoordinateIndexer(cast(CoordinateSelection, selection), shape, chunk_grid) + return CoordinateIndexer(cast("CoordinateSelection", selection), shape, chunk_grid) elif is_mask_selection(new_selection, shape): - return MaskIndexer(cast(MaskSelection, selection), shape, chunk_grid) + return MaskIndexer(cast("MaskSelection", selection), shape, chunk_grid) else: raise VindexInvalidSelectionError(new_selection) elif is_pure_orthogonal_indexing(pure_selection, len(shape)): - return OrthogonalIndexer(cast(OrthogonalSelection, selection), shape, chunk_grid) + return OrthogonalIndexer(cast("OrthogonalSelection", selection), shape, chunk_grid) else: - return BasicIndexer(cast(BasicSelection, selection), shape, chunk_grid) + return BasicIndexer(cast("BasicSelection", selection), shape, chunk_grid) diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 029a3e09a7..a8f4f4abb4 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -378,7 +378,7 @@ def _serialize_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> JSON: # There's a relationship between dtype and fill_value # that mypy isn't aware of. The fact that we have S or V dtype here # means we should have a bytes-type fill_value. - serialized = base64.standard_b64encode(cast(bytes, fill_value)).decode("ascii") + serialized = base64.standard_b64encode(cast("bytes", fill_value)).decode("ascii") elif isinstance(fill_value, np.datetime64): serialized = np.datetime_as_string(fill_value) elif isinstance(fill_value, numbers.Integral): @@ -448,7 +448,7 @@ def _default_compressor( else: raise ValueError(f"Unsupported dtype kind {dtype.kind}") - return cast(dict[str, JSON] | None, default_compressor.get(dtype_key, None)) + return cast("dict[str, JSON] | None", default_compressor.get(dtype_key, None)) def _default_filters( @@ -470,4 +470,4 @@ def _default_filters( else: raise ValueError(f"Unsupported dtype kind {dtype.kind}") - return cast(list[dict[str, JSON]] | None, default_filters.get(dtype_key, None)) + return cast("list[dict[str, JSON]] | None", default_filters.get(dtype_key, None)) diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 63f6515e44..dcbf44f89b 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -273,7 +273,7 @@ def __init__( fill_value = default_fill_value(data_type_parsed) # we pass a string here rather than an enum to make mypy happy fill_value_parsed = parse_fill_value( - fill_value, dtype=cast(ALL_DTYPES, data_type_parsed.value) + fill_value, dtype=cast("ALL_DTYPES", data_type_parsed.value) ) attributes_parsed = parse_attributes(attributes) codecs_parsed_partial = parse_codecs(codecs) @@ -524,7 +524,7 @@ def parse_fill_value( return np.bytes_(fill_value) # the rest are numeric types - np_dtype = cast(np.dtype[Any], data_type.to_numpy()) + np_dtype = cast("np.dtype[Any]", data_type.to_numpy()) if isinstance(fill_value, Sequence) and not isinstance(fill_value, str): if data_type in (DataType.complex64, DataType.complex128): @@ -588,7 +588,7 @@ def default_fill_value(dtype: DataType) -> str | bytes | np.generic: return b"" else: np_dtype = dtype.to_numpy() - np_dtype = cast(np.dtype[Any], np_dtype) + np_dtype = cast("np.dtype[Any]", np_dtype) return np_dtype.type(0) # type: ignore[misc] diff --git a/src/zarr/core/strings.py b/src/zarr/core/strings.py index ffca0c3b0c..15c5fddfee 100644 --- a/src/zarr/core/strings.py +++ b/src/zarr/core/strings.py @@ -30,7 +30,7 @@ def cast_array( data: np.ndarray[Any, np.dtype[Any]], ) -> np.ndarray[Any, np.dtypes.StringDType | np.dtypes.ObjectDType]: out = data.astype(_STRING_DTYPE, copy=False) - return cast(np.ndarray[Any, np.dtypes.StringDType], out) + return cast("np.ndarray[Any, np.dtypes.StringDType]", out) except AttributeError: # if not available, we fall back on an object array of strings, as in Zarr < 3 @@ -41,7 +41,7 @@ def cast_array( data: np.ndarray[Any, np.dtype[Any]], ) -> np.ndarray[Any, Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"]]: out = data.astype(_STRING_DTYPE, copy=False) - return cast(np.ndarray[Any, np.dtypes.ObjectDType], out) + return cast("np.ndarray[Any, np.dtypes.ObjectDType]", out) def cast_to_string_dtype( diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 7cf57ab9d6..afc15d742c 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -30,7 +30,7 @@ def has_cupy() -> bool: try: import cupy - return cast(bool, cupy.cuda.runtime.getDeviceCount() > 0) + return cast("bool", cupy.cuda.runtime.getDeviceCount() > 0) except ImportError: return False except cupy.cuda.runtime.CUDARuntimeError: From ada3b221579b426154d0df9a1681af390164a6cc Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 28 May 2025 10:37:22 +0100 Subject: [PATCH 132/160] Test using latest numpy (#3064) --- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/test.yml | 6 +++--- pyproject.toml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index c7056a2c4b..752440719b 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: python-version: ['3.11'] - numpy-version: ['2.1'] + numpy-version: ['2.2'] dependency-set: ["minimal"] steps: diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 0a320de00b..15adb0d4a8 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -26,7 +26,7 @@ jobs: strategy: matrix: python-version: ['3.11'] - numpy-version: ['2.1'] + numpy-version: ['2.2'] dependency-set: ["optional"] steps: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4160fa3506..ee1adb6b0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: python-version: ['3.11', '3.12', '3.13'] - numpy-version: ['1.25', '2.1'] + numpy-version: ['1.25', '2.2'] dependency-set: ["minimal", "optional"] os: ["ubuntu-latest"] include: @@ -30,7 +30,7 @@ jobs: dependency-set: 'optional' os: 'macos-latest' - python-version: '3.13' - numpy-version: '2.1' + numpy-version: '2.2' dependency-set: 'optional' os: 'macos-latest' - python-version: '3.11' @@ -38,7 +38,7 @@ jobs: dependency-set: 'optional' os: 'windows-latest' - python-version: '3.13' - numpy-version: '2.1' + numpy-version: '2.2' dependency-set: 'optional' os: 'windows-latest' runs-on: ${{ matrix.os }} diff --git a/pyproject.toml b/pyproject.toml index a43e51abd2..1f270b435f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,7 +149,7 @@ features = ["test"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] +numpy = ["1.25", "2.2"] deps = ["minimal", "optional"] [tool.hatch.envs.test.overrides] @@ -185,7 +185,7 @@ features = ["test", "gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] +numpy = ["1.25", "2.2"] version = ["minimal"] [tool.hatch.envs.gputest.scripts] From feb4aa283295530a127c481558e810831ea2d6ef Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 28 May 2025 19:45:51 +0200 Subject: [PATCH 133/160] remove insertion of vlen-string codec for v2 metadata creation (#3100) * remove insertion of vlen-string codec for v2 metadata creation * changelog * Update changes/3100.bugfix.rst Co-authored-by: David Stansby * Update changes/3100.bugfix.rst Co-authored-by: David Stansby * add dtype strings to test_v2_chunk_encoding --------- Co-authored-by: David Stansby --- changes/3100.bugfix.rst | 3 +++ src/zarr/core/array.py | 8 -------- tests/test_array.py | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) create mode 100644 changes/3100.bugfix.rst diff --git a/changes/3100.bugfix.rst b/changes/3100.bugfix.rst new file mode 100644 index 0000000000..11f06628c0 --- /dev/null +++ b/changes/3100.bugfix.rst @@ -0,0 +1,3 @@ +For Zarr format 2, allow fixed-length string arrays to be created without automatically inserting a +``Vlen-UT8`` codec in the array of filters. Fixed-length string arrays do not need this codec. This +change fixes a regression where fixed-length string arrays created with Zarr Python 3 could not be read with Zarr Python 2.18. \ No newline at end of file diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 62af644f7d..b4e8ac0ff6 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -768,14 +768,6 @@ def _create_metadata_v2( dtype = parse_dtype(dtype, zarr_format=2) - # inject VLenUTF8 for str dtype if not already present - if np.issubdtype(dtype, np.str_): - filters = filters or [] - from numcodecs.vlen import VLenUTF8 - - if not any(isinstance(x, VLenUTF8) or x["id"] == "vlen-utf8" for x in filters): - filters = list(filters) + [VLenUTF8()] - return ArrayV2Metadata( shape=shape, dtype=np.dtype(dtype), diff --git a/tests/test_array.py b/tests/test_array.py index a6bcd17c4b..3fc7b3938c 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1245,7 +1245,7 @@ async def test_invalid_v3_arguments( zarr.create(store=store, dtype="uint8", shape=(10,), zarr_format=3, **kwargs) @staticmethod - @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + @pytest.mark.parametrize("dtype", ["uint8", "float32", "str", "U10", "S10", ">M8[10s]"]) @pytest.mark.parametrize( "compressors", [ From af55fcfaefa42b5ef556b1b5be33dcdd06a7fd0b Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 29 May 2025 10:06:43 +0100 Subject: [PATCH 134/160] Add a GroupNotFoundError (#3066) * Add a GroupNotFoundError * Fixup tests * Add changelog entries * Remove inheritance change --- changes/3066.feature.rst | 1 + src/zarr/api/asynchronous.py | 4 ++-- src/zarr/errors.py | 8 ++++++++ tests/test_metadata/test_v3.py | 8 +++++--- 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 changes/3066.feature.rst diff --git a/changes/3066.feature.rst b/changes/3066.feature.rst new file mode 100644 index 0000000000..89d5ddb1c6 --- /dev/null +++ b/changes/3066.feature.rst @@ -0,0 +1 @@ +Added `~zarr.errors.GroupNotFoundError`, which is raised when attempting to open a group that does not exist. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 5b9c0bee3d..aae7d28d15 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -39,7 +39,7 @@ ) from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v2 import _default_compressor, _default_filters -from zarr.errors import NodeTypeValidationError +from zarr.errors import GroupNotFoundError, NodeTypeValidationError from zarr.storage._common import make_store_path if TYPE_CHECKING: @@ -836,7 +836,7 @@ async def open_group( overwrite=overwrite, attributes=attributes, ) - raise FileNotFoundError(f"Unable to find group: {store_path}") + raise GroupNotFoundError(store, store_path.path) async def create( diff --git a/src/zarr/errors.py b/src/zarr/errors.py index 441cdab9a3..4d3140a4a9 100644 --- a/src/zarr/errors.py +++ b/src/zarr/errors.py @@ -21,6 +21,14 @@ def __init__(self, *args: Any) -> None: super().__init__(self._msg.format(*args)) +class GroupNotFoundError(BaseZarrError, FileNotFoundError): + """ + Raised when a group isn't found at a certain path. + """ + + _msg = "No group found in store {!r} at path {!r}" + + class ContainsGroupError(BaseZarrError): """Raised when a group already exists at a certain path.""" diff --git a/tests/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py index a47cbf43bb..13549b10a4 100644 --- a/tests/test_metadata/test_v3.py +++ b/tests/test_metadata/test_v3.py @@ -20,7 +20,7 @@ parse_fill_value, parse_zarr_format, ) -from zarr.errors import MetadataValidationError +from zarr.errors import MetadataValidationError, NodeTypeValidationError if TYPE_CHECKING: from collections.abc import Sequence @@ -62,7 +62,8 @@ @pytest.mark.parametrize("data", [None, 1, 2, 4, 5, "3"]) def test_parse_zarr_format_invalid(data: Any) -> None: with pytest.raises( - ValueError, match=f"Invalid value for 'zarr_format'. Expected '3'. Got '{data}'." + MetadataValidationError, + match=f"Invalid value for 'zarr_format'. Expected '3'. Got '{data}'.", ): parse_zarr_format(data) @@ -88,7 +89,8 @@ def test_parse_node_type_invalid(node_type: Any) -> None: @pytest.mark.parametrize("data", [None, "group"]) def test_parse_node_type_array_invalid(data: Any) -> None: with pytest.raises( - ValueError, match=f"Invalid value for 'node_type'. Expected 'array'. Got '{data}'." + NodeTypeValidationError, + match=f"Invalid value for 'node_type'. Expected 'array'. Got '{data}'.", ): parse_node_type_array(data) From 36ca49724c29b3fd4319dd57e08c881095fa4b9f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 30 May 2025 13:42:34 +0100 Subject: [PATCH 135/160] Use automatic chunking in array creation routines (#3103) * Delegate logic for chunks to AsyncArray._create * Fix changelog number * Test public API * Add todo --- changes/3103.bugfix.rst | 7 +++++++ src/zarr/api/asynchronous.py | 8 -------- src/zarr/core/chunk_grids.py | 3 +++ tests/test_api.py | 38 ++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 changes/3103.bugfix.rst diff --git a/changes/3103.bugfix.rst b/changes/3103.bugfix.rst new file mode 100644 index 0000000000..93aecce908 --- /dev/null +++ b/changes/3103.bugfix.rst @@ -0,0 +1,7 @@ +When creating arrays without explicitly specifying a chunk size using `zarr.create` and other +array creation routines, the chunk size will now set automatically instead of defaulting to the data shape. +For large arrays this will result in smaller default chunk sizes. +To retain previous behaviour, explicitly set the chunk shape to the data shape. + +This fix matches the existing chunking behaviour of +`zarr.save_array` and `zarr.api.asynchronous.AsyncArray.create`. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index aae7d28d15..b262ced29b 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -992,19 +992,11 @@ async def create( ) if zarr_format == 2: - if chunks is None: - chunks = shape dtype = parse_dtype(dtype, zarr_format) if not filters: filters = _default_filters(dtype) if compressor == "auto": compressor = _default_compressor(dtype) - elif zarr_format == 3 and chunk_shape is None: # type: ignore[redundant-expr] - if chunks is not None: - chunk_shape = chunks - chunks = None - else: - chunk_shape = shape if synchronizer is not None: warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2) diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index d3e40c26ed..b5a581b8a4 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -64,6 +64,9 @@ def _guess_chunks( if isinstance(shape, int): shape = (shape,) + if typesize == 0: + return shape + ndims = len(shape) # require chunks to have non-zero length for all dimensions chunks = np.maximum(np.array(shape, dtype="=f8"), 1) diff --git a/tests/test_api.py b/tests/test_api.py index 640478e9c1..8cd4ab6b60 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1301,3 +1301,41 @@ def test_no_overwrite_load(tmp_path: Path) -> None: with contextlib.suppress(NotImplementedError): zarr.load(store) assert existing_fpath.exists() + + +@pytest.mark.parametrize( + "f", + [ + zarr.array, + zarr.create, + zarr.create_array, + zarr.ones, + zarr.ones_like, + zarr.empty, + zarr.empty_like, + zarr.full, + zarr.full_like, + zarr.zeros, + zarr.zeros_like, + ], +) +def test_auto_chunks(f: Callable[..., Array]) -> None: + # Make sure chunks are set automatically across the public API + # TODO: test shards with this test too + shape = (1000, 1000) + dtype = np.uint8 + kwargs = {"shape": shape, "dtype": dtype} + array = np.zeros(shape, dtype=dtype) + store = zarr.storage.MemoryStore() + + if f in [zarr.full, zarr.full_like]: + kwargs["fill_value"] = 0 + if f in [zarr.array]: + kwargs["data"] = array + if f in [zarr.empty_like, zarr.full_like, zarr.empty_like, zarr.ones_like, zarr.zeros_like]: + kwargs["a"] = array + if f in [zarr.create_array]: + kwargs["store"] = store + + a = f(**kwargs) + assert a.chunks == (500, 500) From d19f3f0a825b5be986fa6e2f797aa4ddec0b90ed Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Sat, 31 May 2025 03:09:35 -0500 Subject: [PATCH 136/160] Ignore stale children when reconsolidating metadata (#2980) Co-authored-by: David Stansby --- changes/2921.bugfix.rst | 1 + src/zarr/api/asynchronous.py | 5 +- src/zarr/core/group.py | 68 +++++++++++++++++++++--- tests/test_metadata/test_consolidated.py | 43 +++++++++++++++ 4 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 changes/2921.bugfix.rst diff --git a/changes/2921.bugfix.rst b/changes/2921.bugfix.rst new file mode 100644 index 0000000000..65db48654f --- /dev/null +++ b/changes/2921.bugfix.rst @@ -0,0 +1 @@ +Ignore stale child metadata when reconsolidating metadata. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index b262ced29b..7e10350bb9 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -201,7 +201,10 @@ async def consolidate_metadata( group = await AsyncGroup.open(store_path, zarr_format=zarr_format, use_consolidated=False) group.store_path.store._check_writable() - members_metadata = {k: v.metadata async for k, v in group.members(max_depth=None)} + members_metadata = { + k: v.metadata + async for k, v in group.members(max_depth=None, use_consolidated_for_children=False) + } # While consolidating, we want to be explicit about when child groups # are empty by inserting an empty dict for consolidated_metadata.metadata for k, v in members_metadata.items(): diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 86cc6a3c6b..6c6f104605 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1305,6 +1305,8 @@ async def nmembers( async def members( self, max_depth: int | None = 0, + *, + use_consolidated_for_children: bool = True, ) -> AsyncGenerator[ tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], None, @@ -1322,6 +1324,11 @@ async def members( default, (``max_depth=0``) only immediate children are included. Set ``max_depth=None`` to include all nodes, and some positive integer to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. Returns ------- @@ -1332,7 +1339,9 @@ async def members( """ if max_depth is not None and max_depth < 0: raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") - async for item in self._members(max_depth=max_depth): + async for item in self._members( + max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children + ): yield item def _members_consolidated( @@ -1362,7 +1371,7 @@ def _members_consolidated( yield from obj._members_consolidated(new_depth, prefix=key) async def _members( - self, max_depth: int | None + self, max_depth: int | None, *, use_consolidated_for_children: bool = True ) -> AsyncGenerator[ tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None ]: @@ -1392,7 +1401,11 @@ async def _members( # enforce a concurrency limit by passing a semaphore to all the recursive functions semaphore = asyncio.Semaphore(config.get("async.concurrency")) async for member in _iter_members_deep( - self, max_depth=max_depth, skip_keys=skip_keys, semaphore=semaphore + self, + max_depth=max_depth, + skip_keys=skip_keys, + semaphore=semaphore, + use_consolidated_for_children=use_consolidated_for_children, ): yield member @@ -2092,10 +2105,34 @@ def nmembers(self, max_depth: int | None = 0) -> int: return self._sync(self._async_group.nmembers(max_depth=max_depth)) - def members(self, max_depth: int | None = 0) -> tuple[tuple[str, Array | Group], ...]: + def members( + self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True + ) -> tuple[tuple[str, Array | Group], ...]: """ - Return the sub-arrays and sub-groups of this group as a tuple of (name, array | group) - pairs + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. """ _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) @@ -3329,6 +3366,7 @@ async def _iter_members_deep( max_depth: int | None, skip_keys: tuple[str, ...], semaphore: asyncio.Semaphore | None = None, + use_consolidated_for_children: bool = True, ) -> AsyncGenerator[ tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None ]: @@ -3346,6 +3384,11 @@ async def _iter_members_deep( A tuple of keys to skip when iterating over the possible members of the group. semaphore : asyncio.Semaphore | None An optional semaphore to use for concurrency control. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. Yields ------ @@ -3360,8 +3403,19 @@ async def _iter_members_deep( else: new_depth = max_depth - 1 async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): + is_group = isinstance(node, AsyncGroup) + if ( + is_group + and not use_consolidated_for_children + and node.metadata.consolidated_metadata is not None # type: ignore [union-attr] + ): + node = cast("AsyncGroup", node) + # We've decided not to trust consolidated metadata at this point, because we're + # reconsolidating the metadata, for example. + node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) yield name, node - if isinstance(node, AsyncGroup) and do_recursion: + if is_group and do_recursion: + node = cast("AsyncGroup", node) to_recurse[name] = _iter_members_deep( node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore ) diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index a179982e94..9bf1c4e544 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -574,6 +574,49 @@ async def test_use_consolidated_false( assert good.metadata.consolidated_metadata assert sorted(good.metadata.consolidated_metadata.metadata) == ["a", "b"] + async def test_stale_child_metadata_ignored(self, memory_store: zarr.storage.MemoryStore): + # https://github.com/zarr-developers/zarr-python/issues/2921 + # When consolidating metadata, we should ignore any (possibly stale) metadata + # from previous consolidations, *including at child nodes*. + root = await zarr.api.asynchronous.group(store=memory_store, zarr_format=3) + await root.create_group("foo") + await zarr.api.asynchronous.consolidate_metadata(memory_store, path="foo") + await root.create_group("foo/bar/spam") + + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + reopened = await zarr.api.asynchronous.open_consolidated(store=memory_store, zarr_format=3) + result = [x[0] async for x in reopened.members(max_depth=None)] + expected = ["foo", "foo/bar", "foo/bar/spam"] + assert result == expected + + async def test_use_consolidated_for_children_members( + self, memory_store: zarr.storage.MemoryStore + ): + # A test that has *unconsolidated* metadata at the root group, but discovers + # a child group with consolidated metadata. + + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + # Consolidate metadata at "a/b" + await zarr.api.asynchronous.consolidate_metadata(memory_store, path="a/b") + + # Add a new group a/b/c, that's not present in the CM at "a/b" + await root.create_group("a/b/c") + + # Now according to the consolidated metadata, "a" has children ["b"] + # but according to the unconsolidated metadata, "a" has children ["b", "c"] + group = await zarr.api.asynchronous.open_group(store=memory_store, path="a") + with pytest.warns(UserWarning, match="Object at 'c' not found"): + result = sorted([x[0] async for x in group.members(max_depth=None)]) + expected = ["b"] + assert result == expected + + result = sorted( + [x[0] async for x in group.members(max_depth=None, use_consolidated_for_children=False)] + ) + expected = ["b", "b/c"] + assert result == expected @pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf]) async def test_consolidated_metadata_encodes_special_chars( From feb034aebc4a3151e8639b7f3d49219f861c4ec8 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 4 Jun 2025 19:52:12 +0200 Subject: [PATCH 137/160] Add pytest pin (#3113) --- pyproject.toml | 3 ++- tests/test_metadata/test_consolidated.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1f270b435f..0ef78cdaf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,8 @@ gpu = [ # Development extras test = [ "coverage", - "pytest", + # Pin possibly due to https://github.com/pytest-dev/pytest-cov/issues/693 + "pytest<8.4", "pytest-asyncio", "pytest-cov", "pytest-accept", diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 9bf1c4e544..f71a946300 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -618,6 +618,7 @@ async def test_use_consolidated_for_children_members( expected = ["b", "b/c"] assert result == expected + @pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf]) async def test_consolidated_metadata_encodes_special_chars( memory_store: Store, zarr_format: ZarrFormat, fill_value: float From aa844a36114c0079ab4db144ce7eb1a67bf23cc1 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 4 Jun 2025 21:00:01 +0200 Subject: [PATCH 138/160] remove fsspec from upstream tests (#3116) --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0ef78cdaf4..dddcb981bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -210,7 +210,6 @@ dependencies = [ 'packaging @ git+https://github.com/pypa/packaging', 'numpy', # from scientific-python-nightly-wheels 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', - 'fsspec @ git+https://github.com/fsspec/filesystem_spec', 's3fs @ git+https://github.com/fsspec/s3fs', 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', 'typing_extensions @ git+https://github.com/python/typing_extensions', From d103619ea98436cff75a1bbf38a5e5d480c784eb Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 6 Jun 2025 08:53:04 +0200 Subject: [PATCH 139/160] Fix release note links (#3106) Co-authored-by: Davis Bennett --- docs/conf.py | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9bb1c48901..5f2e4ba3ec 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -105,7 +105,8 @@ def skip_submodules( "roadmap": "developers/roadmap.html", "installation": "user-guide/installation.html", "api": "api/zarr/index", - "release": "release-notes" + "release": "release-notes.html", + "release-notes": "release-notes.html" } # The language for content autogenerated by Sphinx. Refer to documentation diff --git a/pyproject.toml b/pyproject.toml index dddcb981bb..d25af7c5fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,7 @@ docs = [ [project.urls] "Bug Tracker" = "https://github.com/zarr-developers/zarr-python/issues" -Changelog = "https://zarr.readthedocs.io/en/stable/release.html" +Changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" Discussions = "https://github.com/zarr-developers/zarr-python/discussions" Documentation = "https://zarr.readthedocs.io/" Homepage = "https://github.com/zarr-developers/zarr-python" From a8d4f42720d9aa698adeaac1836793a43ee2ff87 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:30:10 -0400 Subject: [PATCH 140/160] Use context manager for temp dirs in store tests (#3110) Co-authored-by: Davis Bennett --- tests/test_store/test_core.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 6a94cc0ac2..e9c9319ad3 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -18,6 +18,26 @@ ) +@pytest.fixture( + params=["none", "temp_dir_str", "temp_dir_path", "store_path", "memory_store", "dict"] +) +def store_like(request): + if request.param == "none": + yield None + elif request.param == "temp_dir_str": + with tempfile.TemporaryDirectory() as temp_dir: + yield temp_dir + elif request.param == "temp_dir_path": + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + elif request.param == "store_path": + yield StorePath(store=MemoryStore(store_dict={}), path="/") + elif request.param == "memory_store": + yield MemoryStore(store_dict={}) + elif request.param == "dict": + yield {} + + @pytest.mark.parametrize("path", ["foo", "foo/bar"]) @pytest.mark.parametrize("write_group", [True, False]) @pytest.mark.parametrize("zarr_format", [2, 3]) @@ -134,17 +154,6 @@ async def test_make_store_path_fsspec(monkeypatch) -> None: assert isinstance(store_path.store, FsspecStore) -@pytest.mark.parametrize( - "store_like", - [ - None, - tempfile.TemporaryDirectory().name, - Path(tempfile.TemporaryDirectory().name), - StorePath(store=MemoryStore(store_dict={}), path="/"), - MemoryStore(store_dict={}), - {}, - ], -) async def test_make_store_path_storage_options_raises(store_like: StoreLike) -> None: with pytest.raises(TypeError, match="storage_options"): await make_store_path(store_like, storage_options={"foo": "bar"}) From 402777a57ebef9a2871f02b5f478789cc8ae4af0 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Sat, 7 Jun 2025 12:13:57 +0200 Subject: [PATCH 141/160] Multiple imports for an import name (#3033) --- src/zarr/core/sync.py | 2 +- src/zarr/storage/_obstore.py | 4 +--- tests/test_group.py | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index d9b4839e8e..ffb04e764d 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -6,7 +6,7 @@ import os import threading from concurrent.futures import ThreadPoolExecutor, wait -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, TypeVar from typing_extensions import ParamSpec diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index 738754a8b9..c048721cae 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -4,8 +4,7 @@ import contextlib import pickle from collections import defaultdict -from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, TypedDict +from typing import TYPE_CHECKING, TypedDict from zarr.abc.store import ( ByteRequest, @@ -14,7 +13,6 @@ Store, SuffixByteRequest, ) -from zarr.core.buffer.core import BufferPrototype from zarr.core.config import config if TYPE_CHECKING: diff --git a/tests/test_group.py b/tests/test_group.py index b4dace2568..7cf29c30d9 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1519,7 +1519,6 @@ def test_create_nodes_concurrency_limit(store: MemoryStore) -> None: # if create_nodes is sensitive to IO latency, # this should take (num_groups * get_latency) seconds # otherwise, it should take only marginally more than get_latency seconds - with zarr_config.set({"async.concurrency": 1}): start = time.time() _ = tuple(sync_group.create_nodes(store=latency_store, nodes=groups)) @@ -2024,9 +2023,7 @@ def test_group_members_concurrency_limit(store: MemoryStore) -> None: # if .members is sensitive to IO latency, # this should take (num_groups * get_latency) seconds # otherwise, it should take only marginally more than get_latency seconds - from zarr.core.config import config - - with config.set({"async.concurrency": 1}): + with zarr_config.set({"async.concurrency": 1}): start = time.time() _ = group_read.members() elapsed = time.time() - start From 9a9d3f9509e8e7e54743f2dfb0016ac5d80d8e3e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 7 Jun 2025 13:53:40 +0200 Subject: [PATCH 142/160] Improvements to the release guide (#3070) Co-authored-by: Davis Bennett --- docs/developers/contributing.rst | 50 ++++++++++++-------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index fa65f71d48..03388e1544 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -251,14 +251,11 @@ See the `towncrier`_ docs for more. .. _towncrier: https://towncrier.readthedocs.io/en/stable/tutorial.html -Development best practices, policies and procedures ---------------------------------------------------- - The following information is mainly for core developers, but may also be of interest to contributors. Merging pull requests -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Pull requests submitted by an external contributor should be reviewed and approved by at least one core developer before being merged. Ideally, pull requests submitted by a core developer @@ -268,10 +265,10 @@ Pull requests should not be merged until all CI checks have passed (GitHub Actio Codecov) against code that has had the latest main merged in. Compatibility and versioning policies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------- Versioning -"""""""""" +~~~~~~~~~~ Versions of this library are identified by a triplet of integers with the form ``..``, for example ``3.0.4``. A release of ``zarr-python`` is associated with a new version identifier. That new identifier is generated by incrementing exactly one of the components of @@ -323,7 +320,7 @@ backwards-compatible changes wherever possible. When a backwards-incompatible ch users should be notified well in advance, e.g. via informative deprecation warnings. Data format compatibility -^^^^^^^^^^^^^^^^^^^^^^^^^ +""""""""""""""""""""""""" The Zarr library is an implementation of a file format standard defined externally -- see the `Zarr specifications website `_ for the list of @@ -340,36 +337,28 @@ breaking changes may be more frequent than usual. Release procedure -~~~~~~~~~~~~~~~~~ - -.. note:: - - Most of the release process is now handled by GitHub workflow which should - automatically push a release to PyPI if a tag is pushed. +----------------- Pre-release -""""""""""" +~~~~~~~~~~~ 1. Make sure that all pull requests which will be included in the release - have been properly documented as changelog files in :file:`changes`. -2. Run ``towncrier build --version x.y.z`` to create the changelog. + have been properly documented as changelog files in the :file:`changes/` directory. +2. Run ``towncrier build --version x.y.z`` to create the changelog, and commit the result + to the main branch. Releasing -""""""""" -To make a new release, go to -https://github.com/zarr-developers/zarr-python/releases and -click "Draft a new release". Choose a version number prefixed -with a `v` (e.g. `v0.0.0`). For pre-releases, include the -appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`). - - -Set the description of the release to:: +~~~~~~~~~ +1. Go to https://github.com/zarr-developers/zarr-python/releases +2. Click "Draft a new release". +3. Choose a version number prefixed with a `v` (e.g. `v0.0.0`). + For pre-releases, include the appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`). +4. Set the description of the release to:: See release notes https://zarr.readthedocs.io/en/stable/release-notes.html#release-0-0-0 -replacing the correct version numbers. For pre-release versions, -the URL should omit the pre-release suffix, e.g. "a1" or "rc1". - -Click on "Generate release notes" to auto-file the description. + replacing the correct version numbers. For pre-release versions, + the URL should omit the pre-release suffix, e.g. "a1" or "rc1". +5. Click on "Generate release notes" to auto-fill the description. After creating the release, the documentation will be built on https://readthedocs.io. Full releases will be available under @@ -378,9 +367,8 @@ pre-releases will be available under `/latest `_. Post-release -"""""""""""" +~~~~~~~~~~~~ - Review and merge the pull request on the `conda-forge feedstock `_ that will be automatically generated. -- Create a new "Unreleased" section in the release notes From 6193fd9b3bd23f0aa9676b489e62f224e4325b3e Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 9 Jun 2025 00:17:40 +0200 Subject: [PATCH 143/160] Multiple imports for an import name (#3120) --- tests/test_store/test_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py index 943564abc8..4d9e8fcc1f 100644 --- a/tests/test_store/test_object.py +++ b/tests/test_store/test_object.py @@ -4,7 +4,7 @@ import pytest obstore = pytest.importorskip("obstore") -import pytest + from hypothesis.stateful import ( run_state_machine_as_test, ) From d53d3683362f30add9d2f7e874a636d9691046ad Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 10 Jun 2025 13:11:56 +0200 Subject: [PATCH 144/160] Unused import (#3121) * Unused import * Partially revert "Unused import" This reverts commit ff13739393b2f1926b18b4019cc642f5d9ed3dafi partially. --- tests/test_store/test_memory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py index a090f56951..4fc3f6e698 100644 --- a/tests/test_store/test_memory.py +++ b/tests/test_store/test_memory.py @@ -8,8 +8,6 @@ import pytest import zarr -import zarr.core -import zarr.core.array from zarr.core.buffer import Buffer, cpu, gpu from zarr.storage import GpuMemoryStore, MemoryStore from zarr.testing.store import StoreTests From c5522826109cb813d23bd3780dfe51573906d821 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Tue, 10 Jun 2025 14:04:04 +0200 Subject: [PATCH 145/160] Fix `zarr.save` for given `path` and multiple `args` arrays (#3127) * test and fix path argument in save_group * add tests for load_array * Revert "add tests for load_array" partially This reverts commit 2d0f6e9ce9de44bfbb5a9c4ed54d2b31f7de1b10 partially. * document changes --- changes/3127.bugfix.rst | 2 ++ src/zarr/api/asynchronous.py | 3 +-- tests/test_api.py | 17 +++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 changes/3127.bugfix.rst diff --git a/changes/3127.bugfix.rst b/changes/3127.bugfix.rst new file mode 100644 index 0000000000..35d7f5d329 --- /dev/null +++ b/changes/3127.bugfix.rst @@ -0,0 +1,2 @@ +When `zarr.save` has an argument `path=some/path/` and multiple arrays in `args`, the path resulted in `some/path/some/path` due to using the `path` +argument twice while building the array path. This is now fixed. \ No newline at end of file diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 7e10350bb9..b296d21bd4 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -505,13 +505,12 @@ async def save_group( raise ValueError("at least one array must be provided") aws = [] for i, arr in enumerate(args): - _path = f"{path}/arr_{i}" if path is not None else f"arr_{i}" aws.append( save_array( store_path, arr, zarr_format=zarr_format, - path=_path, + path=f"arr_{i}", storage_options=storage_options, ) ) diff --git a/tests/test_api.py b/tests/test_api.py index 8cd4ab6b60..5d41a37493 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -229,22 +229,23 @@ async def test_open_group_unspecified_version( @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("n_args", [10, 1, 0]) @pytest.mark.parametrize("n_kwargs", [10, 1, 0]) -def test_save(store: Store, n_args: int, n_kwargs: int) -> None: +@pytest.mark.parametrize("path", [None, "some_path"]) +def test_save(store: Store, n_args: int, n_kwargs: int, path: None | str) -> None: data = np.arange(10) args = [np.arange(10) for _ in range(n_args)] kwargs = {f"arg_{i}": data for i in range(n_kwargs)} if n_kwargs == 0 and n_args == 0: with pytest.raises(ValueError): - save(store) + save(store, path=path) elif n_args == 1 and n_kwargs == 0: - save(store, *args) - array = zarr.api.synchronous.open(store) + save(store, *args, path=path) + array = zarr.api.synchronous.open(store, path=path) assert isinstance(array, Array) assert_array_equal(array[:], data) else: - save(store, *args, **kwargs) # type: ignore [arg-type] - group = zarr.api.synchronous.open(store) + save(store, *args, path=path, **kwargs) # type: ignore [arg-type] + group = zarr.api.synchronous.open(store, path=path) assert isinstance(group, Group) for array in group.array_values(): assert_array_equal(array[:], data) @@ -384,8 +385,8 @@ def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) - # assert "LazyLoader: " in repr(loader) -def test_load_array(memory_store: Store) -> None: - store = memory_store +def test_load_array(sync_store: Store) -> None: + store = sync_store foo = np.arange(100) bar = np.arange(100, 0, -1) save(store, foo=foo, bar=bar) From 0713cfa788af006eedd1de5cccc68984b5a902de Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 10 Jun 2025 16:52:09 +0200 Subject: [PATCH 146/160] Fix infinite reload loop on latest release notes page (#3129) --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 5f2e4ba3ec..68bf003ad5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -106,7 +106,6 @@ def skip_submodules( "installation": "user-guide/installation.html", "api": "api/zarr/index", "release": "release-notes.html", - "release-notes": "release-notes.html" } # The language for content autogenerated by Sphinx. Refer to documentation From 229a6c72cf8e9ae14b731cf8a570ef6061df00e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Galkin?= Date: Wed, 11 Jun 2025 12:44:21 -0300 Subject: [PATCH 147/160] Allow Stores to opt out of consolidated metadata. (#3119) * Allow Stores to opt out of consolidated metadata. Some Stores don't benefit from Zarr's consolidated metadata mechanism. These Stores usually implement their own consolidation mechanism, or provide good performance for metadata retrieval out of the box. These Stores can now implement the `supports_consolidated_metadata` property returning `False`. In this situation, Zarr will silently ignore any requests to consolidate the metadata. * typo * PR Feedback * Add warning when consolidation is used with Stores that don't support it * AsyncGroup.open with use_consolidated=None lets the Store select behavior * Typo * TypeError if consolidate_metadata called on non-consolidating stores * Update docstrings Co-authored-by: David Stansby * More docstring improvements * Fix docstrings Co-authored-by: David Stansby --------- Co-authored-by: David Stansby --- docs/user-guide/consolidated_metadata.rst | 20 +++++++++++++ src/zarr/abc/store.py | 12 ++++++++ src/zarr/api/asynchronous.py | 13 +++++++-- src/zarr/api/synchronous.py | 8 ++++-- src/zarr/core/group.py | 17 +++++++++-- tests/test_metadata/test_consolidated.py | 35 +++++++++++++++++++++++ 6 files changed, 98 insertions(+), 7 deletions(-) diff --git a/docs/user-guide/consolidated_metadata.rst b/docs/user-guide/consolidated_metadata.rst index 3c015dcfca..edd5bafc8d 100644 --- a/docs/user-guide/consolidated_metadata.rst +++ b/docs/user-guide/consolidated_metadata.rst @@ -114,3 +114,23 @@ removed, or modified, consolidated metadata may not be desirable. metadata. .. _Consolidated Metadata: https://github.com/zarr-developers/zarr-specs/pull/309 + +Stores Without Support for Consolidated Metadata +------------------------------------------------ + +Some stores may want to opt out of the consolidated metadata mechanism. This +may be for several reasons like: + +* They want to maintain read-write consistency, which is challenging with + consolidated metadata. +* They have their own consolidated metadata mechanism. +* They offer good enough performance without need for consolidation. + +This type of store can declare it doesn't want consolidation by implementing +`Store.supports_consolidated_metadata` and returning `False`. For stores that don't support +consolidation, Zarr will: + +* Raise an error on `consolidate_metadata` calls, maintaining the store in + its unconsolidated state. +* Raise an error in `AsyncGroup.open(..., use_consolidated=True)` +* Not use consolidated metadata in `AsyncGroup.open(..., use_consolidated=None)` diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 96165f8ba0..db4dee8cdd 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -264,6 +264,18 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: """ await gather(*starmap(self.set, values)) + @property + def supports_consolidated_metadata(self) -> bool: + """ + Does the store support consolidated metadata?. + + If it doesn't an error will be raised on requests to consolidate the metadata. + Returning `False` can be useful for stores which implement their own + consolidation mechanism outside of the zarr-python implementation. + """ + + return True + @property @abstractmethod def supports_deletes(self) -> bool: diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index b296d21bd4..3ef9903c57 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -174,7 +174,8 @@ async def consolidate_metadata( Consolidate the metadata of all nodes in a hierarchy. Upon completion, the metadata of the root node in the Zarr hierarchy will be - updated to include all the metadata of child nodes. + updated to include all the metadata of child nodes. For Stores that do + not support consolidated metadata, this operation raises a ``TypeError``. Parameters ---------- @@ -194,10 +195,18 @@ async def consolidate_metadata( ------- group: AsyncGroup The group, with the ``consolidated_metadata`` field set to include - the metadata of each child node. + the metadata of each child node. If the Store doesn't support + consolidated metadata, this function raises a `TypeError`. + See ``Store.supports_consolidated_metadata``. """ store_path = await make_store_path(store, path=path) + if not store_path.store.supports_consolidated_metadata: + store_name = type(store_path.store).__name__ + raise TypeError( + f"The Zarr Store in use ({store_name}) doesn't support consolidated metadata", + ) + group = await AsyncGroup.open(store_path, zarr_format=zarr_format, use_consolidated=False) group.store_path.store._check_writable() diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index d4b652ad6e..6aa1cc4de7 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -81,7 +81,8 @@ def consolidate_metadata( Consolidate the metadata of all nodes in a hierarchy. Upon completion, the metadata of the root node in the Zarr hierarchy will be - updated to include all the metadata of child nodes. + updated to include all the metadata of child nodes. For Stores that do + not use consolidated metadata, this operation raises a `TypeError`. Parameters ---------- @@ -101,7 +102,10 @@ def consolidate_metadata( ------- group: Group The group, with the ``consolidated_metadata`` field set to include - the metadata of each child node. + the metadata of each child node. If the Store doesn't support + consolidated metadata, this function raises a `TypeError`. + See ``Store.supports_consolidated_metadata``. + """ return Group(sync(async_api.consolidate_metadata(store, path=path, zarr_format=zarr_format))) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 6c6f104605..3ce46ec97b 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -490,10 +490,11 @@ async def open( By default, consolidated metadata is used if it's present in the store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file - for Zarr format 2). + for Zarr format 2) and the Store supports it. - To explicitly require consolidated metadata, set ``use_consolidated=True``, - which will raise an exception if consolidated metadata is not found. + To explicitly require consolidated metadata, set ``use_consolidated=True``. + In this case, if the Store doesn't support consolidation or consolidated metadata is + not found, a ``ValueError`` exception is raised. To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, which will fall back to using the regular, non consolidated metadata. @@ -503,6 +504,16 @@ async def open( to load consolidated metadata from a non-default key. """ store_path = await make_store_path(store) + if not store_path.store.supports_consolidated_metadata: + # Fail if consolidated metadata was requested but the Store doesn't support it + if use_consolidated: + store_name = type(store_path.store).__name__ + raise ValueError( + f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." + ) + + # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation + use_consolidated = False consolidated_key = ZMETADATA_V2_JSON diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index f71a946300..ff4fe6a780 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -651,3 +651,38 @@ async def test_consolidated_metadata_encodes_special_chars( elif zarr_format == 3: assert root_metadata["child"]["attributes"]["test"] == expected_fill_value assert root_metadata["time"]["fill_value"] == expected_fill_value + + +class NonConsolidatedStore(zarr.storage.MemoryStore): + """A store that doesn't support consolidated metadata""" + + @property + def supports_consolidated_metadata(self) -> bool: + return False + + +async def test_consolidate_metadata_raises_for_self_consolidating_stores(): + """Verify calling consolidate_metadata on a non supporting stores raises an error.""" + + memory_store = NonConsolidatedStore() + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + + with pytest.raises(TypeError, match="doesn't support consolidated metadata"): + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + +async def test_open_group_in_non_consolidating_stores(): + memory_store = NonConsolidatedStore() + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + + # Opening a group without consolidatedion works as expected + await AsyncGroup.open(memory_store, use_consolidated=False) + + # let the Store opt out of consolidation + await AsyncGroup.open(memory_store, use_consolidated=None) + + # Opening a group with use_consolidated=True should fail + with pytest.raises(ValueError, match="doesn't support consolidated metadata"): + await AsyncGroup.open(memory_store, use_consolidated=True) From ba1f71a2af6f5d15eab71f43bdb0b85c76898ebe Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Thu, 12 Jun 2025 15:54:58 +0200 Subject: [PATCH 148/160] Fix `zarr.open` default for argument `mode` when `store` is `read_only` (#3128) * fix `zarr.open` default for argument `mode` when store is read_only * document changes and format * assert type in test_load_zip and test_load_local * default mode=None for `zarr.synchronous.open` * update docstring * also check for read_only if store is a StorePath --- changes/3128.bugfix.rst | 1 + src/zarr/api/asynchronous.py | 11 +++++++++-- src/zarr/api/synchronous.py | 3 ++- tests/test_api.py | 34 +++++++++++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 changes/3128.bugfix.rst diff --git a/changes/3128.bugfix.rst b/changes/3128.bugfix.rst new file mode 100644 index 0000000000..b93416070e --- /dev/null +++ b/changes/3128.bugfix.rst @@ -0,0 +1 @@ +Fix `zarr.open` default for argument `mode` when `store` is `read_only` \ No newline at end of file diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3ef9903c57..54bddd80a8 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -9,6 +9,7 @@ import numpy.typing as npt from typing_extensions import deprecated +from zarr.abc.store import Store from zarr.core.array import ( Array, AsyncArray, @@ -40,6 +41,7 @@ from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v2 import _default_compressor, _default_filters from zarr.errors import GroupNotFoundError, NodeTypeValidationError +from zarr.storage import StorePath from zarr.storage._common import make_store_path if TYPE_CHECKING: @@ -298,7 +300,7 @@ async def load( async def open( *, store: StoreLike | None = None, - mode: AccessModeLiteral = "a", + mode: AccessModeLiteral | None = None, zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, @@ -316,6 +318,7 @@ async def open( read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). + If the store is read-only, the default is 'r'; otherwise, it is 'a'. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional @@ -333,7 +336,11 @@ async def open( Return type depends on what exists in the given store. """ zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) - + if mode is None: + if isinstance(store, (Store, StorePath)) and store.read_only: + mode = "r" + else: + mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) # TODO: the mode check below seems wrong! diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 6aa1cc4de7..a7f7cfda35 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -162,7 +162,7 @@ def load( def open( store: StoreLike | None = None, *, - mode: AccessModeLiteral = "a", + mode: AccessModeLiteral | None = None, zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, @@ -180,6 +180,7 @@ def open( read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). + If the store is read-only, the default is 'r'; otherwise, it is 'a'. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional diff --git a/tests/test_api.py b/tests/test_api.py index 5d41a37493..2a95d7b97c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -39,7 +39,7 @@ ) from zarr.core.buffer import NDArrayLike from zarr.errors import MetadataValidationError -from zarr.storage import MemoryStore +from zarr.storage import LocalStore, MemoryStore, ZipStore from zarr.storage._utils import normalize_path from zarr.testing.utils import gpu_test @@ -401,6 +401,38 @@ def test_load_array(sync_store: Store) -> None: assert_array_equal(bar, array) +@pytest.mark.parametrize("path", ["data", None]) +@pytest.mark.parametrize("load_read_only", [True, False, None]) +def test_load_zip(tmp_path: pathlib.Path, path: str | None, load_read_only: bool | None) -> None: + file = tmp_path / "test.zip" + data = np.arange(100).reshape(10, 10) + + with ZipStore(file, mode="w", read_only=False) as zs: + save(zs, data, path=path) + with ZipStore(file, mode="r", read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + with ZipStore(file, read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + + +@pytest.mark.parametrize("path", ["data", None]) +@pytest.mark.parametrize("load_read_only", [True, False]) +def test_load_local(tmp_path: pathlib.Path, path: str | None, load_read_only: bool) -> None: + file = tmp_path / "test.zip" + data = np.arange(100).reshape(10, 10) + + with LocalStore(file, read_only=False) as zs: + save(zs, data, path=path) + with LocalStore(file, read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + + def test_tree() -> None: pytest.importorskip("rich") g1 = zarr.group() From c972f7f7cbafe2a023073d2e7afa7c558b55a676 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 13 Jun 2025 09:30:53 -0600 Subject: [PATCH 149/160] Port more stateful test actions from icechunk (#3130) * Port more stateful test actions from icechunk * Parallelize with pytest-xdist * minor edit * Tweak profiles Closes #3010 * one more tweak --- .github/workflows/hypothesis.yaml | 10 ++- .github/workflows/test.yml | 2 + changes/3130.feature.rst | 1 + pyproject.toml | 5 +- src/zarr/testing/stateful.py | 135 +++++++++++++++++++++++++++++- src/zarr/testing/strategies.py | 17 +++- tests/conftest.py | 25 ++++-- 7 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 changes/3130.feature.rst diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 15adb0d4a8..776f859d6e 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -25,12 +25,19 @@ jobs: strategy: matrix: - python-version: ['3.11'] + python-version: ['3.12'] numpy-version: ['2.2'] dependency-set: ["optional"] steps: - uses: actions/checkout@v4 + - name: Set HYPOTHESIS_PROFILE based on trigger + run: | + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "HYPOTHESIS_PROFILE=nightly" >> $GITHUB_ENV + else + echo "HYPOTHESIS_PROFILE=ci" >> $GITHUB_ENV + fi - name: Set up Python uses: actions/setup-python@v5 with: @@ -58,6 +65,7 @@ jobs: if: success() id: status run: | + echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-hypothesis # explicitly save the cache so it gets updated, also do this even if it fails. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee1adb6b0f..ac36562a2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,6 +61,8 @@ jobs: hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env - name: Run Tests + env: + HYPOTHESIS_PROFILE: ci run: | hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage - name: Upload coverage diff --git a/changes/3130.feature.rst b/changes/3130.feature.rst new file mode 100644 index 0000000000..7a64582f06 --- /dev/null +++ b/changes/3130.feature.rst @@ -0,0 +1 @@ +Port more stateful testing actions from `Icechunk `_. diff --git a/pyproject.toml b/pyproject.toml index d25af7c5fc..d9264fcb6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ test = [ "rich", "mypy", "hypothesis", + "pytest-xdist", ] remote_tests = [ 'zarr[remote]', @@ -165,7 +166,7 @@ run = "run-coverage --no-cov" run-pytest = "run" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" +run-hypothesis = "run-coverage -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.doctest] @@ -194,7 +195,7 @@ run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.docs] diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py index f4f7b33318..f83d942549 100644 --- a/src/zarr/testing/stateful.py +++ b/src/zarr/testing/stateful.py @@ -21,7 +21,14 @@ from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype from zarr.core.sync import SyncMixin from zarr.storage import LocalStore, MemoryStore -from zarr.testing.strategies import key_ranges, node_names, np_array_and_chunks, numpy_arrays +from zarr.testing.strategies import ( + basic_indices, + chunk_paths, + key_ranges, + node_names, + np_array_and_chunks, + numpy_arrays, +) from zarr.testing.strategies import keys as zarr_keys MAX_BINARY_SIZE = 100 @@ -120,6 +127,120 @@ def add_array( ) self.all_arrays.add(path) + @rule() + def clear(self) -> None: + note("clearing") + import zarr + + self._sync(self.store.clear()) + self._sync(self.model.clear()) + + assert self._sync(self.store.is_empty("/")) + assert self._sync(self.model.is_empty("/")) + + self.all_groups.clear() + self.all_arrays.clear() + + zarr.group(store=self.store) + zarr.group(store=self.model) + + # TODO: MemoryStore is broken? + # assert not self._sync(self.store.is_empty("/")) + # assert not self._sync(self.model.is_empty("/")) + + def draw_directory(self, data: DataObject) -> str: + group_st = st.sampled_from(sorted(self.all_groups)) if self.all_groups else st.nothing() + array_st = st.sampled_from(sorted(self.all_arrays)) if self.all_arrays else st.nothing() + array_or_group = data.draw(st.one_of(group_st, array_st)) + if data.draw(st.booleans()) and array_or_group in self.all_arrays: + arr = zarr.open_array(path=array_or_group, store=self.model) + path = data.draw( + st.one_of( + st.sampled_from([array_or_group]), + chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape).map( + lambda x: f"{array_or_group}/c/" + ), + ) + ) + else: + path = array_or_group + return path + + @precondition(lambda self: bool(self.all_groups)) + @rule(data=st.data()) + def check_list_dir(self, data: DataObject) -> None: + path = self.draw_directory(data) + note(f"list_dir for {path=!r}") + # Consider .list_dir("path/to/array") for an array with a single chunk. + # The MemoryStore model will return `"c", "zarr.json"` only if the chunk exists + # If that chunk was deleted, then `"c"` is not returned. + # LocalStore will not have this behaviour :/ + # There are similar consistency issues with delete_dir("/path/to/array/c/0/0") + assume(not isinstance(self.store, LocalStore)) + model_ls = sorted(self._sync_iter(self.model.list_dir(path))) + store_ls = sorted(self._sync_iter(self.store.list_dir(path))) + assert model_ls == store_ls, (model_ls, store_ls) + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def delete_chunk(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + arr = zarr.open_array(path=array, store=self.model) + chunk_path = data.draw(chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape, subset=False)) + path = f"{array}/c/{chunk_path}" + note(f"deleting chunk {path=!r}") + self._sync(self.model.delete(path)) + self._sync(self.store.delete(path)) + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def overwrite_array_basic_indexing(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + model_array = zarr.open_array(path=array, store=self.model) + store_array = zarr.open_array(path=array, store=self.store) + slicer = data.draw(basic_indices(shape=model_array.shape)) + note(f"overwriting array with basic indexer: {slicer=}") + new_data = data.draw( + npst.arrays(shape=np.shape(model_array[slicer]), dtype=model_array.dtype) + ) + model_array[slicer] = new_data + store_array[slicer] = new_data + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def resize_array(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + model_array = zarr.open_array(path=array, store=self.model) + store_array = zarr.open_array(path=array, store=self.store) + ndim = model_array.ndim + new_shape = tuple( + 0 if oldsize == 0 else newsize + for newsize, oldsize in zip( + data.draw(npst.array_shapes(max_dims=ndim, min_dims=ndim, min_side=0)), + model_array.shape, + strict=True, + ) + ) + + note(f"resizing array from {model_array.shape} to {new_shape}") + model_array.resize(new_shape) + store_array.resize(new_shape) + + @precondition(lambda self: bool(self.all_arrays) or bool(self.all_groups)) + @rule(data=st.data()) + def delete_dir(self, data: DataObject) -> None: + path = self.draw_directory(data) + note(f"delete_dir with {path=!r}") + self._sync(self.model.delete_dir(path)) + self._sync(self.store.delete_dir(path)) + + matches = set() + for node in self.all_groups | self.all_arrays: + if node.startswith(path): + matches.add(node) + self.all_groups = self.all_groups - matches + self.all_arrays = self.all_arrays - matches + # @precondition(lambda self: bool(self.all_groups)) # @precondition(lambda self: bool(self.all_arrays)) # @rule(data=st.data()) @@ -230,13 +351,19 @@ def delete_group_using_del(self, data: DataObject) -> None: # self.check_group_arrays(group) # t1 = time.time() # note(f"Checks took {t1 - t0} sec.") - @invariant() def check_list_prefix_from_root(self) -> None: model_list = self._sync_iter(self.model.list_prefix("")) store_list = self._sync_iter(self.store.list_prefix("")) - note(f"Checking {len(model_list)} keys") - assert sorted(model_list) == sorted(store_list) + note(f"Checking {len(model_list)} expected keys vs {len(store_list)} actual keys") + assert sorted(model_list) == sorted(store_list), ( + sorted(model_list), + sorted(store_list), + ) + + # check that our internal state matches that of the store and model + assert all(f"{path}/zarr.json" in model_list for path in self.all_groups | self.all_arrays) + assert all(f"{path}/zarr.json" in store_list for path in self.all_groups | self.all_arrays) class SyncStoreWrapper(zarr.core.sync.SyncMixin): diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 3b10592ec0..0cb992a4f2 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -77,7 +77,7 @@ def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]: return st.text( alphabet=st.characters( - blacklist_categories=["Cs"], # Avoid *technically allowed* surrogates + exclude_categories=["Cs"], # Avoid *technically allowed* surrogates min_codepoint=32, ), min_size=1, @@ -324,7 +324,7 @@ def is_negative_slice(idx: Any) -> bool: @st.composite -def end_slices(draw: st.DrawFn, *, shape: tuple[int]) -> Any: +def end_slices(draw: st.DrawFn, *, shape: tuple[int, ...]) -> Any: """ A strategy that slices ranges that include the last chunk. This is intended to stress-test handling of a possibly smaller last chunk. @@ -342,7 +342,7 @@ def end_slices(draw: st.DrawFn, *, shape: tuple[int]) -> Any: def basic_indices( draw: st.DrawFn, *, - shape: tuple[int], + shape: tuple[int, ...], min_dims: int = 0, max_dims: int | None = None, allow_newaxis: bool = False, @@ -370,7 +370,7 @@ def basic_indices( @st.composite def orthogonal_indices( - draw: st.DrawFn, *, shape: tuple[int] + draw: st.DrawFn, *, shape: tuple[int, ...] ) -> tuple[tuple[np.ndarray[Any, Any], ...], tuple[np.ndarray[Any, Any], ...]]: """ Strategy that returns @@ -426,3 +426,12 @@ def make_request(start: int, length: int) -> RangeByteRequest: ) key_tuple = st.tuples(keys, byte_ranges) return st.lists(key_tuple, min_size=1, max_size=10) + + +@st.composite +def chunk_paths(draw: st.DrawFn, ndim: int, numblocks: tuple[int, ...], subset: bool = True) -> str: + blockidx = draw( + st.tuples(*tuple(st.integers(min_value=0, max_value=max(0, b - 1)) for b in numblocks)) + ) + subset_slicer = slice(draw(st.integers(min_value=0, max_value=ndim))) if subset else slice(None) + return "/".join(map(str, blockidx[subset_slicer])) diff --git a/tests/conftest.py b/tests/conftest.py index 948d3cd055..30d7eec4d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import pathlib from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -188,17 +189,31 @@ def pytest_collection_modifyitems(config: Any, items: Any) -> None: settings.register_profile( - "ci", - max_examples=1000, - deadline=None, + "default", + parent=settings.get_profile("default"), + max_examples=300, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], + deadline=None, + verbosity=Verbosity.verbose, ) settings.register_profile( - "local", + "ci", + parent=settings.get_profile("ci"), max_examples=300, + derandomize=True, # more like regression testing + deadline=None, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], - verbosity=Verbosity.verbose, ) +settings.register_profile( + "nightly", + max_examples=500, + parent=settings.get_profile("ci"), + derandomize=False, + stateful_step_count=100, +) + +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) + # TODO: uncomment these overrides when we can get mypy to accept them """ From 11d488d731cd537b84ae394a943c39cb5b62638f Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 16 Jun 2025 02:23:55 -0400 Subject: [PATCH 150/160] Support async FSMap objects in zarr.open (#2774) * WIP: Support fsspec mutable mapping objects in zarr.open * Simplify library availability checking * Improve test coverage * Improve error messages * Consolidate code * Make test more readable * Make async instances from sync fsmap objects * Move test to fsspec store * Re-add type ignore * "Update docstring" * Add another test * Require auto_mkdir for LocalFileSystem * Update test location * Convert older filesystems to async * Use if on fsspec versions rather than try; else * Always use asynchronous=True in _make_async * Improve tests * Apply suggestions from code review Co-authored-by: Martin Durant * Apply more code suggestions * Fix typing error * Test remote stores in min_deps env * Remove redundant import * Test warning * Lint * Add pytest pin * Add release note * Generate coverage on min_deps and upstream jobs * Update src/zarr/storage/_fsspec.py Co-authored-by: Davis Bennett * More useful error messages * Add TypeAlias * Fix typing for no fsspec installation * Move imports * Don't mutate FSMap object --------- Co-authored-by: Davis Bennett Co-authored-by: Deepak Cherian Co-authored-by: Martin Durant Co-authored-by: David Stansby --- .github/workflows/test.yml | 2 +- changes/2774.feature.rst | 1 + pyproject.toml | 1 + src/zarr/storage/_common.py | 24 +++++- src/zarr/storage/_fsspec.py | 84 ++++++++++++++++++--- tests/test_store/test_fsspec.py | 127 ++++++++++++++++++++++++++++++-- 6 files changed, 216 insertions(+), 23 deletions(-) create mode 100644 changes/2774.feature.rst diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac36562a2a..7cfce41312 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,7 +104,7 @@ jobs: hatch env run -e ${{ matrix.dependency-set }} list-env - name: Run Tests run: | - hatch env run --env ${{ matrix.dependency-set }} run + hatch env run --env ${{ matrix.dependency-set }} run-coverage - name: Upload coverage uses: codecov/codecov-action@v5 with: diff --git a/changes/2774.feature.rst b/changes/2774.feature.rst new file mode 100644 index 0000000000..4df83f54ec --- /dev/null +++ b/changes/2774.feature.rst @@ -0,0 +1 @@ +Add `zarr.storage.FsspecStore.from_mapper()` so that `zarr.open()` supports stores of type `fsspec.mapping.FSMap`. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d9264fcb6b..8141374d5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -253,6 +253,7 @@ dependencies = [ 'obstore==0.5.*', # test deps 'zarr[test]', + 'zarr[remote_tests]', ] [tool.hatch.envs.min_deps.scripts] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index b2fefe96d7..f264728cf2 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -1,8 +1,9 @@ from __future__ import annotations +import importlib.util import json from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, Self +from typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, default_buffer_prototype @@ -12,6 +13,12 @@ from zarr.storage._memory import MemoryStore from zarr.storage._utils import normalize_path +_has_fsspec = importlib.util.find_spec("fsspec") +if _has_fsspec: + from fsspec.mapping import FSMap +else: + FSMap = None + if TYPE_CHECKING: from zarr.core.buffer import BufferPrototype @@ -227,7 +234,7 @@ def __eq__(self, other: object) -> bool: return False -StoreLike = Store | StorePath | Path | str | dict[str, Buffer] +StoreLike: TypeAlias = Store | StorePath | FSMap | Path | str | dict[str, Buffer] async def make_store_path( @@ -314,9 +321,18 @@ async def make_store_path( # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. store = await MemoryStore.open(store_dict=store_like, read_only=_read_only) + elif _has_fsspec and isinstance(store_like, FSMap): + if path: + raise ValueError( + "'path' was provided but is not used for FSMap store_like objects. Specify the path when creating the FSMap instance instead." + ) + if storage_options: + raise ValueError( + "'storage_options was provided but is not used for FSMap store_like objects. Specify the storage options when creating the FSMap instance instead." + ) + store = FsspecStore.from_mapper(store_like, read_only=_read_only) else: - msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] - raise TypeError(msg) + raise TypeError(f"Unsupported type for store_like: '{type(store_like).__name__}'") result = await StorePath.open(store, path=path_normalized, mode=mode) diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 40f1b2fbc0..ba673056a3 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -1,9 +1,12 @@ from __future__ import annotations +import json import warnings from contextlib import suppress from typing import TYPE_CHECKING, Any +from packaging.version import parse as parse_version + from zarr.abc.store import ( ByteRequest, OffsetByteRequest, @@ -17,7 +20,9 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable + from fsspec import AbstractFileSystem from fsspec.asyn import AsyncFileSystem + from fsspec.mapping import FSMap from zarr.core.buffer import BufferPrototype from zarr.core.common import BytesLike @@ -30,6 +35,42 @@ ) +def _make_async(fs: AbstractFileSystem) -> AsyncFileSystem: + """Convert a sync FSSpec filesystem to an async FFSpec filesystem + + If the filesystem class supports async operations, a new async instance is created + from the existing instance. + + If the filesystem class does not support async operations, the existing instance + is wrapped with AsyncFileSystemWrapper. + """ + import fsspec + + fsspec_version = parse_version(fsspec.__version__) + if fs.async_impl and fs.asynchronous: + # Already an async instance of an async filesystem, nothing to do + return fs + if fs.async_impl: + # Convert sync instance of an async fs to an async instance + fs_dict = json.loads(fs.to_json()) + fs_dict["asynchronous"] = True + return fsspec.AbstractFileSystem.from_json(json.dumps(fs_dict)) + + # Wrap sync filesystems with the async wrapper + if type(fs) is fsspec.implementations.local.LocalFileSystem and not fs.auto_mkdir: + raise ValueError( + f"LocalFilesystem {fs} was created with auto_mkdir=False but Zarr requires the filesystem to automatically create directories" + ) + if fsspec_version < parse_version("2024.12.0"): + raise ImportError( + f"The filesystem '{fs}' is synchronous, and the required " + "AsyncFileSystemWrapper is not available. Upgrade fsspec to version " + "2024.12.0 or later to enable this functionality." + ) + + return fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper(fs, asynchronous=True) + + class FsspecStore(Store): """ Store for remote data based on FSSpec. @@ -137,6 +178,38 @@ def from_upath( allowed_exceptions=allowed_exceptions, ) + @classmethod + def from_mapper( + cls, + fs_map: FSMap, + read_only: bool = False, + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> FsspecStore: + """ + Create a FsspecStore from a FSMap object. + + Parameters + ---------- + fs_map : FSMap + Fsspec mutable mapping object. + read_only : bool + Whether the store is read-only, defaults to False. + allowed_exceptions : tuple, optional + The exceptions that are allowed to be raised when accessing the + store. Defaults to ALLOWED_EXCEPTIONS. + + Returns + ------- + FsspecStore + """ + fs = _make_async(fs_map.fs) + return cls( + fs=fs, + path=fs_map.root, + read_only=read_only, + allowed_exceptions=allowed_exceptions, + ) + @classmethod def from_url( cls, @@ -175,16 +248,7 @@ def from_url( fs, path = url_to_fs(url, **opts) if not fs.async_impl: - try: - from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper - - fs = AsyncFileSystemWrapper(fs, asynchronous=True) - except ImportError as e: - raise ImportError( - f"The filesystem for URL '{url}' is synchronous, and the required " - "AsyncFileSystemWrapper is not available. Upgrade fsspec to version " - "2024.12.0 or later to enable this functionality." - ) from e + fs = _make_async(fs) # fsspec is not consistent about removing the scheme from the path, so check and strip it here # https://github.com/fsspec/filesystem_spec/issues/1722 diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index c10471809c..1a989525e3 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -5,17 +5,21 @@ import re from typing import TYPE_CHECKING, Any +import numpy as np import pytest from packaging.version import parse as parse_version import zarr.api.asynchronous +from zarr import Array from zarr.abc.store import OffsetByteRequest from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.sync import _collect_aiterator, sync from zarr.storage import FsspecStore +from zarr.storage._fsspec import _make_async from zarr.testing.store import StoreTests if TYPE_CHECKING: + import pathlib from collections.abc import Generator from pathlib import Path @@ -191,7 +195,11 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value"} - meta["attributes"]["key"] = "value-2" # type: ignore[index] + meta = { + "attributes": {"key": "value-2"}, + "zarr_format": 3, + "node_type": "group", + } await store.set( "directory-2/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), @@ -201,7 +209,11 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value-2"} - meta["attributes"]["key"] = "value-3" # type: ignore[index] + meta = { + "attributes": {"key": "value-3"}, + "zarr_format": 3, + "node_type": "group", + } await store.set( "directory-3/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), @@ -264,18 +276,44 @@ async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: await store.delete_dir("test_prefix") +def array_roundtrip(store: FsspecStore) -> None: + """ + Round trip an array using a Zarr store + + Args: + store: FsspecStore + """ + data = np.ones((3, 3)) + arr = zarr.create_array(store=store, overwrite=True, data=data) + assert isinstance(arr, Array) + # Read set values + arr2 = zarr.open_array(store=store) + assert isinstance(arr2, Array) + np.testing.assert_array_equal(arr[:], data) + + @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) -def test_wrap_sync_filesystem() -> None: +def test_wrap_sync_filesystem(tmp_path: pathlib.Path) -> None: """The local fs is not async so we should expect it to be wrapped automatically""" from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper - store = FsspecStore.from_url("local://test/path") - + store = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) assert isinstance(store.fs, AsyncFileSystemWrapper) assert store.fs.async_impl + array_roundtrip(store) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) >= parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_wrap_sync_filesystem_raises(tmp_path: pathlib.Path) -> None: + """The local fs is not async so we should expect it to be wrapped automatically""" + with pytest.raises(ImportError, match="The filesystem .*"): + FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) @pytest.mark.skipif( @@ -283,13 +321,86 @@ def test_wrap_sync_filesystem() -> None: reason="No AsyncFileSystemWrapper", ) def test_no_wrap_async_filesystem() -> None: - """An async fs should not be wrapped automatically; fsspec's https filesystem is such an fs""" + """An async fs should not be wrapped automatically; fsspec's s3 filesystem is such an fs""" from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper - store = FsspecStore.from_url("https://test/path") - + store = FsspecStore.from_url( + f"s3://{test_bucket_name}/foo/spam/", + storage_options={"endpoint_url": endpoint_url, "anon": False, "asynchronous": True}, + read_only=False, + ) assert not isinstance(store.fs, AsyncFileSystemWrapper) assert store.fs.async_impl + array_roundtrip(store) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_open_fsmap_file(tmp_path: pathlib.Path) -> None: + min_fsspec_with_async_wrapper = parse_version("2024.12.0") + current_version = parse_version(fsspec.__version__) + + fs = fsspec.filesystem("file", auto_mkdir=True) + mapper = fs.get_mapper(tmp_path) + + if current_version < min_fsspec_with_async_wrapper: + # Expect ImportError for older versions + with pytest.raises( + ImportError, + match=r"The filesystem .* is synchronous, and the required AsyncFileSystemWrapper is not available.*", + ): + array_roundtrip(mapper) + else: + # Newer versions should work + array_roundtrip(mapper) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_open_fsmap_file_raises(tmp_path: pathlib.Path) -> None: + fsspec = pytest.importorskip("fsspec.implementations.local") + fs = fsspec.LocalFileSystem(auto_mkdir=False) + mapper = fs.get_mapper(tmp_path) + with pytest.raises(ValueError, match="LocalFilesystem .*"): + array_roundtrip(mapper) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +def test_open_fsmap_s3(asynchronous: bool) -> None: + s3_filesystem = s3fs.S3FileSystem( + asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False + ) + mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") + array_roundtrip(mapper) + + +def test_open_s3map_raises() -> None: + with pytest.raises(TypeError, match="Unsupported type for store_like:.*"): + zarr.open(store=0, mode="w", shape=(3, 3)) + s3_filesystem = s3fs.S3FileSystem(asynchronous=True, endpoint_url=endpoint_url, anon=False) + mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") + with pytest.raises( + ValueError, match="'path' was provided but is not used for FSMap store_like objects" + ): + zarr.open(store=mapper, path="bar", mode="w", shape=(3, 3)) + with pytest.raises( + ValueError, + match="'storage_options was provided but is not used for FSMap store_like objects", + ): + zarr.open(store=mapper, storage_options={"anon": True}, mode="w", shape=(3, 3)) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +def test_make_async(asynchronous: bool) -> None: + s3_filesystem = s3fs.S3FileSystem( + asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False + ) + fs = _make_async(s3_filesystem) + assert fs.asynchronous @pytest.mark.skipif( From 67984660834f2095d966b0c1ec92154020f933e1 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 16 Jun 2025 10:00:42 +0200 Subject: [PATCH 151/160] refactor v3 data types (#2874) * modernize typing * lint * new dtypes * rename base dtype, change type to kind * start working on JSON serialization * get json de/serialization largely working, and start making tests pass * tweak json type guards * fix dtype sizes, adjust fill value parsing in from_dict, fix tests * mid-refactor commit * working form for dtype classes * remove unused code * use wrap / unwrap instead of to_dtype / from_dtype; push into v2 codebase * push into v2 * remove endianness kwarg to methods, make it an instance variable instead * make wrapping safe by default * dtype-specific tests * more tests, fix void type default value logic * fix dtype mechanics in bytescodec * remove __post_init__ magic in favor of more explicit declaration * fix tests * refactor data types * start design doc * more design doc * update docs * fix sphinx warnings * tweak docs * info about v3 data types * adjust note * fix: use unparametrized types in direct assignment * start fixing config * Update src/zarr/core/_info.py Co-authored-by: Joe Hamman * add placeholder disclaimer to v3 data types summary * make example runnable * placeholder section for adding a custom dtype * define native data type and native scalar * update data type names * fix config test failures * call to_dtype once in blosc evolve_from_array_spec * refactor dtypewrapper -> zdtype * update code examples in docs; remove native endianness * adjust type annotations * fix info tests to use zdtype * remove dead code and add code coverage exemption to zarr format checks * fix: add special check for resolving int32 on windows * add dtype entry point test * remove default parameters for parametric dtypes; add mixin classes for numpy dtypes; define zdtypelike * Update docs/user-guide/data_types.rst Co-authored-by: Ilan Gold * refactor: use inheritance to remove boilerplate in dtype definitions * update data types documentation, and expose core/dtype module to autodoc * add failing endianness round-trip test * fix endianness * additional check in test_explicit_endianness * add failing test for round-tripping vlen strings * route object dtype arrays to vlen string dtype when numpy > 2 * relax endianness mismatch to a warning instead of an error * use public dtype module for docs instead of special-casing the core dype module * use public dtype module for docs instead of special-casing the core dype module * silence mypy error about array indexing * add release note * fix doctests, excluding config tests * revert addition of linkage between dtype endianness and bytes codec endianness * remove Any types * add docstring for wrapper module * simplify config and docs * update config test * fix S dtype test for v2 * fully remove v3jsonencoder * refactor dtype module structure * add timedelta64 * refactor time dtypes * widen dtype test strategies * modify structured dtype fill value rt to avoid to_dict * wip: begin creating isomorphic test suite for dtypes * finish common tests * wip: test infrastructure for dtypes * wip: use class-based tests for all dtypes * fill out more tests, and adjust sized dtypes * wip: json schema test * add casting tests * use relative link for changes * typo * make bytes codec dtype logic a bit more literate * increase deadline to 500ms * fewer commented sections of problematic lru_store_cache section of the sharding codecs * add link to gh issue about lru_cache for sharding codec * attempt to speed up hypothesis tests by reducing max array size * clean up docs * remove placeholder * make final example section doctested and more readable * revert change to auto chunking * revert quotation of literal type * lint * fix broken code block * specialize test to handle stringdtype changes coming in numpy 2.3 * add docstring to _TestZDType class * type hints * expand changelog * tweak docstring * support v3 nan strings in JSON for float dtypes * revert removal of metadata chunk grid attribute * use none to denote default fill value; remove old structured tests; use cast_value where appropriate * add item size abstraction * rename fixed-length string dtypes, and be strict about the numpy object dtype (i.e., refuse to match it) * remove vestigial use of to_dtype().itemsize() * remove another vestigial use of to_dtype().itemsize() * emit warning about unstable dtype when serializing Structured dtype to JSON * put string dtypes in the strings module * make tests isomorphic to source code * remove old string logic * use scale_factor and unit in cast_value for datetime * add regression testing against v2.18 * truncate U and S scalars in _cast_value_unsafe * docstrings and simplification for regression tests * changes necessary for linting with regression tests * improve method names, refactor type hints with typeddictionaries, fix registry load frequency, add object_codec_id for v2 json deserialization * fix storage info discrepancy in docs * fix docstring that was troubling sphinx * wip: add vlen-bytes * add vlen-bytes * replace placeholder text with links to a github issue * refactor fixed-length bytes dtypes * more v3 unstable dtype warnings, and their exemptions from tests * clean up typeddicts * update docstrings * Update docs/user-guide/data_types.rst Co-authored-by: Ryan Abernathey * refactor wrapper to allow subclasses to freely define their own type guards for native dtype and json input * make method definition order consistent * allow structured scalars to be np.void * use a common function signature for from_json by packing the object_codec_id in a typeddict for zarr v2 metadata * fix dtype doc example --------- Co-authored-by: Joe Hamman Co-authored-by: Ilan Gold Co-authored-by: Ryan Abernathey --- changes/2874.feature.rst | 9 + docs/user-guide/arrays.rst | 14 +- docs/user-guide/config.rst | 59 +- docs/user-guide/consolidated_metadata.rst | 6 +- docs/user-guide/data_types.rst | 172 +++++ docs/user-guide/groups.rst | 4 +- docs/user-guide/index.rst | 1 + docs/user-guide/performance.rst | 8 +- pyproject.toml | 3 +- src/zarr/abc/codec.py | 17 +- src/zarr/api/asynchronous.py | 20 +- src/zarr/api/synchronous.py | 9 +- src/zarr/codecs/_v2.py | 8 +- src/zarr/codecs/blosc.py | 9 +- src/zarr/codecs/bytes.py | 16 +- src/zarr/codecs/sharding.py | 30 +- src/zarr/codecs/transpose.py | 10 +- src/zarr/codecs/vlen_utf8.py | 5 +- src/zarr/core/_info.py | 16 +- src/zarr/core/array.py | 229 +++--- src/zarr/core/array_spec.py | 10 +- src/zarr/core/buffer/core.py | 4 +- src/zarr/core/chunk_grids.py | 3 +- src/zarr/core/codec_pipeline.py | 16 +- src/zarr/core/common.py | 23 +- src/zarr/core/config.py | 51 +- src/zarr/core/dtype/__init__.py | 162 +++++ src/zarr/core/dtype/common.py | 224 ++++++ src/zarr/core/dtype/npy/__init__.py | 0 src/zarr/core/dtype/npy/bool.py | 163 +++++ src/zarr/core/dtype/npy/bytes.py | 369 ++++++++++ src/zarr/core/dtype/npy/common.py | 503 +++++++++++++ src/zarr/core/dtype/npy/complex.py | 213 ++++++ src/zarr/core/dtype/npy/float.py | 222 ++++++ src/zarr/core/dtype/npy/int.py | 686 ++++++++++++++++++ src/zarr/core/dtype/npy/string.py | 302 ++++++++ src/zarr/core/dtype/npy/structured.py | 206 ++++++ src/zarr/core/dtype/npy/time.py | 359 +++++++++ src/zarr/core/dtype/registry.py | 90 +++ src/zarr/core/dtype/wrapper.py | 297 ++++++++ src/zarr/core/group.py | 20 +- src/zarr/core/metadata/v2.py | 332 +++------ src/zarr/core/metadata/v3.py | 488 ++----------- src/zarr/core/strings.py | 86 --- src/zarr/dtype.py | 3 + src/zarr/registry.py | 13 +- src/zarr/testing/strategies.py | 20 +- tests/conftest.py | 35 +- .../entry_points.txt | 2 + tests/package_with_entrypoint/__init__.py | 40 +- tests/test_array.py | 265 ++++--- tests/test_codecs/test_endian.py | 2 + tests/test_codecs/test_vlen.py | 64 +- tests/test_config.py | 154 ++-- tests/test_dtype/__init__.py | 0 tests/test_dtype/conftest.py | 71 ++ tests/test_dtype/test_npy/test_bool.py | 41 ++ tests/test_dtype/test_npy/test_bytes.py | 154 ++++ tests/test_dtype/test_npy/test_common.py | 342 +++++++++ tests/test_dtype/test_npy/test_complex.py | 100 +++ tests/test_dtype/test_npy/test_float.py | 169 +++++ tests/test_dtype/test_npy/test_int.py | 273 +++++++ tests/test_dtype/test_npy/test_string.py | 130 ++++ tests/test_dtype/test_npy/test_structured.py | 108 +++ tests/test_dtype/test_npy/test_time.py | 163 +++++ tests/test_dtype/test_wrapper.py | 136 ++++ tests/test_dtype_registry.py | 198 +++++ tests/test_group.py | 8 +- tests/test_info.py | 10 +- tests/test_metadata/test_consolidated.py | 13 +- tests/test_metadata/test_v2.py | 39 +- tests/test_metadata/test_v3.py | 194 ++--- tests/test_properties.py | 50 +- tests/test_regression/__init__.py | 0 tests/test_regression/scripts/__init__.py | 0 tests/test_regression/scripts/v2.18.py | 81 +++ tests/test_regression/test_regression.py | 156 ++++ tests/test_store/test_stateful.py | 1 + tests/test_strings.py | 35 - tests/test_v2.py | 170 +---- 80 files changed, 7067 insertions(+), 1647 deletions(-) create mode 100644 changes/2874.feature.rst create mode 100644 docs/user-guide/data_types.rst create mode 100644 src/zarr/core/dtype/__init__.py create mode 100644 src/zarr/core/dtype/common.py create mode 100644 src/zarr/core/dtype/npy/__init__.py create mode 100644 src/zarr/core/dtype/npy/bool.py create mode 100644 src/zarr/core/dtype/npy/bytes.py create mode 100644 src/zarr/core/dtype/npy/common.py create mode 100644 src/zarr/core/dtype/npy/complex.py create mode 100644 src/zarr/core/dtype/npy/float.py create mode 100644 src/zarr/core/dtype/npy/int.py create mode 100644 src/zarr/core/dtype/npy/string.py create mode 100644 src/zarr/core/dtype/npy/structured.py create mode 100644 src/zarr/core/dtype/npy/time.py create mode 100644 src/zarr/core/dtype/registry.py create mode 100644 src/zarr/core/dtype/wrapper.py delete mode 100644 src/zarr/core/strings.py create mode 100644 src/zarr/dtype.py create mode 100644 tests/test_dtype/__init__.py create mode 100644 tests/test_dtype/conftest.py create mode 100644 tests/test_dtype/test_npy/test_bool.py create mode 100644 tests/test_dtype/test_npy/test_bytes.py create mode 100644 tests/test_dtype/test_npy/test_common.py create mode 100644 tests/test_dtype/test_npy/test_complex.py create mode 100644 tests/test_dtype/test_npy/test_float.py create mode 100644 tests/test_dtype/test_npy/test_int.py create mode 100644 tests/test_dtype/test_npy/test_string.py create mode 100644 tests/test_dtype/test_npy/test_structured.py create mode 100644 tests/test_dtype/test_npy/test_time.py create mode 100644 tests/test_dtype/test_wrapper.py create mode 100644 tests/test_dtype_registry.py create mode 100644 tests/test_regression/__init__.py create mode 100644 tests/test_regression/scripts/__init__.py create mode 100644 tests/test_regression/scripts/v2.18.py create mode 100644 tests/test_regression/test_regression.py delete mode 100644 tests/test_strings.py diff --git a/changes/2874.feature.rst b/changes/2874.feature.rst new file mode 100644 index 0000000000..4c50532ae0 --- /dev/null +++ b/changes/2874.feature.rst @@ -0,0 +1,9 @@ +Adds zarr-specific data type classes. This replaces the internal use of numpy data types for zarr +v2 and a fixed set of string enums for zarr v3. This change is largely internal, but it does +change the type of the ``dtype`` and ``data_type`` fields on the ``ArrayV2Metadata`` and +``ArrayV3Metadata`` classes. It also changes the JSON metadata representation of the +variable-length string data type, but the old metadata representation can still be +used when reading arrays. The logic for automatically choosing the chunk encoding for a given data +type has also changed, and this necessitated changes to the ``config`` API. + +For more on this new feature, see the `documentation `_ \ No newline at end of file diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst index 5bd6b1500f..c27f1296b9 100644 --- a/docs/user-guide/arrays.rst +++ b/docs/user-guide/arrays.rst @@ -182,7 +182,7 @@ which can be used to print useful diagnostics, e.g.:: >>> z.info Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) @@ -200,7 +200,7 @@ prints additional diagnostics, e.g.:: >>> z.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) @@ -248,7 +248,7 @@ built-in delta filter:: The default compressor can be changed by setting the value of the using Zarr's :ref:`user-guide-config`, e.g.:: - >>> with zarr.config.set({'array.v2_default_compressor.numeric': {'id': 'blosc'}}): + >>> with zarr.config.set({'array.v2_default_compressor.default': {'id': 'blosc'}}): ... z = zarr.create_array(store={}, shape=(100000000,), chunks=(1000000,), dtype='int32', zarr_format=2) >>> z.filters () @@ -288,7 +288,7 @@ Here is an example using a delta filter with the Blosc compressor:: >>> z.info Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) @@ -603,7 +603,7 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za >>> a.info_complete() Type : Array Zarr format : 3 - Data type : DataType.uint8 + Data type : UInt8() Fill value : 0 Shape : (10000, 10000) Shard shape : (1000, 1000) @@ -612,10 +612,10 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za Read-only : False Store type : LocalStore Filters : () - Serializer : BytesCodec(endian=) + Serializer : BytesCodec(endian=None) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 100000000 (95.4M) - No. bytes stored : 3981552 + No. bytes stored : 3981473 Storage ratio : 25.1 Shards Initialized : 100 diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst index 91ffe50b91..4479e30619 100644 --- a/docs/user-guide/config.rst +++ b/docs/user-guide/config.rst @@ -43,39 +43,30 @@ This is the current default configuration:: >>> zarr.config.pprint() {'array': {'order': 'C', - 'v2_default_compressor': {'bytes': {'checksum': False, - 'id': 'zstd', - 'level': 0}, - 'numeric': {'checksum': False, - 'id': 'zstd', - 'level': 0}, - 'string': {'checksum': False, + 'v2_default_compressor': {'default': {'checksum': False, 'id': 'zstd', - 'level': 0}}, - 'v2_default_filters': {'bytes': [{'id': 'vlen-bytes'}], - 'numeric': None, - 'raw': None, - 'string': [{'id': 'vlen-utf8'}]}, - 'v3_default_compressors': {'bytes': [{'configuration': {'checksum': False, - 'level': 0}, - 'name': 'zstd'}], - 'numeric': [{'configuration': {'checksum': False, + 'level': 0}, + 'variable-length-string': {'checksum': False, + 'id': 'zstd', + 'level': 0}}, + 'v2_default_filters': {'default': None, + 'variable-length-string': [{'id': 'vlen-utf8'}]}, + 'v3_default_compressors': {'default': [{'configuration': {'checksum': False, 'level': 0}, 'name': 'zstd'}], - 'string': [{'configuration': {'checksum': False, - 'level': 0}, - 'name': 'zstd'}]}, - 'v3_default_filters': {'bytes': [], 'numeric': [], 'string': []}, - 'v3_default_serializer': {'bytes': {'name': 'vlen-bytes'}, - 'numeric': {'configuration': {'endian': 'little'}, - 'name': 'bytes'}, - 'string': {'name': 'vlen-utf8'}}, - 'write_empty_chunks': False}, - 'async': {'concurrency': 10, 'timeout': None}, - 'buffer': 'zarr.core.buffer.cpu.Buffer', - 'codec_pipeline': {'batch_size': 1, - 'path': 'zarr.core.codec_pipeline.BatchedCodecPipeline'}, - 'codecs': {'blosc': 'zarr.codecs.blosc.BloscCodec', + 'variable-length-string': [{'configuration': {'checksum': False, + 'level': 0}, + 'name': 'zstd'}]}, + 'v3_default_filters': {'default': [], 'variable-length-string': []}, + 'v3_default_serializer': {'default': {'configuration': {'endian': 'little'}, + 'name': 'bytes'}, + 'variable-length-string': {'name': 'vlen-utf8'}}, + 'write_empty_chunks': False}, + 'async': {'concurrency': 10, 'timeout': None}, + 'buffer': 'zarr.core.buffer.cpu.Buffer', + 'codec_pipeline': {'batch_size': 1, + 'path': 'zarr.core.codec_pipeline.BatchedCodecPipeline'}, + 'codecs': {'blosc': 'zarr.codecs.blosc.BloscCodec', 'bytes': 'zarr.codecs.bytes.BytesCodec', 'crc32c': 'zarr.codecs.crc32c_.Crc32cCodec', 'endian': 'zarr.codecs.bytes.BytesCodec', @@ -85,7 +76,7 @@ This is the current default configuration:: 'vlen-bytes': 'zarr.codecs.vlen_utf8.VLenBytesCodec', 'vlen-utf8': 'zarr.codecs.vlen_utf8.VLenUTF8Codec', 'zstd': 'zarr.codecs.zstd.ZstdCodec'}, - 'default_zarr_format': 3, - 'json_indent': 2, - 'ndbuffer': 'zarr.core.buffer.cpu.NDBuffer', - 'threading': {'max_workers': None}} + 'default_zarr_format': 3, + 'json_indent': 2, + 'ndbuffer': 'zarr.core.buffer.cpu.NDBuffer', + 'threading': {'max_workers': None}} diff --git a/docs/user-guide/consolidated_metadata.rst b/docs/user-guide/consolidated_metadata.rst index edd5bafc8d..4cd72dbc74 100644 --- a/docs/user-guide/consolidated_metadata.rst +++ b/docs/user-guide/consolidated_metadata.rst @@ -47,7 +47,7 @@ that can be used.: >>> from pprint import pprint >>> pprint(dict(sorted(consolidated_metadata.items()))) {'a': ArrayV3Metadata(shape=(1,), - data_type=, + data_type=Float64(endianness='little'), chunk_grid=RegularChunkGrid(chunk_shape=(1,)), chunk_key_encoding=DefaultChunkKeyEncoding(name='default', separator='/'), @@ -60,7 +60,7 @@ that can be used.: node_type='array', storage_transformers=()), 'b': ArrayV3Metadata(shape=(2, 2), - data_type=, + data_type=Float64(endianness='little'), chunk_grid=RegularChunkGrid(chunk_shape=(2, 2)), chunk_key_encoding=DefaultChunkKeyEncoding(name='default', separator='/'), @@ -73,7 +73,7 @@ that can be used.: node_type='array', storage_transformers=()), 'c': ArrayV3Metadata(shape=(3, 3, 3), - data_type=, + data_type=Float64(endianness='little'), chunk_grid=RegularChunkGrid(chunk_shape=(3, 3, 3)), chunk_key_encoding=DefaultChunkKeyEncoding(name='default', separator='/'), diff --git a/docs/user-guide/data_types.rst b/docs/user-guide/data_types.rst new file mode 100644 index 0000000000..87c8efc1f5 --- /dev/null +++ b/docs/user-guide/data_types.rst @@ -0,0 +1,172 @@ +Data types +========== + +Zarr's data type model +---------------------- + +Every Zarr array has a "data type", which defines the meaning and physical layout of the +array's elements. As Zarr Python is tightly integrated with `NumPy `_, +it's easy to create arrays with NumPy data types: + +.. code-block:: python + + >>> import zarr + >>> import numpy as np + >>> z = zarr.create_array(store={}, shape=(10,), dtype=np.dtype('uint8')) + >>> z + + +Unlike NumPy arrays, Zarr arrays are designed to accessed by Zarr +implementations in different programming languages. This means Zarr data types must be interpreted +correctly when clients read an array. Each Zarr data type defines procedures for +encoding and decoding both the data type itself, and scalars from that data type to and from Zarr array metadata. And these serialization procedures +depend on the Zarr format. + +Data types in Zarr version 2 +----------------------------- + +Version 2 of the Zarr format defined its data types relative to +`NumPy's data types `_, +and added a few non-NumPy data types as well. Thus the JSON identifier for a NumPy-compatible data +type is just the NumPy ``str`` attribute of that data type: + +.. code-block:: python + + >>> import zarr + >>> import numpy as np + >>> import json + >>> + >>> store = {} + >>> np_dtype = np.dtype('int64') + >>> z = zarr.create_array(store=store, shape=(1,), dtype=np_dtype, zarr_format=2) + >>> dtype_meta = json.loads(store['.zarray'].to_bytes())["dtype"] + >>> dtype_meta + '>> assert dtype_meta == np_dtype.str + +.. note:: + The ``<`` character in the data type metadata encodes the + `endianness `_, + or "byte order", of the data type. Following NumPy's example, + in Zarr version 2 each data type has an endianness where applicable. + However, Zarr version 3 data types do not store endianness information. + +In addition to defining a representation of the data type itself (which in the example above was +just a simple string ``"M[10s]"`` in + Zarr V2. This is more compact, but can be harder to parse. + +For more about data types in Zarr V3, see the +`V3 specification `_. + +Data types in Zarr Python +------------------------- + +The two Zarr formats that Zarr Python supports specify data types in two different ways: +data types in Zarr version 2 are encoded as NumPy-compatible strings, while data types in Zarr version +3 are encoded as either strings or ``JSON`` objects, +and the Zarr V3 data types don't have any associated endianness information, unlike Zarr V2 data types. + +To abstract over these syntactical and semantic differences, Zarr Python uses a class called +`ZDType <../api/zarr/dtype/index.html#zarr.dtype.ZDType>`_ provide Zarr V2 and Zarr V3 compatibility +routines for ""native" data types. In this context, a "native" data type is a Python class, +typically defined in another library, that models an array's data type. For example, ``np.uint8`` is a native +data type defined in NumPy, which Zarr Python wraps with a ``ZDType`` instance called +`UInt8 <../api/zarr/dtype/index.html#zarr.dtype.ZDType>`_. + +Each data type supported by Zarr Python is modeled by ``ZDType`` subclass, which provides an +API for the following operations: + +- Wrapping / unwrapping a native data type +- Encoding / decoding a data type to / from Zarr V2 and Zarr V3 array metadata. +- Encoding / decoding a scalar value to / from Zarr V2 and Zarr V3 array metadata. + + +Example Usage +~~~~~~~~~~~~~ + +Create a ``ZDType`` from a native data type: + +.. code-block:: python + + >>> from zarr.core.dtype import Int8 + >>> import numpy as np + >>> int8 = Int8.from_native_dtype(np.dtype('int8')) + +Convert back to native data type: + +.. code-block:: python + + >>> native_dtype = int8.to_native_dtype() + >>> assert native_dtype == np.dtype('int8') + +Get the default scalar value for the data type: + +.. code-block:: python + + >>> default_value = int8.default_scalar() + >>> assert default_value == np.int8(0) + + +Serialize to JSON for Zarr V2 and V3 + +.. code-block:: python + + >>> json_v2 = int8.to_json(zarr_format=2) + >>> json_v2 + {'name': '|i1', 'object_codec_id': None} + >>> json_v3 = int8.to_json(zarr_format=3) + >>> json_v3 + 'int8' + +Serialize a scalar value to JSON: + +.. code-block:: python + + >>> json_value = int8.to_json_scalar(42, zarr_format=3) + >>> json_value + 42 + +Deserialize a scalar value from JSON: + +.. code-block:: python + + >>> scalar_value = int8.from_json_scalar(42, zarr_format=3) + >>> assert scalar_value == np.int8(42) diff --git a/docs/user-guide/groups.rst b/docs/user-guide/groups.rst index 99234bad4e..4237a9df50 100644 --- a/docs/user-guide/groups.rst +++ b/docs/user-guide/groups.rst @@ -128,7 +128,7 @@ property. E.g.:: >>> bar.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int64 + Data type : Int64(endianness='little') Fill value : 0 Shape : (1000000,) Chunk shape : (100000,) @@ -145,7 +145,7 @@ property. E.g.:: >>> baz.info Type : Array Zarr format : 3 - Data type : DataType.float32 + Data type : Float32(endianness='little') Fill value : 0.0 Shape : (1000, 1000) Chunk shape : (100, 100) diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index c50713332b..ea34ac2561 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -8,6 +8,7 @@ User guide installation arrays + data_types groups attributes storage diff --git a/docs/user-guide/performance.rst b/docs/user-guide/performance.rst index 88329f11b8..7d24c87373 100644 --- a/docs/user-guide/performance.rst +++ b/docs/user-guide/performance.rst @@ -91,7 +91,7 @@ To use sharding, you need to specify the ``shards`` parameter when creating the >>> z6.info Type : Array Zarr format : 3 - Data type : DataType.uint8 + Data type : UInt8() Fill value : 0 Shape : (10000, 10000, 1000) Shard shape : (1000, 1000, 1000) @@ -100,7 +100,7 @@ To use sharding, you need to specify the ``shards`` parameter when creating the Read-only : False Store type : MemoryStore Filters : () - Serializer : BytesCodec(endian=) + Serializer : BytesCodec(endian=None) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 100000000000 (93.1G) @@ -122,7 +122,7 @@ ratios, depending on the correlation structure within the data. E.g.:: >>> c.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) @@ -142,7 +142,7 @@ ratios, depending on the correlation structure within the data. E.g.:: >>> f.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) diff --git a/pyproject.toml b/pyproject.toml index 8141374d5e..2680396e7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -285,6 +285,7 @@ extend-exclude = [ "notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6 "venv", "docs", + "tests/test_regression/scripts/", # these are scripts that use a different version of python "src/zarr/v2/", "tests/v2/", ] @@ -355,7 +356,6 @@ strict = true warn_unreachable = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] - [[tool.mypy.overrides]] module = [ "tests.package_with_entrypoint.*", @@ -385,6 +385,7 @@ module = [ "tests.test_properties", "tests.test_sync", "tests.test_v2", + "tests.test_regression.scripts.*" ] ignore_errors = true diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index 16400f5f4b..d9e3520d42 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import abstractmethod -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar from zarr.abc.metadata import Metadata from zarr.core.buffer import Buffer, NDBuffer @@ -12,11 +12,10 @@ from collections.abc import Awaitable, Callable, Iterable from typing import Self - import numpy as np - from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.indexing import SelectorTuple __all__ = [ @@ -93,7 +92,13 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: """ return self - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, + *, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: """Validates that the codec configuration is compatible with the array metadata. Raises errors when the codec configuration is not compatible. @@ -285,7 +290,9 @@ def supports_partial_decode(self) -> bool: ... def supports_partial_encode(self) -> bool: ... @abstractmethod - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, *, shape: ChunkCoords, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGrid + ) -> None: """Validates that all codec configurations are compatible with the array metadata. Raises errors when a codec configuration is not compatible. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 54bddd80a8..3b53095636 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -14,6 +14,7 @@ Array, AsyncArray, CompressorLike, + _get_default_chunk_encoding_v2, create_array, from_array, get_array_metadata, @@ -30,8 +31,8 @@ _default_zarr_format, _warn_order_kwarg, _warn_write_empty_chunks_kwarg, - parse_dtype, ) +from zarr.core.dtype import ZDTypeLike, get_data_type_from_native_dtype, parse_data_type from zarr.core.group import ( AsyncGroup, ConsolidatedMetadata, @@ -39,7 +40,6 @@ create_hierarchy, ) from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata -from zarr.core.metadata.v2 import _default_compressor, _default_filters from zarr.errors import GroupNotFoundError, NodeTypeValidationError from zarr.storage import StorePath from zarr.storage._common import make_store_path @@ -239,7 +239,6 @@ async def consolidate_metadata( group, metadata=metadata, ) - await group._save_metadata() return group @@ -457,11 +456,12 @@ async def save_array( shape = arr.shape chunks = getattr(arr, "chunks", None) # for array-likes with chunks attribute overwrite = kwargs.pop("overwrite", None) or _infer_overwrite(mode) + zarr_dtype = get_data_type_from_native_dtype(arr.dtype) new = await AsyncArray._create( store_path, zarr_format=zarr_format, shape=shape, - dtype=arr.dtype, + dtype=zarr_dtype, chunks=chunks, overwrite=overwrite, **kwargs, @@ -861,7 +861,7 @@ async def create( shape: ChunkCoords | int, *, # Note: this is a change from v2 chunks: ChunkCoords | int | None = None, # TODO: v2 allowed chunks=True - dtype: npt.DTypeLike | None = None, + dtype: ZDTypeLike | None = None, compressor: CompressorLike = "auto", fill_value: Any | None = 0, # TODO: need type order: MemoryOrder | None = None, @@ -1008,13 +1008,13 @@ async def create( _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) or _default_zarr_format() ) - + zdtype = parse_data_type(dtype, zarr_format=zarr_format) if zarr_format == 2: - dtype = parse_dtype(dtype, zarr_format) + default_filters, default_compressor = _get_default_chunk_encoding_v2(zdtype) if not filters: - filters = _default_filters(dtype) + filters = default_filters # type: ignore[assignment] if compressor == "auto": - compressor = _default_compressor(dtype) + compressor = default_compressor if synchronizer is not None: warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2) @@ -1066,7 +1066,7 @@ async def create( store_path, shape=shape, chunks=chunks, - dtype=dtype, + dtype=zdtype, compressor=compressor, fill_value=fill_value, overwrite=overwrite, diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index a7f7cfda35..f2dc8757d6 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -38,6 +38,7 @@ ShapeLike, ZarrFormat, ) + from zarr.core.dtype import ZDTypeLike from zarr.storage import StoreLike __all__ = [ @@ -603,9 +604,9 @@ def create( shape: ChunkCoords | int, *, # Note: this is a change from v2 chunks: ChunkCoords | int | bool | None = None, - dtype: npt.DTypeLike | None = None, + dtype: ZDTypeLike | None = None, compressor: CompressorLike = "auto", - fill_value: Any | None = 0, # TODO: need type + fill_value: Any | None = None, # TODO: need type order: MemoryOrder | None = None, store: str | StoreLike | None = None, synchronizer: Any | None = None, @@ -755,7 +756,7 @@ def create_array( *, name: str | None = None, shape: ShapeLike | None = None, - dtype: npt.DTypeLike | None = None, + dtype: ZDTypeLike | None = None, data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunkCoords | Literal["auto"] = "auto", shards: ShardsLike | None = None, @@ -786,7 +787,7 @@ def create_array( at the root of the store. shape : ChunkCoords, optional Shape of the array. Can be ``None`` if ``data`` is provided. - dtype : npt.DTypeLike, optional + dtype : ZDTypeLike, optional Data type of the array. Can be ``None`` if ``data`` is provided. data : np.ndarray, optional Array-like data to use for initializing the array. If this parameter is provided, the diff --git a/src/zarr/codecs/_v2.py b/src/zarr/codecs/_v2.py index 53edc1f4a1..08853f27f1 100644 --- a/src/zarr/codecs/_v2.py +++ b/src/zarr/codecs/_v2.py @@ -46,9 +46,9 @@ async def _decode_single( chunk = ensure_ndarray_like(chunk) # special case object dtype, because incorrect handling can lead to # segfaults and other bad things happening - if chunk_spec.dtype != object: + if chunk_spec.dtype.dtype_cls is not np.dtypes.ObjectDType: try: - chunk = chunk.view(chunk_spec.dtype) + chunk = chunk.view(chunk_spec.dtype.to_native_dtype()) except TypeError: # this will happen if the dtype of the chunk # does not match the dtype of the array spec i.g. if @@ -56,7 +56,7 @@ async def _decode_single( # is an object array. In this case, we need to convert the object # array to the correct dtype. - chunk = np.array(chunk).astype(chunk_spec.dtype) + chunk = np.array(chunk).astype(chunk_spec.dtype.to_native_dtype()) elif chunk.dtype != object: # If we end up here, someone must have hacked around with the filters. @@ -80,7 +80,7 @@ async def _encode_single( chunk = chunk_array.as_ndarray_like() # ensure contiguous and correct order - chunk = chunk.astype(chunk_spec.dtype, order=chunk_spec.order, copy=False) + chunk = chunk.astype(chunk_spec.dtype.to_native_dtype(), order=chunk_spec.order, copy=False) # apply filters if self.filters: diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index 9a999e10d7..1c5e52e9a4 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -13,6 +13,7 @@ from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_enum, parse_named_configuration +from zarr.core.dtype.common import HasItemSize from zarr.registry import register_codec if TYPE_CHECKING: @@ -137,14 +138,16 @@ def to_dict(self) -> dict[str, JSON]: } def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: - dtype = array_spec.dtype + item_size = 1 + if isinstance(array_spec.dtype, HasItemSize): + item_size = array_spec.dtype.item_size new_codec = self if new_codec.typesize is None: - new_codec = replace(new_codec, typesize=dtype.itemsize) + new_codec = replace(new_codec, typesize=item_size) if new_codec.shuffle is None: new_codec = replace( new_codec, - shuffle=(BloscShuffle.bitshuffle if dtype.itemsize == 1 else BloscShuffle.shuffle), + shuffle=(BloscShuffle.bitshuffle if item_size == 1 else BloscShuffle.shuffle), ) return new_codec diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 750707d36a..d663a3b2cc 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -10,6 +10,7 @@ from zarr.abc.codec import ArrayBytesCodec from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer from zarr.core.common import JSON, parse_enum, parse_named_configuration +from zarr.core.dtype.common import HasEndianness from zarr.registry import register_codec if TYPE_CHECKING: @@ -56,7 +57,7 @@ def to_dict(self) -> dict[str, JSON]: return {"name": "bytes", "configuration": {"endian": self.endian.value}} def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: - if array_spec.dtype.itemsize == 0: + if not isinstance(array_spec.dtype, HasEndianness): if self.endian is not None: return replace(self, endian=None) elif self.endian is None: @@ -71,15 +72,12 @@ async def _decode_single( chunk_spec: ArraySpec, ) -> NDBuffer: assert isinstance(chunk_bytes, Buffer) - if chunk_spec.dtype.itemsize > 0: - if self.endian == Endian.little: - prefix = "<" - else: - prefix = ">" - dtype = np.dtype(f"{prefix}{chunk_spec.dtype.str[1:]}") + # TODO: remove endianness enum in favor of literal union + endian_str = self.endian.value if self.endian is not None else None + if isinstance(chunk_spec.dtype, HasEndianness): + dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] else: - dtype = np.dtype(f"|{chunk_spec.dtype.str[1:]}") - + dtype = chunk_spec.dtype.to_native_dtype() as_array_like = chunk_bytes.as_array_like() if isinstance(as_array_like, NDArrayLike): as_nd_array_like = as_array_like diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 4638d973cb..cd8676b4d1 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -43,6 +43,7 @@ parse_shapelike, product, ) +from zarr.core.dtype.npy.int import UInt64 from zarr.core.indexing import ( BasicIndexer, SelectorTuple, @@ -58,6 +59,7 @@ from typing import Self from zarr.core.common import JSON + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType MAX_UINT_64 = 2**64 - 1 ShardMapping = Mapping[ChunkCoords, Buffer] @@ -355,7 +357,11 @@ def __init__( object.__setattr__(self, "index_location", index_location_parsed) # Use instance-local lru_cache to avoid memory leaks - object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) + + # numpy void scalars are not hashable, which means an array spec with a fill value that is + # a numpy void scalar will break the lru_cache. This is commented for now but should be + # fixed. See https://github.com/zarr-developers/zarr-python/issues/3054 + # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) @@ -371,7 +377,7 @@ def __setstate__(self, state: dict[str, Any]) -> None: object.__setattr__(self, "index_location", parse_index_location(config["index_location"])) # Use instance-local lru_cache to avoid memory leaks - object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) + # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) @@ -402,7 +408,13 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: return replace(self, codecs=evolved_codecs) return self - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, + *, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: if len(self.chunk_shape) != len(shape): raise ValueError( "The shard's `chunk_shape` and array's `shape` need to have the same number of dimensions." @@ -439,7 +451,10 @@ async def _decode_single( # setup output array out = chunk_spec.prototype.nd_buffer.create( - shape=shard_shape, dtype=shard_spec.dtype, order=shard_spec.order, fill_value=0 + shape=shard_shape, + dtype=shard_spec.dtype.to_native_dtype(), + order=shard_spec.order, + fill_value=0, ) shard_dict = await _ShardReader.from_bytes(shard_bytes, self, chunks_per_shard) @@ -483,7 +498,10 @@ async def _decode_partial_single( # setup output array out = shard_spec.prototype.nd_buffer.create( - shape=indexer.shape, dtype=shard_spec.dtype, order=shard_spec.order, fill_value=0 + shape=indexer.shape, + dtype=shard_spec.dtype.to_native_dtype(), + order=shard_spec.order, + fill_value=0, ) indexed_chunks = list(indexer) @@ -678,7 +696,7 @@ def _shard_index_size(self, chunks_per_shard: ChunkCoords) -> int: def _get_index_chunk_spec(self, chunks_per_shard: ChunkCoords) -> ArraySpec: return ArraySpec( shape=chunks_per_shard + (2,), - dtype=np.dtype(" tuple[int, ...]: @@ -45,7 +46,12 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: def to_dict(self) -> dict[str, JSON]: return {"name": "transpose", "configuration": {"order": tuple(self.order)}} - def validate(self, shape: tuple[int, ...], dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, + shape: tuple[int, ...], + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: if len(self.order) != len(shape): raise ValueError( f"The `order` tuple needs have as many entries as there are dimensions in the array. Got {self.order}." diff --git a/src/zarr/codecs/vlen_utf8.py b/src/zarr/codecs/vlen_utf8.py index 0ef423793d..b7c0418b2e 100644 --- a/src/zarr/codecs/vlen_utf8.py +++ b/src/zarr/codecs/vlen_utf8.py @@ -10,7 +10,6 @@ from zarr.abc.codec import ArrayBytesCodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.common import JSON, parse_named_configuration -from zarr.core.strings import cast_to_string_dtype from zarr.registry import register_codec if TYPE_CHECKING: @@ -49,6 +48,7 @@ def to_dict(self) -> dict[str, JSON]: def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: return self + # TODO: expand the tests for this function async def _decode_single( self, chunk_bytes: Buffer, @@ -60,8 +60,7 @@ async def _decode_single( decoded = _vlen_utf8_codec.decode(raw_bytes) assert decoded.dtype == np.object_ decoded.shape = chunk_spec.shape - # coming out of the code, we know this is safe, so don't issue a warning - as_string_dtype = cast_to_string_dtype(decoded, safe=True) + as_string_dtype = decoded.astype(chunk_spec.dtype.to_native_dtype(), copy=False) return chunk_spec.prototype.nd_buffer.from_numpy_array(as_string_dtype) async def _encode_single( diff --git a/src/zarr/core/_info.py b/src/zarr/core/_info.py index ee953d4591..d57d17f934 100644 --- a/src/zarr/core/_info.py +++ b/src/zarr/core/_info.py @@ -1,13 +1,15 @@ +from __future__ import annotations + import dataclasses import textwrap -from typing import Any, Literal +from typing import TYPE_CHECKING, Literal -import numcodecs.abc -import numpy as np +if TYPE_CHECKING: + import numcodecs.abc -from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec -from zarr.core.common import ZarrFormat -from zarr.core.metadata.v3 import DataType + from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec + from zarr.core.common import ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType @dataclasses.dataclass(kw_only=True) @@ -78,7 +80,7 @@ class ArrayInfo: _type: Literal["Array"] = "Array" _zarr_format: ZarrFormat - _data_type: np.dtype[Any] | DataType + _data_type: ZDType[TBaseDType, TBaseScalar] _fill_value: object _shape: tuple[int, ...] _shard_shape: tuple[int, ...] | None = None diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index b4e8ac0ff6..cd6b33a28c 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -22,7 +22,6 @@ import numcodecs import numcodecs.abc import numpy as np -import numpy.typing as npt from typing_extensions import deprecated import zarr @@ -30,6 +29,7 @@ from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.abc.store import Store, set_or_delete from zarr.codecs._v2 import V2Codec +from zarr.codecs.bytes import BytesCodec from zarr.core._info import ArrayInfo from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, parse_array_config from zarr.core.attributes import Attributes @@ -61,12 +61,18 @@ _default_zarr_format, _warn_order_kwarg, concurrent_map, - parse_dtype, parse_order, parse_shapelike, product, ) +from zarr.core.config import categorize_data_type from zarr.core.config import config as zarr_config +from zarr.core.dtype import ( + ZDType, + ZDTypeLike, + parse_data_type, +) +from zarr.core.dtype.common import HasEndianness, HasItemSize from zarr.core.indexing import ( BasicIndexer, BasicSelection, @@ -103,12 +109,10 @@ ) from zarr.core.metadata.v2 import ( CompressorLikev2, - _default_compressor, - _default_filters, parse_compressor, parse_filters, ) -from zarr.core.metadata.v3 import DataType, parse_node_type_array +from zarr.core.metadata.v3 import parse_node_type_array from zarr.core.sync import sync from zarr.errors import MetadataValidationError from zarr.registry import ( @@ -124,8 +128,11 @@ from collections.abc import Iterator, Sequence from typing import Self + import numpy.typing as npt + from zarr.abc.codec import CodecPipeline from zarr.codecs.sharding import ShardingCodecIndexLocation + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar from zarr.core.group import AsyncGroup from zarr.storage import StoreLike @@ -287,7 +294,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: Literal[2], fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -311,7 +318,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: Literal[3], fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -339,7 +346,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: Literal[3] = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -367,7 +374,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -402,7 +409,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -438,7 +445,7 @@ async def create( The store where the array will be created. shape : ShapeLike The shape of the array. - dtype : npt.DTypeLike + dtype : ZDTypeLike The data type of the array. zarr_format : ZarrFormat, optional The Zarr format version (default is 3). @@ -543,7 +550,7 @@ async def _create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike | ZDType[TBaseDType, TBaseScalar], zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -572,18 +579,21 @@ async def _create( See :func:`AsyncArray.create` for more details. Deprecated in favor of :func:`zarr.api.asynchronous.create_array`. """ + + dtype_parsed = parse_data_type(dtype, zarr_format=zarr_format) store_path = await make_store_path(store) - dtype_parsed = parse_dtype(dtype, zarr_format) shape = parse_shapelike(shape) if chunks is not None and chunk_shape is not None: raise ValueError("Only one of chunk_shape or chunks can be provided.") - + item_size = 1 + if isinstance(dtype_parsed, HasItemSize): + item_size = dtype_parsed.item_size if chunks: - _chunks = normalize_chunks(chunks, shape, dtype_parsed.itemsize) + _chunks = normalize_chunks(chunks, shape, item_size) else: - _chunks = normalize_chunks(chunk_shape, shape, dtype_parsed.itemsize) + _chunks = normalize_chunks(chunk_shape, shape, item_size) config_parsed = parse_array_config(config) result: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] @@ -661,7 +671,7 @@ async def _create( @staticmethod def _create_metadata_v3( shape: ShapeLike, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], chunk_shape: ChunkCoords, fill_value: Any | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, @@ -672,30 +682,36 @@ def _create_metadata_v3( """ Create an instance of ArrayV3Metadata. """ + filters: tuple[ArrayArrayCodec, ...] + compressors: tuple[BytesBytesCodec, ...] shape = parse_shapelike(shape) - codecs = list(codecs) if codecs is not None else _get_default_codecs(np.dtype(dtype)) + if codecs is None: + filters, serializer, compressors = _get_default_chunk_encoding_v3(dtype) + codecs_parsed = (*filters, serializer, *compressors) + else: + codecs_parsed = tuple(codecs) + chunk_key_encoding_parsed: ChunkKeyEncodingLike if chunk_key_encoding is None: chunk_key_encoding_parsed = {"name": "default", "separator": "/"} else: chunk_key_encoding_parsed = chunk_key_encoding - if dtype.kind in "UTS": - warn( - f"The dtype `{dtype}` is currently not part in the Zarr format 3 specification. It " - "may not be supported by other zarr implementations and may change in the future.", - category=UserWarning, - stacklevel=2, - ) + if fill_value is None: + # v3 spec will not allow a null fill value + fill_value_parsed = dtype.default_scalar() + else: + fill_value_parsed = fill_value + chunk_grid_parsed = RegularChunkGrid(chunk_shape=chunk_shape) return ArrayV3Metadata( shape=shape, data_type=dtype, chunk_grid=chunk_grid_parsed, chunk_key_encoding=chunk_key_encoding_parsed, - fill_value=fill_value, - codecs=codecs, + fill_value=fill_value_parsed, + codecs=codecs_parsed, # type: ignore[arg-type] dimension_names=tuple(dimension_names) if dimension_names else None, attributes=attributes or {}, ) @@ -706,7 +722,7 @@ async def _create_v3( store_path: StorePath, *, shape: ShapeLike, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], chunk_shape: ChunkCoords, config: ArrayConfig, fill_value: Any | None = None, @@ -754,7 +770,7 @@ async def _create_v3( @staticmethod def _create_metadata_v2( shape: ChunkCoords, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], chunks: ChunkCoords, order: MemoryOrder, dimension_separator: Literal[".", "/"] | None = None, @@ -765,12 +781,11 @@ def _create_metadata_v2( ) -> ArrayV2Metadata: if dimension_separator is None: dimension_separator = "." - - dtype = parse_dtype(dtype, zarr_format=2) - + if fill_value is None: + fill_value = dtype.default_scalar() # type: ignore[assignment] return ArrayV2Metadata( shape=shape, - dtype=np.dtype(dtype), + dtype=dtype, chunks=chunks, order=order, dimension_separator=dimension_separator, @@ -786,7 +801,7 @@ async def _create_v2( store_path: StorePath, *, shape: ChunkCoords, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], chunks: ChunkCoords, order: MemoryOrder, config: ArrayConfig, @@ -807,7 +822,7 @@ async def _create_v2( compressor_parsed: CompressorLikev2 if compressor == "auto": - compressor_parsed = _default_compressor(dtype) + _, compressor_parsed = _get_default_chunk_encoding_v2(dtype) elif isinstance(compressor, BytesBytesCodec): raise ValueError( "Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. " @@ -1023,7 +1038,17 @@ def compressors(self) -> tuple[numcodecs.abc.Codec, ...] | tuple[BytesBytesCodec ) @property - def dtype(self) -> np.dtype[Any]: + def _zdtype(self) -> ZDType[TBaseDType, TBaseScalar]: + """ + The zarr-specific representation of the array data type + """ + if self.metadata.zarr_format == 2: + return self.metadata.dtype + else: + return self.metadata.data_type + + @property + def dtype(self) -> TBaseDType: """Returns the data type of the array. Returns @@ -1031,7 +1056,7 @@ def dtype(self) -> np.dtype[Any]: np.dtype Data type of the array """ - return self.metadata.dtype + return self._zdtype.to_native_dtype() @property def order(self) -> MemoryOrder: @@ -1392,20 +1417,21 @@ async def _set_selection( # TODO: need to handle array types that don't support __array_function__ # like PyTorch and JAX array_like_ = cast("np._typing._SupportsArrayFunc", array_like) - value = np.asanyarray(value, dtype=self.metadata.dtype, like=array_like_) + value = np.asanyarray(value, dtype=self.dtype, like=array_like_) else: if not hasattr(value, "shape"): - value = np.asarray(value, self.metadata.dtype) + value = np.asarray(value, self.dtype) # assert ( # value.shape == indexer.shape # ), f"shape of value doesn't match indexer shape. Expected {indexer.shape}, got {value.shape}" - if not hasattr(value, "dtype") or value.dtype.name != self.metadata.dtype.name: + if not hasattr(value, "dtype") or value.dtype.name != self.dtype.name: if hasattr(value, "astype"): # Handle things that are already NDArrayLike more efficiently - value = value.astype(dtype=self.metadata.dtype, order="A") + value = value.astype(dtype=self.dtype, order="A") else: - value = np.array(value, dtype=self.metadata.dtype, order="A") + value = np.array(value, dtype=self.dtype, order="A") value = cast("NDArrayLike", value) + # We accept any ndarray like object from the user and convert it # to a NDBuffer (or subclass). From this point onwards, we only pass # Buffer and NDBuffer between components. @@ -1685,15 +1711,9 @@ async def info_complete(self) -> Any: def _info( self, count_chunks_initialized: int | None = None, count_bytes_stored: int | None = None ) -> Any: - _data_type: np.dtype[Any] | DataType - if isinstance(self.metadata, ArrayV2Metadata): - _data_type = self.metadata.dtype - else: - _data_type = self.metadata.data_type - return ArrayInfo( _zarr_format=self.metadata.zarr_format, - _data_type=_data_type, + _data_type=self._zdtype, _fill_value=self.metadata.fill_value, _shape=self.shape, _order=self.order, @@ -1728,7 +1748,7 @@ def create( *, # v2 and v3 shape: ChunkCoords, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -1763,7 +1783,7 @@ def create( The array store that has already been initialized. shape : ChunkCoords The shape of the array. - dtype : npt.DTypeLike + dtype : ZDTypeLike The data type of the array. chunk_shape : ChunkCoords, optional The shape of the Array's chunks. @@ -1857,7 +1877,7 @@ def _create( *, # v2 and v3 shape: ChunkCoords, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -3773,13 +3793,6 @@ def _build_parents( return parents -def _get_default_codecs( - np_dtype: np.dtype[Any], -) -> tuple[Codec, ...]: - filters, serializer, compressors = _get_default_chunk_encoding_v3(np_dtype) - return filters + (serializer,) + compressors - - FiltersLike: TypeAlias = ( Iterable[dict[str, JSON] | ArrayArrayCodec | numcodecs.abc.Codec] | ArrayArrayCodec @@ -4079,7 +4092,7 @@ async def init_array( *, store_path: StorePath, shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, chunks: ChunkCoords | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", @@ -4102,7 +4115,7 @@ async def init_array( StorePath instance. The path attribute is the name of the array to initialize. shape : ChunkCoords Shape of the array. - dtype : npt.DTypeLike + dtype : ZDTypeLike Data type of the array. chunks : ChunkCoords, optional Chunk shape of the array. @@ -4186,7 +4199,7 @@ async def init_array( from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation - dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) + zdtype = parse_data_type(dtype, zarr_format=zarr_format) shape_parsed = parse_shapelike(shape) chunk_key_encoding_parsed = _parse_chunk_key_encoding( chunk_key_encoding, zarr_format=zarr_format @@ -4200,8 +4213,15 @@ async def init_array( else: await ensure_no_existing_node(store_path, zarr_format=zarr_format) + item_size = 1 + if isinstance(zdtype, HasItemSize): + item_size = zdtype.item_size + shard_shape_parsed, chunk_shape_parsed = _auto_partition( - array_shape=shape_parsed, shard_shape=shards, chunk_shape=chunks, dtype=dtype_parsed + array_shape=shape_parsed, + shard_shape=shards, + chunk_shape=chunks, + item_size=item_size, ) chunks_out: tuple[int, ...] meta: ArrayV2Metadata | ArrayV3Metadata @@ -4217,9 +4237,8 @@ async def init_array( raise ValueError("Zarr format 2 arrays do not support `serializer`.") filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( - compressor=compressors, filters=filters, dtype=np.dtype(dtype) + compressor=compressors, filters=filters, dtype=zdtype ) - if dimension_names is not None: raise ValueError("Zarr format 2 arrays do not support dimension names.") if order is None: @@ -4229,7 +4248,7 @@ async def init_array( meta = AsyncArray._create_metadata_v2( shape=shape_parsed, - dtype=dtype_parsed, + dtype=zdtype, chunks=chunk_shape_parsed, dimension_separator=chunk_key_encoding_parsed.separator, fill_value=fill_value, @@ -4243,7 +4262,7 @@ async def init_array( compressors=compressors, filters=filters, serializer=serializer, - dtype=dtype_parsed, + dtype=zdtype, ) sub_codecs = cast("tuple[Codec, ...]", (*array_array, array_bytes, *bytes_bytes)) codecs_out: tuple[Codec, ...] @@ -4258,7 +4277,7 @@ async def init_array( ) sharding_codec.validate( shape=chunk_shape_parsed, - dtype=dtype_parsed, + dtype=zdtype, chunk_grid=RegularChunkGrid(chunk_shape=shard_shape_parsed), ) codecs_out = (sharding_codec,) @@ -4274,7 +4293,7 @@ async def init_array( meta = AsyncArray._create_metadata_v3( shape=shape_parsed, - dtype=dtype_parsed, + dtype=zdtype, fill_value=fill_value, chunk_shape=chunks_out, chunk_key_encoding=chunk_key_encoding_parsed, @@ -4293,7 +4312,7 @@ async def create_array( *, name: str | None = None, shape: ShapeLike | None = None, - dtype: npt.DTypeLike | None = None, + dtype: ZDTypeLike | None = None, data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunkCoords | Literal["auto"] = "auto", shards: ShardsLike | None = None, @@ -4322,7 +4341,7 @@ async def create_array( at the root of the store. shape : ChunkCoords, optional Shape of the array. Can be ``None`` if ``data`` is provided. - dtype : npt.DTypeLike | None + dtype : ZDTypeLike | None Data type of the array. Can be ``None`` if ``data`` is provided. data : Array-like data to use for initializing the array. If this parameter is provided, the ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``, @@ -4582,62 +4601,50 @@ def _parse_chunk_key_encoding( def _get_default_chunk_encoding_v3( - np_dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: """ Get the default ArrayArrayCodecs, ArrayBytesCodec, and BytesBytesCodec for a given dtype. """ - dtype = DataType.from_numpy(np_dtype) - if dtype == DataType.string: - dtype_key = "string" - elif dtype == DataType.bytes: - dtype_key = "bytes" - else: - dtype_key = "numeric" - default_filters = zarr_config.get("array.v3_default_filters").get(dtype_key) - default_serializer = zarr_config.get("array.v3_default_serializer").get(dtype_key) - default_compressors = zarr_config.get("array.v3_default_compressors").get(dtype_key) + dtype_category = categorize_data_type(dtype) - filters = tuple(_parse_array_array_codec(codec_dict) for codec_dict in default_filters) - serializer = _parse_array_bytes_codec(default_serializer) - compressors = tuple(_parse_bytes_bytes_codec(codec_dict) for codec_dict in default_compressors) + filters = zarr_config.get("array.v3_default_filters").get(dtype_category) + compressors = zarr_config.get("array.v3_default_compressors").get(dtype_category) + serializer = zarr_config.get("array.v3_default_serializer").get(dtype_category) - return filters, serializer, compressors + return ( + tuple(_parse_array_array_codec(f) for f in filters), + _parse_array_bytes_codec(serializer), + tuple(_parse_bytes_bytes_codec(c) for c in compressors), + ) def _get_default_chunk_encoding_v2( - np_dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[numcodecs.abc.Codec, ...] | None, numcodecs.abc.Codec | None]: """ Get the default chunk encoding for Zarr format 2 arrays, given a dtype """ + dtype_category = categorize_data_type(dtype) + filters = zarr_config.get("array.v2_default_filters").get(dtype_category) + compressor = zarr_config.get("array.v2_default_compressor").get(dtype_category) + if filters is not None: + filters = tuple(numcodecs.get_codec(f) for f in filters) - compressor_dict = _default_compressor(np_dtype) - filter_dicts = _default_filters(np_dtype) - - compressor = None - if compressor_dict is not None: - compressor = numcodecs.get_codec(compressor_dict) - - filters = None - if filter_dicts is not None: - filters = tuple(numcodecs.get_codec(f) for f in filter_dicts) - - return filters, compressor + return filters, numcodecs.get_codec(compressor) def _parse_chunk_encoding_v2( *, compressor: CompressorsLike, filters: FiltersLike, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[numcodecs.abc.Codec, ...] | None, numcodecs.abc.Codec | None]: """ Generate chunk encoding classes for Zarr format 2 arrays with optional defaults. """ default_filters, default_compressor = _get_default_chunk_encoding_v2(dtype) - _filters: tuple[numcodecs.abc.Codec, ...] | None _compressor: numcodecs.abc.Codec | None @@ -4676,7 +4683,7 @@ def _parse_chunk_encoding_v3( compressors: CompressorsLike, filters: FiltersLike, serializer: SerializerLike, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: """ Generate chunk encoding classes for v3 arrays with optional defaults. @@ -4700,6 +4707,9 @@ def _parse_chunk_encoding_v3( if serializer == "auto": out_array_bytes = default_array_bytes else: + # TODO: ensure that the serializer is compatible with the ndarray produced by the + # array-array codecs. For example, if a sequence of array-array codecs produces an + # array with a single-byte data type, then the serializer should not specify endiannesss. out_array_bytes = _parse_array_bytes_codec(serializer) if compressors is None: @@ -4715,6 +4725,17 @@ def _parse_chunk_encoding_v3( out_bytes_bytes = tuple(_parse_bytes_bytes_codec(c) for c in maybe_bytes_bytes) + # specialize codecs as needed given the dtype + + # TODO: refactor so that the config only contains the name of the codec, and we use the dtype + # to create the codec instance, instead of storing a dict representation of a full codec. + + # TODO: ensure that the serializer is compatible with the ndarray produced by the + # array-array codecs. For example, if a sequence of array-array codecs produces an + # array with a single-byte data type, then the serializer should not specify endiannesss. + if isinstance(out_array_bytes, BytesCodec) and not isinstance(dtype, HasEndianness): + # The default endianness in the bytescodec might not be None, so we need to replace it + out_array_bytes = replace(out_array_bytes, endian=None) return out_array_array, out_array_bytes, out_bytes_bytes @@ -4744,8 +4765,8 @@ def _parse_data_params( *, data: np.ndarray[Any, np.dtype[Any]] | None, shape: ShapeLike | None, - dtype: npt.DTypeLike | None, -) -> tuple[np.ndarray[Any, np.dtype[Any]] | None, ShapeLike, npt.DTypeLike]: + dtype: ZDTypeLike | None, +) -> tuple[np.ndarray[Any, np.dtype[Any]] | None, ShapeLike, ZDTypeLike]: """ Ensure an array-like ``data`` parameter is consistent with the ``dtype`` and ``shape`` parameters. diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py index 6cd27b30eb..279bf6edf0 100644 --- a/src/zarr/core/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -3,8 +3,6 @@ from dataclasses import dataclass, fields from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast -import numpy as np - from zarr.core.common import ( MemoryOrder, parse_bool, @@ -19,6 +17,7 @@ from zarr.core.buffer import BufferPrototype from zarr.core.common import ChunkCoords + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType class ArrayConfigParams(TypedDict): @@ -90,7 +89,7 @@ def parse_array_config(data: ArrayConfigLike | None) -> ArrayConfig: @dataclass(frozen=True) class ArraySpec: shape: ChunkCoords - dtype: np.dtype[Any] + dtype: ZDType[TBaseDType, TBaseScalar] fill_value: Any config: ArrayConfig prototype: BufferPrototype @@ -98,17 +97,16 @@ class ArraySpec: def __init__( self, shape: ChunkCoords, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], fill_value: Any, config: ArrayConfig, prototype: BufferPrototype, ) -> None: shape_parsed = parse_shapelike(shape) - dtype_parsed = np.dtype(dtype) fill_value_parsed = parse_fill_value(fill_value) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "dtype", dtype_parsed) + object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "fill_value", fill_value_parsed) object.__setattr__(self, "config", config) object.__setattr__(self, "prototype", prototype) diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index d0a2d992d2..0e24c5b326 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -495,7 +495,9 @@ def all_equal(self, other: Any, equal_nan: bool = True) -> bool: return np.array_equal( self._data, other, - equal_nan=equal_nan if self._data.dtype.kind not in "USTOV" else False, + equal_nan=equal_nan + if self._data.dtype.kind not in ("U", "S", "T", "O", "V") + else False, ) def fill(self, value: Any) -> None: diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index b5a581b8a4..4bf03c89de 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -207,7 +207,7 @@ def _auto_partition( array_shape: tuple[int, ...], chunk_shape: tuple[int, ...] | Literal["auto"], shard_shape: ShardsLike | None, - dtype: np.dtype[Any], + item_size: int, ) -> tuple[tuple[int, ...] | None, tuple[int, ...]]: """ Automatically determine the shard shape and chunk shape for an array, given the shape and dtype of the array. @@ -217,7 +217,6 @@ def _auto_partition( of the array; if the `chunk_shape` is also "auto", then the chunks will be set heuristically as well, given the dtype and shard shape. Otherwise, the chunks will be returned as-is. """ - item_size = dtype.itemsize if shard_shape is None: _shards_out: None | tuple[int, ...] = None if chunk_shape == "auto": diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index 628a7e0487..23c27e40c6 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -17,19 +17,17 @@ from zarr.core.common import ChunkCoords, concurrent_map from zarr.core.config import config from zarr.core.indexing import SelectorTuple, is_scalar -from zarr.core.metadata.v2 import _default_fill_value from zarr.registry import register_pipeline if TYPE_CHECKING: from collections.abc import Iterable, Iterator from typing import Self - import numpy as np - from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType T = TypeVar("T") U = TypeVar("U") @@ -64,7 +62,7 @@ def fill_value_or_default(chunk_spec: ArraySpec) -> Any: # validated when decoding the metadata, but we support reading # Zarr V2 data and need to support the case where fill_value # is None. - return _default_fill_value(dtype=chunk_spec.dtype) + return chunk_spec.dtype.default_scalar() else: return fill_value @@ -134,7 +132,9 @@ def __iter__(self) -> Iterator[Codec]: yield self.array_bytes_codec yield from self.bytes_bytes_codecs - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, *, shape: ChunkCoords, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGrid + ) -> None: for codec in self: codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid) @@ -296,7 +296,9 @@ def _merge_chunk_array( is_complete_chunk: bool, drop_axes: tuple[int, ...], ) -> NDBuffer: - if chunk_selection == () or is_scalar(value.as_ndarray_like(), chunk_spec.dtype): + if chunk_selection == () or is_scalar( + value.as_ndarray_like(), chunk_spec.dtype.to_native_dtype() + ): chunk_value = value else: chunk_value = value[out_selection] @@ -317,7 +319,7 @@ def _merge_chunk_array( if existing_chunk_array is None: chunk_array = chunk_spec.prototype.nd_buffer.create( shape=chunk_spec.shape, - dtype=chunk_spec.dtype, + dtype=chunk_spec.dtype.to_native_dtype(), order=chunk_spec.order, fill_value=fill_value_or_default(chunk_spec), ) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index be37dc5109..2ba5914ea5 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -10,16 +10,15 @@ from typing import ( TYPE_CHECKING, Any, + Generic, Literal, + TypedDict, TypeVar, cast, overload, ) -import numpy as np - from zarr.core.config import config as zarr_config -from zarr.core.strings import _STRING_DTYPE if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator @@ -42,6 +41,14 @@ AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] DimensionNames = Iterable[str | None] | None +TName = TypeVar("TName", bound=str) +TConfig = TypeVar("TConfig", bound=Mapping[str, object]) + + +class NamedConfig(TypedDict, Generic[TName, TConfig]): + name: TName + configuration: TConfig + def product(tup: ChunkCoords) -> int: return functools.reduce(operator.mul, tup, 1) @@ -168,16 +175,6 @@ def parse_bool(data: Any) -> bool: raise ValueError(f"Expected bool, got {data} instead.") -def parse_dtype(dtype: Any, zarr_format: ZarrFormat) -> np.dtype[Any]: - if dtype is str or dtype == "str": - if zarr_format == 2: - # special case as object - return np.dtype("object") - else: - return _STRING_DTYPE - return np.dtype(dtype) - - def _warn_write_empty_chunks_kwarg() -> None: # TODO: link to docs page on array configuration in this message msg = ( diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 2a10943d80..74e9bdd8dd 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -36,11 +36,21 @@ if TYPE_CHECKING: from donfig.config_obj import ConfigSet + from zarr.core.dtype.wrapper import ZDType + class BadConfigError(ValueError): _msg = "bad Config: %r" +# These values are used for rough categorization of data types +# we use this for choosing a default encoding scheme based on the data type. Specifically, +# these categories are keys in a configuration dictionary. +# it is not a part of the ZDType class because these categories are more of an implementation detail +# of our config system rather than a useful attribute of any particular data type. +DTypeCategory = Literal["variable-length-string", "default"] + + class Config(DConfig): # type: ignore[misc] """The Config will collect configuration from config files and environment variables @@ -78,31 +88,24 @@ def enable_gpu(self) -> ConfigSet: "order": "C", "write_empty_chunks": False, "v2_default_compressor": { - "numeric": {"id": "zstd", "level": 0, "checksum": False}, - "string": {"id": "zstd", "level": 0, "checksum": False}, - "bytes": {"id": "zstd", "level": 0, "checksum": False}, + "default": {"id": "zstd", "level": 0, "checksum": False}, + "variable-length-string": {"id": "zstd", "level": 0, "checksum": False}, }, "v2_default_filters": { - "numeric": None, - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], - "raw": None, + "default": None, + "variable-length-string": [{"id": "vlen-utf8"}], }, - "v3_default_filters": {"numeric": [], "string": [], "bytes": []}, + "v3_default_filters": {"default": [], "variable-length-string": []}, "v3_default_serializer": { - "numeric": {"name": "bytes", "configuration": {"endian": "little"}}, - "string": {"name": "vlen-utf8"}, - "bytes": {"name": "vlen-bytes"}, + "default": {"name": "bytes", "configuration": {"endian": "little"}}, + "variable-length-string": {"name": "vlen-utf8"}, }, "v3_default_compressors": { - "numeric": [ + "default": [ {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, ], - "string": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], - "bytes": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, + "variable-length-string": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}} ], }, }, @@ -137,3 +140,17 @@ def parse_indexing_order(data: Any) -> Literal["C", "F"]: return cast("Literal['C', 'F']", data) msg = f"Expected one of ('C', 'F'), got {data} instead." raise ValueError(msg) + + +def categorize_data_type(dtype: ZDType[Any, Any]) -> DTypeCategory: + """ + Classify a ZDType. The return value is a string which belongs to the type ``DTypeCategory``. + + This is used by the config system to determine how to encode arrays with the associated data type + when the user has not specified a particular serialization scheme. + """ + from zarr.core.dtype import VariableLengthUTF8 + + if isinstance(dtype, VariableLengthUTF8): + return "variable-length-string" + return "default" diff --git a/src/zarr/core/dtype/__init__.py b/src/zarr/core/dtype/__init__.py new file mode 100644 index 0000000000..735690d4bc --- /dev/null +++ b/src/zarr/core/dtype/__init__.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, TypeAlias + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeJSON, +) +from zarr.core.dtype.npy.bool import Bool +from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes +from zarr.core.dtype.npy.complex import Complex64, Complex128 +from zarr.core.dtype.npy.float import Float16, Float32, Float64 +from zarr.core.dtype.npy.int import Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 +from zarr.core.dtype.npy.structured import ( + Structured, +) +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 + +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + +from collections.abc import Mapping + +import numpy as np +import numpy.typing as npt + +from zarr.core.common import JSON +from zarr.core.dtype.npy.string import ( + FixedLengthUTF32, + VariableLengthUTF8, +) +from zarr.core.dtype.registry import DataTypeRegistry +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +__all__ = [ + "Bool", + "Complex64", + "Complex128", + "DataTypeRegistry", + "DataTypeValidationError", + "DateTime64", + "FixedLengthUTF32", + "Float16", + "Float32", + "Float64", + "Int8", + "Int16", + "Int32", + "Int64", + "NullTerminatedBytes", + "RawBytes", + "Structured", + "TBaseDType", + "TBaseScalar", + "TimeDelta64", + "TimeDelta64", + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "VariableLengthUTF8", + "ZDType", + "data_type_registry", + "parse_data_type", +] + +data_type_registry = DataTypeRegistry() + +IntegerDType = Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 +INTEGER_DTYPE: Final = Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 + +FloatDType = Float16 | Float32 | Float64 +FLOAT_DTYPE: Final = Float16, Float32, Float64 + +ComplexFloatDType = Complex64 | Complex128 +COMPLEX_FLOAT_DTYPE: Final = Complex64, Complex128 + +StringDType = FixedLengthUTF32 | VariableLengthUTF8 +STRING_DTYPE: Final = FixedLengthUTF32, VariableLengthUTF8 + +TimeDType = DateTime64 | TimeDelta64 +TIME_DTYPE: Final = DateTime64, TimeDelta64 + +BytesDType = RawBytes | NullTerminatedBytes | VariableLengthBytes +BYTES_DTYPE: Final = RawBytes, NullTerminatedBytes, VariableLengthBytes + +AnyDType = ( + Bool + | IntegerDType + | FloatDType + | ComplexFloatDType + | StringDType + | BytesDType + | Structured + | TimeDType + | VariableLengthBytes +) +# mypy has trouble inferring the type of variablelengthstring dtype, because its class definition +# depends on the installed numpy version. That's why the type: ignore statement is needed here. +ANY_DTYPE: Final = ( + Bool, + *INTEGER_DTYPE, + *FLOAT_DTYPE, + *COMPLEX_FLOAT_DTYPE, + *STRING_DTYPE, + *BYTES_DTYPE, + Structured, + *TIME_DTYPE, + VariableLengthBytes, +) + +# This type models inputs that can be coerced to a ZDType +ZDTypeLike: TypeAlias = npt.DTypeLike | ZDType[TBaseDType, TBaseScalar] | Mapping[str, JSON] | str + +for dtype in ANY_DTYPE: + # mypy does not know that all the elements of ANY_DTYPE are subclasses of ZDType + data_type_registry.register(dtype._zarr_v3_name, dtype) # type: ignore[arg-type] + + +# TODO: find a better name for this function +def get_data_type_from_native_dtype(dtype: npt.DTypeLike) -> ZDType[TBaseDType, TBaseScalar]: + """ + Get a data type wrapper (an instance of ``ZDType``) from a native data type, e.g. a numpy dtype. + """ + if not isinstance(dtype, np.dtype): + na_dtype: np.dtype[np.generic] + if isinstance(dtype, list): + # this is a valid _VoidDTypeLike check + na_dtype = np.dtype([tuple(d) for d in dtype]) + else: + na_dtype = np.dtype(dtype) + else: + na_dtype = dtype + return data_type_registry.match_dtype(dtype=na_dtype) + + +def get_data_type_from_json( + dtype_spec: DTypeJSON, *, zarr_format: ZarrFormat +) -> ZDType[TBaseDType, TBaseScalar]: + """ + Given a JSON representation of a data type and a Zarr format version, + attempt to create a ZDType instance from the registered ZDType classes. + """ + return data_type_registry.match_json(dtype_spec, zarr_format=zarr_format) + + +def parse_data_type( + dtype_spec: ZDTypeLike, + *, + zarr_format: ZarrFormat, +) -> ZDType[TBaseDType, TBaseScalar]: + """ + Interpret the input as a ZDType instance. + """ + if isinstance(dtype_spec, ZDType): + return dtype_spec + # dict and zarr_format 3 means that we have a JSON object representation of the dtype + if zarr_format == 3 and isinstance(dtype_spec, Mapping): + return get_data_type_from_json(dtype_spec, zarr_format=3) + # otherwise, we have either a numpy dtype string, or a zarr v3 dtype string, and in either case + # we can create a numpy dtype from it, and do the dtype inference from that + return get_data_type_from_native_dtype(dtype_spec) # type: ignore[arg-type] diff --git a/src/zarr/core/dtype/common.py b/src/zarr/core/dtype/common.py new file mode 100644 index 0000000000..6f61b6775e --- /dev/null +++ b/src/zarr/core/dtype/common.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import warnings +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import ( + ClassVar, + Final, + Generic, + Literal, + TypedDict, + TypeGuard, + TypeVar, +) + +from zarr.core.common import NamedConfig + +EndiannessStr = Literal["little", "big"] +ENDIANNESS_STR: Final = "little", "big" + +SpecialFloatStrings = Literal["NaN", "Infinity", "-Infinity"] +SPECIAL_FLOAT_STRINGS: Final = ("NaN", "Infinity", "-Infinity") + +JSONFloatV2 = float | SpecialFloatStrings +JSONFloatV3 = float | SpecialFloatStrings | str + +ObjectCodecID = Literal["vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2"] +# These are the ids of the known object codecs for zarr v2. +OBJECT_CODEC_IDS: Final = ("vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2") + +# This is a wider type than our standard JSON type because we need +# to work with typeddict objects which are assignable to Mapping[str, object] +DTypeJSON = str | int | float | Sequence["DTypeJSON"] | None | Mapping[str, object] + +# The DTypeJSON_V2 type exists because ZDType.from_json takes a single argument, which must contain +# all the information necessary to decode the data type. Zarr v2 supports multiple distinct +# data types that all used the "|O" data type identifier. These data types can only be +# discriminated on the basis of their "object codec", i.e. a special data type specific +# compressor or filter. So to figure out what data type a zarr v2 array has, we need the +# data type identifier from metadata, as well as an object codec id if the data type identifier +# is "|O". +# So we will pack the name of the dtype alongside the name of the object codec id, if applicable, +# in a single dict, and pass that to the data type inference logic. +# These type variables have a very wide bound because the individual zdtype +# classes can perform a very specific type check. + +# This is the JSON representation of a structured dtype in zarr v2 +StructuredName_V2 = Sequence["str | StructuredName_V2"] + +# This models the type of the name a dtype might have in zarr v2 array metadata +DTypeName_V2 = StructuredName_V2 | str + +TDTypeNameV2_co = TypeVar("TDTypeNameV2_co", bound=DTypeName_V2, covariant=True) +TObjectCodecID_co = TypeVar("TObjectCodecID_co", bound=None | str, covariant=True) + + +class DTypeConfig_V2(TypedDict, Generic[TDTypeNameV2_co, TObjectCodecID_co]): + name: TDTypeNameV2_co + object_codec_id: TObjectCodecID_co + + +DTypeSpec_V2 = DTypeConfig_V2[DTypeName_V2, None | str] + + +def check_structured_dtype_v2_inner(data: object) -> TypeGuard[StructuredName_V2]: + """ + A type guard for the inner elements of a structured dtype. This is a recursive check because + the type is itself recursive. + + This check ensures that all the elements are 2-element sequences beginning with a string + and ending with either another string or another 2-element sequence beginning with a string and + ending with another instance of that type. + """ + if isinstance(data, (str, Mapping)): + return False + if not isinstance(data, Sequence): + return False + if len(data) != 2: + return False + if not (isinstance(data[0], str)): + return False + if isinstance(data[-1], str): + return True + elif isinstance(data[-1], Sequence): + return check_structured_dtype_v2_inner(data[-1]) + return False + + +def check_structured_dtype_name_v2(data: Sequence[object]) -> TypeGuard[StructuredName_V2]: + return all(check_structured_dtype_v2_inner(d) for d in data) + + +def check_dtype_name_v2(data: object) -> TypeGuard[DTypeName_V2]: + """ + Type guard for narrowing the type of a python object to an valid zarr v2 dtype name. + """ + if isinstance(data, str): + return True + elif isinstance(data, Sequence): + return check_structured_dtype_name_v2(data) + return False + + +def check_dtype_spec_v2(data: object) -> TypeGuard[DTypeSpec_V2]: + """ + Type guard for narrowing a python object to an instance of DTypeSpec_V2 + """ + if not isinstance(data, Mapping): + return False + if set(data.keys()) != {"name", "object_codec_id"}: + return False + if not check_dtype_name_v2(data["name"]): + return False + return isinstance(data["object_codec_id"], str | None) + + +# By comparison, The JSON representation of a dtype in zarr v3 is much simpler. +# It's either a string, or a structured dict +DTypeSpec_V3 = str | NamedConfig[str, Mapping[str, object]] + + +def check_dtype_spec_v3(data: object) -> TypeGuard[DTypeSpec_V3]: + """ + Type guard for narrowing the type of a python object to an instance of + DTypeSpec_V3, i.e either a string or a dict with a "name" field that's a string and a + "configuration" field that's a mapping with string keys. + """ + if isinstance(data, str) or ( # noqa: SIM103 + isinstance(data, Mapping) + and set(data.keys()) == {"name", "configuration"} + and isinstance(data["configuration"], Mapping) + and all(isinstance(k, str) for k in data["configuration"]) + ): + return True + return False + + +def unpack_dtype_json(data: DTypeSpec_V2 | DTypeSpec_V3) -> DTypeJSON: + """ + Return the array metadata form of the dtype JSON representation. For the Zarr V3 form of dtype + metadata, this is a no-op. For the Zarr V2 form of dtype metadata, this unpacks the dtype name. + """ + if isinstance(data, Mapping) and set(data.keys()) == {"name", "object_codec_id"}: + return data["name"] + return data + + +class DataTypeValidationError(ValueError): ... + + +class ScalarTypeValidationError(ValueError): ... + + +@dataclass(frozen=True) +class HasLength: + """ + A mix-in class for data types with a length attribute, such as fixed-size collections + of unicode strings, or bytes. + """ + + length: int + + +@dataclass(frozen=True) +class HasEndianness: + """ + A mix-in class for data types with an endianness attribute + """ + + endianness: EndiannessStr = "little" + + +@dataclass(frozen=True) +class HasItemSize: + """ + A mix-in class for data types with an item size attribute. + This mix-in bears a property ``item_size``, which denotes the size of each element of the data + type, in bytes. + """ + + @property + def item_size(self) -> int: + raise NotImplementedError + + +@dataclass(frozen=True) +class HasObjectCodec: + """ + A mix-in class for data types that require an object codec id. + This class bears the property ``object_codec_id``, which is the string name of an object + codec that is required to encode and decode the data type. + + In zarr-python 2.x certain data types like variable-length strings or variable-length arrays + used the catch-all numpy "object" data type for their in-memory representation. But these data + types cannot be stored as numpy object data types, because the object data type does not define + a fixed memory layout. So these data types required a special codec, called an "object codec", + that effectively defined a compact representation for the data type, which was used to encode + and decode the data type. + + Zarr-python 2.x would not allow the creation of arrays with the "object" data type if an object + codec was not specified, and thus the name of the object codec is effectively part of the data + type model. + """ + + object_codec_id: ClassVar[str] + + +class UnstableSpecificationWarning(FutureWarning): ... + + +def v3_unstable_dtype_warning(dtype: object) -> None: + """ + Emit this warning when a data type does not have a stable zarr v3 spec + """ + msg = ( + f"The data type ({dtype}) does not have a Zarr V3 specification. " + "That means that the representation of array saved with this data type may change without " + "warning in a future version of Zarr Python. " + "Arrays stored with this data type may be unreadable by other Zarr libraries. " + "Use this data type at your own risk! " + "Check https://github.com/zarr-developers/zarr-extensions/tree/main/data-types for the " + "status of data type specifications for Zarr V3." + ) + warnings.warn(msg, category=UnstableSpecificationWarning, stacklevel=2) diff --git a/src/zarr/core/dtype/npy/__init__.py b/src/zarr/core/dtype/npy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/zarr/core/dtype/npy/bool.py b/src/zarr/core/dtype/npy/bool.py new file mode 100644 index 0000000000..d8d52468bf --- /dev/null +++ b/src/zarr/core/dtype/npy/bool.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True, kw_only=True, slots=True) +class Bool(ZDType[np.dtypes.BoolDType, np.bool_], HasItemSize): + """ + Wrapper for numpy boolean dtype. + + Attributes + ---------- + name : str + The name of the dtype. + dtype_cls : ClassVar[type[np.dtypes.BoolDType]] + The numpy dtype class. + """ + + _zarr_v3_name: ClassVar[Literal["bool"]] = "bool" + _zarr_v2_name: ClassVar[Literal["|b1"]] = "|b1" + dtype_cls = np.dtypes.BoolDType + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Bool from a np.dtype('bool') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.BoolDType: + """ + Create a NumPy boolean dtype instance from this ZDType + """ + return self.dtype_cls() + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|b1"], None]]: + """ + Check that the input is a valid JSON representation of a Bool. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == cls._zarr_v2_name + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["bool"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_name!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|b1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["bool"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|b1"], None] | Literal["bool"]: + if zarr_format == 2: + return {"name": self._zarr_v2_name, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> bool: + # Anything can become a bool + return True + + def cast_scalar(self, data: object) -> np.bool_: + if self._check_scalar(data): + return np.bool_(data) + msg = f"Cannot convert object with type {type(data)} to a numpy boolean." + raise TypeError(msg) + + def default_scalar(self) -> np.bool_: + """ + Get the default value for the boolean dtype. + + Returns + ------- + np.bool_ + The default value. + """ + return np.False_ + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> bool: + """ + Convert a scalar to a python bool. + + Parameters + ---------- + data : object + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + bool + The JSON-serializable format. + """ + return bool(data) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bool_: + """ + Read a JSON-serializable value as a numpy boolean scalar. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + np.bool_ + The numpy boolean scalar. + """ + if self._check_scalar(data): + return np.bool_(data) + raise TypeError(f"Invalid type: {data}. Expected a boolean.") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 diff --git a/src/zarr/core/dtype/npy/bytes.py b/src/zarr/core/dtype/npy/bytes.py new file mode 100644 index 0000000000..e363c75053 --- /dev/null +++ b/src/zarr/core/dtype/npy/bytes.py @@ -0,0 +1,369 @@ +from __future__ import annotations + +import base64 +import re +from dataclasses import dataclass +from typing import Any, ClassVar, Literal, Self, TypedDict, TypeGuard, cast, overload + +import numpy as np + +from zarr.core.common import JSON, NamedConfig, ZarrFormat +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasItemSize, + HasLength, + HasObjectCodec, + check_dtype_spec_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import check_json_str +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +BytesLike = np.bytes_ | str | bytes | int + + +class FixedLengthBytesConfig(TypedDict): + length_bytes: int + + +NullTerminatedBytesJSONV3 = NamedConfig[Literal["null_terminated_bytes"], FixedLengthBytesConfig] +RawBytesJSONV3 = NamedConfig[Literal["raw_bytes"], FixedLengthBytesConfig] + + +@dataclass(frozen=True, kw_only=True) +class NullTerminatedBytes(ZDType[np.dtypes.BytesDType[int], np.bytes_], HasLength, HasItemSize): + dtype_cls = np.dtypes.BytesDType + _zarr_v3_name: ClassVar[Literal["null_terminated_bytes"]] = "null_terminated_bytes" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(length=dtype.itemsize) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.BytesDType[int]: + return self.dtype_cls(self.length) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid representation of a numpy S dtype. We expect + something like ``{"name": "|S10", "object_codec_id": None}`` + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^\|S\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[NullTerminatedBytesJSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and "length_bytes" in data["configuration"] + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls(length=int(name[2:])) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|S1', '|S2', etc" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"]) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> NullTerminatedBytesJSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | NullTerminatedBytesJSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return { + "name": self._zarr_v3_name, + "configuration": {"length_bytes": self.length}, + } + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: + # this is generous for backwards compatibility + return isinstance(data, BytesLike) + + def _cast_scalar_unchecked(self, data: BytesLike) -> np.bytes_: + # We explicitly truncate the result because of the following numpy behavior: + # >>> x = np.dtype('S3').type('hello world') + # >>> x + # np.bytes_(b'hello world') + # >>> x.dtype + # dtype('S11') + + if isinstance(data, int): + return self.to_native_dtype().type(str(data)[: self.length]) + else: + return self.to_native_dtype().type(data[: self.length]) + + def cast_scalar(self, data: object) -> np.bytes_: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy bytes scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.bytes_: + return np.bytes_(b"") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + as_bytes = self.cast_scalar(data) + return base64.standard_b64encode(as_bytes).decode("ascii") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bytes_: + if check_json_str(data): + return self.to_native_dtype().type(base64.standard_b64decode(data.encode("ascii"))) + raise TypeError( + f"Invalid type: {data}. Expected a base64-encoded string." + ) # pragma: no cover + + @property + def item_size(self) -> int: + return self.length + + +@dataclass(frozen=True, kw_only=True) +class RawBytes(ZDType[np.dtypes.VoidDType[int], np.void], HasLength, HasItemSize): + # np.dtypes.VoidDType is specified in an odd way in numpy + # it cannot be used to create instances of the dtype + # so we have to tell mypy to ignore this here + dtype_cls = np.dtypes.VoidDType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["raw_bytes"]] = "raw_bytes" + + @classmethod + def _check_native_dtype( + cls: type[Self], dtype: TBaseDType + ) -> TypeGuard[np.dtypes.VoidDType[Any]]: + """ + Numpy void dtype comes in two forms: + * If the ``fields`` attribute is ``None``, then the dtype represents N raw bytes. + * If the ``fields`` attribute is not ``None``, then the dtype represents a structured dtype, + + In this check we ensure that ``fields`` is ``None``. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return cls.dtype_cls is type(dtype) and dtype.fields is None # type: ignore[has-type] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(length=dtype.itemsize) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + + def to_native_dtype(self) -> np.dtypes.VoidDType[int]: + # Numpy does not allow creating a void type + # by invoking np.dtypes.VoidDType directly + return cast("np.dtypes.VoidDType[int]", np.dtype(f"V{self.length}")) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid representation of a numpy S dtype. We expect + something like ``{"name": "|V10", "object_codec_id": None}`` + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^\|V\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[RawBytesJSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"length_bytes"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls(length=int(name[2:])) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|V1', '|V2', etc" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"]) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> RawBytesJSONV3: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | RawBytesJSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return {"name": self._zarr_v3_name, "configuration": {"length_bytes": self.length}} + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> bool: + return isinstance(data, np.bytes_ | str | bytes | np.void) + + def _cast_scalar_unchecked(self, data: object) -> np.void: + native_dtype = self.to_native_dtype() + # Without the second argument, numpy will return a void scalar for dtype V1. + # The second argument ensures that, if native_dtype is something like V10, + # the result will actually be a V10 scalar. + return native_dtype.type(data, native_dtype) + + def cast_scalar(self, data: object) -> np.void: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy void scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.void: + return self.to_native_dtype().type(("\x00" * self.length).encode("ascii")) + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return base64.standard_b64encode(self.cast_scalar(data).tobytes()).decode("ascii") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: + if check_json_str(data): + return self.to_native_dtype().type(base64.standard_b64decode(data)) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + @property + def item_size(self) -> int: + return self.length + + +@dataclass(frozen=True, kw_only=True) +class VariableLengthBytes(ZDType[np.dtypes.ObjectDType, bytes], HasObjectCodec): + dtype_cls = np.dtypes.ObjectDType + _zarr_v3_name: ClassVar[Literal["variable_length_bytes"]] = "variable_length_bytes" + object_codec_id: ClassVar[Literal["vlen-bytes"]] = "vlen-bytes" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.ObjectDType: + return self.dtype_cls() + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]]]: + """ + Check that the input is a valid JSON representation of a numpy O dtype, and that the + object codec id is appropriate for variable-length UTF-8 strings. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == "|O" + and data["object_codec_id"] == cls.object_codec_id + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_bytes"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O' and an object_codec_id of {cls.object_codec_id}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json( + self, zarr_format: Literal[2] + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_bytes"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]] | Literal["variable_length_bytes"]: + if zarr_format == 2: + return {"name": "|O", "object_codec_id": self.object_codec_id} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def default_scalar(self) -> bytes: + return b"" + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return base64.standard_b64encode(data).decode("ascii") # type: ignore[arg-type] + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> bytes: + if check_json_str(data): + return base64.standard_b64decode(data.encode("ascii")) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: + return isinstance(data, BytesLike) + + def _cast_scalar_unchecked(self, data: BytesLike) -> bytes: + if isinstance(data, str): + return bytes(data, encoding="utf-8") + return bytes(data) + + def cast_scalar(self, data: object) -> bytes: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to bytes." + raise TypeError(msg) diff --git a/src/zarr/core/dtype/npy/common.py b/src/zarr/core/dtype/npy/common.py new file mode 100644 index 0000000000..264561f25c --- /dev/null +++ b/src/zarr/core/dtype/npy/common.py @@ -0,0 +1,503 @@ +from __future__ import annotations + +import base64 +import struct +import sys +from collections.abc import Sequence +from typing import ( + TYPE_CHECKING, + Any, + Final, + Literal, + SupportsComplex, + SupportsFloat, + SupportsIndex, + SupportsInt, + TypeGuard, + TypeVar, +) + +import numpy as np + +from zarr.core.dtype.common import ( + ENDIANNESS_STR, + SPECIAL_FLOAT_STRINGS, + EndiannessStr, + JSONFloatV2, + JSONFloatV3, +) + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +IntLike = SupportsInt | SupportsIndex | bytes | str +FloatLike = SupportsIndex | SupportsFloat | bytes | str +ComplexLike = SupportsFloat | SupportsIndex | SupportsComplex | bytes | str | None +DateTimeUnit = Literal[ + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" +] +DATETIME_UNIT: Final = ( + "Y", + "M", + "W", + "D", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", + "generic", +) + +NumpyEndiannessStr = Literal[">", "<", "="] +NUMPY_ENDIANNESS_STR: Final = ">", "<", "=" + +TFloatDType_co = TypeVar( + "TFloatDType_co", + bound=np.dtypes.Float16DType | np.dtypes.Float32DType | np.dtypes.Float64DType, + covariant=True, +) +TFloatScalar_co = TypeVar( + "TFloatScalar_co", bound=np.float16 | np.float32 | np.float64, covariant=True +) + +TComplexDType_co = TypeVar( + "TComplexDType_co", bound=np.dtypes.Complex64DType | np.dtypes.Complex128DType, covariant=True +) +TComplexScalar_co = TypeVar("TComplexScalar_co", bound=np.complex64 | np.complex128, covariant=True) + + +def endianness_from_numpy_str(endianness: NumpyEndiannessStr) -> EndiannessStr: + """ + Convert a numpy endianness string literal to a human-readable literal value. + + Parameters + ---------- + endianness : Literal[">", "<", "="] + The numpy string representation of the endianness. + + Returns + ------- + Endianness + The human-readable representation of the endianness. + + Raises + ------ + ValueError + If the endianness is invalid. + """ + match endianness: + case "=": + # Use the local system endianness + return sys.byteorder + case "<": + return "little" + case ">": + return "big" + raise ValueError(f"Invalid endianness: {endianness!r}. Expected one of {NUMPY_ENDIANNESS_STR}") + + +def endianness_to_numpy_str(endianness: EndiannessStr) -> NumpyEndiannessStr: + """ + Convert an endianness literal to its numpy string representation. + + Parameters + ---------- + endianness : Endianness + The endianness to convert. + + Returns + ------- + Literal[">", "<"] + The numpy string representation of the endianness. + + Raises + ------ + ValueError + If the endianness is invalid. + """ + match endianness: + case "little": + return "<" + case "big": + return ">" + raise ValueError( + f"Invalid endianness: {endianness!r}. Expected one of {ENDIANNESS_STR} or None" + ) + + +def get_endianness_from_numpy_dtype(dtype: np.dtype[np.generic]) -> EndiannessStr: + """ + Gets the endianness from a numpy dtype that has an endianness. This function will + raise a ValueError if the numpy data type does not have a concrete endianness. + """ + endianness = dtype.byteorder + if dtype.byteorder in NUMPY_ENDIANNESS_STR: + return endianness_from_numpy_str(endianness) # type: ignore [arg-type] + raise ValueError(f"The dtype {dtype} has an unsupported endianness: {endianness}") + + +def float_from_json_v2(data: JSONFloatV2) -> float: + """ + Convert a JSON float to a float (Zarr v2). + + Parameters + ---------- + data : JSONFloat + The JSON float to convert. + + Returns + ------- + float + The float value. + """ + match data: + case "NaN": + return float("nan") + case "Infinity": + return float("inf") + case "-Infinity": + return float("-inf") + case _: + return float(data) + + +def float_from_json_v3(data: JSONFloatV3) -> float: + """ + Convert a JSON float to a float (v3). + + Parameters + ---------- + data : JSONFloat + The JSON float to convert. + + Returns + ------- + float + The float value. + + Notes + ----- + Zarr V3 allows floats to be stored as hex strings. To quote the spec: + "...for float32, "NaN" is equivalent to "0x7fc00000". + This representation is the only way to specify a NaN value other than the specific NaN value + denoted by "NaN"." + """ + + if isinstance(data, str): + if data in SPECIAL_FLOAT_STRINGS: + return float_from_json_v2(data) # type: ignore[arg-type] + if not data.startswith("0x"): + msg = ( + f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" + " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." + ) + raise ValueError(msg) + if len(data[2:]) == 4: + dtype_code = ">e" + elif len(data[2:]) == 8: + dtype_code = ">f" + elif len(data[2:]) == 16: + dtype_code = ">d" + else: + msg = ( + f"Invalid hexadecimal float value: {data!r}. " + "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" + ) + raise ValueError(msg) + return float(struct.unpack(dtype_code, bytes.fromhex(data[2:]))[0]) + return float_from_json_v2(data) + + +def bytes_from_json(data: str, *, zarr_format: ZarrFormat) -> bytes: + """ + Convert a JSON string to bytes + + Parameters + ---------- + data : str + The JSON string to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + bytes + The bytes. + """ + if zarr_format == 2: + return base64.b64decode(data.encode("ascii")) + # TODO: differentiate these as needed. This is a spec question. + if zarr_format == 3: + return base64.b64decode(data.encode("ascii")) + raise ValueError(f"Invalid zarr format: {zarr_format}. Expected 2 or 3.") # pragma: no cover + + +def bytes_to_json(data: bytes, zarr_format: ZarrFormat) -> str: + """ + Convert bytes to JSON. + + Parameters + ---------- + data : bytes + The bytes to store. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The bytes encoded as ascii using the base64 alphabet. + """ + # TODO: decide if we are going to make this implementation zarr format-specific + return base64.b64encode(data).decode("ascii") + + +def float_to_json_v2(data: float | np.floating[Any]) -> JSONFloatV2: + """ + Convert a float to JSON (v2). + + Parameters + ---------- + data : float or np.floating + The float value to convert. + + Returns + ------- + JSONFloat + The JSON representation of the float. + """ + if np.isnan(data): + return "NaN" + elif np.isinf(data): + return "Infinity" if data > 0 else "-Infinity" + return float(data) + + +def float_to_json_v3(data: float | np.floating[Any]) -> JSONFloatV3: + """ + Convert a float to JSON (v3). + + Parameters + ---------- + data : float or np.floating + The float value to convert. + + Returns + ------- + JSONFloat + The JSON representation of the float. + """ + # v3 can in principle handle distinct NaN values, but numpy does not represent these explicitly + # so we just reuse the v2 routine here + return float_to_json_v2(data) + + +def complex_float_to_json_v3( + data: complex | np.complexfloating[Any, Any], +) -> tuple[JSONFloatV3, JSONFloatV3]: + """ + Convert a complex number to JSON as defined by the Zarr V3 spec. + + Parameters + ---------- + data : complex or np.complexfloating + The complex value to convert. + + Returns + ------- + tuple[JSONFloat, JSONFloat] + The JSON representation of the complex number. + """ + return float_to_json_v3(data.real), float_to_json_v3(data.imag) + + +def complex_float_to_json_v2( + data: complex | np.complexfloating[Any, Any], +) -> tuple[JSONFloatV2, JSONFloatV2]: + """ + Convert a complex number to JSON as defined by the Zarr V2 spec. + + Parameters + ---------- + data : complex | np.complexfloating + The complex value to convert. + + Returns + ------- + tuple[JSONFloat, JSONFloat] + The JSON representation of the complex number. + """ + return float_to_json_v2(data.real), float_to_json_v2(data.imag) + + +def complex_float_from_json_v2(data: tuple[JSONFloatV2, JSONFloatV2]) -> complex: + """ + Convert a JSON complex float to a complex number (v2). + + Parameters + ---------- + data : tuple[JSONFloat, JSONFloat] + The JSON complex float to convert. + + Returns + ------- + np.complexfloating + The complex number. + """ + return complex(float_from_json_v2(data[0]), float_from_json_v2(data[1])) + + +def complex_float_from_json_v3(data: tuple[JSONFloatV3, JSONFloatV3]) -> complex: + """ + Convert a JSON complex float to a complex number (v3). + + Parameters + ---------- + data : tuple[JSONFloat, JSONFloat] + The JSON complex float to convert. + + Returns + ------- + np.complexfloating + The complex number. + """ + return complex(float_from_json_v3(data[0]), float_from_json_v3(data[1])) + + +def check_json_float_v2(data: JSON) -> TypeGuard[JSONFloatV2]: + """ + Check if a JSON value represents a float (v2). + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a float, False otherwise. + """ + if data == "NaN" or data == "Infinity" or data == "-Infinity": + return True + return isinstance(data, float | int) + + +def check_json_float_v3(data: JSON) -> TypeGuard[JSONFloatV3]: + """ + Check if a JSON value represents a float (v3). + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a float, False otherwise. + """ + return check_json_float_v2(data) or (isinstance(data, str) and data.startswith("0x")) + + +def check_json_complex_float_v2(data: JSON) -> TypeGuard[tuple[JSONFloatV2, JSONFloatV2]]: + """ + Check if a JSON value represents a complex float, as per the behavior of zarr-python 2.x + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a complex float, False otherwise. + """ + return ( + not isinstance(data, str) + and isinstance(data, Sequence) + and len(data) == 2 + and check_json_float_v2(data[0]) + and check_json_float_v2(data[1]) + ) + + +def check_json_complex_float_v3(data: JSON) -> TypeGuard[tuple[JSONFloatV3, JSONFloatV3]]: + """ + Check if a JSON value represents a complex float, as per the zarr v3 spec + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a complex float, False otherwise. + """ + return ( + not isinstance(data, str) + and isinstance(data, Sequence) + and len(data) == 2 + and check_json_float_v3(data[0]) + and check_json_float_v3(data[1]) + ) + + +def check_json_int(data: JSON) -> TypeGuard[int]: + """ + Check if a JSON value is an integer. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is an integer, False otherwise. + """ + return bool(isinstance(data, int)) + + +def check_json_str(data: JSON) -> TypeGuard[str]: + """ + Check if a JSON value is a string. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a string, False otherwise. + """ + return bool(isinstance(data, str)) + + +def check_json_bool(data: JSON) -> TypeGuard[bool]: + """ + Check if a JSON value is a boolean. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a boolean, False otherwise. + """ + return isinstance(data, bool) diff --git a/src/zarr/core/dtype/npy/complex.py b/src/zarr/core/dtype/npy/complex.py new file mode 100644 index 0000000000..38e506f1bc --- /dev/null +++ b/src/zarr/core/dtype/npy/complex.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + TypeGuard, + overload, +) + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + ComplexLike, + TComplexDType_co, + TComplexScalar_co, + check_json_complex_float_v2, + check_json_complex_float_v3, + complex_float_from_json_v2, + complex_float_from_json_v3, + complex_float_to_json_v2, + complex_float_to_json_v3, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True) +class BaseComplex(ZDType[TComplexDType_co, TComplexScalar_co], HasEndianness, HasItemSize): + # This attribute holds the possible zarr v2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> TComplexDType_co: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> str: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[ComplexLike]: + return isinstance(data, ComplexLike) + + def _cast_scalar_unchecked(self, data: ComplexLike) -> TComplexScalar_co: + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TComplexScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy float scalar." + raise TypeError(msg) + + def default_scalar(self) -> TComplexScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TComplexScalar_co: + """ + Read a JSON-serializable value as a numpy float. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy float. + """ + if zarr_format == 2: + if check_json_complex_float_v2(data): + return self._cast_scalar_unchecked(complex_float_from_json_v2(data)) + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + elif zarr_format == 3: + if check_json_complex_float_v3(data): + return self._cast_scalar_unchecked(complex_float_from_json_v3(data)) + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: + """ + Convert an object to a JSON-serializable float. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + JSON + The JSON-serializable form of the complex number, which is a list of two floats, + each of which is encoding according to a zarr-format-specific encoding. + """ + if zarr_format == 2: + return complex_float_to_json_v2(self.cast_scalar(data)) + elif zarr_format == 3: + return complex_float_to_json_v3(self.cast_scalar(data)) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True) +class Complex64(BaseComplex[np.dtypes.Complex64DType, np.complex64]): + dtype_cls = np.dtypes.Complex64DType + _zarr_v3_name: ClassVar[Literal["complex64"]] = "complex64" + _zarr_v2_names: ClassVar[tuple[str, ...]] = (">c8", " int: + return 8 + + +@dataclass(frozen=True, kw_only=True) +class Complex128(BaseComplex[np.dtypes.Complex128DType, np.complex128], HasEndianness): + dtype_cls = np.dtypes.Complex128DType + _zarr_v3_name: ClassVar[Literal["complex128"]] = "complex128" + _zarr_v2_names: ClassVar[tuple[str, ...]] = (">c16", " int: + return 16 diff --git a/src/zarr/core/dtype/npy/float.py b/src/zarr/core/dtype/npy/float.py new file mode 100644 index 0000000000..7b7243993f --- /dev/null +++ b/src/zarr/core/dtype/npy/float.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + ScalarTypeValidationError, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + FloatLike, + TFloatDType_co, + TFloatScalar_co, + check_json_float_v2, + check_json_float_v3, + endianness_to_numpy_str, + float_from_json_v2, + float_from_json_v3, + float_to_json_v2, + float_to_json_v3, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True) +class BaseFloat(ZDType[TFloatDType_co, TFloatScalar_co], HasEndianness, HasItemSize): + # This attribute holds the possible zarr v2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> TFloatDType_co: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> str: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[FloatLike]: + return isinstance(data, FloatLike) + + def _cast_scalar_unchecked(self, data: FloatLike) -> TFloatScalar_co: + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TFloatScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy float scalar." + raise ScalarTypeValidationError(msg) + + def default_scalar(self) -> TFloatScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TFloatScalar_co: + """ + Read a JSON-serializable value as a numpy float. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy float. + """ + if zarr_format == 2: + if check_json_float_v2(data): + return self._cast_scalar_unchecked(float_from_json_v2(data)) + else: + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + elif zarr_format == 3: + if check_json_float_v3(data): + return self._cast_scalar_unchecked(float_from_json_v3(data)) + else: + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> float | str: + """ + Convert an object to a JSON-serializable float. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + JSON + The JSON-serializable form of the float, which is potentially a number or a string. + See the zarr specifications for details on the JSON encoding for floats. + """ + if zarr_format == 2: + return float_to_json_v2(self.cast_scalar(data)) + elif zarr_format == 3: + return float_to_json_v3(self.cast_scalar(data)) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True) +class Float16(BaseFloat[np.dtypes.Float16DType, np.float16]): + dtype_cls = np.dtypes.Float16DType + _zarr_v3_name = "float16" + _zarr_v2_names: ClassVar[tuple[Literal[">f2"], Literal["f2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class Float32(BaseFloat[np.dtypes.Float32DType, np.float32]): + dtype_cls = np.dtypes.Float32DType + _zarr_v3_name = "float32" + _zarr_v2_names: ClassVar[tuple[Literal[">f4"], Literal["f4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class Float64(BaseFloat[np.dtypes.Float64DType, np.float64]): + dtype_cls = np.dtypes.Float64DType + _zarr_v3_name = "float64" + _zarr_v2_names: ClassVar[tuple[Literal[">f8"], Literal["f8", " int: + return 8 diff --git a/src/zarr/core/dtype/npy/int.py b/src/zarr/core/dtype/npy/int.py new file mode 100644 index 0000000000..79d3ce2d47 --- /dev/null +++ b/src/zarr/core/dtype/npy/int.py @@ -0,0 +1,686 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + SupportsIndex, + SupportsInt, + TypeGuard, + TypeVar, + overload, +) + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + check_json_int, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +_NumpyIntDType = ( + np.dtypes.Int8DType + | np.dtypes.Int16DType + | np.dtypes.Int32DType + | np.dtypes.Int64DType + | np.dtypes.UInt8DType + | np.dtypes.UInt16DType + | np.dtypes.UInt32DType + | np.dtypes.UInt64DType +) +_NumpyIntScalar = ( + np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64 +) +TIntDType_co = TypeVar("TIntDType_co", bound=_NumpyIntDType, covariant=True) +TIntScalar_co = TypeVar("TIntScalar_co", bound=_NumpyIntScalar, covariant=True) +IntLike = SupportsInt | SupportsIndex | bytes | str + + +@dataclass(frozen=True) +class BaseInt(ZDType[TIntDType_co, TIntScalar_co], HasItemSize): + # This attribute holds the possible zarr V2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def _check_json_v2(cls, data: object) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: object) -> TypeGuard[str]: + """ + Check that a JSON value is consistent with the zarr v3 spec for this data type. + """ + return data == cls._zarr_v3_name + + def _check_scalar(self, data: object) -> TypeGuard[IntLike]: + """ + Check that a python object is IntLike + """ + return isinstance(data, IntLike) + + def _cast_scalar_unchecked(self, data: IntLike) -> TIntScalar_co: + """ + Create an integer without any type checking of the input. + """ + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TIntScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy integer." + raise TypeError(msg) + + def default_scalar(self) -> TIntScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TIntScalar_co: + """ + Read a JSON-serializable value as a numpy int scalar. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy scalar. + """ + if check_json_int(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected an integer.") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: + """ + Convert an object to JSON-serializable scalar. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + int + The JSON-serializable form of the scalar. + """ + return int(self.cast_scalar(data)) + + +@dataclass(frozen=True, kw_only=True) +class Int8(BaseInt[np.dtypes.Int8DType, np.int8]): + dtype_cls = np.dtypes.Int8DType + _zarr_v3_name: ClassVar[Literal["int8"]] = "int8" + _zarr_v2_names: ClassVar[tuple[Literal["|i1"]]] = ("|i1",) + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Int8 from a np.dtype('int8') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.Int8DType: + return self.dtype_cls() + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|i1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["int8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|i1"], None] | Literal["int8"]: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self._zarr_v2_names[0], "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 + + +@dataclass(frozen=True, kw_only=True) +class UInt8(BaseInt[np.dtypes.UInt8DType, np.uint8]): + dtype_cls = np.dtypes.UInt8DType + _zarr_v3_name: ClassVar[Literal["uint8"]] = "uint8" + _zarr_v2_names: ClassVar[tuple[Literal["|u1"]]] = ("|u1",) + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Bool from a np.dtype('uint8') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.UInt8DType: + return self.dtype_cls() + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|u1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["uint8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|u1"], None] | Literal["uint8"]: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self._zarr_v2_names[0], "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 + + +@dataclass(frozen=True, kw_only=True) +class Int16(BaseInt[np.dtypes.Int16DType, np.int16], HasEndianness): + dtype_cls = np.dtypes.Int16DType + _zarr_v3_name: ClassVar[Literal["int16"]] = "int16" + _zarr_v2_names: ClassVar[tuple[Literal[">i2"], Literal["i2", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int16DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names!r}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i2", " Literal["int16"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class UInt16(BaseInt[np.dtypes.UInt16DType, np.uint16], HasEndianness): + dtype_cls = np.dtypes.UInt16DType + _zarr_v3_name: ClassVar[Literal["uint16"]] = "uint16" + _zarr_v2_names: ClassVar[tuple[Literal[">u2"], Literal["u2", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.UInt16DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u2", " Literal["uint16"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class Int32(BaseInt[np.dtypes.Int32DType, np.int32], HasEndianness): + dtype_cls = np.dtypes.Int32DType + _zarr_v3_name: ClassVar[Literal["int32"]] = "int32" + _zarr_v2_names: ClassVar[tuple[Literal[">i4"], Literal["i4", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int32DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i4", " Literal["int32"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class UInt32(BaseInt[np.dtypes.UInt32DType, np.uint32], HasEndianness): + dtype_cls = np.dtypes.UInt32DType + _zarr_v3_name: ClassVar[Literal["uint32"]] = "uint32" + _zarr_v2_names: ClassVar[tuple[Literal[">u4"], Literal["u4", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.UInt32DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u4", " Literal["uint32"]: ... + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class Int64(BaseInt[np.dtypes.Int64DType, np.int64], HasEndianness): + dtype_cls = np.dtypes.Int64DType + _zarr_v3_name: ClassVar[Literal["int64"]] = "int64" + _zarr_v2_names: ClassVar[tuple[Literal[">i8"], Literal["i8", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int64DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i8", " Literal["int64"]: ... + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i8", " int: + return 8 + + +@dataclass(frozen=True, kw_only=True) +class UInt64(BaseInt[np.dtypes.UInt64DType, np.uint64], HasEndianness): + dtype_cls = np.dtypes.UInt64DType + _zarr_v3_name: ClassVar[Literal["uint64"]] = "uint64" + _zarr_v2_names: ClassVar[tuple[Literal[">u8"], Literal["u8", " np.dtypes.UInt64DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u8", " Literal["uint64"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u8", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + @property + def item_size(self) -> int: + return 8 diff --git a/src/zarr/core/dtype/npy/string.py b/src/zarr/core/dtype/npy/string.py new file mode 100644 index 0000000000..4a1114617a --- /dev/null +++ b/src/zarr/core/dtype/npy/string.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Protocol, + Self, + TypedDict, + TypeGuard, + overload, + runtime_checkable, +) + +import numpy as np + +from zarr.core.common import NamedConfig +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + HasLength, + HasObjectCodec, + check_dtype_spec_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import ( + check_json_str, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TDType_co, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType + +_NUMPY_SUPPORTS_VLEN_STRING = hasattr(np.dtypes, "StringDType") + + +@runtime_checkable +class SupportsStr(Protocol): + def __str__(self) -> str: ... + + +class LengthBytesConfig(TypedDict): + length_bytes: int + + +# TODO: Fix this terrible name +FixedLengthUTF32JSONV3 = NamedConfig[Literal["fixed_length_utf32"], LengthBytesConfig] + + +@dataclass(frozen=True, kw_only=True) +class FixedLengthUTF32( + ZDType[np.dtypes.StrDType[int], np.str_], HasEndianness, HasLength, HasItemSize +): + dtype_cls = np.dtypes.StrDType + _zarr_v3_name: ClassVar[Literal["fixed_length_utf32"]] = "fixed_length_utf32" + code_point_bytes: ClassVar[int] = 4 # utf32 is 4 bytes per code point + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + endianness = get_endianness_from_numpy_dtype(dtype) + return cls( + length=dtype.itemsize // (cls.code_point_bytes), + endianness=endianness, + ) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.StrDType[int]: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls(self.length).newbyteorder(byte_order) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of a numpy U dtype. + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^[><]U\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[FixedLengthUTF32JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and "configuration" in data + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"length_bytes"} + and isinstance(data["configuration"]["length_bytes"], int) + ) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> FixedLengthUTF32JSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | FixedLengthUTF32JSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return { + "name": self._zarr_v3_name, + "configuration": {"length_bytes": self.length * self.code_point_bytes}, + } + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Construct the numpy dtype instead of string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + raise DataTypeValidationError( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string representation of a numpy U dtype." + ) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"] // cls.code_point_bytes) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + def default_scalar(self) -> np.str_: + return np.str_("") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return str(data) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.str_: + if check_json_str(data): + return self.to_native_dtype().type(data) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[str | np.str_ | bytes | int]: + # this is generous for backwards compatibility + return isinstance(data, str | np.str_ | bytes | int) + + def cast_scalar(self, data: object) -> np.str_: + if self._check_scalar(data): + # We explicitly truncate before casting because of the following numpy behavior: + # >>> x = np.dtype('U3').type('hello world') + # >>> x + # np.str_('hello world') + # >>> x.dtype + # dtype('U11') + + if isinstance(data, int): + return self.to_native_dtype().type(str(data)[: self.length]) + else: + return self.to_native_dtype().type(data[: self.length]) + raise TypeError( + f"Cannot convert object with type {type(data)} to a numpy unicode string scalar." + ) + + @property + def item_size(self) -> int: + return self.length * self.code_point_bytes + + +def check_vlen_string_json_scalar(data: object) -> TypeGuard[int | str | float]: + """ + This function checks the type of JSON-encoded variable length strings. It is generous for + backwards compatibility, as zarr-python v2 would use ints for variable length strings + fill values + """ + return isinstance(data, int | str | float) + + +# VariableLengthUTF8 is defined in two places, conditioned on the version of numpy. +# If numpy 2 is installed, then VariableLengthUTF8 is defined with the numpy variable length +# string dtype as the native dtype. Otherwise, VariableLengthUTF8 is defined with the numpy object +# dtype as the native dtype. +class UTF8Base(ZDType[TDType_co, str], HasObjectCodec): + """ + A base class for the variable length UTF-8 string data type. This class should not be used + as data type, but as a base class for other variable length string data types. + """ + + _zarr_v3_name: ClassVar[Literal["variable_length_utf8"]] = "variable_length_utf8" + object_codec_id: ClassVar[Literal["vlen-utf8"]] = "vlen-utf8" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]]: + """ + Check that the input is a valid JSON representation of a numpy O dtype, and that the + object codec id is appropriate for variable-length UTF-8 strings. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == "|O" + and data["object_codec_id"] == cls.object_codec_id + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_utf8"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O'" + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json( + self, zarr_format: Literal[2] + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_utf8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]] | Literal["variable_length_utf8"]: + if zarr_format == 2: + return {"name": "|O", "object_codec_id": self.object_codec_id} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def default_scalar(self) -> str: + return "" + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected a string.") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> str: + if not check_vlen_string_json_scalar(data): + raise TypeError(f"Invalid type: {data}. Expected a string or number.") + return str(data) + + def _check_scalar(self, data: object) -> TypeGuard[SupportsStr]: + return isinstance(data, SupportsStr) + + def _cast_scalar_unchecked(self, data: SupportsStr) -> str: + return str(data) + + def cast_scalar(self, data: object) -> str: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Cannot convert object with type {type(data)} to a python string.") + + +if _NUMPY_SUPPORTS_VLEN_STRING: + + @dataclass(frozen=True, kw_only=True) + class VariableLengthUTF8(UTF8Base[np.dtypes.StringDType]): # type: ignore[type-var] + dtype_cls = np.dtypes.StringDType + + def to_native_dtype(self) -> np.dtypes.StringDType: + return self.dtype_cls() + +else: + # Numpy pre-2 does not have a variable length string dtype, so we use the Object dtype instead. + @dataclass(frozen=True, kw_only=True) + class VariableLengthUTF8(UTF8Base[np.dtypes.ObjectDType]): # type: ignore[no-redef] + dtype_cls = np.dtypes.ObjectDType + + def to_native_dtype(self) -> np.dtypes.ObjectDType: + return self.dtype_cls() diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py new file mode 100644 index 0000000000..d9e1ff55ae --- /dev/null +++ b/src/zarr/core/dtype/npy/structured.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Self, TypeGuard, cast, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + DTypeSpec_V3, + HasItemSize, + StructuredName_V2, + check_dtype_spec_v2, + check_structured_dtype_name_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import ( + bytes_from_json, + bytes_to_json, + check_json_str, +) +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr.core.common import JSON, NamedConfig, ZarrFormat + +StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int + + +@dataclass(frozen=True, kw_only=True) +class Structured(ZDType[np.dtypes.VoidDType[int], np.void], HasItemSize): + dtype_cls = np.dtypes.VoidDType # type: ignore[assignment] + _zarr_v3_name = "structured" + fields: tuple[tuple[str, ZDType[TBaseDType, TBaseScalar]], ...] + + @classmethod + def _check_native_dtype(cls, dtype: TBaseDType) -> TypeGuard[np.dtypes.VoidDType[int]]: + """ + Check that this dtype is a numpy structured dtype + + Parameters + ---------- + dtype : np.dtypes.DTypeLike + The dtype to check. + + Returns + ------- + TypeGuard[np.dtypes.VoidDType] + True if the dtype matches, False otherwise. + """ + return isinstance(dtype, cls.dtype_cls) and dtype.fields is not None # type: ignore[has-type] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + from zarr.core.dtype import get_data_type_from_native_dtype + + fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] + if cls._check_native_dtype(dtype): + # fields of a structured numpy dtype are either 2-tuples or 3-tuples. we only + # care about the first element in either case. + for key, (dtype_instance, *_) in dtype.fields.items(): # type: ignore[union-attr] + dtype_wrapped = get_data_type_from_native_dtype(dtype_instance) + fields.append((key, dtype_wrapped)) + + return cls(fields=tuple(fields)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + + def to_native_dtype(self) -> np.dtypes.VoidDType[int]: + return cast( + "np.dtypes.VoidDType[int]", + np.dtype([(key, dtype.to_native_dtype()) for (key, dtype) in self.fields]), + ) + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[StructuredName_V2, None]]: + return ( + check_dtype_spec_v2(data) + and not isinstance(data["name"], str) + and check_structured_dtype_name_v2(data["name"]) + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3( + cls, data: DTypeJSON + ) -> TypeGuard[NamedConfig[Literal["structured"], dict[str, Sequence[tuple[str, DTypeJSON]]]]]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"fields"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + # avoid circular import + from zarr.core.dtype import get_data_type_from_json + + if cls._check_json_v2(data): + # structured dtypes are constructed directly from a list of lists + # note that we do not handle the object codec here! this will prevent structured + # dtypes from containing object dtypes. + return cls( + fields=tuple( # type: ignore[misc] + ( # type: ignore[misc] + f_name, + get_data_type_from_json( + {"name": f_dtype, "object_codec_id": None}, zarr_format=2 + ), + ) + for f_name, f_dtype in data["name"] + ) + ) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON array of arrays" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + # avoid circular import + from zarr.core.dtype import get_data_type_from_json + + if cls._check_json_v3(data): + config = data["configuration"] + meta_fields = config["fields"] + return cls( + fields=tuple( + (f_name, get_data_type_from_json(f_dtype, zarr_format=3)) + for f_name, f_dtype in meta_fields + ) + ) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON object with the key {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[StructuredName_V2, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> DTypeSpec_V3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[StructuredName_V2, None] | DTypeSpec_V3: + if zarr_format == 2: + fields = [ + [f_name, f_dtype.to_json(zarr_format=zarr_format)["name"]] + for f_name, f_dtype in self.fields + ] + return {"name": fields, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + fields = [ + [f_name, f_dtype.to_json(zarr_format=zarr_format)] # type: ignore[list-item] + for f_name, f_dtype in self.fields + ] + base_dict = {"name": self._zarr_v3_name} + base_dict["configuration"] = {"fields": fields} # type: ignore[assignment] + return cast("DTypeSpec_V3", base_dict) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[StructuredScalarLike]: + # TODO: implement something more precise here! + return isinstance(data, (bytes, list, tuple, int, np.void)) + + def _cast_scalar_unchecked(self, data: StructuredScalarLike) -> np.void: + na_dtype = self.to_native_dtype() + if isinstance(data, bytes): + res = np.frombuffer(data, dtype=na_dtype)[0] + elif isinstance(data, list | tuple): + res = np.array([tuple(data)], dtype=na_dtype)[0] + else: + res = np.array([data], dtype=na_dtype)[0] + return cast("np.void", res) + + def cast_scalar(self, data: object) -> np.void: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy structured scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.void: + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: + if check_json_str(data): + as_bytes = bytes_from_json(data, zarr_format=zarr_format) + dtype = self.to_native_dtype() + return cast("np.void", np.array([as_bytes]).view(dtype)[0]) + raise TypeError(f"Invalid type: {data}. Expected a string.") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return bytes_to_json(self.cast_scalar(data).tobytes(), zarr_format) + + @property + def item_size(self) -> int: + # Lets have numpy do the arithmetic here + return self.to_native_dtype().itemsize diff --git a/src/zarr/core/dtype/npy/time.py b/src/zarr/core/dtype/npy/time.py new file mode 100644 index 0000000000..1f9080475c --- /dev/null +++ b/src/zarr/core/dtype/npy/time.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + TypedDict, + TypeGuard, + TypeVar, + cast, + get_args, + overload, +) + +import numpy as np + +from zarr.core.common import NamedConfig +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + DATETIME_UNIT, + DateTimeUnit, + check_json_int, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +_DTypeName = Literal["datetime64", "timedelta64"] +TimeDeltaLike = str | int | bytes | np.timedelta64 | timedelta | None +DateTimeLike = str | int | bytes | np.datetime64 | datetime | None + + +def datetime_from_int(data: int, *, unit: DateTimeUnit, scale_factor: int) -> np.datetime64: + """ + Convert an integer to a datetime64. + + Parameters + ---------- + data : int + The integer to convert. + unit : DateTimeUnit + The unit of the datetime64. + scale_factor : int + The scale factor of the datetime64. + + Returns + ------- + np.datetime64 + The datetime64 value. + """ + dtype_name = f"datetime64[{scale_factor}{unit}]" + return cast("np.datetime64", np.int64(data).view(dtype_name)) + + +def datetimelike_to_int(data: np.datetime64 | np.timedelta64) -> int: + """ + Convert a datetime64 or a timedelta64 to an integer. + + Parameters + ---------- + data : np.datetime64 | np.timedelta64 + The value to convert. + + Returns + ------- + int + An integer representation of the scalar. + """ + return data.view(np.int64).item() + + +def check_json_time(data: JSON) -> TypeGuard[Literal["NaT"] | int]: + """ + Type guard to check if the input JSON data is the literal string "NaT" + or an integer. + """ + return check_json_int(data) or data == "NaT" + + +BaseTimeDType_co = TypeVar( + "BaseTimeDType_co", + bound=np.dtypes.TimeDelta64DType | np.dtypes.DateTime64DType, + covariant=True, +) +BaseTimeScalar_co = TypeVar( + "BaseTimeScalar_co", bound=np.timedelta64 | np.datetime64, covariant=True +) + + +class TimeConfig(TypedDict): + unit: DateTimeUnit + scale_factor: int + + +DateTime64JSONV3 = NamedConfig[Literal["numpy.datetime64"], TimeConfig] +TimeDelta64JSONV3 = NamedConfig[Literal["numpy.timedelta64"], TimeConfig] + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TimeDTypeBase(ZDType[BaseTimeDType_co, BaseTimeScalar_co], HasEndianness, HasItemSize): + _zarr_v2_names: ClassVar[tuple[str, ...]] + # this attribute exists so that we can programmatically create a numpy dtype instance + # because the particular numpy dtype we are wrapping does not allow direct construction via + # cls.dtype_cls() + _numpy_name: ClassVar[_DTypeName] + scale_factor: int + unit: DateTimeUnit + + def __post_init__(self) -> None: + if self.scale_factor < 1: + raise ValueError(f"scale_factor must be > 0, got {self.scale_factor}.") + if self.scale_factor >= 2**31: + raise ValueError(f"scale_factor must be < 2147483648, got {self.scale_factor}.") + if self.unit not in get_args(DateTimeUnit): + raise ValueError(f"unit must be one of {get_args(DateTimeUnit)}, got {self.unit!r}.") + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + unit, scale_factor = np.datetime_data(dtype.name) + unit = cast("DateTimeUnit", unit) + return cls( + unit=unit, + scale_factor=scale_factor, + endianness=get_endianness_from_numpy_dtype(dtype), + ) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> BaseTimeDType_co: + # Numpy does not allow creating datetime64 or timedelta64 via + # np.dtypes.{dtype_name}() + # so we use np.dtype with a formatted string. + dtype_string = f"{self._numpy_name}[{self.scale_factor}{self.unit}]" + return np.dtype(dtype_string).newbyteorder(endianness_to_numpy_str(self.endianness)) # type: ignore[return-value] + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> DateTime64JSONV3 | TimeDelta64JSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | DateTime64JSONV3 | TimeDelta64JSONV3: + if zarr_format == 2: + name = self.to_native_dtype().str + return {"name": name, "object_codec_id": None} + elif zarr_format == 3: + return cast( + "DateTime64JSONV3 | TimeDelta64JSONV3", + { + "name": self._zarr_v3_name, + "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, + }, + ) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: + return datetimelike_to_int(data) # type: ignore[arg-type] + + @property + def item_size(self) -> int: + return 8 + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TimeDelta64(TimeDTypeBase[np.dtypes.TimeDelta64DType, np.timedelta64], HasEndianness): + """ + A wrapper for the ``TimeDelta64`` data type defined in numpy. + Scalars of this type can be created by performing arithmetic with ``DateTime64`` scalars. + Like ``DateTime64``, ``TimeDelta64`` is parametrized by a unit, but unlike ``DateTime64``, the + unit for ``TimeDelta64`` is optional. + """ + + # mypy infers the type of np.dtypes.TimeDelta64DType to be + # "Callable[[Literal['Y', 'M', 'W', 'D'] | Literal['h', 'm', 's', 'ms', 'us', 'ns', 'ps', 'fs', 'as']], Never]" + dtype_cls = np.dtypes.TimeDelta64DType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["numpy.timedelta64"]] = "numpy.timedelta64" + _zarr_v2_names = (">m8", " TypeGuard[DTypeConfig_V2[str, None]]: + if not check_dtype_spec_v2(data): + return False + name = data["name"] + # match m[M], etc + # consider making this a standalone function + if not isinstance(name, str): + return False + if not name.startswith(cls._zarr_v2_names): + return False + if len(name) == 3: + # no unit, and + # we already checked that this string is either m8 + return True + else: + return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"unit", "scale_factor"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " + f"representation of an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + unit = data["configuration"]["unit"] + scale_factor = data["configuration"]["scale_factor"] + return cls(unit=unit, scale_factor=scale_factor) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " + f"with a 'name' key with the value 'numpy.timedelta64', " + "and a 'configuration' key with a value of a dict with a 'unit' key and a " + "'scale_factor' key" + ) + raise DataTypeValidationError(msg) + + def _check_scalar(self, data: object) -> TypeGuard[TimeDeltaLike]: + if data is None: + return True + return isinstance(data, str | int | bytes | np.timedelta64 | timedelta) + + def _cast_scalar_unchecked(self, data: TimeDeltaLike) -> np.timedelta64: + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + + def cast_scalar(self, data: object) -> np.timedelta64: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy timedelta64 scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.timedelta64: + return np.timedelta64("NaT") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.timedelta64: + if check_json_time(data): + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DateTime64(TimeDTypeBase[np.dtypes.DateTime64DType, np.datetime64], HasEndianness): + dtype_cls = np.dtypes.DateTime64DType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["numpy.datetime64"]] = "numpy.datetime64" + _zarr_v2_names = (">M8", " TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that JSON input is a string representation of a NumPy datetime64 data type, like "M8[10s]". This function can be used as a type guard to narrow the type of unknown JSON + input. + """ + if not check_dtype_spec_v2(data): + return False + name = data["name"] + if not isinstance(name, str): + return False + if not name.startswith(cls._zarr_v2_names): + return False + if len(name) == 3: + # no unit, and + # we already checked that this string is either M8 + return True + else: + return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"unit", "scale_factor"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " + f"representation of an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + unit = data["configuration"]["unit"] + scale_factor = data["configuration"]["scale_factor"] + return cls(unit=unit, scale_factor=scale_factor) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " + f"with a 'name' key with the value 'numpy.datetime64', " + "and a 'configuration' key with a value of a dict with a 'unit' key and a " + "'scale_factor' key" + ) + raise DataTypeValidationError(msg) + + def _check_scalar(self, data: object) -> TypeGuard[DateTimeLike]: + if data is None: + return True + return isinstance(data, str | int | bytes | np.datetime64 | datetime) + + def _cast_scalar_unchecked(self, data: DateTimeLike) -> np.datetime64: + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + + def cast_scalar(self, data: object) -> np.datetime64: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy datetime scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.datetime64: + return np.datetime64("NaT") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.datetime64: + if check_json_time(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover diff --git a/src/zarr/core/dtype/registry.py b/src/zarr/core/dtype/registry.py new file mode 100644 index 0000000000..1d2a97a90a --- /dev/null +++ b/src/zarr/core/dtype/registry.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import contextlib +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Self + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeJSON, +) + +if TYPE_CHECKING: + from importlib.metadata import EntryPoint + + from zarr.core.common import ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +# This class is different from the other registry classes, which inherit from +# dict. IMO it's simpler to just do a dataclass. But long-term we should +# have just 1 registry class in use. +@dataclass(frozen=True, kw_only=True) +class DataTypeRegistry: + contents: dict[str, type[ZDType[TBaseDType, TBaseScalar]]] = field( + default_factory=dict, init=False + ) + + lazy_load_list: list[EntryPoint] = field(default_factory=list, init=False) + + def lazy_load(self) -> None: + for e in self.lazy_load_list: + self.register(e.load()._zarr_v3_name, e.load()) + + self.lazy_load_list.clear() + + def register(self: Self, key: str, cls: type[ZDType[TBaseDType, TBaseScalar]]) -> None: + # don't register the same dtype twice + if key not in self.contents or self.contents[key] != cls: + self.contents[key] = cls + + def unregister(self, key: str) -> None: + """Unregister a data type by its key.""" + if key in self.contents: + del self.contents[key] + else: + raise KeyError(f"Data type '{key}' not found in registry.") + + def get(self, key: str) -> type[ZDType[TBaseDType, TBaseScalar]]: + return self.contents[key] + + def match_dtype(self, dtype: TBaseDType) -> ZDType[TBaseDType, TBaseScalar]: + if dtype == np.dtype("O"): + msg = ( + f"Zarr data type resolution from {dtype} failed. " + 'Attempted to resolve a zarr data type from a numpy "Object" data type, which is ' + 'ambiguous, as multiple zarr data types can be represented by the numpy "Object" ' + "data type. " + "In this case you should construct your array by providing a specific Zarr data " + 'type. For a list of Zarr data types that are compatible with the numpy "Object"' + "data type, see https://github.com/zarr-developers/zarr-python/issues/3117" + ) + raise ValueError(msg) + matched: list[ZDType[TBaseDType, TBaseScalar]] = [] + for val in self.contents.values(): + with contextlib.suppress(DataTypeValidationError): + matched.append(val.from_native_dtype(dtype)) + if len(matched) == 1: + return matched[0] + elif len(matched) > 1: + msg = ( + f"Zarr data type resolution from {dtype} failed. " + f"Multiple data type wrappers found that match dtype '{dtype}': {matched}. " + "You should unregister one of these data types, or avoid Zarr data type inference " + "entirely by providing a specific Zarr data type when creating your array." + "For more information, see https://github.com/zarr-developers/zarr-python/issues/3117" + ) + raise ValueError(msg) + raise ValueError(f"No Zarr data type found that matches dtype '{dtype!r}'") + + def match_json( + self, data: DTypeJSON, *, zarr_format: ZarrFormat + ) -> ZDType[TBaseDType, TBaseScalar]: + for val in self.contents.values(): + try: + return val.from_json(data, zarr_format=zarr_format) + except DataTypeValidationError: + pass + raise ValueError(f"No Zarr data type found that matches {data!r}") diff --git a/src/zarr/core/dtype/wrapper.py b/src/zarr/core/dtype/wrapper.py new file mode 100644 index 0000000000..e974712e38 --- /dev/null +++ b/src/zarr/core/dtype/wrapper.py @@ -0,0 +1,297 @@ +""" +Wrapper for native array data types. + +The ``ZDType`` class is an abstract base class for wrapping native array data types, e.g. NumPy dtypes. +``ZDType`` provides a common interface for working with data types in a way that is independent of the +underlying data type system. + +The wrapper class encapsulates a native data type. Instances of the class can be created from a +native data type instance, and a native data type instance can be created from an instance of the +wrapper class. + +The wrapper class is responsible for: +- Serializing and deserializing a native data type to Zarr V2 or Zarr V3 metadata. + This ensures that the data type can be properly stored and retrieved from array metadata. +- Serializing and deserializing scalar values to Zarr V2 or Zarr V3 metadata. This is important for + storing a fill value for an array in a manner that is valid for the data type. + +You can add support for a new data type in Zarr by subclassing ``ZDType`` wrapper class and adapt its methods +to support your native data type. The wrapper class must be added to a data type registry +(defined elsewhere) before array creation routines or array reading routines can use your new data +type. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Generic, + Literal, + Self, + TypeGuard, + TypeVar, + overload, +) + +import numpy as np + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2, DTypeSpec_V3 + +# This the upper bound for the scalar types we support. It's numpy scalars + str, +# because the new variable-length string dtype in numpy does not have a corresponding scalar type +TBaseScalar = np.generic | str | bytes +# This is the bound for the dtypes that we support. If we support non-numpy dtypes, +# then this bound will need to be widened. +TBaseDType = np.dtype[np.generic] + +# These two type parameters are covariant because we want +# x : ZDType[BaseDType, BaseScalar] = ZDType[SubDType, SubScalar] +# to type check +TScalar_co = TypeVar("TScalar_co", bound=TBaseScalar, covariant=True) +TDType_co = TypeVar("TDType_co", bound=TBaseDType, covariant=True) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ZDType(Generic[TDType_co, TScalar_co], ABC): + """ + Abstract base class for wrapping native array data types, e.g. numpy dtypes + + Attributes + ---------- + dtype_cls : ClassVar[type[TDType]] + The wrapped dtype class. This is a class variable. + _zarr_v3_name : ClassVar[str] + The name given to the data type by a Zarr v3 data type specification. This is a + class variable, and it should generally be unique across different data types. + """ + + # this class will create a native data type + # mypy currently disallows class variables to contain type parameters + # but it seems OK for us to use it here: + # https://github.com/python/typing/discussions/1424#discussioncomment-7989934 + dtype_cls: ClassVar[type[TDType_co]] # type: ignore[misc] + _zarr_v3_name: ClassVar[str] + + @classmethod + def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[TDType_co]: + """ + Check that a native data type matches the dtype_cls class attribute. Used as a type guard. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return type(dtype) is cls.dtype_cls + + @classmethod + @abstractmethod + def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self: + """ + Create a ZDType instance from a native data type. The default implementation first performs + a type check via ``cls._check_native_dtype``. If that type check succeeds, the ZDType class + instance is created. + + This method is used when taking a user-provided native data type, like a NumPy data type, + and creating the corresponding ZDType instance from them. + + Parameters + ---------- + dtype : TDType + The native data type object to wrap. + + Returns + ------- + Self + The ZDType that wraps the native data type. + + Raises + ------ + TypeError + If the native data type is not consistent with the wrapped data type. + """ + ... + + @abstractmethod + def to_native_dtype(self: Self) -> TDType_co: + """ + Return an instance of the wrapped data type. This operation inverts ``from_native_dtype``. + + Returns + ------- + TDType + The native data type wrapped by this ZDType. + """ + ... + + @classmethod + @abstractmethod + def _from_json_v2(cls: type[Self], data: DTypeJSON) -> Self: ... + + @classmethod + @abstractmethod + def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: ... + + @classmethod + def from_json(cls: type[Self], data: DTypeJSON, *, zarr_format: ZarrFormat) -> Self: + """ + Create an instance of this ZDType from JSON data. + + Parameters + ---------- + data : DTypeJSON + The JSON representation of the data type. The type annotation includes + Mapping[str, object] to accommodate typed dictionaries. + + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + Self + The wrapped data type. + """ + if zarr_format == 2: + return cls._from_json_v2(data) + if zarr_format == 3: + return cls._from_json_v3(data) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @overload + def to_json(self, zarr_format: Literal[2]) -> DTypeSpec_V2: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> DTypeSpec_V3: ... + + @abstractmethod + def to_json(self, zarr_format: ZarrFormat) -> DTypeSpec_V2 | DTypeSpec_V3: + """ + Serialize this ZDType to JSON. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + DTypeJSON_V2 | DTypeJSON_V3 + The JSON-serializable representation of the wrapped data type + """ + ... + + @abstractmethod + def _check_scalar(self, data: object) -> bool: + """ + Check that an python object is a valid scalar value for the wrapped data type. + + Parameters + ---------- + data : object + A value to check. + + Returns + ------- + Bool + True if the object is valid, False otherwise. + """ + ... + + @abstractmethod + def cast_scalar(self, data: object) -> TScalar_co: + """ + Cast a python object to the wrapped scalar type. + The type of the provided scalar is first checked for compatibility. + If it's incompatible with the associated scalar type, a ``TypeError`` will be raised. + + Parameters + ---------- + data : object + The python object to cast. + + Returns + ------- + TScalar + The cast value. + """ + + @abstractmethod + def default_scalar(self) -> TScalar_co: + """ + Get the default scalar value for the wrapped data type. This is a method, rather than an + attribute, because the default value for some data types depends on parameters that are + not known until a concrete data type is wrapped. For example, data types parametrized by a + length like fixed-length strings or bytes will generate scalars consistent with that length. + + Returns + ------- + TScalar + The default value for this data type. + """ + ... + + @abstractmethod + def from_json_scalar(self: Self, data: JSON, *, zarr_format: ZarrFormat) -> TScalar_co: + """ + Read a JSON-serializable value as a scalar. + + Parameters + ---------- + data : JSON + A JSON representation of a scalar value. + zarr_format : ZarrFormat + The zarr format version. This is specified because the JSON serialization of scalars + differs between Zarr V2 and Zarr V3. + + Returns + ------- + TScalar + The deserialized scalar value. + """ + ... + + @abstractmethod + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: + """ + Serialize a python object to the JSON representation of a scalar. The value will first be + cast to the scalar type associated with this ZDType, then serialized to JSON. + + Parameters + ---------- + data : object + The value to convert. + zarr_format : ZarrFormat + The zarr format version. This is specified because the JSON serialization of scalars + differs between Zarr V2 and Zarr V3. + + Returns + ------- + JSON + The JSON-serialized scalar. + """ + ... + + +def scalar_failed_type_check_msg( + cls_instance: ZDType[TBaseDType, TBaseScalar], bad_scalar: object +) -> str: + """ + Generate an error message reporting that a particular value failed a type check when attempting + to cast that value to a scalar. + """ + return ( + f"The value {bad_scalar!r} failed a type check. " + f"It cannot be safely cast to a scalar compatible with {cls_instance}. " + f"Consult the documentation for {cls_instance} to determine the possible values that can " + "be cast to scalars of the wrapped data type." + ) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 3ce46ec97b..b50bce3aef 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import base64 import itertools import json import logging @@ -50,7 +49,6 @@ ) from zarr.core.config import config from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata -from zarr.core.metadata.v3 import V3JsonEncoder, _replace_special_floats from zarr.core.sync import SyncMixin, sync from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError from zarr.storage import StoreLike, StorePath @@ -337,7 +335,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: if self.zarr_format == 3: return { ZARR_JSON: prototype.buffer.from_bytes( - json.dumps(_replace_special_floats(self.to_dict()), cls=V3JsonEncoder).encode() + json.dumps(self.to_dict(), indent=json_indent, allow_nan=False).encode() ) } else: @@ -346,7 +344,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: json.dumps({"zarr_format": self.zarr_format}, indent=json_indent).encode() ), ZATTRS_JSON: prototype.buffer.from_bytes( - json.dumps(self.attributes, indent=json_indent).encode() + json.dumps(self.attributes, indent=json_indent, allow_nan=False).encode() ), } if self.consolidated_metadata: @@ -357,16 +355,10 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] assert isinstance(consolidated_metadata, dict) for k, v in consolidated_metadata.items(): - attrs = v.pop("attributes", None) - d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats(attrs) + attrs = v.pop("attributes", {}) + d[f"{k}/{ZATTRS_JSON}"] = attrs if "shape" in v: # it's an array - if isinstance(v.get("fill_value", None), np.void): - v["fill_value"] = base64.standard_b64encode( - cast("bytes", v["fill_value"]) - ).decode("ascii") - else: - v = _replace_special_floats(v) d[f"{k}/{ZARRAY_JSON}"] = v else: d[f"{k}/{ZGROUP_JSON}"] = { @@ -380,8 +372,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: items[ZMETADATA_V2_JSON] = prototype.buffer.from_bytes( json.dumps( - {"metadata": d, "zarr_consolidated_format": 1}, - cls=V3JsonEncoder, + {"metadata": d, "zarr_consolidated_format": 1}, allow_nan=False ).encode() ) @@ -631,6 +622,7 @@ def _from_bytes_v2( consolidated_metadata[path].update(v) else: raise ValueError(f"Invalid file type '{kind}' at path '{path}") + group_metadata["consolidated_metadata"] = { "metadata": dict(consolidated_metadata), "kind": "inline", diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index a8f4f4abb4..3ac75e0418 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -1,15 +1,16 @@ from __future__ import annotations -import base64 import warnings from collections.abc import Iterable, Sequence -from enum import Enum from functools import cached_property from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict, cast import numcodecs.abc from zarr.abc.metadata import Metadata +from zarr.core.chunk_grids import RegularChunkGrid +from zarr.core.dtype import get_data_type_from_json +from zarr.core.dtype.common import OBJECT_CODEC_IDS, DTypeSpec_V2 if TYPE_CHECKING: from typing import Literal, Self @@ -18,18 +19,29 @@ from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.common import ChunkCoords + from zarr.core.dtype.wrapper import ( + TBaseDType, + TBaseScalar, + TDType_co, + TScalar_co, + ZDType, + ) import json -import numbers from dataclasses import dataclass, field, fields, replace import numcodecs import numpy as np from zarr.core.array_spec import ArrayConfig, ArraySpec -from zarr.core.chunk_grids import RegularChunkGrid from zarr.core.chunk_key_encodings import parse_separator -from zarr.core.common import JSON, ZARRAY_JSON, ZATTRS_JSON, MemoryOrder, parse_shapelike +from zarr.core.common import ( + JSON, + ZARRAY_JSON, + ZATTRS_JSON, + MemoryOrder, + parse_shapelike, +) from zarr.core.config import config, parse_indexing_order from zarr.core.metadata.common import parse_attributes @@ -51,8 +63,8 @@ class ArrayV2MetadataDict(TypedDict): class ArrayV2Metadata(Metadata): shape: ChunkCoords chunks: ChunkCoords - dtype: np.dtype[Any] - fill_value: int | float | str | bytes | None = 0 + dtype: ZDType[TBaseDType, TBaseScalar] + fill_value: int | float | str | bytes | None = None order: MemoryOrder = "C" filters: tuple[numcodecs.abc.Codec, ...] | None = None dimension_separator: Literal[".", "/"] = "." @@ -64,7 +76,7 @@ def __init__( self, *, shape: ChunkCoords, - dtype: npt.DTypeLike, + dtype: ZDType[TDType_co, TScalar_co], chunks: ChunkCoords, fill_value: Any, order: MemoryOrder, @@ -77,18 +89,20 @@ def __init__( Metadata for a Zarr format 2 array. """ shape_parsed = parse_shapelike(shape) - dtype_parsed = parse_dtype(dtype) chunks_parsed = parse_shapelike(chunks) - compressor_parsed = parse_compressor(compressor) order_parsed = parse_indexing_order(order) dimension_separator_parsed = parse_separator(dimension_separator) filters_parsed = parse_filters(filters) - fill_value_parsed = parse_fill_value(fill_value, dtype=dtype_parsed) + fill_value_parsed: TBaseScalar | None + if fill_value is not None: + fill_value_parsed = dtype.cast_scalar(fill_value) + else: + fill_value_parsed = fill_value attributes_parsed = parse_attributes(attributes) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "dtype", dtype_parsed) + object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "chunks", chunks_parsed) object.__setattr__(self, "compressor", compressor_parsed) object.__setattr__(self, "order", order_parsed) @@ -113,52 +127,12 @@ def shards(self) -> ChunkCoords | None: return None def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - def _json_convert( - o: Any, - ) -> Any: - if isinstance(o, np.dtype): - if o.fields is None: - return o.str - else: - return o.descr - if isinstance(o, numcodecs.abc.Codec): - codec_config = o.get_config() - - # Hotfix for https://github.com/zarr-developers/zarr-python/issues/2647 - if codec_config["id"] == "zstd" and not codec_config.get("checksum", False): - codec_config.pop("checksum", None) - - return codec_config - if np.isscalar(o): - out: Any - if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): - # https://github.com/zarr-developers/zarr-python/issues/2119 - # `.item()` on a datetime type might or might not return an - # integer, depending on the value. - # Explicitly cast to an int first, and then grab .item() - out = o.view("i8").item() - else: - # convert numpy scalar to python type, and pass - # python types through - out = getattr(o, "item", lambda: o)() - if isinstance(out, complex): - # python complex types are not JSON serializable, so we use the - # serialization defined in the zarr v3 spec - return [out.real, out.imag] - return out - if isinstance(o, Enum): - return o.name - raise TypeError - zarray_dict = self.to_dict() - zarray_dict["fill_value"] = _serialize_fill_value(self.fill_value, self.dtype) zattrs_dict = zarray_dict.pop("attributes", {}) json_indent = config.get("json_indent") return { ZARRAY_JSON: prototype.buffer.from_bytes( - json.dumps( - zarray_dict, default=_json_convert, indent=json_indent, allow_nan=False - ).encode() + json.dumps(zarray_dict, indent=json_indent, allow_nan=False).encode() ), ZATTRS_JSON: prototype.buffer.from_bytes( json.dumps(zattrs_dict, indent=json_indent, allow_nan=False).encode() @@ -172,8 +146,33 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: # Check that the zarr_format attribute is correct. _ = parse_zarr_format(_data.pop("zarr_format")) - # zarr v2 allowed arbitrary keys in the metadata. - # Filter the keys to only those expected by the constructor. + # To resolve a numpy object dtype array, we need to search for an object codec, + # which could be in filters or as a compressor. + # we will reference a hard-coded collection of object codec ids for this search. + + _filters, _compressor = (data.get("filters"), data.get("compressor")) + if _filters is not None: + _filters = cast("tuple[dict[str, JSON], ...]", _filters) + object_codec_id = get_object_codec_id(tuple(_filters) + (_compressor,)) + else: + object_codec_id = get_object_codec_id((_compressor,)) + # we add a layer of indirection here around the dtype attribute of the array metadata + # because we also need to know the object codec id, if any, to resolve the data type + dtype_spec: DTypeSpec_V2 = { + "name": data["dtype"], + "object_codec_id": object_codec_id, + } + dtype = get_data_type_from_json(dtype_spec, zarr_format=2) + + _data["dtype"] = dtype + fill_value_encoded = _data.get("fill_value") + if fill_value_encoded is not None: + fill_value = dtype.from_json_scalar(fill_value_encoded, zarr_format=2) + _data["fill_value"] = fill_value + + # zarr v2 allowed arbitrary keys here. + # We don't want the ArrayV2Metadata constructor to fail just because someone put an + # extra key in the metadata. expected = {x.name for x in fields(cls)} expected |= {"dtype", "chunks"} @@ -198,16 +197,34 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: def to_dict(self) -> dict[str, JSON]: zarray_dict = super().to_dict() + if isinstance(zarray_dict["compressor"], numcodecs.abc.Codec): + codec_config = zarray_dict["compressor"].get_config() + # Hotfix for https://github.com/zarr-developers/zarr-python/issues/2647 + if codec_config["id"] == "zstd" and not codec_config.get("checksum", False): + codec_config.pop("checksum") + zarray_dict["compressor"] = codec_config + + if zarray_dict["filters"] is not None: + raw_filters = zarray_dict["filters"] + # TODO: remove this when we can stratically type the output JSON data structure + # entirely + if not isinstance(raw_filters, list | tuple): + raise TypeError("Invalid type for filters. Expected a list or tuple.") + new_filters = [] + for f in raw_filters: + if isinstance(f, numcodecs.abc.Codec): + new_filters.append(f.get_config()) + else: + new_filters.append(f) + zarray_dict["filters"] = new_filters - _ = zarray_dict.pop("dtype") - dtype_json: JSON - # In the case of zarr v2, the simplest i.e., '|VXX' dtype is represented as a string - dtype_descr = self.dtype.descr - if self.dtype.kind == "V" and dtype_descr[0][0] != "" and len(dtype_descr) != 0: - dtype_json = tuple(self.dtype.descr) - else: - dtype_json = self.dtype.str - zarray_dict["dtype"] = dtype_json + # serialize the fill value after dtype-specific JSON encoding + if self.fill_value is not None: + fill_value = self.dtype.to_json_scalar(self.fill_value, zarr_format=2) + zarray_dict["fill_value"] = fill_value + + # pull the "name" attribute out of the dtype spec returned by self.dtype.to_json + zarray_dict["dtype"] = self.dtype.to_json(zarr_format=2)["name"] return zarray_dict @@ -296,178 +313,19 @@ def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: return data -def _parse_structured_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: - """Handle structured dtype/fill value pairs""" - try: - if isinstance(fill_value, list): - return np.array([tuple(fill_value)], dtype=dtype)[0] - elif isinstance(fill_value, tuple): - return np.array([fill_value], dtype=dtype)[0] - elif isinstance(fill_value, bytes): - return np.frombuffer(fill_value, dtype=dtype)[0] - elif isinstance(fill_value, str): - decoded = base64.standard_b64decode(fill_value) - return np.frombuffer(decoded, dtype=dtype)[0] - else: - return np.array(fill_value, dtype=dtype)[()] - except Exception as e: - raise ValueError(f"Fill_value {fill_value} is not valid for dtype {dtype}.") from e - - -def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: +def get_object_codec_id(maybe_object_codecs: Sequence[JSON]) -> str | None: """ - Parse a potential fill value into a value that is compatible with the provided dtype. - - Parameters - ---------- - fill_value : Any - A potential fill value. - dtype : np.dtype[Any] - A numpy dtype. - - Returns - ------- - An instance of `dtype`, or `None`, or any python object (in the case of an object dtype) + Inspect a sequence of codecs / filters for an "object codec", i.e. a codec + that can serialize object arrays to contiguous bytes. Zarr python + maintains a hard-coded set of object codec ids. If any element from the input + has an id that matches one of the hard-coded object codec ids, that id + is returned immediately. """ - - if fill_value is None or dtype.hasobject: - pass - elif dtype.fields is not None: - # the dtype is structured (has multiple fields), so the fill_value might be a - # compound value (e.g., a tuple or dict) that needs field-wise processing. - # We use parse_structured_fill_value to correctly convert each component. - fill_value = _parse_structured_fill_value(fill_value, dtype) - elif not isinstance(fill_value, np.void) and fill_value == 0: - # this should be compatible across numpy versions for any array type, including - # structured arrays - fill_value = np.zeros((), dtype=dtype)[()] - elif dtype.kind == "U": - # special case unicode because of encoding issues on Windows if passed through numpy - # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713 - - if not isinstance(fill_value, str): - raise ValueError( - f"fill_value {fill_value!r} is not valid for dtype {dtype}; must be a unicode string" - ) - elif dtype.kind in "SV" and isinstance(fill_value, str): - fill_value = base64.standard_b64decode(fill_value) - elif dtype.kind == "c" and isinstance(fill_value, list) and len(fill_value) == 2: - complex_val = complex(float(fill_value[0]), float(fill_value[1])) - fill_value = np.array(complex_val, dtype=dtype)[()] - else: - try: - if isinstance(fill_value, bytes) and dtype.kind == "V": - # special case for numpy 1.14 compatibility - fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()] - else: - fill_value = np.array(fill_value, dtype=dtype)[()] - - except Exception as e: - msg = f"Fill_value {fill_value} is not valid for dtype {dtype}." - raise ValueError(msg) from e - - return fill_value - - -def _serialize_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> JSON: - serialized: JSON - - if fill_value is None: - serialized = None - elif dtype.kind in "SV": - # There's a relationship between dtype and fill_value - # that mypy isn't aware of. The fact that we have S or V dtype here - # means we should have a bytes-type fill_value. - serialized = base64.standard_b64encode(cast("bytes", fill_value)).decode("ascii") - elif isinstance(fill_value, np.datetime64): - serialized = np.datetime_as_string(fill_value) - elif isinstance(fill_value, numbers.Integral): - serialized = int(fill_value) - elif isinstance(fill_value, numbers.Real): - float_fv = float(fill_value) - if np.isnan(float_fv): - serialized = "NaN" - elif np.isinf(float_fv): - serialized = "Infinity" if float_fv > 0 else "-Infinity" - else: - serialized = float_fv - elif isinstance(fill_value, numbers.Complex): - serialized = [ - _serialize_fill_value(fill_value.real, dtype), - _serialize_fill_value(fill_value.imag, dtype), - ] - else: - serialized = fill_value - - return serialized - - -def _default_fill_value(dtype: np.dtype[Any]) -> Any: - """ - Get the default fill value for a type. - - Notes - ----- - This differs from :func:`parse_fill_value`, which parses a fill value - stored in the Array metadata into an in-memory value. This only gives - the default fill value for some type. - - This is useful for reading Zarr format 2 arrays, which allow the fill - value to be unspecified. - """ - if dtype.kind == "S": - return b"" - elif dtype.kind in "UO": - return "" - elif dtype.kind in "Mm": - return dtype.type("nat") - elif dtype.kind == "V": - if dtype.fields is not None: - default = tuple(_default_fill_value(field[0]) for field in dtype.fields.values()) - return np.array([default], dtype=dtype) - else: - return np.zeros(1, dtype=dtype) - else: - return dtype.type(0) - - -def _default_compressor( - dtype: np.dtype[Any], -) -> dict[str, JSON] | None: - """Get the default filters and compressor for a dtype. - - https://numpy.org/doc/2.1/reference/generated/numpy.dtype.kind.html - """ - default_compressor = config.get("array.v2_default_compressor") - if dtype.kind in "biufcmM": - dtype_key = "numeric" - elif dtype.kind in "U": - dtype_key = "string" - elif dtype.kind in "OSV": - dtype_key = "bytes" - else: - raise ValueError(f"Unsupported dtype kind {dtype.kind}") - - return cast("dict[str, JSON] | None", default_compressor.get(dtype_key, None)) - - -def _default_filters( - dtype: np.dtype[Any], -) -> list[dict[str, JSON]] | None: - """Get the default filters and compressor for a dtype. - - https://numpy.org/doc/2.1/reference/generated/numpy.dtype.kind.html - """ - default_filters = config.get("array.v2_default_filters") - if dtype.kind in "biufcmM": - dtype_key = "numeric" - elif dtype.kind in "U": - dtype_key = "string" - elif dtype.kind in "OS": - dtype_key = "bytes" - elif dtype.kind == "V": - dtype_key = "raw" - else: - raise ValueError(f"Unsupported dtype kind {dtype.kind}") - - return cast("list[dict[str, JSON]] | None", default_filters.get(dtype_key, None)) + object_codec_id = None + for maybe_object_codec in maybe_object_codecs: + if ( + isinstance(maybe_object_codec, dict) + and maybe_object_codec.get("id") in OBJECT_CODEC_IDS + ): + return cast("str", maybe_object_codec["id"]) + return object_codec_id diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index dcbf44f89b..84872d3dbd 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -1,28 +1,25 @@ from __future__ import annotations -import warnings -from typing import TYPE_CHECKING, TypedDict, overload +from typing import TYPE_CHECKING, TypedDict from zarr.abc.metadata import Metadata from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype import VariableLengthUTF8, ZDType, get_data_type_from_json +from zarr.core.dtype.common import check_dtype_spec_v3 if TYPE_CHECKING: - from collections.abc import Callable from typing import Self from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import JSON, ChunkCoords + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar + import json -from collections.abc import Iterable, Sequence +from collections.abc import Iterable from dataclasses import dataclass, field, replace -from enum import Enum -from typing import Any, Literal, cast - -import numcodecs.abc -import numpy as np -import numpy.typing as npt +from typing import Any, Literal from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.core.array_spec import ArrayConfig, ArraySpec @@ -38,20 +35,9 @@ ) from zarr.core.config import config from zarr.core.metadata.common import parse_attributes -from zarr.core.strings import _NUMPY_SUPPORTS_VLEN_STRING -from zarr.core.strings import _STRING_DTYPE as STRING_NP_DTYPE from zarr.errors import MetadataValidationError, NodeTypeValidationError from zarr.registry import get_codec_class -DEFAULT_DTYPE = "float64" - -# Keep in sync with _replace_special_floats -SPECIAL_FLOATS_ENCODED = { - "Infinity": np.inf, - "-Infinity": -np.inf, - "NaN": np.nan, -} - def parse_zarr_format(data: object) -> Literal[3]: if data == 3: @@ -94,7 +80,7 @@ def validate_array_bytes_codec(codecs: tuple[Codec, ...]) -> ArrayBytesCodec: return abcs[0] -def validate_codecs(codecs: tuple[Codec, ...], dtype: DataType) -> None: +def validate_codecs(codecs: tuple[Codec, ...], dtype: ZDType[TBaseDType, TBaseScalar]) -> None: """Check that the codecs are valid for the given dtype""" from zarr.codecs.sharding import ShardingCodec @@ -107,14 +93,11 @@ def validate_codecs(codecs: tuple[Codec, ...], dtype: DataType) -> None: # we need to have special codecs if we are decoding vlen strings or bytestrings # TODO: use codec ID instead of class name codec_class_name = abc.__class__.__name__ - if dtype == DataType.string and not codec_class_name == "VLenUTF8Codec": + # TODO: Fix typing here + if isinstance(dtype, VariableLengthUTF8) and not codec_class_name == "VLenUTF8Codec": # type: ignore[unreachable] raise ValueError( f"For string dtype, ArrayBytesCodec must be `VLenUTF8Codec`, got `{codec_class_name}`." ) - if dtype == DataType.bytes and not codec_class_name == "VLenBytesCodec": - raise ValueError( - f"For bytes dtype, ArrayBytesCodec must be `VLenBytesCodec`, got `{codec_class_name}`." - ) def parse_dimension_names(data: object) -> tuple[str | None, ...] | None: @@ -144,87 +127,6 @@ def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]: ) -class V3JsonEncoder(json.JSONEncoder): - def __init__( - self, - *, - skipkeys: bool = False, - ensure_ascii: bool = True, - check_circular: bool = True, - allow_nan: bool = True, - sort_keys: bool = False, - indent: int | None = None, - separators: tuple[str, str] | None = None, - default: Callable[[object], object] | None = None, - ) -> None: - if indent is None: - indent = config.get("json_indent") - super().__init__( - skipkeys=skipkeys, - ensure_ascii=ensure_ascii, - check_circular=check_circular, - allow_nan=allow_nan, - sort_keys=sort_keys, - indent=indent, - separators=separators, - default=default, - ) - - def default(self, o: object) -> Any: - if isinstance(o, np.dtype): - return str(o) - if np.isscalar(o): - out: Any - if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): - # https://github.com/zarr-developers/zarr-python/issues/2119 - # `.item()` on a datetime type might or might not return an - # integer, depending on the value. - # Explicitly cast to an int first, and then grab .item() - out = o.view("i8").item() - else: - # convert numpy scalar to python type, and pass - # python types through - out = getattr(o, "item", lambda: o)() - if isinstance(out, complex): - # python complex types are not JSON serializable, so we use the - # serialization defined in the zarr v3 spec - return _replace_special_floats([out.real, out.imag]) - elif np.isnan(out): - return "NaN" - elif np.isinf(out): - return "Infinity" if out > 0 else "-Infinity" - return out - elif isinstance(o, Enum): - return o.name - # this serializes numcodecs compressors - # todo: implement to_dict for codecs - elif isinstance(o, numcodecs.abc.Codec): - config: dict[str, Any] = o.get_config() - return config - else: - return super().default(o) - - -def _replace_special_floats(obj: object) -> Any: - """Helper function to replace NaN/Inf/-Inf values with special strings - - Note: this cannot be done in the V3JsonEncoder because Python's `json.dumps` optimistically - converts NaN/Inf values to special types outside of the encoding step. - """ - if isinstance(obj, float): - if np.isnan(obj): - return "NaN" - elif np.isinf(obj): - return "Infinity" if obj > 0 else "-Infinity" - elif isinstance(obj, dict): - # Recursively replace in dictionaries - return {k: _replace_special_floats(v) for k, v in obj.items()} - elif isinstance(obj, list): - # Recursively replace in lists - return [_replace_special_floats(item) for item in obj] - return obj - - class ArrayV3MetadataDict(TypedDict): """ A typed dictionary model for zarr v3 metadata. @@ -237,7 +139,7 @@ class ArrayV3MetadataDict(TypedDict): @dataclass(frozen=True, kw_only=True) class ArrayV3Metadata(Metadata): shape: ChunkCoords - data_type: DataType + data_type: ZDType[TBaseDType, TBaseScalar] chunk_grid: ChunkGrid chunk_key_encoding: ChunkKeyEncoding fill_value: Any @@ -252,10 +154,10 @@ def __init__( self, *, shape: Iterable[int], - data_type: npt.DTypeLike | DataType, + data_type: ZDType[TBaseDType, TBaseScalar], chunk_grid: dict[str, JSON] | ChunkGrid, chunk_key_encoding: ChunkKeyEncodingLike, - fill_value: Any, + fill_value: object, codecs: Iterable[Codec | dict[str, JSON]], attributes: dict[str, JSON] | None, dimension_names: DimensionNames, @@ -264,33 +166,29 @@ def __init__( """ Because the class is a frozen dataclass, we set attributes using object.__setattr__ """ + shape_parsed = parse_shapelike(shape) - data_type_parsed = DataType.parse(data_type) chunk_grid_parsed = ChunkGrid.from_dict(chunk_grid) chunk_key_encoding_parsed = ChunkKeyEncoding.from_dict(chunk_key_encoding) dimension_names_parsed = parse_dimension_names(dimension_names) - if fill_value is None: - fill_value = default_fill_value(data_type_parsed) - # we pass a string here rather than an enum to make mypy happy - fill_value_parsed = parse_fill_value( - fill_value, dtype=cast("ALL_DTYPES", data_type_parsed.value) - ) + # Note: relying on a type method is numpy-specific + fill_value_parsed = data_type.cast_scalar(fill_value) attributes_parsed = parse_attributes(attributes) codecs_parsed_partial = parse_codecs(codecs) storage_transformers_parsed = parse_storage_transformers(storage_transformers) array_spec = ArraySpec( shape=shape_parsed, - dtype=data_type_parsed.to_numpy(), + dtype=data_type, fill_value=fill_value_parsed, config=ArrayConfig.from_dict({}), # TODO: config is not needed here. prototype=default_buffer_prototype(), # TODO: prototype is not needed here. ) codecs_parsed = tuple(c.evolve_from_array_spec(array_spec) for c in codecs_parsed_partial) - validate_codecs(codecs_parsed_partial, data_type_parsed) + validate_codecs(codecs_parsed_partial, data_type) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "data_type", data_type_parsed) + object.__setattr__(self, "data_type", data_type) object.__setattr__(self, "chunk_grid", chunk_grid_parsed) object.__setattr__(self, "chunk_key_encoding", chunk_key_encoding_parsed) object.__setattr__(self, "codecs", codecs_parsed) @@ -315,19 +213,16 @@ def _validate_metadata(self) -> None: if self.fill_value is None: raise ValueError("`fill_value` is required.") for codec in self.codecs: - codec.validate( - shape=self.shape, dtype=self.data_type.to_numpy(), chunk_grid=self.chunk_grid - ) - - @property - def dtype(self) -> np.dtype[Any]: - """Interpret Zarr dtype as NumPy dtype""" - return self.data_type.to_numpy() + codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid) @property def ndim(self) -> int: return len(self.shape) + @property + def dtype(self) -> ZDType[TBaseDType, TBaseScalar]: + return self.data_type + @property def chunks(self) -> ChunkCoords: if isinstance(self.chunk_grid, RegularChunkGrid): @@ -389,8 +284,13 @@ def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: return self.chunk_key_encoding.encode_chunk_key(chunk_coords) def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - d = _replace_special_floats(self.to_dict()) - return {ZARR_JSON: prototype.buffer.from_bytes(json.dumps(d, cls=V3JsonEncoder).encode())} + json_indent = config.get("json_indent") + d = self.to_dict() + return { + ZARR_JSON: prototype.buffer.from_bytes( + json.dumps(d, allow_nan=False, indent=json_indent).encode() + ) + } @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: @@ -402,18 +302,31 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: # check that the node_type attribute is correct _ = parse_node_type_array(_data.pop("node_type")) - # check that the data_type attribute is valid - data_type = DataType.parse(_data.pop("data_type")) + data_type_json = _data.pop("data_type") + if not check_dtype_spec_v3(data_type_json): + raise ValueError(f"Invalid data_type: {data_type_json!r}") + data_type = get_data_type_from_json(data_type_json, zarr_format=3) + + # check that the fill value is consistent with the data type + try: + fill = _data.pop("fill_value") + fill_value_parsed = data_type.from_json_scalar(fill, zarr_format=3) + except ValueError as e: + raise TypeError(f"Invalid fill_value: {fill!r}") from e # dimension_names key is optional, normalize missing to `None` _data["dimension_names"] = _data.pop("dimension_names", None) + # attributes key is optional, normalize missing to `None` _data["attributes"] = _data.pop("attributes", None) - return cls(**_data, data_type=data_type) # type: ignore[arg-type] + + return cls(**_data, fill_value=fill_value_parsed, data_type=data_type) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: out_dict = super().to_dict() - + out_dict["fill_value"] = self.data_type.to_json_scalar( + self.fill_value, zarr_format=self.zarr_format + ) if not isinstance(out_dict, dict): raise TypeError(f"Expected dict. Got {type(out_dict)}.") @@ -421,6 +334,15 @@ def to_dict(self) -> dict[str, JSON]: # the metadata document if out_dict["dimension_names"] is None: out_dict.pop("dimension_names") + + # TODO: replace the `to_dict` / `from_dict` on the `Metadata`` class with + # to_json, from_json, and have ZDType inherit from `Metadata` + # until then, we have this hack here, which relies on the fact that to_dict will pass through + # any non-`Metadata` fields as-is. + dtype_meta = out_dict["data_type"] + if isinstance(dtype_meta, ZDType): + out_dict["data_type"] = dtype_meta.to_json(zarr_format=3) # type: ignore[unreachable] + return out_dict def update_shape(self, shape: ChunkCoords) -> Self: @@ -428,299 +350,3 @@ def update_shape(self, shape: ChunkCoords) -> Self: def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) - - -# enum Literals can't be used in typing, so we have to restate all of the V3 dtypes as types -# https://github.com/python/typing/issues/781 - -BOOL_DTYPE = Literal["bool"] -BOOL = np.bool_ -INTEGER_DTYPE = Literal["int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64"] -INTEGER = np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64 -FLOAT_DTYPE = Literal["float16", "float32", "float64"] -FLOAT = np.float16 | np.float32 | np.float64 -COMPLEX_DTYPE = Literal["complex64", "complex128"] -COMPLEX = np.complex64 | np.complex128 -STRING_DTYPE = Literal["string"] -STRING = np.str_ -BYTES_DTYPE = Literal["bytes"] -BYTES = np.bytes_ - -ALL_DTYPES = BOOL_DTYPE | INTEGER_DTYPE | FLOAT_DTYPE | COMPLEX_DTYPE | STRING_DTYPE | BYTES_DTYPE - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: BOOL_DTYPE, -) -> BOOL: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: INTEGER_DTYPE, -) -> INTEGER: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: FLOAT_DTYPE, -) -> FLOAT: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: COMPLEX_DTYPE, -) -> COMPLEX: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: STRING_DTYPE, -) -> STRING: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: BYTES_DTYPE, -) -> BYTES: ... - - -def parse_fill_value( - fill_value: Any, - dtype: ALL_DTYPES, -) -> Any: - """ - Parse `fill_value`, a potential fill value, into an instance of `dtype`, a data type. - If `fill_value` is `None`, then this function will return the result of casting the value 0 - to the provided data type. Otherwise, `fill_value` will be cast to the provided data type. - - Note that some numpy dtypes use very permissive casting rules. For example, - `np.bool_({'not remotely a bool'})` returns `True`. Thus this function should not be used for - validating that the provided fill value is a valid instance of the data type. - - Parameters - ---------- - fill_value : Any - A potential fill value. - dtype : str - A valid Zarr format 3 DataType. - - Returns - ------- - A scalar instance of `dtype` - """ - data_type = DataType(dtype) - if fill_value is None: - raise ValueError("Fill value cannot be None") - if data_type == DataType.string: - return np.str_(fill_value) - if data_type == DataType.bytes: - return np.bytes_(fill_value) - - # the rest are numeric types - np_dtype = cast("np.dtype[Any]", data_type.to_numpy()) - - if isinstance(fill_value, Sequence) and not isinstance(fill_value, str): - if data_type in (DataType.complex64, DataType.complex128): - if len(fill_value) == 2: - decoded_fill_value = tuple( - SPECIAL_FLOATS_ENCODED.get(value, value) for value in fill_value - ) - # complex datatypes serialize to JSON arrays with two elements - return np_dtype.type(complex(*decoded_fill_value)) - else: - msg = ( - f"Got an invalid fill value for complex data type {data_type.value}." - f"Expected a sequence with 2 elements, but {fill_value!r} has " - f"length {len(fill_value)}." - ) - raise ValueError(msg) - msg = f"Cannot parse non-string sequence {fill_value!r} as a scalar with type {data_type.value}." - raise TypeError(msg) - - # Cast the fill_value to the given dtype - try: - # This warning filter can be removed after Zarr supports numpy>=2.0 - # The warning is saying that the future behavior of out of bounds casting will be to raise - # an OverflowError. In the meantime, we allow overflow and catch cases where - # fill_value != casted_value below. - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) - casted_value = np.dtype(np_dtype).type(fill_value) - except (ValueError, OverflowError, TypeError) as e: - raise ValueError(f"fill value {fill_value!r} is not valid for dtype {data_type}") from e - # Check if the value is still representable by the dtype - if (fill_value == "NaN" and np.isnan(casted_value)) or ( - fill_value in ["Infinity", "-Infinity"] and not np.isfinite(casted_value) - ): - pass - elif np_dtype.kind == "f": - # float comparison is not exact, especially when dtype str | bytes | np.generic: - if dtype == DataType.string: - return "" - elif dtype == DataType.bytes: - return b"" - else: - np_dtype = dtype.to_numpy() - np_dtype = cast("np.dtype[Any]", np_dtype) - return np_dtype.type(0) # type: ignore[misc] - - -# For type checking -_bool = bool - - -class DataType(Enum): - bool = "bool" - int8 = "int8" - int16 = "int16" - int32 = "int32" - int64 = "int64" - uint8 = "uint8" - uint16 = "uint16" - uint32 = "uint32" - uint64 = "uint64" - float16 = "float16" - float32 = "float32" - float64 = "float64" - complex64 = "complex64" - complex128 = "complex128" - string = "string" - bytes = "bytes" - - @property - def byte_count(self) -> int | None: - data_type_byte_counts = { - DataType.bool: 1, - DataType.int8: 1, - DataType.int16: 2, - DataType.int32: 4, - DataType.int64: 8, - DataType.uint8: 1, - DataType.uint16: 2, - DataType.uint32: 4, - DataType.uint64: 8, - DataType.float16: 2, - DataType.float32: 4, - DataType.float64: 8, - DataType.complex64: 8, - DataType.complex128: 16, - } - try: - return data_type_byte_counts[self] - except KeyError: - # string and bytes have variable length - return None - - @property - def has_endianness(self) -> _bool: - return self.byte_count is not None and self.byte_count != 1 - - def to_numpy_shortname(self) -> str: - data_type_to_numpy = { - DataType.bool: "bool", - DataType.int8: "i1", - DataType.int16: "i2", - DataType.int32: "i4", - DataType.int64: "i8", - DataType.uint8: "u1", - DataType.uint16: "u2", - DataType.uint32: "u4", - DataType.uint64: "u8", - DataType.float16: "f2", - DataType.float32: "f4", - DataType.float64: "f8", - DataType.complex64: "c8", - DataType.complex128: "c16", - } - return data_type_to_numpy[self] - - def to_numpy(self) -> np.dtypes.StringDType | np.dtypes.ObjectDType | np.dtype[Any]: - # note: it is not possible to round trip DataType <-> np.dtype - # due to the fact that DataType.string and DataType.bytes both - # generally return np.dtype("O") from this function, even though - # they can originate as fixed-length types (e.g. " DataType: - if dtype.kind in "UT": - return DataType.string - elif dtype.kind == "S": - return DataType.bytes - elif not _NUMPY_SUPPORTS_VLEN_STRING and dtype.kind == "O": - # numpy < 2.0 does not support vlen string dtype - # so we fall back on object array of strings - return DataType.string - dtype_to_data_type = { - "|b1": "bool", - "bool": "bool", - "|i1": "int8", - " DataType: - if dtype is None: - return DataType[DEFAULT_DTYPE] - if isinstance(dtype, DataType): - return dtype - try: - return DataType(dtype) - except ValueError: - pass - try: - dtype = np.dtype(dtype) - except (ValueError, TypeError) as e: - raise ValueError(f"Invalid Zarr format 3 data_type: {dtype}") from e - # check that this is a valid v3 data_type - try: - data_type = DataType.from_numpy(dtype) - except KeyError as e: - raise ValueError(f"Invalid Zarr format 3 data_type: {dtype}") from e - return data_type diff --git a/src/zarr/core/strings.py b/src/zarr/core/strings.py deleted file mode 100644 index 15c5fddfee..0000000000 --- a/src/zarr/core/strings.py +++ /dev/null @@ -1,86 +0,0 @@ -"""This module contains utilities for working with string arrays across -different versions of Numpy. -""" - -from typing import Any, Union, cast -from warnings import warn - -import numpy as np - -# _STRING_DTYPE is the in-memory datatype that will be used for V3 string arrays -# when reading data back from Zarr. -# Any valid string-like datatype should be fine for *setting* data. - -_STRING_DTYPE: Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"] -_NUMPY_SUPPORTS_VLEN_STRING: bool - - -def cast_array( - data: np.ndarray[Any, np.dtype[Any]], -) -> np.ndarray[Any, Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"]]: - raise NotImplementedError - - -try: - # this new vlen string dtype was added in NumPy 2.0 - _STRING_DTYPE = np.dtypes.StringDType() - _NUMPY_SUPPORTS_VLEN_STRING = True - - def cast_array( - data: np.ndarray[Any, np.dtype[Any]], - ) -> np.ndarray[Any, np.dtypes.StringDType | np.dtypes.ObjectDType]: - out = data.astype(_STRING_DTYPE, copy=False) - return cast("np.ndarray[Any, np.dtypes.StringDType]", out) - -except AttributeError: - # if not available, we fall back on an object array of strings, as in Zarr < 3 - _STRING_DTYPE = np.dtypes.ObjectDType() - _NUMPY_SUPPORTS_VLEN_STRING = False - - def cast_array( - data: np.ndarray[Any, np.dtype[Any]], - ) -> np.ndarray[Any, Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"]]: - out = data.astype(_STRING_DTYPE, copy=False) - return cast("np.ndarray[Any, np.dtypes.ObjectDType]", out) - - -def cast_to_string_dtype( - data: np.ndarray[Any, np.dtype[Any]], safe: bool = False -) -> np.ndarray[Any, Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"]]: - """Take any data and attempt to cast to to our preferred string dtype. - - data : np.ndarray - The data to cast - - safe : bool - If True, do not issue a warning if the data is cast from object to string dtype. - - """ - if np.issubdtype(data.dtype, np.str_): - # legacy fixed-width string type (e.g. "= 2.", - stacklevel=2, - ) - return cast_array(data) - raise ValueError(f"Cannot cast dtype {data.dtype} to string dtype") diff --git a/src/zarr/dtype.py b/src/zarr/dtype.py new file mode 100644 index 0000000000..6e3789543b --- /dev/null +++ b/src/zarr/dtype.py @@ -0,0 +1,3 @@ +from zarr.core.dtype import ZDType, data_type_registry + +__all__ = ["ZDType", "data_type_registry"] diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 704db3f704..d1fe1d181c 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from zarr.core.config import BadConfigError, config +from zarr.core.dtype import data_type_registry if TYPE_CHECKING: from importlib.metadata import EntryPoint @@ -43,6 +44,7 @@ def __init__(self) -> None: def lazy_load(self) -> None: for e in self.lazy_load_list: self.register(e.load()) + self.lazy_load_list.clear() def register(self, cls: type[T]) -> None: @@ -58,12 +60,14 @@ def register(self, cls: type[T]) -> None: The registry module is responsible for managing implementations of codecs, pipelines, buffers and ndbuffers and collecting them from entrypoints. The implementation used is determined by the config. + +The registry module is also responsible for managing dtypes. """ def _collect_entrypoints() -> list[Registry[Any]]: """ - Collects codecs, pipelines, buffers and ndbuffers from entrypoints. + Collects codecs, pipelines, dtypes, buffers and ndbuffers from entrypoints. Entry points can either be single items or groups of items. Allowed syntax for entry_points.txt is e.g. @@ -86,6 +90,10 @@ def _collect_entrypoints() -> list[Registry[Any]]: __buffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="buffer")) __ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr.ndbuffer")) __ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="ndbuffer")) + + data_type_registry.lazy_load_list.extend(entry_points.select(group="zarr.data_type")) + data_type_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="data_type")) + __pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) __pipeline_registry.lazy_load_list.extend( entry_points.select(group="zarr", name="codec_pipeline") @@ -148,7 +156,8 @@ def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]: if len(codec_classes) == 1: return next(iter(codec_classes.values())) warnings.warn( - f"Codec '{key}' not configured in config. Selecting any implementation.", stacklevel=2 + f"Codec '{key}' not configured in config. Selecting any implementation.", + stacklevel=2, ) return list(codec_classes.values())[-1] selected_codec_cls = codec_classes[config_entry] diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 0cb992a4f2..5e070b5387 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -17,6 +17,7 @@ from zarr.core.chunk_grids import RegularChunkGrid from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding from zarr.core.common import JSON, ZarrFormat +from zarr.core.dtype import get_data_type_from_native_dtype from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync from zarr.storage import MemoryStore, StoreLike @@ -49,10 +50,10 @@ def v3_dtypes() -> st.SearchStrategy[np.dtype[Any]]: | npst.unsigned_integer_dtypes(endianness="=") | npst.floating_dtypes(endianness="=") | npst.complex_number_dtypes(endianness="=") - # | npst.byte_string_dtypes(endianness="=") - # | npst.unicode_string_dtypes() - # | npst.datetime64_dtypes() - # | npst.timedelta64_dtypes() + | npst.byte_string_dtypes(endianness="=") + | npst.unicode_string_dtypes(endianness="=") + | npst.datetime64_dtypes(endianness="=") + | npst.timedelta64_dtypes(endianness="=") ) @@ -66,7 +67,7 @@ def v2_dtypes() -> st.SearchStrategy[np.dtype[Any]]: | npst.byte_string_dtypes(endianness="=") | npst.unicode_string_dtypes(endianness="=") | npst.datetime64_dtypes(endianness="=") - # | npst.timedelta64_dtypes() + | npst.timedelta64_dtypes(endianness="=") ) @@ -119,7 +120,9 @@ def clear_store(x: Store) -> Store: compressors = st.sampled_from([None, "default"]) zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([3, 2]) # We de-prioritize arrays having dim sizes 0, 1, 2 -array_shapes = npst.array_shapes(max_dims=4, min_side=3) | npst.array_shapes(max_dims=4, min_side=0) +array_shapes = npst.array_shapes(max_dims=4, min_side=3, max_side=5) | npst.array_shapes( + max_dims=4, min_side=0 +) @st.composite @@ -141,8 +144,9 @@ def array_metadata( shape = draw(array_shapes()) ndim = len(shape) chunk_shape = draw(array_shapes(min_dims=ndim, max_dims=ndim)) - dtype = draw(v3_dtypes()) - fill_value = draw(npst.from_dtype(dtype)) + np_dtype = draw(v3_dtypes()) + dtype = get_data_type_from_native_dtype(np_dtype) + fill_value = draw(npst.from_dtype(np_dtype)) if zarr_format == 2: return ArrayV2Metadata( shape=shape, diff --git a/tests/conftest.py b/tests/conftest.py index 30d7eec4d4..4d300a1fd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,8 +19,12 @@ _parse_chunk_key_encoding, ) from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition -from zarr.core.common import JSON, DimensionNames, parse_dtype, parse_shapelike +from zarr.core.common import JSON, DimensionNames, parse_shapelike from zarr.core.config import config as zarr_config +from zarr.core.dtype import ( + get_data_type_from_native_dtype, +) +from zarr.core.dtype.common import HasItemSize from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import sync @@ -36,6 +40,7 @@ from zarr.core.array import CompressorsLike, FiltersLike, SerializerLike, ShardsLike from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike from zarr.core.common import ChunkCoords, MemoryOrder, ShapeLike, ZarrFormat + from zarr.core.dtype.wrapper import ZDType async def parse_store( @@ -265,7 +270,7 @@ def create_array_metadata( filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", serializer: SerializerLike = "auto", - fill_value: Any | None = None, + fill_value: Any = 0, order: MemoryOrder | None = None, zarr_format: ZarrFormat, attributes: dict[str, JSON] | None = None, @@ -275,14 +280,19 @@ def create_array_metadata( """ Create array metadata """ - dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) + dtype_parsed = get_data_type_from_native_dtype(dtype) shape_parsed = parse_shapelike(shape) chunk_key_encoding_parsed = _parse_chunk_key_encoding( chunk_key_encoding, zarr_format=zarr_format ) - + item_size = 1 + if isinstance(dtype_parsed, HasItemSize): + item_size = dtype_parsed.item_size shard_shape_parsed, chunk_shape_parsed = _auto_partition( - array_shape=shape_parsed, shard_shape=shards, chunk_shape=chunks, dtype=dtype_parsed + array_shape=shape_parsed, + shard_shape=shards, + chunk_shape=chunks, + item_size=item_size, ) if order is None: @@ -293,11 +303,11 @@ def create_array_metadata( if zarr_format == 2: filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( - compressor=compressors, filters=filters, dtype=np.dtype(dtype) + compressor=compressors, filters=filters, dtype=dtype_parsed ) return ArrayV2Metadata( shape=shape_parsed, - dtype=np.dtype(dtype), + dtype=dtype_parsed, chunks=chunk_shape_parsed, order=order_parsed, dimension_separator=chunk_key_encoding_parsed.separator, @@ -398,7 +408,7 @@ def meta_from_array( filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", serializer: SerializerLike = "auto", - fill_value: Any | None = None, + fill_value: Any = 0, order: MemoryOrder | None = None, zarr_format: ZarrFormat = 3, attributes: dict[str, JSON] | None = None, @@ -423,3 +433,12 @@ def meta_from_array( chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, ) + + +def skip_object_dtype(dtype: ZDType[Any, Any]) -> None: + if dtype.dtype_cls is type(np.dtype("O")): + msg = ( + f"{dtype} uses the numpy object data type, which is not a valid target for data " + "type resolution" + ) + pytest.skip(msg) diff --git a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt index eee724c912..7eb0eb7c86 100644 --- a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +++ b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt @@ -12,3 +12,5 @@ another_buffer = package_with_entrypoint:TestEntrypointGroup.Buffer another_ndbuffer = package_with_entrypoint:TestEntrypointGroup.NDBuffer [zarr.codec_pipeline] another_pipeline = package_with_entrypoint:TestEntrypointGroup.Pipeline +[zarr.data_type] +new_data_type = package_with_entrypoint:TestDataType \ No newline at end of file diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py index cfbd4f23a9..e0d8a52c4d 100644 --- a/tests/package_with_entrypoint/__init__.py +++ b/tests/package_with_entrypoint/__init__.py @@ -1,5 +1,6 @@ -from collections.abc import Iterable -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal, Self import numpy as np import numpy.typing as npt @@ -7,8 +8,16 @@ import zarr.core.buffer from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline from zarr.codecs import BytesCodec -from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer +from zarr.core.dtype.common import DataTypeValidationError, DTypeJSON, DTypeSpec_V2 +from zarr.core.dtype.npy.bool import Bool + +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import ClassVar, Literal + + from zarr.core.array_spec import ArraySpec + from zarr.core.common import ZarrFormat class TestEntrypointCodec(ArrayBytesCodec): @@ -65,3 +74,28 @@ class NDBuffer(zarr.core.buffer.NDBuffer): class Pipeline(CodecPipeline): pass + + +class TestDataType(Bool): + """ + This is a "data type" that serializes to "test" + """ + + _zarr_v3_name: ClassVar[Literal["test"]] = "test" # type: ignore[assignment] + + @classmethod + def from_json(cls, data: DTypeJSON, *, zarr_format: Literal[2, 3]) -> Self: + if zarr_format == 2 and data == {"name": cls._zarr_v3_name, "object_codec_id": None}: + return cls() + if zarr_format == 3 and data == cls._zarr_v3_name: + return cls() + raise DataTypeValidationError( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}" + ) + + def to_json(self, zarr_format: ZarrFormat) -> str | DTypeSpec_V2: # type: ignore[override] + if zarr_format == 2: + return {"name": self._zarr_v3_name, "object_codec_id": None} + if zarr_format == 3: + return self._zarr_v3_name + raise ValueError("zarr_format must be 2 or 3") diff --git a/tests/test_array.py b/tests/test_array.py index 3fc7b3938c..28ea812967 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1,4 +1,5 @@ import dataclasses +import inspect import json import math import multiprocessing as mp @@ -17,22 +18,19 @@ import zarr.api.asynchronous import zarr.api.synchronous as sync_api +from tests.conftest import skip_object_dtype from zarr import Array, AsyncArray, Group from zarr.abc.store import Store from zarr.codecs import ( BytesCodec, GzipCodec, TransposeCodec, - VLenBytesCodec, - VLenUTF8Codec, ZstdCodec, ) from zarr.core._info import ArrayInfo from zarr.core.array import ( CompressorsLike, FiltersLike, - _get_default_chunk_encoding_v2, - _get_default_chunk_encoding_v3, _parse_chunk_encoding_v2, _parse_chunk_encoding_v3, chunks_initialized, @@ -43,16 +41,29 @@ from zarr.core.chunk_grids import _auto_partition from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams from zarr.core.common import JSON, MemoryOrder, ZarrFormat +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype.common import ENDIANNESS_STR, EndiannessStr +from zarr.core.dtype.npy.common import NUMPY_ENDIANNESS_STR, endianness_from_numpy_str +from zarr.core.dtype.npy.float import Float32, Float64 +from zarr.core.dtype.npy.int import Int16, UInt8 +from zarr.core.dtype.npy.string import VariableLengthUTF8 +from zarr.core.dtype.npy.structured import ( + Structured, +) +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 +from zarr.core.dtype.wrapper import ZDType from zarr.core.group import AsyncGroup from zarr.core.indexing import BasicIndexer, ceildiv -from zarr.core.metadata.v3 import ArrayV3Metadata, DataType +from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.storage import LocalStore, MemoryStore, StorePath +from .test_dtype.conftest import zdtype_examples + if TYPE_CHECKING: from zarr.core.array_spec import ArrayConfigLike -from zarr.core.metadata.v2 import ArrayV2Metadata + from zarr.core.metadata.v3 import ArrayV3Metadata @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @@ -152,7 +163,7 @@ def test_array_name_properties_no_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: arr = zarr.create_array( - store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" + store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype=">i4" ) assert arr.path == "" assert arr.name == "/" @@ -178,34 +189,45 @@ def test_array_name_properties_with_group( assert spam.basename == "spam" +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("specifiy_fill_value", [True, False]) -@pytest.mark.parametrize("dtype_str", ["bool", "uint8", "complex64"]) -def test_array_v3_fill_value_default( - store: MemoryStore, specifiy_fill_value: bool, dtype_str: str +@pytest.mark.parametrize( + "zdtype", zdtype_examples, ids=tuple(str(type(v)) for v in zdtype_examples) +) +def test_array_fill_value_default( + store: MemoryStore, specifiy_fill_value: bool, zdtype: ZDType[Any, Any] ) -> None: """ Test that creating an array with the fill_value parameter set to None, or unspecified, - results in the expected fill_value attribute of the array, i.e. 0 cast to the array's dtype. + results in the expected fill_value attribute of the array, i.e. the default value of the dtype """ shape = (10,) - default_fill_value = 0 if specifiy_fill_value: arr = zarr.create_array( store=store, shape=shape, - dtype=dtype_str, + dtype=zdtype, zarr_format=3, chunks=shape, fill_value=None, ) else: - arr = zarr.create_array( - store=store, shape=shape, dtype=dtype_str, zarr_format=3, chunks=shape - ) + arr = zarr.create_array(store=store, shape=shape, dtype=zdtype, zarr_format=3, chunks=shape) + expected_fill_value = zdtype.default_scalar() + if isinstance(expected_fill_value, np.datetime64 | np.timedelta64): + if np.isnat(expected_fill_value): + assert np.isnat(arr.fill_value) + elif isinstance(expected_fill_value, np.floating | np.complexfloating): + if np.isnan(expected_fill_value): + assert np.isnan(arr.fill_value) + else: + assert arr.fill_value == expected_fill_value + # A simpler check would be to ensure that arr.fill_value.dtype == arr.dtype + # But for some numpy data types (namely, U), scalars might not have length. An empty string + # scalar from a `>U4` array would have dtype `>U`, and arr.fill_value.dtype == arr.dtype will fail. - assert arr.fill_value == np.dtype(dtype_str).type(default_fill_value) - assert arr.fill_value.dtype == arr.dtype + assert type(arr.fill_value) is type(np.array([arr.fill_value], dtype=arr.dtype)[0]) @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -348,7 +370,7 @@ def test_storage_transformers(store: MemoryStore, zarr_format: ZarrFormat | str) "zarr_format": zarr_format, "shape": (10,), "chunks": (1,), - "dtype": "uint8", + "dtype": "|u1", "dimension_separator": ".", "codecs": (BytesCodec().to_dict(),), "fill_value": 0, @@ -458,48 +480,6 @@ async def test_nbytes_stored_async() -> None: assert result == 902 # the size with all chunks filled. -def test_default_fill_values() -> None: - a = zarr.Array.create(MemoryStore(), shape=5, chunk_shape=5, dtype=" None: - with pytest.raises(ValueError, match="At least one ArrayBytesCodec is required."): - Array.create(MemoryStore(), shape=5, chunks=5, dtype=" None: # regression test for https://github.com/zarr-developers/zarr-python/issues/2328 @@ -521,7 +501,7 @@ def test_info_v2(self, chunks: tuple[int, int], shards: tuple[int, int] | None) result = arr.info expected = ArrayInfo( _zarr_format=2, - _data_type=np.dtype("float64"), + _data_type=arr._async_array._zdtype, _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, @@ -539,7 +519,7 @@ def test_info_v3(self, chunks: tuple[int, int], shards: tuple[int, int] | None) result = arr.info expected = ArrayInfo( _zarr_format=3, - _data_type=DataType.parse("float64"), + _data_type=arr._async_array._zdtype, _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, @@ -565,7 +545,7 @@ def test_info_complete(self, chunks: tuple[int, int], shards: tuple[int, int] | result = arr.info_complete() expected = ArrayInfo( _zarr_format=3, - _data_type=DataType.parse("float64"), + _data_type=arr._async_array._zdtype, _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, @@ -601,7 +581,7 @@ async def test_info_v2_async( result = arr.info expected = ArrayInfo( _zarr_format=2, - _data_type=np.dtype("float64"), + _data_type=Float64(), _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=(2, 2), @@ -627,7 +607,7 @@ async def test_info_v3_async( result = arr.info expected = ArrayInfo( _zarr_format=3, - _data_type=DataType.parse("float64"), + _data_type=arr._zdtype, _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=chunks, @@ -655,7 +635,7 @@ async def test_info_complete_async( result = await arr.info_complete() expected = ArrayInfo( _zarr_format=3, - _data_type=DataType.parse("float64"), + _data_type=arr._zdtype, _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=chunks, @@ -982,7 +962,10 @@ def test_auto_partition_auto_shards( expected_shards += (cs,) auto_shards, _ = _auto_partition( - array_shape=array_shape, chunk_shape=chunk_shape, shard_shape="auto", dtype=dtype + array_shape=array_shape, + chunk_shape=chunk_shape, + shard_shape="auto", + item_size=dtype.itemsize, ) assert auto_shards == expected_shards @@ -1017,53 +1000,81 @@ def test_chunks_and_shards(store: Store) -> None: assert arr_v2.shards is None @staticmethod - @pytest.mark.parametrize( - ("dtype", "fill_value_expected"), [(" None: + @pytest.mark.parametrize("dtype", zdtype_examples) + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + def test_default_fill_value(dtype: ZDType[Any, Any], store: Store) -> None: + """ + Test that the fill value of an array is set to the default value for the dtype object + """ a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype) - assert a.fill_value == fill_value_expected + if isinstance(dtype, DateTime64 | TimeDelta64) and np.isnat(a.fill_value): + assert np.isnat(dtype.default_scalar()) + else: + assert a.fill_value == dtype.default_scalar() @staticmethod - @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) - @pytest.mark.parametrize("empty_value", [None, ()]) - async def test_no_filters_compressors( - store: MemoryStore, dtype: str, empty_value: object, zarr_format: ZarrFormat - ) -> None: + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", zdtype_examples) + def test_dtype_forms(dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat) -> None: """ - Test that the default ``filters`` and ``compressors`` are removed when ``create_array`` is invoked. + Test that the same array is produced from a ZDType instance, a numpy dtype, or a numpy string """ + skip_object_dtype(dtype) + a = zarr.create_array( + store, name="a", shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format + ) - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), + b = zarr.create_array( + store, + name="b", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype(), zarr_format=zarr_format, - compressors=empty_value, - filters=empty_value, ) - # Test metadata explicitly - if zarr_format == 2: - assert arr.metadata.zarr_format == 2 # guard for mypy - # v2 spec requires that filters be either a collection with at least one filter, or None - assert arr.metadata.filters is None - # Compressor is a single element in v2 metadata; the absence of a compressor is encoded - # as None - assert arr.metadata.compressor is None - - assert arr.filters == () - assert arr.compressors == () - else: - assert arr.metadata.zarr_format == 3 # guard for mypy - if dtype == "str": - assert arr.metadata.codecs == (VLenUTF8Codec(),) - assert arr.serializer == VLenUTF8Codec() + assert a.dtype == b.dtype + + # Structured dtypes do not have a numpy string representation that uniquely identifies them + if not isinstance(dtype, Structured): + if isinstance(dtype, VariableLengthUTF8): + # in numpy 2.3, StringDType().str becomes the string 'StringDType()' which numpy + # does not accept as a string representation of the dtype. + c = zarr.create_array( + store, + name="c", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype().char, + zarr_format=zarr_format, + ) else: - assert arr.metadata.codecs == (BytesCodec(),) - assert arr.serializer == BytesCodec() + c = zarr.create_array( + store, + name="c", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype().str, + zarr_format=zarr_format, + ) + assert a.dtype == c.dtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", zdtype_examples) + def test_dtype_roundtrip( + dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat + ) -> None: + """ + Test that creating an array, then opening it, gets the same array. + """ + skip_object_dtype(dtype) + a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format) + b = zarr.open_array(store) + assert a.dtype == b.dtype @staticmethod - @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", ["uint8", "float32", "U3", "S4", "V1"]) @pytest.mark.parametrize( "compressors", [ @@ -1134,7 +1145,10 @@ async def test_v3_chunk_encoding( compressors=compressors, ) filters_expected, _, compressors_expected = _parse_chunk_encoding_v3( - filters=filters, compressors=compressors, serializer="auto", dtype=np.dtype(dtype) + filters=filters, + compressors=compressors, + serializer="auto", + dtype=arr._zdtype, ) assert arr.filters == filters_expected assert arr.compressors == compressors_expected @@ -1271,7 +1285,7 @@ async def test_v2_chunk_encoding( filters=filters, ) filters_expected, compressor_expected = _parse_chunk_encoding_v2( - filters=filters, compressor=compressors, dtype=np.dtype(dtype) + filters=filters, compressor=compressors, dtype=get_data_type_from_native_dtype(dtype) ) assert arr.metadata.zarr_format == 2 # guard for mypy assert arr.metadata.compressor == compressor_expected @@ -1285,27 +1299,37 @@ async def test_v2_chunk_encoding( assert arr.filters == filters_expected @staticmethod - @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + @pytest.mark.parametrize("dtype", [UInt8(), Float32(), VariableLengthUTF8()]) + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") async def test_default_filters_compressors( - store: MemoryStore, dtype: str, zarr_format: ZarrFormat + store: MemoryStore, dtype: UInt8 | Float32 | VariableLengthUTF8, zarr_format: ZarrFormat ) -> None: """ Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with ``filters`` and ``compressors`` unspecified. """ + arr = await create_array( store=store, - dtype=dtype, + dtype=dtype, # type: ignore[arg-type] shape=(10,), zarr_format=zarr_format, ) + + sig = inspect.signature(create_array) + if zarr_format == 3: - expected_filters, expected_serializer, expected_compressors = ( - _get_default_chunk_encoding_v3(np_dtype=np.dtype(dtype)) + expected_filters, expected_serializer, expected_compressors = _parse_chunk_encoding_v3( + compressors=sig.parameters["compressors"].default, + filters=sig.parameters["filters"].default, + serializer=sig.parameters["serializer"].default, + dtype=dtype, # type: ignore[arg-type] ) elif zarr_format == 2: - default_filters, default_compressors = _get_default_chunk_encoding_v2( - np_dtype=np.dtype(dtype) + default_filters, default_compressors = _parse_chunk_encoding_v2( + compressor=sig.parameters["compressors"].default, + filters=sig.parameters["filters"].default, + dtype=dtype, # type: ignore[arg-type] ) if default_filters is None: expected_filters = () @@ -1482,9 +1506,24 @@ async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) -> store=store, path=parent_path, zarr_format=zarr_format ) + @staticmethod + @pytest.mark.parametrize("endianness", ENDIANNESS_STR) + def test_default_endianness( + store: Store, zarr_format: ZarrFormat, endianness: EndiannessStr + ) -> None: + """ + Test that that endianness is correctly set when creating an array when not specifying a serializer + """ + dtype = Int16(endianness=endianness) + arr = zarr.create_array(store=store, shape=(1,), dtype=dtype, zarr_format=zarr_format) + byte_order: str = arr[:].dtype.byteorder # type: ignore[union-attr] + assert byte_order in NUMPY_ENDIANNESS_STR + assert endianness_from_numpy_str(byte_order) == endianness # type: ignore[arg-type] + @pytest.mark.parametrize("value", [1, 1.4, "a", b"a", np.array(1)]) @pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_scalar_array(value: Any, zarr_format: ZarrFormat) -> None: arr = zarr.array(value, zarr_format=zarr_format) assert arr[...] == value diff --git a/tests/test_codecs/test_endian.py b/tests/test_codecs/test_endian.py index c0c4dd4e75..ab64afb1b8 100644 --- a/tests/test_codecs/test_endian.py +++ b/tests/test_codecs/test_endian.py @@ -11,6 +11,7 @@ from .test_codecs import _AsyncArrayProxy +@pytest.mark.filterwarnings("ignore:The endianness of the requested serializer") @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("endian", ["big", "little"]) async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: @@ -32,6 +33,7 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: assert np.array_equal(data, readback_data) +@pytest.mark.filterwarnings("ignore:The endianness of the requested serializer") @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("dtype_input_endian", [">u2", " None: - bstrings = [b"hello", b"world", b"this", b"is", b"a", b"test"] - data = np.array(bstrings).reshape((2, 3)) - assert data.dtype == "|S5" - - sp = StorePath(store, path="string") - a = zarr.create_array( - sp, - shape=data.shape, - chunks=data.shape, - dtype=data.dtype, - fill_value=b"", - compressors=compressor, - ) - assert isinstance(a.metadata, ArrayV3Metadata) # needed for mypy - - # should also work if input array is an object array, provided we explicitly specified - # a bytesting-like dtype when creating the Array - if as_object_array: - data = data.astype("O") - a[:, :] = data - assert np.array_equal(data, a[:, :]) - assert a.metadata.data_type == DataType.bytes - assert a.dtype == "O" - - # test round trip - b = Array.open(sp) - assert isinstance(b.metadata, ArrayV3Metadata) # needed for mypy - assert np.array_equal(data, b[:, :]) - assert b.metadata.data_type == DataType.bytes - assert a.dtype == "O" + assert b.metadata.data_type == get_data_type_from_native_dtype(data.dtype) + assert a.dtype == data.dtype diff --git a/tests/test_config.py b/tests/test_config.py index 2cbf172752..1dc6f8bf4f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ import os from collections.abc import Iterable -from typing import Any +from typing import TYPE_CHECKING, Any from unittest import mock from unittest.mock import Mock @@ -19,11 +19,13 @@ GzipCodec, ShardingCodec, ) +from zarr.core.array import create_array from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer from zarr.core.buffer.core import Buffer from zarr.core.codec_pipeline import BatchedCodecPipeline from zarr.core.config import BadConfigError, config +from zarr.core.dtype import Int8, VariableLengthUTF8 from zarr.core.indexing import SelectorTuple from zarr.registry import ( fully_qualified_name, @@ -44,67 +46,66 @@ TestNDArrayLike, ) +if TYPE_CHECKING: + from zarr.core.dtype.wrapper import ZDType + def test_config_defaults_set() -> None: # regression test for available defaults - assert config.defaults == [ - { - "default_zarr_format": 3, - "array": { - "order": "C", - "write_empty_chunks": False, - "v2_default_compressor": { - "numeric": {"id": "zstd", "level": 0, "checksum": False}, - "string": {"id": "zstd", "level": 0, "checksum": False}, - "bytes": {"id": "zstd", "level": 0, "checksum": False}, - }, - "v2_default_filters": { - "numeric": None, - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], - "raw": None, + assert ( + config.defaults + == [ + { + "default_zarr_format": 3, + "array": { + "order": "C", + "write_empty_chunks": False, + "v2_default_compressor": { + "default": {"id": "zstd", "level": 0, "checksum": False}, + "variable-length-string": {"id": "zstd", "level": 0, "checksum": False}, + }, + "v2_default_filters": { + "default": None, + "variable-length-string": [{"id": "vlen-utf8"}], + }, + "v3_default_filters": {"default": [], "variable-length-string": []}, + "v3_default_serializer": { + "default": {"name": "bytes", "configuration": {"endian": "little"}}, + "variable-length-string": {"name": "vlen-utf8"}, + }, + "v3_default_compressors": { + "default": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, + ], + "variable-length-string": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}} + ], + }, }, - "v3_default_filters": {"numeric": [], "string": [], "bytes": []}, - "v3_default_serializer": { - "numeric": {"name": "bytes", "configuration": {"endian": "little"}}, - "string": {"name": "vlen-utf8"}, - "bytes": {"name": "vlen-bytes"}, + "async": {"concurrency": 10, "timeout": None}, + "threading": {"max_workers": None}, + "json_indent": 2, + "codec_pipeline": { + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", + "batch_size": 1, }, - "v3_default_compressors": { - "numeric": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], - "string": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], - "bytes": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], + "codecs": { + "blosc": "zarr.codecs.blosc.BloscCodec", + "gzip": "zarr.codecs.gzip.GzipCodec", + "zstd": "zarr.codecs.zstd.ZstdCodec", + "bytes": "zarr.codecs.bytes.BytesCodec", + "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 + "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", + "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", + "transpose": "zarr.codecs.transpose.TransposeCodec", + "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", + "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", }, - }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, - "json_indent": 2, - "codec_pipeline": { - "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", - "batch_size": 1, - }, - "buffer": "zarr.core.buffer.cpu.Buffer", - "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", - "codecs": { - "blosc": "zarr.codecs.blosc.BloscCodec", - "gzip": "zarr.codecs.gzip.GzipCodec", - "zstd": "zarr.codecs.zstd.ZstdCodec", - "bytes": "zarr.codecs.bytes.BytesCodec", - "endian": "zarr.codecs.bytes.BytesCodec", - "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", - "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", - "transpose": "zarr.codecs.transpose.TransposeCodec", - "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", - "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", - }, - } - ] + "buffer": "zarr.core.buffer.cpu.Buffer", + "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", + } + ] + ) assert config.get("array.order") == "C" assert config.get("async.concurrency") == 10 assert config.get("async.timeout") is None @@ -304,28 +305,29 @@ class NewCodec2(BytesCodec): get_codec_class("new_codec") -@pytest.mark.parametrize("dtype", ["int", "bytes", "str"]) -async def test_default_codecs(dtype: str) -> None: - with config.set( - { - "array.v3_default_compressors": { # test setting non-standard codecs - "numeric": [ - {"name": "gzip", "configuration": {"level": 5}}, - ], - "string": [ - {"name": "gzip", "configuration": {"level": 5}}, - ], - "bytes": [ - {"name": "gzip", "configuration": {"level": 5}}, - ], - } - } - ): - arr = await zarr.api.asynchronous.create_array( +@pytest.mark.parametrize("dtype_category", ["variable-length-string", "default"]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +async def test_default_codecs(dtype_category: str) -> None: + """ + Test that the default compressors are sensitive to the current setting of the config. + """ + zdtype: ZDType[Any, Any] + if dtype_category == "variable-length-string": + zdtype = VariableLengthUTF8() + else: + zdtype = Int8() + expected_compressors = (GzipCodec(),) + new_conf = { + f"array.v3_default_compressors.{dtype_category}": [ + c.to_dict() for c in expected_compressors + ] + } + with config.set(new_conf): + arr = await create_array( shape=(100,), chunks=(100,), - dtype=np.dtype(dtype), + dtype=zdtype, zarr_format=3, store=MemoryStore(), ) - assert arr.compressors == (GzipCodec(),) + assert arr.compressors == expected_compressors diff --git a/tests/test_dtype/__init__.py b/tests/test_dtype/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_dtype/conftest.py b/tests/test_dtype/conftest.py new file mode 100644 index 0000000000..0be1c60088 --- /dev/null +++ b/tests/test_dtype/conftest.py @@ -0,0 +1,71 @@ +# Generate a collection of zdtype instances for use in testing. +import warnings +from typing import Any + +import numpy as np + +from zarr.core.dtype import data_type_registry +from zarr.core.dtype.common import HasLength +from zarr.core.dtype.npy.structured import Structured +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 +from zarr.core.dtype.wrapper import ZDType + +zdtype_examples: tuple[ZDType[Any, Any], ...] = () +for wrapper_cls in data_type_registry.contents.values(): + # The Structured dtype has to be constructed with some actual fields + if wrapper_cls is Structured: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zdtype_examples += ( + wrapper_cls.from_native_dtype(np.dtype([("a", np.float64), ("b", np.int8)])), + ) + elif issubclass(wrapper_cls, HasLength): + zdtype_examples += (wrapper_cls(length=1),) + elif issubclass(wrapper_cls, DateTime64 | TimeDelta64): + zdtype_examples += (wrapper_cls(unit="s", scale_factor=10),) + else: + zdtype_examples += (wrapper_cls(),) + + +def pytest_generate_tests(metafunc: Any) -> None: + """ + This is a pytest hook to parametrize class-scoped fixtures. + + This hook allows us to define class-scoped fixtures as class attributes and then + generate the parametrize calls for pytest. This allows the fixtures to be + reused across multiple tests within the same class. + + For example, if you had a regular pytest class like this: + + class TestClass: + @pytest.mark.parametrize("param_a", [1, 2, 3]) + def test_method(self, param_a): + ... + + Child classes inheriting from ``TestClass`` would not be able to override the ``param_a`` fixture + + this implementation of ``pytest_generate_tests`` allows you to define class-scoped fixtures as + class attributes, which allows the following to work: + + class TestExample: + param_a = [1, 2, 3] + + def test_example(self, param_a): + ... + + # this class will have its test_example method parametrized with the values of TestB.param_a + class TestB(TestExample): + param_a = [1, 2, 100, 10] + + """ + # Iterate over all the fixtures defined in the class + # and parametrize them with the values defined in the class + # This allows us to define class-scoped fixtures as class attributes + # and then generate the parametrize calls for pytest + for fixture_name in metafunc.fixturenames: + if hasattr(metafunc.cls, fixture_name): + params = getattr(metafunc.cls, fixture_name) + if len(params) == 0: + msg = f"{metafunc.cls}.{fixture_name} is empty. Please provide a non-empty sequence of values." + raise ValueError(msg) + metafunc.parametrize(fixture_name, params, scope="class") diff --git a/tests/test_dtype/test_npy/test_bool.py b/tests/test_dtype/test_npy/test_bool.py new file mode 100644 index 0000000000..010dec2e47 --- /dev/null +++ b/tests/test_dtype/test_npy/test_bool.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.bool import Bool + + +class TestBool(BaseTestZDType): + test_cls = Bool + + valid_dtype = (np.dtype(np.bool_),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype(np.uint16), + ) + valid_json_v2 = ({"name": "|b1", "object_codec_id": None},) + valid_json_v3 = ("bool",) + invalid_json_v2 = ( + "|b1", + "bool", + "|f8", + ) + invalid_json_v3 = ( + "|b1", + "|f8", + {"name": "bool", "configuration": {"endianness": "little"}}, + ) + + scalar_v2_params = ((Bool(), True), (Bool(), False)) + scalar_v3_params = ((Bool(), True), (Bool(), False)) + + cast_value_params = ( + (Bool(), "true", np.True_), + (Bool(), True, np.True_), + (Bool(), False, np.False_), + (Bool(), np.True_, np.True_), + (Bool(), np.False_, np.False_), + ) + item_size_params = (Bool(),) diff --git a/tests/test_dtype/test_npy/test_bytes.py b/tests/test_dtype/test_npy/test_bytes.py new file mode 100644 index 0000000000..b7c16f573e --- /dev/null +++ b/tests/test_dtype/test_npy/test_bytes.py @@ -0,0 +1,154 @@ +import numpy as np +import pytest + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.common import UnstableSpecificationWarning +from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes + + +class TestNullTerminatedBytes(BaseTestZDType): + test_cls = NullTerminatedBytes + valid_dtype = (np.dtype("|S10"), np.dtype("|S4")) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|U10"), + ) + valid_json_v2 = ( + {"name": "|S0", "object_codec_id": None}, + {"name": "|S2", "object_codec_id": None}, + {"name": "|S4", "object_codec_id": None}, + ) + valid_json_v3 = ({"name": "null_terminated_bytes", "configuration": {"length_bytes": 10}},) + invalid_json_v2 = ( + "|S", + "|U10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, + {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, + ) + + scalar_v2_params = ( + (NullTerminatedBytes(length=0), ""), + (NullTerminatedBytes(length=2), "YWI="), + (NullTerminatedBytes(length=4), "YWJjZA=="), + ) + scalar_v3_params = ( + (NullTerminatedBytes(length=0), ""), + (NullTerminatedBytes(length=2), "YWI="), + (NullTerminatedBytes(length=4), "YWJjZA=="), + ) + cast_value_params = ( + (NullTerminatedBytes(length=0), "", np.bytes_("")), + (NullTerminatedBytes(length=2), "ab", np.bytes_("ab")), + (NullTerminatedBytes(length=4), "abcdefg", np.bytes_("abcd")), + ) + item_size_params = ( + NullTerminatedBytes(length=0), + NullTerminatedBytes(length=4), + NullTerminatedBytes(length=10), + ) + + +class TestRawBytes(BaseTestZDType): + test_cls = RawBytes + valid_dtype = (np.dtype("|V10"),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|S10"), + ) + valid_json_v2 = ({"name": "|V10", "object_codec_id": None},) + valid_json_v3 = ( + {"name": "raw_bytes", "configuration": {"length_bytes": 0}}, + {"name": "raw_bytes", "configuration": {"length_bytes": 8}}, + ) + + invalid_json_v2 = ( + "|V", + "|S10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "r10"}, + {"name": "r-80"}, + ) + + scalar_v2_params = ( + (RawBytes(length=0), ""), + (RawBytes(length=2), "YWI="), + (RawBytes(length=4), "YWJjZA=="), + ) + scalar_v3_params = ( + (RawBytes(length=0), ""), + (RawBytes(length=2), "YWI="), + (RawBytes(length=4), "YWJjZA=="), + ) + cast_value_params = ( + (RawBytes(length=0), b"", np.void(b"")), + (RawBytes(length=2), b"ab", np.void(b"ab")), + (RawBytes(length=4), b"abcd", np.void(b"abcd")), + ) + item_size_params = ( + RawBytes(length=0), + RawBytes(length=4), + RawBytes(length=10), + ) + + +class TestVariableLengthBytes(BaseTestZDType): + test_cls = VariableLengthBytes + valid_dtype = (np.dtype("|O"),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|U10"), + ) + valid_json_v2 = ({"name": "|O", "object_codec_id": "vlen-bytes"},) + valid_json_v3 = ("variable_length_bytes",) + invalid_json_v2 = ( + "|S", + "|U10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, + {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, + ) + + scalar_v2_params = ( + (VariableLengthBytes(), ""), + (VariableLengthBytes(), "YWI="), + (VariableLengthBytes(), "YWJjZA=="), + ) + scalar_v3_params = ( + (VariableLengthBytes(), ""), + (VariableLengthBytes(), "YWI="), + (VariableLengthBytes(), "YWJjZA=="), + ) + cast_value_params = ( + (VariableLengthBytes(), "", b""), + (VariableLengthBytes(), "ab", b"ab"), + (VariableLengthBytes(), "abcdefg", b"abcdefg"), + ) + item_size_params = ( + VariableLengthBytes(), + VariableLengthBytes(), + VariableLengthBytes(), + ) + + +@pytest.mark.parametrize( + "zdtype", [NullTerminatedBytes(length=10), RawBytes(length=10), VariableLengthBytes()] +) +def test_unstable_dtype_warning( + zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, +) -> None: + """ + Test that we get a warning when serializing a dtype without a zarr v3 spec to json + when zarr_format is 3 + """ + with pytest.raises(UnstableSpecificationWarning): + zdtype.to_json(zarr_format=3) diff --git a/tests/test_dtype/test_npy/test_common.py b/tests/test_dtype/test_npy/test_common.py new file mode 100644 index 0000000000..d39d308112 --- /dev/null +++ b/tests/test_dtype/test_npy/test_common.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import base64 +import math +import re +import sys +from typing import TYPE_CHECKING, Any, get_args + +import numpy as np +import pytest + +from zarr.core.dtype.common import ENDIANNESS_STR, JSONFloatV2, SpecialFloatStrings +from zarr.core.dtype.npy.common import ( + NumpyEndiannessStr, + bytes_from_json, + bytes_to_json, + check_json_bool, + check_json_complex_float_v2, + check_json_complex_float_v3, + check_json_float_v2, + check_json_float_v3, + check_json_int, + check_json_str, + complex_float_to_json_v2, + complex_float_to_json_v3, + endianness_from_numpy_str, + endianness_to_numpy_str, + float_from_json_v2, + float_from_json_v3, + float_to_json_v2, + float_to_json_v3, +) + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +def nan_equal(a: object, b: object) -> bool: + """ + Convenience function for equality comparison between two values ``a`` and ``b``, that might both + be NaN. Returns True if both ``a`` and ``b`` are NaN, otherwise returns a == b + """ + if math.isnan(a) and math.isnan(b): # type: ignore[arg-type] + return True + return a == b + + +json_float_v2_roundtrip_cases: tuple[tuple[JSONFloatV2, float | np.floating[Any]], ...] = ( + ("Infinity", float("inf")), + ("Infinity", np.inf), + ("-Infinity", float("-inf")), + ("-Infinity", -np.inf), + ("NaN", float("nan")), + ("NaN", np.nan), + (1.0, 1.0), +) + +json_float_v3_cases = json_float_v2_roundtrip_cases + + +@pytest.mark.parametrize( + ("data", "expected"), + [(">", "big"), ("<", "little"), ("=", sys.byteorder), ("|", None), ("err", "")], +) +def test_endianness_from_numpy_str(data: str, expected: str | None) -> None: + """ + Test that endianness_from_numpy_str correctly converts a numpy str literal to a human-readable literal value. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data in get_args(NumpyEndiannessStr): + assert endianness_from_numpy_str(data) == expected # type: ignore[arg-type] + else: + msg = f"Invalid endianness: {data!r}. Expected one of {get_args(NumpyEndiannessStr)}" + with pytest.raises(ValueError, match=re.escape(msg)): + endianness_from_numpy_str(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), + [("big", ">"), ("little", "<"), (None, "|"), ("err", "")], +) +def test_endianness_to_numpy_str(data: str | None, expected: str) -> None: + """ + Test that endianness_to_numpy_str correctly converts a human-readable literal value to a numpy str literal. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data in ENDIANNESS_STR: + assert endianness_to_numpy_str(data) == expected # type: ignore[arg-type] + else: + msg = f"Invalid endianness: {data!r}. Expected one of {ENDIANNESS_STR}" + with pytest.raises(ValueError, match=re.escape(msg)): + endianness_to_numpy_str(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), json_float_v2_roundtrip_cases + (("SHOULD_ERR", ""),) +) +def test_float_from_json_v2(data: JSONFloatV2 | str, expected: float | str) -> None: + """ + Test that float_from_json_v2 correctly converts a JSON string representation of a float to a float. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data != "SHOULD_ERR": + assert nan_equal(float_from_json_v2(data), expected) # type: ignore[arg-type] + else: + msg = f"could not convert string to float: {data!r}" + with pytest.raises(ValueError, match=msg): + float_from_json_v2(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), json_float_v3_cases + (("SHOULD_ERR", ""), ("0x", "")) +) +def test_float_from_json_v3(data: JSONFloatV2 | str, expected: float | str) -> None: + """ + Test that float_from_json_v3 correctly converts a JSON string representation of a float to a float. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data == "SHOULD_ERR": + msg = ( + f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" + " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." + ) + with pytest.raises(ValueError, match=msg): + float_from_json_v3(data) + elif data == "0x": + msg = ( + f"Invalid hexadecimal float value: {data!r}. " + "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" + ) + + with pytest.raises(ValueError, match=msg): + float_from_json_v3(data) + else: + assert nan_equal(float_from_json_v3(data), expected) + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("expected", "data"), json_float_v2_roundtrip_cases) +def test_float_to_json_v2(data: float | np.floating[Any], expected: JSONFloatV2) -> None: + """ + Test that floats are JSON-encoded properly for zarr v2 + """ + observed = float_to_json_v2(data) + assert observed == expected + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("expected", "data"), json_float_v3_cases) +def test_float_to_json_v3(data: float | np.floating[Any], expected: JSONFloatV2) -> None: + """ + Test that floats are JSON-encoded properly for zarr v3 + """ + observed = float_to_json_v3(data) + assert observed == expected + + +def test_bytes_from_json(zarr_format: ZarrFormat) -> None: + """ + Test that a string is interpreted as base64-encoded bytes using the ascii alphabet. + This test takes zarr_format as a parameter but doesn't actually do anything with it, because at + present there is no zarr-format-specific logic in the code being tested, but such logic may + exist in the future. + """ + data = "\00" + assert bytes_from_json(data, zarr_format=zarr_format) == base64.b64decode(data.encode("ascii")) + + +def test_bytes_to_json(zarr_format: ZarrFormat) -> None: + """ + Test that bytes are encoded with base64 using the ascii alphabet. + + This test takes zarr_format as a parameter but doesn't actually do anything with it, because at + present there is no zarr-format-specific logic in the code being tested, but such logic may + exist in the future. + """ + + data = b"asdas" + assert bytes_to_json(data, zarr_format=zarr_format) == base64.b64encode(data).decode("ascii") + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v2_roundtrip_cases) +def test_complex_to_json_v2( + float_data: float | np.floating[Any], json_expected: JSONFloatV2 +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v2 format. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v2(cplx_npy) == (json_expected, json_expected) + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) +def test_complex_to_json_v3( + float_data: float | np.floating[Any], json_expected: JSONFloatV2 +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v3 format. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v3(cplx_npy) == (json_expected, json_expected) + + +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) +def test_complex_float_to_json( + float_data: float | np.floating[Any], json_expected: JSONFloatV2, zarr_format: ZarrFormat +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v2 or v3 formats, depending + on the ``zarr_format`` keyword argument. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + if zarr_format == 2: + assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v2(cplx_npy) == ( + json_expected, + json_expected, + ) + elif zarr_format == 3: + assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v3(cplx_npy) == ( + json_expected, + json_expected, + ) + else: + raise ValueError("zarr_format must be 2 or 3") # pragma: no cover + + +check_json_float_cases = get_args(SpecialFloatStrings) + (1.0, 2) + + +@pytest.mark.parametrize("data", check_json_float_cases) +def test_check_json_float_v2_valid(data: JSONFloatV2 | int) -> None: + assert check_json_float_v2(data) + + +def test_check_json_float_v2_invalid() -> None: + assert not check_json_float_v2("invalid") + + +@pytest.mark.parametrize("data", check_json_float_cases) +def test_check_json_float_v3_valid(data: JSONFloatV2 | int) -> None: + assert check_json_float_v3(data) + + +def test_check_json_float_v3_invalid() -> None: + assert not check_json_float_v3("invalid") + + +check_json_complex_float_true_cases: tuple[list[JSONFloatV2], ...] = ( + [0.0, 1.0], + [0.0, 1.0], + [-1.0, "NaN"], + ["Infinity", 1.0], + ["Infinity", "NaN"], +) + +check_json_complex_float_false_cases: tuple[object, ...] = ( + 0.0, + "foo", + [0.0], + [1.0, 2.0, 3.0], + [1.0, "_infinity_"], + {"hello": 1.0}, +) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_v2_true(data: JSON) -> None: + assert check_json_complex_float_v2(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_v2_false(data: JSON) -> None: + assert not check_json_complex_float_v2(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_v3_true(data: JSON) -> None: + assert check_json_complex_float_v3(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_v3_false(data: JSON) -> None: + assert not check_json_complex_float_v3(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_true(data: JSON, zarr_format: ZarrFormat) -> None: + if zarr_format == 2: + assert check_json_complex_float_v2(data) + elif zarr_format == 3: + assert check_json_complex_float_v3(data) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_false(data: JSON, zarr_format: ZarrFormat) -> None: + if zarr_format == 2: + assert not check_json_complex_float_v2(data) + elif zarr_format == 3: + assert not check_json_complex_float_v3(data) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +def test_check_json_int() -> None: + assert check_json_int(0) + assert not check_json_int(1.0) + + +def test_check_json_str() -> None: + assert check_json_str("0") + assert not check_json_str(1.0) + + +def test_check_json_bool() -> None: + assert check_json_bool(True) + assert check_json_bool(False) + assert not check_json_bool(1.0) + assert not check_json_bool("True") diff --git a/tests/test_dtype/test_npy/test_complex.py b/tests/test_dtype/test_npy/test_complex.py new file mode 100644 index 0000000000..b6a1e799eb --- /dev/null +++ b/tests/test_dtype/test_npy/test_complex.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import math + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.complex import Complex64, Complex128 + + +class _BaseTestFloat(BaseTestZDType): + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + +class TestComplex64(_BaseTestFloat): + test_cls = Complex64 + valid_dtype = (np.dtype(">c8"), np.dtype("c8", "object_codec_id": None}, + {"name": "c16"), np.dtype("c16", "object_codec_id": None}, + {"name": " bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + hex_string_params: tuple[tuple[str, float], ...] = () + + def test_hex_encoding(self, hex_string_params: tuple[str, float]) -> None: + """ + Test that hexadecimal strings can be read as NaN values + """ + hex_string, expected = hex_string_params + zdtype = self.test_cls() + observed = zdtype.from_json_scalar(hex_string, zarr_format=3) + assert self.scalar_equals(observed, expected) + + +class TestFloat16(_BaseTestFloat): + test_cls = Float16 + valid_dtype = (np.dtype(">f2"), np.dtype("f2", "object_codec_id": None}, + {"name": "f4"), np.dtype("f4", "object_codec_id": None}, + {"name": "f8"), np.dtype("f8", "object_codec_id": None}, + {"name": "i1", + "int8", + "|f8", + ) + invalid_json_v3 = ( + "|i1", + "|f8", + {"name": "int8", "configuration": {"endianness": "little"}}, + ) + + scalar_v2_params = ((Int8(), 1), (Int8(), -1)) + scalar_v3_params = ((Int8(), 1), (Int8(), -1)) + cast_value_params = ( + (Int8(), 1, np.int8(1)), + (Int8(), -1, np.int8(-1)), + ) + item_size_params = (Int8(),) + + +class TestInt16(BaseTestZDType): + test_cls = Int16 + scalar_type = np.int16 + valid_dtype = (np.dtype(">i2"), np.dtype("i2", "object_codec_id": None}, + {"name": "i4"), np.dtype("i4", "object_codec_id": None}, + {"name": "i8"), np.dtype("i8", "object_codec_id": None}, + {"name": "u2"), np.dtype("u2", "object_codec_id": None}, + {"name": "u4"), np.dtype("u4", "object_codec_id": None}, + {"name": "u8"), np.dtype("u8", "object_codec_id": None}, + {"name": "U10"), np.dtype("U10", "object_codec_id": None}, + {"name": " None: + """ + Test that we get a warning when serializing a dtype without a zarr v3 spec to json + when zarr_format is 3 + """ + with pytest.raises(UnstableSpecificationWarning): + zdtype.to_json(zarr_format=3) diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py new file mode 100644 index 0000000000..e9c9ab11d0 --- /dev/null +++ b/tests/test_dtype/test_npy/test_structured.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype import ( + Float16, + Float64, + Int32, + Int64, + Structured, +) + + +class TestStructured(BaseTestZDType): + test_cls = Structured + valid_dtype = ( + np.dtype([("field1", np.int32), ("field2", np.float64)]), + np.dtype([("field1", np.int64), ("field2", np.int32)]), + ) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|S10"), + ) + valid_json_v2 = ( + {"name": [["field1", ">i4"], ["field2", ">f8"]], "object_codec_id": None}, + {"name": [["field1", ">i8"], ["field2", ">i4"]], "object_codec_id": None}, + ) + valid_json_v3 = ( + { + "name": "structured", + "configuration": { + "fields": [ + ["field1", "int32"], + ["field2", "float64"], + ] + }, + }, + { + "name": "structured", + "configuration": { + "fields": [ + [ + "field1", + { + "name": "numpy.datetime64", + "configuration": {"unit": "s", "scale_factor": 1}, + }, + ], + [ + "field2", + {"name": "fixed_length_utf32", "configuration": {"length_bytes": 32}}, + ], + ] + }, + }, + ) + invalid_json_v2 = ( + [("field1", "|i1"), ("field2", "|f8")], + [("field1", "|S10"), ("field2", "|f8")], + ) + invalid_json_v3 = ( + { + "name": "structured", + "configuration": { + "fields": [ + ("field1", {"name": "int32", "configuration": {"endianness": "invalid"}}), + ("field2", {"name": "float64", "configuration": {"endianness": "big"}}), + ] + }, + }, + {"name": "invalid_name"}, + ) + + scalar_v2_params = ( + (Structured(fields=(("field1", Int32()), ("field2", Float64()))), "AQAAAAAAAAAAAPA/"), + (Structured(fields=(("field1", Float16()), ("field2", Int32()))), "AQAAAAAA"), + ) + scalar_v3_params = ( + (Structured(fields=(("field1", Int32()), ("field2", Float64()))), "AQAAAAAAAAAAAPA/"), + (Structured(fields=(("field1", Int64()), ("field2", Int32()))), "AQAAAAAAAAAAAPA/"), + ) + + cast_value_params = ( + ( + Structured(fields=(("field1", Int32()), ("field2", Float64()))), + (1, 2.0), + np.array((1, 2.0), dtype=[("field1", np.int32), ("field2", np.float64)]), + ), + ( + Structured(fields=(("field1", Int64()), ("field2", Int32()))), + (3, 4.5), + np.array((3, 4.5), dtype=[("field1", np.int64), ("field2", np.int32)]), + ), + ) + + def scalar_equals(self, scalar1: Any, scalar2: Any) -> bool: + if hasattr(scalar1, "shape") and hasattr(scalar2, "shape"): + return np.array_equal(scalar1, scalar2) + return super().scalar_equals(scalar1, scalar2) + + item_size_params = ( + Structured(fields=(("field1", Int32()), ("field2", Float64()))), + Structured(fields=(("field1", Int64()), ("field2", Int32()))), + ) diff --git a/tests/test_dtype/test_npy/test_time.py b/tests/test_dtype/test_npy/test_time.py new file mode 100644 index 0000000000..e201be5cf6 --- /dev/null +++ b/tests/test_dtype/test_npy/test_time.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import re +from typing import get_args + +import numpy as np +import pytest + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.common import DateTimeUnit +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64, datetime_from_int + + +class _TestTimeBase(BaseTestZDType): + def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # This method gets overridden here to support the equivalency between NaT and + # -9223372036854775808 fill values + nat_scalars = (-9223372036854775808, "NaT") + if scalar1 in nat_scalars and scalar2 in nat_scalars: + return True + return scalar1 == scalar2 + + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + +class TestDateTime64(_TestTimeBase): + test_cls = DateTime64 + valid_dtype = (np.dtype("datetime64[10ns]"), np.dtype("datetime64[us]"), np.dtype("datetime64")) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("timedelta64[ns]"), + ) + valid_json_v2 = ( + {"name": ">M8", "object_codec_id": None}, + {"name": ">M8[s]", "object_codec_id": None}, + {"name": "m8", "object_codec_id": None}, + {"name": ">m8[s]", "object_codec_id": None}, + {"name": " None: + """ + Test that an invalid unit raises a ValueError. + """ + unit = "invalid" + msg = f"unit must be one of ('Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'μs', 'ns', 'ps', 'fs', 'as', 'generic'), got {unit!r}." + with pytest.raises(ValueError, match=re.escape(msg)): + DateTime64(unit=unit) # type: ignore[arg-type] + with pytest.raises(ValueError, match=re.escape(msg)): + TimeDelta64(unit=unit) # type: ignore[arg-type] + + +def test_time_scale_factor_too_low() -> None: + """ + Test that an invalid unit raises a ValueError. + """ + scale_factor = 0 + msg = f"scale_factor must be > 0, got {scale_factor}." + with pytest.raises(ValueError, match=msg): + DateTime64(scale_factor=scale_factor) + with pytest.raises(ValueError, match=msg): + TimeDelta64(scale_factor=scale_factor) + + +def test_time_scale_factor_too_high() -> None: + """ + Test that an invalid unit raises a ValueError. + """ + scale_factor = 2**31 + msg = f"scale_factor must be < 2147483648, got {scale_factor}." + with pytest.raises(ValueError, match=msg): + DateTime64(scale_factor=scale_factor) + with pytest.raises(ValueError, match=msg): + TimeDelta64(scale_factor=scale_factor) + + +@pytest.mark.parametrize("unit", get_args(DateTimeUnit)) +@pytest.mark.parametrize("scale_factor", [1, 10]) +@pytest.mark.parametrize("value", [0, 1, 10]) +def test_datetime_from_int(unit: DateTimeUnit, scale_factor: int, value: int) -> None: + """ + Test datetime_from_int. + """ + expected = np.int64(value).view(f"datetime64[{scale_factor}{unit}]") + assert datetime_from_int(value, unit=unit, scale_factor=scale_factor) == expected diff --git a/tests/test_dtype/test_wrapper.py b/tests/test_dtype/test_wrapper.py new file mode 100644 index 0000000000..8f461f1a77 --- /dev/null +++ b/tests/test_dtype/test_wrapper.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +import pytest + +from zarr.core.dtype.common import DTypeSpec_V2, DTypeSpec_V3, HasItemSize + +if TYPE_CHECKING: + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +""" +class _TestZDTypeSchema: + # subclasses define the URL for the schema, if available + schema_url: ClassVar[str] = "" + + @pytest.fixture(scope="class") + def get_schema(self) -> object: + response = requests.get(self.schema_url) + response.raise_for_status() + return json_schema.loads(response.text) + + def test_schema(self, schema: json_schema.Schema) -> None: + assert schema.is_valid(self.test_cls.to_json(zarr_format=2)) +""" + + +class BaseTestZDType: + """ + A base class for testing ZDType subclasses. This class works in conjunction with the custom + pytest collection function ``pytest_generate_tests`` defined in conftest.py, which applies the + following procedure when generating tests: + + At test generation time, for each test fixture referenced by a method on this class + pytest will look for an attribute with the same name as that fixture. Pytest will assume that + this class attribute is a tuple of values to be used for generating a parametrized test fixture. + + This means that child classes can, by using different values for these class attributes, have + customized test parametrization. + + Attributes + ---------- + test_cls : type[ZDType[TBaseDType, TBaseScalar]] + The ZDType subclass being tested. + scalar_type : ClassVar[type[TBaseScalar]] + The expected scalar type for the ZDType. + valid_dtype : ClassVar[tuple[TBaseDType, ...]] + A tuple of valid numpy dtypes for the ZDType. + invalid_dtype : ClassVar[tuple[TBaseDType, ...]] + A tuple of invalid numpy dtypes for the ZDType. + valid_json_v2 : ClassVar[tuple[str | dict[str, object] | list[object], ...]] + A tuple of valid JSON representations for Zarr format version 2. + invalid_json_v2 : ClassVar[tuple[str | dict[str, object] | list[object], ...]] + A tuple of invalid JSON representations for Zarr format version 2. + valid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] + A tuple of valid JSON representations for Zarr format version 3. + invalid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] + A tuple of invalid JSON representations for Zarr format version 3. + cast_value_params : ClassVar[tuple[tuple[Any, Any, Any], ...]] + A tuple of (dtype, value, expected) tuples for testing ZDType.cast_value. + """ + + test_cls: type[ZDType[TBaseDType, TBaseScalar]] + scalar_type: ClassVar[type[TBaseScalar]] + valid_dtype: ClassVar[tuple[TBaseDType, ...]] = () + invalid_dtype: ClassVar[tuple[TBaseDType, ...]] = () + + valid_json_v2: ClassVar[tuple[DTypeSpec_V2, ...]] = () + invalid_json_v2: ClassVar[tuple[str | dict[str, object] | list[object], ...]] = () + + valid_json_v3: ClassVar[tuple[DTypeSpec_V3, ...]] = () + invalid_json_v3: ClassVar[tuple[str | dict[str, object], ...]] = () + + # for testing scalar round-trip serialization, we need a tuple of (data type json, scalar json) + # pairs. the first element of the pair is used to create a dtype instance, and the second + # element is the json serialization of the scalar that we want to round-trip. + + scalar_v2_params: ClassVar[tuple[tuple[Any, Any], ...]] = () + scalar_v3_params: ClassVar[tuple[tuple[Any, Any], ...]] = () + cast_value_params: ClassVar[tuple[tuple[Any, Any, Any], ...]] + item_size_params: ClassVar[tuple[ZDType[Any, Any], ...]] + + def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # An equality check for json-encoded scalars. This defaults to regular equality, + # but some classes may need to override this for special cases + return scalar1 == scalar2 + + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # An equality check for scalars. This defaults to regular equality, + # but some classes may need to override this for special cases + return scalar1 == scalar2 + + def test_check_dtype_valid(self, valid_dtype: TBaseDType) -> None: + assert self.test_cls._check_native_dtype(valid_dtype) + + def test_check_dtype_invalid(self, invalid_dtype: object) -> None: + assert not self.test_cls._check_native_dtype(invalid_dtype) # type: ignore[arg-type] + + def test_from_dtype_roundtrip(self, valid_dtype: Any) -> None: + zdtype = self.test_cls.from_native_dtype(valid_dtype) + assert zdtype.to_native_dtype() == valid_dtype + + def test_from_json_roundtrip_v2(self, valid_json_v2: DTypeSpec_V2) -> None: + zdtype = self.test_cls.from_json(valid_json_v2, zarr_format=2) + assert zdtype.to_json(zarr_format=2) == valid_json_v2 + + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + def test_from_json_roundtrip_v3(self, valid_json_v3: DTypeSpec_V3) -> None: + zdtype = self.test_cls.from_json(valid_json_v3, zarr_format=3) + assert zdtype.to_json(zarr_format=3) == valid_json_v3 + + def test_scalar_roundtrip_v2(self, scalar_v2_params: tuple[ZDType[Any, Any], Any]) -> None: + zdtype, scalar_json = scalar_v2_params + scalar = zdtype.from_json_scalar(scalar_json, zarr_format=2) + assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=2)) + + def test_scalar_roundtrip_v3(self, scalar_v3_params: tuple[ZDType[Any, Any], Any]) -> None: + zdtype, scalar_json = scalar_v3_params + scalar = zdtype.from_json_scalar(scalar_json, zarr_format=3) + assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=3)) + + def test_cast_value(self, cast_value_params: tuple[ZDType[Any, Any], Any, Any]) -> None: + zdtype, value, expected = cast_value_params + observed = zdtype.cast_scalar(value) + assert self.scalar_equals(expected, observed) + + def test_item_size(self, item_size_params: ZDType[Any, Any]) -> None: + """ + Test that the item_size attribute matches the numpy dtype itemsize attribute, for dtypes + with a fixed scalar size. + """ + if isinstance(item_size_params, HasItemSize): + assert item_size_params.item_size == item_size_params.to_native_dtype().itemsize + else: + pytest.skip(f"Dtype {item_size_params} does not implement HasItemSize") diff --git a/tests/test_dtype_registry.py b/tests/test_dtype_registry.py new file mode 100644 index 0000000000..c7d5f90065 --- /dev/null +++ b/tests/test_dtype_registry.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any, get_args + +import numpy as np +import pytest + +import zarr +from tests.conftest import skip_object_dtype +from zarr.core.config import config +from zarr.core.dtype import ( + AnyDType, + Bool, + DataTypeRegistry, + DateTime64, + FixedLengthUTF32, + Int8, + Int16, + TBaseDType, + TBaseScalar, + ZDType, + data_type_registry, + get_data_type_from_json, + parse_data_type, +) + +if TYPE_CHECKING: + from collections.abc import Generator + + from zarr.core.common import ZarrFormat + +from .test_dtype.conftest import zdtype_examples + + +@pytest.fixture +def data_type_registry_fixture() -> DataTypeRegistry: + return DataTypeRegistry() + + +class TestRegistry: + @staticmethod + def test_register(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that registering a dtype in a data type registry works. + """ + data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) + assert data_type_registry_fixture.get(Bool._zarr_v3_name) == Bool + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), Bool) + + @staticmethod + def test_override(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that registering a new dtype with the same name works (overriding the previous one). + """ + data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) + + class NewBool(Bool): + def default_scalar(self) -> np.bool_: + return np.True_ + + data_type_registry_fixture.register(NewBool._zarr_v3_name, NewBool) + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), NewBool) + + @staticmethod + @pytest.mark.parametrize( + ("wrapper_cls", "dtype_str"), [(Bool, "bool"), (FixedLengthUTF32, "|U4")] + ) + def test_match_dtype( + data_type_registry_fixture: DataTypeRegistry, + wrapper_cls: type[ZDType[TBaseDType, TBaseScalar]], + dtype_str: str, + ) -> None: + """ + Test that match_dtype resolves a numpy dtype into an instance of the correspond wrapper for that dtype. + """ + data_type_registry_fixture.register(wrapper_cls._zarr_v3_name, wrapper_cls) + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype(dtype_str)), wrapper_cls) + + @staticmethod + def test_unregistered_dtype(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that match_dtype raises an error if the dtype is not registered. + """ + outside_dtype_name = "int8" + outside_dtype = np.dtype(outside_dtype_name) + msg = f"No Zarr data type found that matches dtype '{outside_dtype!r}'" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_dtype(outside_dtype) + + with pytest.raises(KeyError): + data_type_registry_fixture.get(outside_dtype_name) + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_registered_dtypes_match_dtype(zdtype: ZDType[TBaseDType, TBaseScalar]) -> None: + """ + Test that the registered dtypes can be retrieved from the registry. + """ + skip_object_dtype(zdtype) + assert data_type_registry.match_dtype(zdtype.to_native_dtype()) == zdtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_registered_dtypes_match_json( + zdtype: ZDType[TBaseDType, TBaseScalar], zarr_format: ZarrFormat + ) -> None: + assert ( + data_type_registry.match_json( + zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format + ) + == zdtype + ) + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_match_dtype_unique( + zdtype: ZDType[Any, Any], + data_type_registry_fixture: DataTypeRegistry, + zarr_format: ZarrFormat, + ) -> None: + """ + Test that the match_dtype method uniquely specifies a registered data type. We create a local registry + that excludes the data type class being tested, and ensure that an instance of the wrapped data type + fails to match anything in the registry + """ + skip_object_dtype(zdtype) + for _cls in get_args(AnyDType): + if _cls is not type(zdtype): + data_type_registry_fixture.register(_cls._zarr_v3_name, _cls) + + dtype_instance = zdtype.to_native_dtype() + + msg = f"No Zarr data type found that matches dtype '{dtype_instance!r}'" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_dtype(dtype_instance) + + instance_dict = zdtype.to_json(zarr_format=zarr_format) + msg = f"No Zarr data type found that matches {instance_dict!r}" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_json(instance_dict, zarr_format=zarr_format) + + +# this is copied from the registry tests -- we should deduplicate +here = str(Path(__file__).parent.absolute()) + + +@pytest.fixture +def set_path() -> Generator[None, None, None]: + sys.path.append(here) + zarr.registry._collect_entrypoints() + yield + sys.path.remove(here) + registries = zarr.registry._collect_entrypoints() + for registry in registries: + registry.lazy_load_list.clear() + config.reset() + + +@pytest.mark.usefixtures("set_path") +def test_entrypoint_dtype(zarr_format: ZarrFormat) -> None: + from package_with_entrypoint import TestDataType + + data_type_registry.lazy_load() + instance = TestDataType() + dtype_json = instance.to_json(zarr_format=zarr_format) + assert get_data_type_from_json(dtype_json, zarr_format=zarr_format) == instance + data_type_registry.unregister(TestDataType._zarr_v3_name) + + +@pytest.mark.parametrize( + ("dtype_params", "expected", "zarr_format"), + [ + ("int8", Int8(), 3), + (Int8(), Int8(), 3), + (">i2", Int16(endianness="big"), 2), + ("datetime64[10s]", DateTime64(unit="s", scale_factor=10), 2), + ( + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, + DateTime64(unit="s", scale_factor=10), + 3, + ), + ], +) +def test_parse_data_type( + dtype_params: Any, expected: ZDType[Any, Any], zarr_format: ZarrFormat +) -> None: + """ + Test that parse_data_type accepts alternative representations of ZDType instances, and resolves + those inputs to the expected ZDType instance. + """ + observed = parse_data_type(dtype_params, zarr_format=zarr_format) + assert observed == expected diff --git a/tests/test_group.py b/tests/test_group.py index 7cf29c30d9..60a1fcb9bf 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -23,6 +23,8 @@ from zarr.core._info import GroupInfo from zarr.core.buffer import default_buffer_prototype from zarr.core.config import config as zarr_config +from zarr.core.dtype.common import unpack_dtype_json +from zarr.core.dtype.npy.int import UInt8 from zarr.core.group import ( ConsolidatedMetadata, GroupMetadata, @@ -494,7 +496,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat expected_groups = list(zip(expected_group_keys, expected_group_values, strict=False)) fill_value = 3 - dtype = "uint8" + dtype = UInt8() expected_group_values[0].create_group("subgroup") expected_group_values[0].create_array( @@ -515,7 +517,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat metadata = { "subarray": { "attributes": {}, - "dtype": dtype, + "dtype": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), "fill_value": fill_value, "shape": (1,), "chunks": (1,), @@ -551,7 +553,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat {"configuration": {"endian": "little"}, "name": "bytes"}, {"configuration": {}, "name": "zstd"}, ), - "data_type": dtype, + "data_type": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), "fill_value": fill_value, "node_type": "array", "shape": (1,), diff --git a/tests/test_info.py b/tests/test_info.py index 04e6339092..0abaff9ae7 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -1,11 +1,11 @@ import textwrap -import numpy as np import pytest from zarr.codecs.bytes import BytesCodec from zarr.core._info import ArrayInfo, GroupInfo, human_readable_size from zarr.core.common import ZarrFormat +from zarr.core.dtype.npy.int import Int32 ZARR_FORMATS = [2, 3] @@ -53,7 +53,7 @@ def test_group_info_complete(zarr_format: ZarrFormat) -> None: def test_array_info(zarr_format: ZarrFormat) -> None: info = ArrayInfo( _zarr_format=zarr_format, - _data_type=np.dtype("int32"), + _data_type=Int32(), _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), @@ -66,7 +66,7 @@ def test_array_info(zarr_format: ZarrFormat) -> None: assert result == textwrap.dedent(f"""\ Type : Array Zarr format : {zarr_format} - Data type : int32 + Data type : Int32(endianness='little') Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) @@ -93,7 +93,7 @@ def test_array_info_complete( ) = bytes_things info = ArrayInfo( _zarr_format=zarr_format, - _data_type=np.dtype("int32"), + _data_type=Int32(), _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), @@ -109,7 +109,7 @@ def test_array_info_complete( assert result == textwrap.dedent(f"""\ Type : Array Zarr format : {zarr_format} - Data type : int32 + Data type : Int32(endianness='little') Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index ff4fe6a780..395e036db2 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -18,6 +18,7 @@ open_consolidated, ) from zarr.core.buffer import cpu, default_buffer_prototype +from zarr.core.dtype import parse_data_type from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV3Metadata from zarr.core.metadata.v2 import ArrayV2Metadata @@ -503,7 +504,7 @@ async def test_consolidated_metadata_backwards_compatibility( async def test_consolidated_metadata_v2(self): store = zarr.storage.MemoryStore() g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2) - dtype = "uint8" + dtype = parse_data_type("uint8", zarr_format=2) await g.create_array(name="a", shape=(1,), attributes={"key": "a"}, dtype=dtype) g1 = await g.create_group(name="g1", attributes={"key": "g1"}) await g1.create_group(name="g2", attributes={"key": "g2"}) @@ -624,7 +625,6 @@ async def test_consolidated_metadata_encodes_special_chars( memory_store: Store, zarr_format: ZarrFormat, fill_value: float ): root = await group(store=memory_store, zarr_format=zarr_format) - _child = await root.create_group("child", attributes={"test": fill_value}) _time = await root.create_array("time", shape=(12,), dtype=np.float64, fill_value=fill_value) await zarr.api.asynchronous.consolidate_metadata(memory_store) @@ -638,18 +638,11 @@ async def test_consolidated_metadata_encodes_special_chars( "consolidated_metadata" ]["metadata"] - if np.isnan(fill_value): - expected_fill_value = "NaN" - elif np.isneginf(fill_value): - expected_fill_value = "-Infinity" - elif np.isinf(fill_value): - expected_fill_value = "Infinity" + expected_fill_value = _time._zdtype.to_json_scalar(fill_value, zarr_format=2) if zarr_format == 2: - assert root_metadata["child/.zattrs"]["test"] == expected_fill_value assert root_metadata["time/.zarray"]["fill_value"] == expected_fill_value elif zarr_format == 3: - assert root_metadata["child"]["attributes"]["test"] == expected_fill_value assert root_metadata["time"]["fill_value"] == expected_fill_value diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 08b9cb2507..a2894529aa 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -10,6 +10,8 @@ import zarr.storage from zarr.core.buffer import cpu from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype.npy.float import Float32, Float64 +from zarr.core.dtype.npy.int import Int16 from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV2Metadata from zarr.core.metadata.v2 import parse_zarr_format @@ -19,8 +21,6 @@ from zarr.abc.codec import Codec -import numcodecs - def test_parse_zarr_format_valid() -> None: assert parse_zarr_format(2) == 2 @@ -33,8 +33,8 @@ def test_parse_zarr_format_invalid(data: Any) -> None: @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) -@pytest.mark.parametrize("filters", [None, (numcodecs.GZip(),)]) -@pytest.mark.parametrize("compressor", [None, numcodecs.GZip()]) +@pytest.mark.parametrize("filters", [None, [{"id": "gzip", "level": 1}]]) +@pytest.mark.parametrize("compressor", [None, {"id": "gzip", "level": 1}]) @pytest.mark.parametrize("fill_value", [None, 0, 1]) @pytest.mark.parametrize("order", ["C", "F"]) @pytest.mark.parametrize("dimension_separator", [".", "/", None]) @@ -86,7 +86,7 @@ def test_filters_empty_tuple_warns() -> None: "zarr_format": 2, "shape": (1,), "chunks": (1,), - "dtype": "uint8", + "dtype": "|u1", "order": "C", "compressor": None, "filters": (), @@ -128,7 +128,7 @@ async def v2_consolidated_metadata( "chunks": [730], "compressor": None, "dtype": " None: expected = ArrayV2Metadata( attributes={"key": "value"}, shape=(8,), - dtype="float64", + dtype=Float64(), chunks=(8,), fill_value=0.0, order="C", @@ -318,12 +318,11 @@ def test_zstd_checksum() -> None: assert "checksum" not in metadata["compressor"] -@pytest.mark.parametrize( - "fill_value", [None, np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))] -) +@pytest.mark.parametrize("fill_value", [np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))]) def test_structured_dtype_fill_value_serialization(tmp_path, fill_value): + zarr_format = 2 group_path = tmp_path / "test.zarr" - root_group = zarr.open_group(group_path, mode="w", zarr_format=2) + root_group = zarr.open_group(group_path, mode="w", zarr_format=zarr_format) dtype = np.dtype([("foo", "i4"), ("bar", "i4")]) root_group.create_array( name="structured_dtype", @@ -333,11 +332,7 @@ def test_structured_dtype_fill_value_serialization(tmp_path, fill_value): fill_value=fill_value, ) - zarr.consolidate_metadata(root_group.store, zarr_format=2) + zarr.consolidate_metadata(root_group.store, zarr_format=zarr_format) root_group = zarr.open_group(group_path, mode="r") - assert ( - root_group.metadata.consolidated_metadata.to_dict()["metadata"]["structured_dtype"][ - "fill_value" - ] - == fill_value - ) + observed = root_group.metadata.consolidated_metadata.metadata["structured_dtype"].fill_value + assert observed == fill_value diff --git a/tests/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py index 13549b10a4..4f385afa6d 100644 --- a/tests/test_metadata/test_v3.py +++ b/tests/test_metadata/test_v3.py @@ -11,13 +11,13 @@ from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding from zarr.core.config import config +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype.npy.string import _NUMPY_SUPPORTS_VLEN_STRING +from zarr.core.dtype.npy.time import DateTime64 from zarr.core.group import GroupMetadata, parse_node_type from zarr.core.metadata.v3 import ( ArrayV3Metadata, - DataType, - default_fill_value, parse_dimension_names, - parse_fill_value, parse_zarr_format, ) from zarr.errors import MetadataValidationError, NodeTypeValidationError @@ -54,9 +54,20 @@ ) complex_dtypes = ("complex64", "complex128") -vlen_dtypes = ("string", "bytes") - -dtypes = (*bool_dtypes, *int_dtypes, *float_dtypes, *complex_dtypes, *vlen_dtypes) +flexible_dtypes = ("str", "bytes", "void") +if _NUMPY_SUPPORTS_VLEN_STRING: + vlen_string_dtypes = ("T",) +else: + vlen_string_dtypes = ("O",) + +dtypes = ( + *bool_dtypes, + *int_dtypes, + *float_dtypes, + *complex_dtypes, + *flexible_dtypes, + *vlen_string_dtypes, +) @pytest.mark.parametrize("data", [None, 1, 2, 4, 5, "3"]) @@ -110,90 +121,19 @@ def parse_dimension_names_valid(data: Sequence[str] | None) -> None: assert parse_dimension_names(data) == data -@pytest.mark.parametrize("dtype_str", dtypes) -def test_default_fill_value(dtype_str: str) -> None: - """ - Test that parse_fill_value(None, dtype) results in the 0 value for the given dtype. - """ - dtype = DataType(dtype_str) - fill_value = default_fill_value(dtype) - if dtype == DataType.string: - assert fill_value == "" - elif dtype == DataType.bytes: - assert fill_value == b"" - else: - assert fill_value == dtype.to_numpy().type(0) - - -@pytest.mark.parametrize( - ("fill_value", "dtype_str"), - [ - (True, "bool"), - (False, "bool"), - (-8, "int8"), - (0, "int16"), - (1e10, "uint64"), - (-999, "float32"), - (1e32, "float64"), - (float("NaN"), "float64"), - (np.nan, "float64"), - (np.inf, "float64"), - (-1 * np.inf, "float64"), - (0j, "complex64"), - ], -) -def test_parse_fill_value_valid(fill_value: Any, dtype_str: str) -> None: - """ - Test that parse_fill_value(fill_value, dtype) casts fill_value to the given dtype. - """ - parsed = parse_fill_value(fill_value, dtype_str) - - if np.isnan(fill_value): - assert np.isnan(parsed) - else: - assert parsed == DataType(dtype_str).to_numpy().type(fill_value) - - -@pytest.mark.parametrize("fill_value", ["not a valid value"]) -@pytest.mark.parametrize("dtype_str", [*int_dtypes, *float_dtypes, *complex_dtypes]) -def test_parse_fill_value_invalid_value(fill_value: Any, dtype_str: str) -> None: - """ - Test that parse_fill_value(fill_value, dtype) raises ValueError for invalid values. - This test excludes bool because the bool constructor takes anything. - """ - with pytest.raises(ValueError): - parse_fill_value(fill_value, dtype_str) - - -@pytest.mark.parametrize("fill_value", [[1.0, 0.0], [0, 1], complex(1, 1), np.complex64(0)]) +@pytest.mark.parametrize("fill_value", [[1.0, 0.0], [0, 1]]) @pytest.mark.parametrize("dtype_str", [*complex_dtypes]) -def test_parse_fill_value_complex(fill_value: Any, dtype_str: str) -> None: +def test_jsonify_fill_value_complex(fill_value: Any, dtype_str: str) -> None: """ Test that parse_fill_value(fill_value, dtype) correctly handles complex values represented as length-2 sequences """ - dtype = DataType(dtype_str) - if isinstance(fill_value, list): - expected = dtype.to_numpy().type(complex(*fill_value)) - else: - expected = dtype.to_numpy().type(fill_value) - assert expected == parse_fill_value(fill_value, dtype_str) - - -@pytest.mark.parametrize("fill_value", [[1.0, 0.0, 3.0], [0, 1, 3], [1]]) -@pytest.mark.parametrize("dtype_str", [*complex_dtypes]) -def test_parse_fill_value_complex_invalid(fill_value: Any, dtype_str: str) -> None: - """ - Test that parse_fill_value(fill_value, dtype) correctly rejects sequences with length not - equal to 2 - """ - match = ( - f"Got an invalid fill value for complex data type {dtype_str}." - f"Expected a sequence with 2 elements, but {fill_value} has " - f"length {len(fill_value)}." - ) - with pytest.raises(ValueError, match=re.escape(match)): - parse_fill_value(fill_value=fill_value, dtype=dtype_str) + zarr_format = 3 + dtype = get_data_type_from_native_dtype(dtype_str) + expected = dtype.to_native_dtype().type(complex(*fill_value)) + observed = dtype.from_json_scalar(fill_value, zarr_format=zarr_format) + assert observed == expected + assert dtype.to_json_scalar(observed, zarr_format=zarr_format) == tuple(fill_value) @pytest.mark.parametrize("fill_value", [{"foo": 10}]) @@ -203,8 +143,9 @@ def test_parse_fill_value_invalid_type(fill_value: Any, dtype_str: str) -> None: Test that parse_fill_value(fill_value, dtype) raises TypeError for invalid non-sequential types. This test excludes bool because the bool constructor takes anything. """ - with pytest.raises(ValueError, match=r"fill value .* is not valid for dtype .*"): - parse_fill_value(fill_value, dtype_str) + dtype_instance = get_data_type_from_native_dtype(dtype_str) + with pytest.raises(TypeError, match=f"Invalid type: {fill_value}"): + dtype_instance.from_json_scalar(fill_value, zarr_format=3) @pytest.mark.parametrize( @@ -223,14 +164,14 @@ def test_parse_fill_value_invalid_type_sequence(fill_value: Any, dtype_str: str) This test excludes bool because the bool constructor takes anything, and complex because complex values can be created from length-2 sequences. """ - match = f"Cannot parse non-string sequence {fill_value} as a scalar with type {dtype_str}" - with pytest.raises(TypeError, match=re.escape(match)): - parse_fill_value(fill_value, dtype_str) + dtype_instance = get_data_type_from_native_dtype(dtype_str) + with pytest.raises(TypeError, match=re.escape(f"Invalid type: {fill_value}")): + dtype_instance.from_json_scalar(fill_value, zarr_format=3) @pytest.mark.parametrize("chunk_grid", ["regular"]) @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) -@pytest.mark.parametrize("codecs", [[BytesCodec()]]) +@pytest.mark.parametrize("codecs", [[BytesCodec(endian=None)]]) @pytest.mark.parametrize("fill_value", [0, 1]) @pytest.mark.parametrize("chunk_key_encoding", ["v2", "default"]) @pytest.mark.parametrize("dimension_separator", [".", "/", None]) @@ -247,7 +188,7 @@ def test_metadata_to_dict( storage_transformers: tuple[dict[str, JSON]] | None, ) -> None: shape = (1, 2, 3) - data_type = DataType.uint8 + data_type_str = "uint8" if chunk_grid == "regular": cgrid = {"name": "regular", "configuration": {"chunk_shape": (1, 1, 1)}} @@ -271,7 +212,7 @@ def test_metadata_to_dict( "node_type": "array", "shape": shape, "chunk_grid": cgrid, - "data_type": data_type, + "data_type": data_type_str, "chunk_key_encoding": cke, "codecs": tuple(c.to_dict() for c in codecs), "fill_value": fill_value, @@ -315,50 +256,32 @@ def test_json_indent(indent: int): assert d == json.dumps(json.loads(d), indent=indent).encode() -# @pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) -# @pytest.mark.parametrize("precision", ["ns", "D"]) -# async def test_datetime_metadata(fill_value: int, precision: str) -> None: -# metadata_dict = { -# "zarr_format": 3, -# "node_type": "array", -# "shape": (1,), -# "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, -# "data_type": f" None: +@pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) +@pytest.mark.parametrize("precision", ["ns", "D"]) +async def test_datetime_metadata(fill_value: int, precision: str) -> None: + dtype = DateTime64(unit=precision) metadata_dict = { "zarr_format": 3, "node_type": "array", "shape": (1,), "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, - "data_type": " None: metadata_dict = { @@ -368,10 +291,11 @@ async def test_invalid_fill_value_raises(data_type: str, fill_value: float) -> N "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, "data_type": data_type, "chunk_key_encoding": {"name": "default", "separator": "."}, - "codecs": (), + "codecs": ({"name": "bytes"},), "fill_value": fill_value, # this is not a valid fill value for uint8 } - with pytest.raises(ValueError, match=r"fill value .* is not valid for dtype .*"): + # multiple things can go wrong here, so we don't match on the error message. + with pytest.raises(TypeError): ArrayV3Metadata.from_dict(metadata_dict) @@ -399,17 +323,3 @@ async def test_special_float_fill_values(fill_value: str) -> None: elif fill_value == "-Infinity": assert np.isneginf(m.fill_value) assert d["fill_value"] == "-Infinity" - - -@pytest.mark.parametrize("dtype_str", dtypes) -def test_dtypes(dtype_str: str) -> None: - dt = DataType(dtype_str) - np_dtype = dt.to_numpy() - if dtype_str not in vlen_dtypes: - # we can round trip "normal" dtypes - assert dt == DataType.from_numpy(np_dtype) - assert dt.byte_count == np_dtype.itemsize - assert dt.has_endianness == (dt.byte_count > 1) - else: - # return type for vlen types may vary depending on numpy version - assert dt.byte_count is None diff --git a/tests/test_properties.py b/tests/test_properties.py index d48dfe2fef..b8d50ef0b1 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,4 +1,3 @@ -import dataclasses import json import numbers from typing import Any @@ -76,6 +75,7 @@ def deep_equal(a: Any, b: Any) -> bool: return a == b +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data(), zarr_format=zarr_formats) def test_array_roundtrip(data: st.DataObject, zarr_format: int) -> None: nparray = data.draw(numpy_arrays(zarr_formats=st.just(zarr_format))) @@ -83,6 +83,7 @@ def test_array_roundtrip(data: st.DataObject, zarr_format: int) -> None: assert_array_equal(nparray, zarray[:]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(array=arrays()) def test_array_creates_implicit_groups(array): path = array.path @@ -102,7 +103,10 @@ def test_array_creates_implicit_groups(array): # this decorator removes timeout; not ideal but it should avoid intermittent CI failures + + @settings(deadline=None) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data()) def test_basic_indexing(data: st.DataObject) -> None: zarray = data.draw(simple_arrays()) @@ -118,6 +122,7 @@ def test_basic_indexing(data: st.DataObject) -> None: @given(data=st.data()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_oindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) @@ -139,6 +144,7 @@ def test_oindex(data: st.DataObject) -> None: @given(data=st.data()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_vindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) @@ -162,6 +168,7 @@ def test_vindex(data: st.DataObject) -> None: @given(store=stores, meta=array_metadata()) # type: ignore[misc] +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") async def test_roundtrip_array_metadata_from_store( store: Store, meta: ArrayV2Metadata | ArrayV3Metadata ) -> None: @@ -181,6 +188,7 @@ async def test_roundtrip_array_metadata_from_store( @given(data=st.data(), zarr_format=zarr_formats) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: int) -> None: """ Verify that JSON serialization and deserialization of metadata is lossless. @@ -209,8 +217,8 @@ def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: in zarray_dict = json.loads(buffer_dict[ZARR_JSON].to_bytes().decode()) metadata_roundtripped = ArrayV3Metadata.from_dict(zarray_dict) - orig = dataclasses.asdict(metadata) - rt = dataclasses.asdict(metadata_roundtripped) + orig = metadata.to_dict() + rt = metadata_roundtripped.to_dict() assert deep_equal(orig, rt), f"Roundtrip mismatch:\nOriginal: {orig}\nRoundtripped: {rt}" @@ -239,6 +247,29 @@ def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: in # assert_array_equal(nparray, zarray[:]) +def serialized_complex_float_is_valid( + serialized: tuple[numbers.Real | str, numbers.Real | str], +) -> bool: + """ + Validate that the serialized representation of a complex float conforms to the spec. + + The specification requires that a serialized complex float must be either: + - A JSON number, or + - One of the strings "NaN", "Infinity", or "-Infinity". + + Args: + serialized: The value produced by JSON serialization for a complex floating point number. + + Returns: + bool: True if the serialized value is valid according to the spec, False otherwise. + """ + return ( + isinstance(serialized, tuple) + and len(serialized) == 2 + and all(serialized_float_is_valid(x) for x in serialized) + ) + + def serialized_float_is_valid(serialized: numbers.Real | str) -> bool: """ Validate that the serialized representation of a float conforms to the spec. @@ -259,6 +290,7 @@ def serialized_float_is_valid(serialized: numbers.Real | str) -> bool: @given(meta=array_metadata()) # type: ignore[misc] +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_array_metadata_meets_spec(meta: ArrayV2Metadata | ArrayV3Metadata) -> None: """ Validate that the array metadata produced by the library conforms to the relevant spec (V2 vs V3). @@ -294,11 +326,11 @@ def test_array_metadata_meets_spec(meta: ArrayV2Metadata | ArrayV3Metadata) -> N assert asdict_dict["zarr_format"] == 3 # version-agnostic validations - if meta.dtype.kind == "f": + dtype_native = meta.dtype.to_native_dtype() + if dtype_native.kind == "f": assert serialized_float_is_valid(asdict_dict["fill_value"]) - elif meta.dtype.kind == "c": + elif dtype_native.kind == "c": # fill_value should be a two-element array [real, imag]. - assert serialized_float_is_valid(asdict_dict["fill_value"].real) - assert serialized_float_is_valid(asdict_dict["fill_value"].imag) - elif meta.dtype.kind == "M" and np.isnat(meta.fill_value): - assert asdict_dict["fill_value"] == "NaT" + assert serialized_complex_float_is_valid(asdict_dict["fill_value"]) + elif dtype_native.kind in ("M", "m") and np.isnat(meta.fill_value): + assert asdict_dict["fill_value"] == -9223372036854775808 diff --git a/tests/test_regression/__init__.py b/tests/test_regression/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_regression/scripts/__init__.py b/tests/test_regression/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_regression/scripts/v2.18.py b/tests/test_regression/scripts/v2.18.py new file mode 100644 index 0000000000..39e1c5210c --- /dev/null +++ b/tests/test_regression/scripts/v2.18.py @@ -0,0 +1,81 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr==2.18", +# "numcodecs==0.15" +# ] +# /// + +import argparse + +import zarr +from zarr._storage.store import BaseStore + + +def copy_group( + *, node: zarr.hierarchy.Group, store: zarr.storage.BaseStore, path: str, overwrite: bool +) -> zarr.hierarchy.Group: + result = zarr.group(store=store, path=path, overwrite=overwrite) + result.attrs.put(node.attrs.asdict()) + for key, child in node.items(): + child_path = f"{path}/{key}" + if isinstance(child, zarr.hierarchy.Group): + copy_group(node=child, store=store, path=child_path, overwrite=overwrite) + elif isinstance(child, zarr.core.Array): + copy_array(node=child, store=store, overwrite=overwrite, path=child_path) + return result + + +def copy_array( + *, node: zarr.core.Array, store: BaseStore, path: str, overwrite: bool +) -> zarr.core.Array: + result = zarr.create( + shape=node.shape, + dtype=node.dtype, + fill_value=node.fill_value, + chunks=node.chunks, + compressor=node.compressor, + filters=node.filters, + order=node.order, + dimension_separator=node._dimension_separator, + store=store, + path=path, + overwrite=overwrite, + ) + result.attrs.put(node.attrs.asdict()) + result[:] = node[:] + return result + + +def copy_node( + node: zarr.hierarchy.Group | zarr.core.Array, store: BaseStore, path: str, overwrite: bool +) -> zarr.hierarchy.Group | zarr.core.Array: + if isinstance(node, zarr.hierarchy.Group): + return copy_group(node=node, store=store, path=path, overwrite=overwrite) + elif isinstance(node, zarr.core.Array): + return copy_array(node=node, store=store, path=path, overwrite=overwrite) + else: + raise TypeError(f"Unexpected node type: {type(node)}") # pragma: no cover + + +def cli() -> None: + parser = argparse.ArgumentParser( + description="Copy a zarr hierarchy from one location to another" + ) + parser.add_argument("source", type=str, help="Path to the source zarr hierarchy") + parser.add_argument("destination", type=str, help="Path to the destination zarr hierarchy") + args = parser.parse_args() + + src, dst = args.source, args.destination + root_src = zarr.open(src, mode="r") + result = copy_node(node=root_src, store=zarr.NestedDirectoryStore(dst), path="", overwrite=True) + + print(f"successfully created {result} at {dst}") + + +def main() -> None: + cli() + + +if __name__ == "__main__": + main() diff --git a/tests/test_regression/test_regression.py b/tests/test_regression/test_regression.py new file mode 100644 index 0000000000..34c48a6933 --- /dev/null +++ b/tests/test_regression/test_regression.py @@ -0,0 +1,156 @@ +import subprocess +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from typing import TYPE_CHECKING + +import numcodecs +import numpy as np +import pytest +from numcodecs import LZ4, LZMA, Blosc, GZip, VLenBytes, VLenUTF8, Zstd + +import zarr +from zarr.core.array import Array +from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding +from zarr.core.dtype.npy.bytes import VariableLengthBytes +from zarr.core.dtype.npy.string import VariableLengthUTF8 +from zarr.storage import LocalStore + +if TYPE_CHECKING: + from zarr.core.dtype import ZDTypeLike + + +def runner_installed() -> bool: + """ + Check if a PEP-723 compliant python script runner is installed. + """ + try: + subprocess.check_output(["uv", "--version"]) + return True # noqa: TRY300 + except FileNotFoundError: + return False + + +@dataclass(kw_only=True) +class ArrayParams: + values: np.ndarray[tuple[int], np.dtype[np.generic]] + fill_value: np.generic | str | int | bytes + filters: tuple[numcodecs.abc.Codec, ...] = () + compressor: numcodecs.abc.Codec + + +basic_codecs = GZip(), Blosc(), LZ4(), LZMA(), Zstd() +basic_dtypes = "|b", ">i2", ">i4", ">f4", ">f8", "c8", "c16", "M8[10us]", "m8[4ps]" +string_dtypes = "U4" +bytes_dtypes = ">S1", "V10", " Array: + dest = tmp_path / "in" + store = LocalStore(dest) + array_params: ArrayParams = request.param + compressor = array_params.compressor + chunk_key_encoding = V2ChunkKeyEncoding(separator="/") + dtype: ZDTypeLike + if array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenUTF8(),): + dtype = VariableLengthUTF8() # type: ignore[assignment] + elif array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenBytes(),): + dtype = VariableLengthBytes() + else: + dtype = array_params.values.dtype + z = zarr.create_array( + store, + shape=array_params.values.shape, + dtype=dtype, + chunks=array_params.values.shape, + compressors=compressor, + filters=array_params.filters, + fill_value=array_params.fill_value, + order="C", + chunk_key_encoding=chunk_key_encoding, + write_data=True, + zarr_format=2, + ) + z[:] = array_params.values + return z + + +# TODO: make this dynamic based on the installed scripts +script_paths = [Path(__file__).resolve().parent / "scripts" / "v2.18.py"] + + +@pytest.mark.skipif(not runner_installed(), reason="no python script runner installed") +@pytest.mark.parametrize( + "source_array", array_cases, indirect=True, ids=tuple(map(str, array_cases)) +) +@pytest.mark.parametrize("script_path", script_paths) +def test_roundtrip(source_array: Array, tmp_path: Path, script_path: Path) -> None: + out_path = tmp_path / "out" + copy_op = subprocess.run( + [ + "uv", + "run", + script_path, + str(source_array.store).removeprefix("file://"), + str(out_path), + ], + capture_output=True, + text=True, + ) + assert copy_op.returncode == 0 + out_array = zarr.open_array(store=out_path, mode="r", zarr_format=2) + assert source_array.metadata.to_dict() == out_array.metadata.to_dict() + assert np.array_equal(source_array[:], out_array[:]) diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py index a17d7a55be..c0997c3df3 100644 --- a/tests/test_store/test_stateful.py +++ b/tests/test_store/test_stateful.py @@ -15,6 +15,7 @@ ] +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_zarr_hierarchy(sync_store: Store): def mk_test_instance_sync() -> ZarrHierarchyStateMachine: return ZarrHierarchyStateMachine(sync_store) diff --git a/tests/test_strings.py b/tests/test_strings.py deleted file mode 100644 index dca0570a25..0000000000 --- a/tests/test_strings.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Tests for the strings module.""" - -import numpy as np -import pytest - -from zarr.core.strings import _NUMPY_SUPPORTS_VLEN_STRING, _STRING_DTYPE, cast_to_string_dtype - - -def test_string_defaults() -> None: - if _NUMPY_SUPPORTS_VLEN_STRING: - assert _STRING_DTYPE == np.dtypes.StringDType() - else: - assert _STRING_DTYPE == np.dtypes.ObjectDType() - - -def test_cast_to_string_dtype() -> None: - d1 = np.array(["a", "b", "c"]) - assert d1.dtype == np.dtype(" None: assert np.array_equal(data, a[:, :]) -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize( - ("dtype", "fill_value"), - [ - ("bool", False), - ("int64", 0), - ("float64", 0.0), - ("|S1", b""), - ("|U1", ""), - ("object", ""), - (str, ""), - ], -) -def test_implicit_fill_value(store: MemoryStore, dtype: str, fill_value: Any) -> None: - arr = zarr.create(store=store, shape=(4,), fill_value=None, zarr_format=2, dtype=dtype) - assert arr.metadata.fill_value is None - assert arr.metadata.to_dict()["fill_value"] is None - result = arr[:] - if dtype is str: - # special case - numpy_dtype = np.dtype(object) - else: - numpy_dtype = np.dtype(dtype) - expected = np.full(arr.shape, fill_value, dtype=numpy_dtype) - np.testing.assert_array_equal(result, expected) - - def test_codec_pipeline() -> None: # https://github.com/zarr-developers/zarr-python/issues/2243 store = MemoryStore() @@ -86,14 +61,14 @@ def test_codec_pipeline() -> None: @pytest.mark.parametrize( - ("dtype", "expected_dtype", "fill_value", "fill_value_encoding"), + ("dtype", "expected_dtype", "fill_value", "fill_value_json"), [ - ("|S", "|S0", b"X", "WA=="), - ("|V", "|V0", b"X", "WA=="), + ("|S1", "|S1", b"X", "WA=="), + ("|V1", "|V1", b"X", "WA=="), ("|V10", "|V10", b"X", "WAAAAAAAAAAAAA=="), ], ) -async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_encoding) -> None: +async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_json) -> None: with config.set( { "array.v2_default_filters.bytes": [{"id": "vlen-bytes"}], @@ -114,8 +89,8 @@ async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_en "chunks": [3], "compressor": None, "dtype": expected_dtype, - "fill_value": fill_value_encoding, - "filters": [{"id": "vlen-bytes"}] if dtype == "|S" else None, + "fill_value": fill_value_json, + "filters": None, "order": "C", "shape": [3], "zarr_format": 2, @@ -128,37 +103,24 @@ async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_en np.testing.assert_equal(data, expected) -@pytest.mark.parametrize("dtype_value", [["|S", b"Y"], ["|U", "Y"], ["O", b"Y"]]) -def test_v2_encode_decode_with_data(dtype_value): - dtype, value = dtype_value - with config.set( - { - "array.v2_default_filters": { - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], - }, - } - ): - expected = np.full((3,), value, dtype=dtype) - a = zarr.create( - shape=(3,), - zarr_format=2, - dtype=dtype, - ) - a[:] = expected - data = a[:] - np.testing.assert_equal(data, expected) - - -@pytest.mark.parametrize("dtype", [str, "str"]) -async def test_create_dtype_str(dtype: Any) -> None: - arr = zarr.create(shape=3, dtype=dtype, zarr_format=2) - assert arr.dtype.kind == "O" - assert arr.metadata.to_dict()["dtype"] == "|O" - assert arr.metadata.filters == (numcodecs.vlen.VLenBytes(),) - arr[:] = [b"a", b"bb", b"ccc"] - result = arr[:] - np.testing.assert_array_equal(result, np.array([b"a", b"bb", b"ccc"], dtype="object")) +@pytest.mark.parametrize( + ("dtype", "value"), + [ + (NullTerminatedBytes(length=1), b"Y"), + (FixedLengthUTF32(length=1), "Y"), + (VariableLengthUTF8(), "Y"), + ], +) +def test_v2_encode_decode_with_data(dtype: ZDType[Any, Any], value: str): + expected = np.full((3,), value, dtype=dtype.to_native_dtype()) + a = zarr.create( + shape=(3,), + zarr_format=2, + dtype=dtype, + ) + a[:] = expected + data = a[:] + np.testing.assert_equal(data, expected) @pytest.mark.parametrize("filters", [[], [numcodecs.Delta(dtype=" None: - with config.set( - { - "array.v2_default_compressor": { - "numeric": {"id": "zstd", "level": "0"}, - "string": {"id": "zstd", "level": "0"}, - "bytes": {"id": "zstd", "level": "0"}, - }, - "array.v2_default_filters": { - "numeric": [], - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], - }, - } - ): - dtype, expected_compressor, expected_filter = dtype_expected - arr = zarr.create(shape=(3,), path="foo", store={}, zarr_format=2, dtype=dtype) - assert arr.metadata.compressor.codec_id == expected_compressor - if expected_filter is not None: - assert arr.metadata.filters[0].codec_id == expected_filter - - @pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"]) def test_structured_dtype_roundtrip(fill_value, tmp_path) -> None: a = np.array( @@ -339,35 +269,18 @@ def test_structured_dtype_roundtrip(fill_value, tmp_path) -> None: np.dtype([("x", "i4"), ("y", "i4")]), np.array([(1, 2)], dtype=[("x", "i4"), ("y", "i4")])[0], ), - ( - "BQAAAA==", - np.dtype([("val", "i4")]), - np.array([(5,)], dtype=[("val", "i4")])[0], - ), - ( - {"x": 1, "y": 2}, - np.dtype([("location", "O")]), - np.array([({"x": 1, "y": 2},)], dtype=[("location", "O")])[0], - ), - ( - {"x": 1, "y": 2, "z": 3}, - np.dtype([("location", "O")]), - np.array([({"x": 1, "y": 2, "z": 3},)], dtype=[("location", "O")])[0], - ), ], ids=[ "tuple_input", "list_input", "bytes_input", - "string_input", - "dictionary_input", - "dictionary_input_extra_fields", ], ) def test_parse_structured_fill_value_valid( fill_value: Any, dtype: np.dtype[Any], expected_result: Any ) -> None: - result = _parse_structured_fill_value(fill_value, dtype) + zdtype = Structured.from_native_dtype(dtype) + result = zdtype.cast_scalar(fill_value) assert result.dtype == expected_result.dtype assert result == expected_result if isinstance(expected_result, np.void): @@ -375,31 +288,6 @@ def test_parse_structured_fill_value_valid( assert result[name] == expected_result[name] -@pytest.mark.parametrize( - ( - "fill_value", - "dtype", - ), - [ - (("Alice", 30), np.dtype([("name", "U10"), ("age", "i4"), ("city", "U20")])), - (b"\x01\x00\x00\x00", np.dtype([("x", "i4"), ("y", "i4")])), - ("this_is_not_base64", np.dtype([("val", "i4")])), - ("hello", np.dtype([("age", "i4")])), - ({"x": 1, "y": 2}, np.dtype([("location", "i4")])), - ], - ids=[ - "tuple_list_wrong_length", - "bytes_wrong_length", - "invalid_base64", - "wrong_data_type", - "wrong_dictionary", - ], -) -def test_parse_structured_fill_value_invalid(fill_value: Any, dtype: np.dtype[Any]) -> None: - with pytest.raises(ValueError): - _parse_structured_fill_value(fill_value, dtype) - - @pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"]) def test_other_dtype_roundtrip(fill_value, tmp_path) -> None: a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7") From 7f4a681b65505734edaf868c242b461a1215a9e5 Mon Sep 17 00:00:00 2001 From: Lukas Bindreiter Date: Wed, 18 Jun 2025 10:56:28 +0200 Subject: [PATCH 152/160] Fix URL to contributing guide (#3141) --- .github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 29281f5be9..9503f3df8a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,4 +1,4 @@ Contributing ============ -Please see the [project documentation](https://zarr.readthedocs.io/en/stable/contributing.html) for information about contributing to Zarr. +Please see the [project documentation](https://zarr.readthedocs.io/en/stable/developers/contributing.html) for information about contributing to Zarr. From f68bf066f0cbbbce4d3ad60865f2058251e7e5e0 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 18 Jun 2025 15:31:13 -0500 Subject: [PATCH 153/160] Public API for buffer objects (#2876) * Public API for buffer objects This moves the public imports from buffer things out of `zarr.core`. Abstract stuff is availble under `zarr.abc.buffer`. Concrete implementations are available under `zarr.buffer.{cpu,gpu}`. * changelog * absolute imports * fixed warning in doc build * Updated config * Updated config * wording * doc * Updated the test to reflect the break between class name and qualname The config uses the public `zarr.buffer.cpu.Buffer`, which differs from the implementation path `zarr.core.buffer.cpu.Buffer`. This is OK because the public API for getting the buffer doesn't depend on where it's implemented at. * backwards compat --- changes/2871.feature.rst | 8 ++++++++ docs/user-guide/config.rst | 4 ++-- docs/user-guide/extending.rst | 5 ++++- src/zarr/abc/buffer.py | 9 +++++++++ src/zarr/buffer/__init__.py | 12 ++++++++++++ src/zarr/buffer/cpu.py | 15 +++++++++++++++ src/zarr/buffer/gpu.py | 7 +++++++ src/zarr/core/buffer/cpu.py | 9 +++++++-- src/zarr/core/buffer/gpu.py | 8 ++++++-- src/zarr/core/config.py | 6 +++--- src/zarr/core/group.py | 2 +- src/zarr/registry.py | 14 ++++++++------ tests/test_buffer.py | 3 ++- tests/test_config.py | 30 ++++++++++++++++++++++++------ 14 files changed, 108 insertions(+), 24 deletions(-) create mode 100644 changes/2871.feature.rst create mode 100644 src/zarr/abc/buffer.py create mode 100644 src/zarr/buffer/__init__.py create mode 100644 src/zarr/buffer/cpu.py create mode 100644 src/zarr/buffer/gpu.py diff --git a/changes/2871.feature.rst b/changes/2871.feature.rst new file mode 100644 index 0000000000..a39f30c558 --- /dev/null +++ b/changes/2871.feature.rst @@ -0,0 +1,8 @@ +Added public API for Buffer ABCs and implementations. + +Use :mod:`zarr.buffer` to access buffer implementations, and +:mod:`zarr.abc.buffer` for the interface to implement new buffer types. + +Users previously importing buffer from ``zarr.core.buffer`` should update their +imports to use :mod:`zarr.buffer`. As a reminder, all of ``zarr.core`` is +considered a private API that's not covered by zarr-python's versioning policy. \ No newline at end of file diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst index 4479e30619..5a9d26f2b9 100644 --- a/docs/user-guide/config.rst +++ b/docs/user-guide/config.rst @@ -63,7 +63,7 @@ This is the current default configuration:: 'variable-length-string': {'name': 'vlen-utf8'}}, 'write_empty_chunks': False}, 'async': {'concurrency': 10, 'timeout': None}, - 'buffer': 'zarr.core.buffer.cpu.Buffer', + 'buffer': 'zarr.buffer.cpu.Buffer', 'codec_pipeline': {'batch_size': 1, 'path': 'zarr.core.codec_pipeline.BatchedCodecPipeline'}, 'codecs': {'blosc': 'zarr.codecs.blosc.BloscCodec', @@ -78,5 +78,5 @@ This is the current default configuration:: 'zstd': 'zarr.codecs.zstd.ZstdCodec'}, 'default_zarr_format': 3, 'json_indent': 2, - 'ndbuffer': 'zarr.core.buffer.cpu.NDBuffer', + 'ndbuffer': 'zarr.buffer.cpu.NDBuffer', 'threading': {'max_workers': None}} diff --git a/docs/user-guide/extending.rst b/docs/user-guide/extending.rst index 7647703fbb..4487e07ddf 100644 --- a/docs/user-guide/extending.rst +++ b/docs/user-guide/extending.rst @@ -83,7 +83,10 @@ Coming soon. Custom array buffers -------------------- -Coming soon. +Zarr-python provides control over where and how arrays stored in memory through +:mod:`zarr.buffer`. Currently both CPU (the default) and GPU implementations are +provided (see :ref:`user-guide-gpu` for more). You can implement your own buffer +classes by implementing the interface defined in :mod:`zarr.abc.buffer`. Other extensions ---------------- diff --git a/src/zarr/abc/buffer.py b/src/zarr/abc/buffer.py new file mode 100644 index 0000000000..3d5ac07157 --- /dev/null +++ b/src/zarr/abc/buffer.py @@ -0,0 +1,9 @@ +from zarr.core.buffer.core import ArrayLike, Buffer, BufferPrototype, NDArrayLike, NDBuffer + +__all__ = [ + "ArrayLike", + "Buffer", + "BufferPrototype", + "NDArrayLike", + "NDBuffer", +] diff --git a/src/zarr/buffer/__init__.py b/src/zarr/buffer/__init__.py new file mode 100644 index 0000000000..db393f66c7 --- /dev/null +++ b/src/zarr/buffer/__init__.py @@ -0,0 +1,12 @@ +""" +Implementations of the Zarr Buffer interface. + +See Also +======== +zarr.abc.buffer: Abstract base class for the Zarr Buffer interface. +""" + +from zarr.buffer import cpu, gpu +from zarr.core.buffer import default_buffer_prototype + +__all__ = ["cpu", "default_buffer_prototype", "gpu"] diff --git a/src/zarr/buffer/cpu.py b/src/zarr/buffer/cpu.py new file mode 100644 index 0000000000..5307927c06 --- /dev/null +++ b/src/zarr/buffer/cpu.py @@ -0,0 +1,15 @@ +from zarr.core.buffer.cpu import ( + Buffer, + NDBuffer, + as_numpy_array_wrapper, + buffer_prototype, + numpy_buffer_prototype, +) + +__all__ = [ + "Buffer", + "NDBuffer", + "as_numpy_array_wrapper", + "buffer_prototype", + "numpy_buffer_prototype", +] diff --git a/src/zarr/buffer/gpu.py b/src/zarr/buffer/gpu.py new file mode 100644 index 0000000000..dbdc1b1357 --- /dev/null +++ b/src/zarr/buffer/gpu.py @@ -0,0 +1,7 @@ +from zarr.core.buffer.gpu import Buffer, NDBuffer, buffer_prototype + +__all__ = [ + "Buffer", + "NDBuffer", + "buffer_prototype", +] diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 3022bafb6f..3140d75111 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -224,5 +224,10 @@ def numpy_buffer_prototype() -> core.BufferPrototype: return core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) -register_buffer(Buffer) -register_ndbuffer(NDBuffer) +register_buffer(Buffer, qualname="zarr.buffer.cpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.buffer.cpu.NDBuffer") + + +# backwards compatibility +register_buffer(Buffer, qualname="zarr.core.buffer.cpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.cpu.NDBuffer") diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index 88746c5fac..7ea6d53fe3 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -220,5 +220,9 @@ def __setitem__(self, key: Any, value: Any) -> None: buffer_prototype = BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) -register_buffer(Buffer) -register_ndbuffer(NDBuffer) +register_buffer(Buffer, qualname="zarr.buffer.gpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.buffer.gpu.NDBuffer") + +# backwards compatibility +register_buffer(Buffer, qualname="zarr.core.buffer.gpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.gpu.NDBuffer") diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 74e9bdd8dd..05d048ef74 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -74,7 +74,7 @@ def enable_gpu(self) -> ConfigSet: Configure Zarr to use GPUs where possible. """ return self.set( - {"buffer": "zarr.core.buffer.gpu.Buffer", "ndbuffer": "zarr.core.buffer.gpu.NDBuffer"} + {"buffer": "zarr.buffer.gpu.Buffer", "ndbuffer": "zarr.buffer.gpu.NDBuffer"} ) @@ -128,8 +128,8 @@ def enable_gpu(self) -> ConfigSet: "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", }, - "buffer": "zarr.core.buffer.cpu.Buffer", - "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", } ], ) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index b50bce3aef..4c8ced21f4 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1454,7 +1454,7 @@ async def create_hierarchy( group already exists at path ``a``, then this function will leave the group at ``a`` as-is. Yields - ------- + ------ tuple[str, AsyncArray | AsyncGroup]. """ # check that all the nodes have the same zarr_format as Self diff --git a/src/zarr/registry.py b/src/zarr/registry.py index d1fe1d181c..eb345b24b1 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -47,8 +47,10 @@ def lazy_load(self) -> None: self.lazy_load_list.clear() - def register(self, cls: type[T]) -> None: - self[fully_qualified_name(cls)] = cls + def register(self, cls: type[T], qualname: str | None = None) -> None: + if qualname is None: + qualname = fully_qualified_name(cls) + self[qualname] = cls __codec_registries: dict[str, Registry[Codec]] = defaultdict(Registry) @@ -131,12 +133,12 @@ def register_pipeline(pipe_cls: type[CodecPipeline]) -> None: __pipeline_registry.register(pipe_cls) -def register_ndbuffer(cls: type[NDBuffer]) -> None: - __ndbuffer_registry.register(cls) +def register_ndbuffer(cls: type[NDBuffer], qualname: str | None = None) -> None: + __ndbuffer_registry.register(cls, qualname) -def register_buffer(cls: type[Buffer]) -> None: - __buffer_registry.register(cls) +def register_buffer(cls: type[Buffer], qualname: str | None = None) -> None: + __buffer_registry.register(cls, qualname) def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]: diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 11ff7cd96c..93b116e908 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -6,12 +6,13 @@ import pytest import zarr +from zarr.abc.buffer import ArrayLike, BufferPrototype, NDArrayLike +from zarr.buffer import cpu, gpu from zarr.codecs.blosc import BloscCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec -from zarr.core.buffer import ArrayLike, BufferPrototype, NDArrayLike, cpu, gpu from zarr.storage import MemoryStore, StorePath from zarr.testing.buffer import ( NDBufferUsingTestNDArrayLike, diff --git a/tests/test_config.py b/tests/test_config.py index 1dc6f8bf4f..ed778a02ae 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -101,8 +101,8 @@ def test_config_defaults_set() -> None: "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", }, - "buffer": "zarr.core.buffer.cpu.Buffer", - "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", } ] ) @@ -224,9 +224,6 @@ class NewBloscCodec(BloscCodec): @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_ndbuffer_implementation(store: Store) -> None: - # has default value - assert fully_qualified_name(get_ndbuffer_class()) == config.defaults[0]["ndbuffer"] - # set custom ndbuffer with TestNDArrayLike implementation register_ndbuffer(NDBufferUsingTestNDArrayLike) with config.set({"ndbuffer": fully_qualified_name(NDBufferUsingTestNDArrayLike)}): @@ -244,7 +241,7 @@ def test_config_ndbuffer_implementation(store: Store) -> None: def test_config_buffer_implementation() -> None: # has default value - assert fully_qualified_name(get_buffer_class()) == config.defaults[0]["buffer"] + assert config.defaults[0]["buffer"] == "zarr.buffer.cpu.Buffer" arr = zeros(shape=(100,), store=StoreExpectingTestBuffer()) @@ -279,6 +276,27 @@ def test_config_buffer_implementation() -> None: assert np.array_equal(arr_Crc32c[:], data2d) +def test_config_buffer_backwards_compatibility() -> None: + # This should warn once zarr.core is private + # https://github.com/zarr-developers/zarr-python/issues/2621 + with zarr.config.set( + {"buffer": "zarr.core.buffer.cpu.Buffer", "ndbuffer": "zarr.core.buffer.cpu.NDBuffer"} + ): + get_buffer_class() + get_ndbuffer_class() + + +@pytest.mark.gpu +def test_config_buffer_backwards_compatibility_gpu() -> None: + # This should warn once zarr.core is private + # https://github.com/zarr-developers/zarr-python/issues/2621 + with zarr.config.set( + {"buffer": "zarr.core.buffer.gpu.Buffer", "ndbuffer": "zarr.core.buffer.gpu.NDBuffer"} + ): + get_buffer_class() + get_ndbuffer_class() + + @pytest.mark.filterwarnings("error") def test_warning_on_missing_codec_config() -> None: class NewCodec(BytesCodec): From 4d71e93e3db6706ed27bd07eed5b9b658e812bed Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 18 Jun 2025 23:26:34 +0100 Subject: [PATCH 154/160] Add GroupNotFound error to API docs (#3149) --- src/zarr/errors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/zarr/errors.py b/src/zarr/errors.py index 4d3140a4a9..4f972a6703 100644 --- a/src/zarr/errors.py +++ b/src/zarr/errors.py @@ -5,6 +5,7 @@ "ContainsArrayAndGroupError", "ContainsArrayError", "ContainsGroupError", + "GroupNotFoundError", "MetadataValidationError", "NodeTypeValidationError", ] From 72786ea079a02bf539ee1437110e0c11c4841017 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 19 Jun 2025 10:38:06 +0200 Subject: [PATCH 155/160] Get rid of old notebooks (#2897) * Get rid of deprecated stores LMDBStore/DBMStore * Get rid of old notebooks --- notebooks/advanced_indexing.ipynb | 2798 ----------------- notebooks/blosc_microbench.ipynb | 200 -- notebooks/dask_2d_subset.ipynb | 869 ----- notebooks/dask_copy.ipynb | 1518 --------- notebooks/dask_count_alleles.ipynb | 648 ---- .../genotype_benchmark_compressors.ipynb | 548 ---- notebooks/object_arrays.ipynb | 350 --- notebooks/repr_info.ipynb | 365 --- notebooks/store_benchmark.ipynb | 1303 -------- notebooks/zip_benchmark.ipynb | 343 -- 10 files changed, 8942 deletions(-) delete mode 100644 notebooks/advanced_indexing.ipynb delete mode 100644 notebooks/blosc_microbench.ipynb delete mode 100644 notebooks/dask_2d_subset.ipynb delete mode 100644 notebooks/dask_copy.ipynb delete mode 100644 notebooks/dask_count_alleles.ipynb delete mode 100644 notebooks/genotype_benchmark_compressors.ipynb delete mode 100644 notebooks/object_arrays.ipynb delete mode 100644 notebooks/repr_info.ipynb delete mode 100644 notebooks/store_benchmark.ipynb delete mode 100644 notebooks/zip_benchmark.ipynb diff --git a/notebooks/advanced_indexing.ipynb b/notebooks/advanced_indexing.ipynb deleted file mode 100644 index eba6b5880b..0000000000 --- a/notebooks/advanced_indexing.ipynb +++ /dev/null @@ -1,2798 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Advanced indexing" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.1.5.dev144'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "import numpy as np\n", - "np.random.seed(42)\n", - "import cProfile\n", - "zarr.__version__" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Functionality and API" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing a 1D array with a Boolean (mask) array\n", - "\n", - "Supported via ``get/set_mask_selection()`` and ``.vindex[]``. Also supported via ``get/set_orthogonal_selection()`` and ``.oindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)\n", - "ix = [False, True, False, True, False, True, False, True, False, True]" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.oindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 2, 30, 4, 50, 6, 70, 8, 90])" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.vindex[ix] = a[ix] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 100, 2, 300, 4, 500, 6, 700, 8, 900])" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.oindex[ix] = a[ix] * 100\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# if using .oindex, indexing array can be any array-like, e.g., Zarr array\n", - "zix = zarr.array(ix, chunks=2)\n", - "za = zarr.array(a, chunks=2)\n", - "za.oindex[zix] # will not load all zix into memory" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing a 1D array with a 1D integer (coordinate) array\n", - "\n", - "Supported via ``get/set_coordinate_selection()`` and ``.vindex[]``. Also supported via ``get/set_orthogonal_selection()`` and ``.oindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)\n", - "ix = [1, 3, 5, 7, 9]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.oindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 2, 30, 4, 50, 6, 70, 8, 90])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.vindex[ix] = a[ix] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 100, 2, 300, 4, 500, 6, 700, 8, 900])" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.oindex[ix] = a[ix] * 100\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing a 1D array with a multi-dimensional integer (coordinate) array\n", - "\n", - "Supported via ``get/set_coordinate_selection()`` and ``.vindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)\n", - "ix = np.array([[1, 3, 5], [2, 4, 6]])" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[1, 3, 5],\n", - " [2, 4, 6]])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 20, 30, 40, 50, 60, 7, 8, 9])" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.vindex[ix] = a[ix] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Slicing a 1D array with step > 1\n", - "\n", - "Slices with step > 1 are supported via ``get/set_basic_selection()``, ``get/set_orthogonal_selection()``, ``__getitem__`` and ``.oindex[]``. Negative steps are not supported." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za[1::2]" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 2, 30, 4, 50, 6, 70, 8, 90])" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.oindex[1::2] = a[1::2] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Orthogonal (outer) indexing of multi-dimensional arrays\n", - "\n", - "Orthogonal (a.k.a. outer) indexing is supported with either Boolean or integer arrays, in combination with integers and slices. This functionality is provided via the ``get/set_orthogonal_selection()`` methods. For convenience, this functionality is also available via the ``.oindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# orthogonal indexing with Boolean arrays\n", - "ix0 = [False, True, False, True, False]\n", - "ix1 = [True, False, True]\n", - "za.get_orthogonal_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# orthogonal indexing with integer arrays\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "za.get_orthogonal_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 4, 5],\n", - " [ 9, 10, 11]])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# combine with slice\n", - "za.oindex[[1, 3], :]" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 2],\n", - " [ 3, 5],\n", - " [ 6, 8],\n", - " [ 9, 11],\n", - " [12, 14]])" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# combine with slice\n", - "za.oindex[:, [0, 2]]" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 42],\n", - " [ 6, 7, 8],\n", - " [42, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items via Boolean selection\n", - "ix0 = [False, True, False, True, False]\n", - "ix1 = [True, False, True]\n", - "selection = ix0, ix1\n", - "value = 42\n", - "za.set_orthogonal_selection(selection, value)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 44],\n", - " [ 6, 7, 8],\n", - " [44, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [46, 4, 46],\n", - " [ 6, 7, 8],\n", - " [46, 10, 46],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items via integer selection\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "selection = ix0, ix1\n", - "value = 46\n", - "za.set_orthogonal_selection(selection, value)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [48, 4, 48],\n", - " [ 6, 7, 8],\n", - " [48, 10, 48],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1] = 48\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Coordinate indexing of multi-dimensional arrays\n", - "\n", - "Selecting arbitrary points from a multi-dimensional array by indexing with integer (coordinate) arrays is supported. This functionality is provided via the ``get/set_coordinate_selection()`` methods. For convenience, this functionality is also available via the ``.vindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "za.get_coordinate_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.vindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.set_coordinate_selection((ix0, ix1), 42)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.vindex[ix0, ix1] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Mask indexing of multi-dimensional arrays\n", - "\n", - "Selecting arbitrary points from a multi-dimensional array by a Boolean array is supported. This functionality is provided via the ``get/set_mask_selection()`` methods. For convenience, this functionality is also available via the ``.vindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix = np.zeros_like(a, dtype=bool)\n", - "ix[1, 0] = True\n", - "ix[3, 2] = True\n", - "za.get_mask_selection(ix)" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.set_mask_selection(ix, 42)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.vindex[ix] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Selecting fields from arrays with a structured dtype\n", - "\n", - "All ``get/set_selection_...()`` methods support a ``fields`` argument which allows retrieving/replacing data for a specific field or fields. Also h5py-like API is supported where fields can be provided within ``__getitem__``, ``.oindex[]`` and ``.vindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([(b'aaa', 1, 4.2), (b'bbb', 2, 8.4), (b'ccc', 3, 12.6)],\n", - " dtype=[('foo', 'S3'), ('bar', '\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'foo'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'baz'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices" - ] - } - ], - "source": [ - "a['foo', 'baz']" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([(b'aaa', 4.2), (b'bbb', 8.4), (b'ccc', 12.6)],\n", - " dtype=[('foo', 'S3'), ('baz', '", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mza\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'foo'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'baz'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 537\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 538\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 539\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEllipsis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mget_basic_selection\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 661\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 663\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_basic_selection_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 664\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 665\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_basic_selection_nd\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 702\u001b[0m \u001b[0;31m# setup indexer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 703\u001b[0;31m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 704\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 705\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/indexing.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, selection, array)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 276\u001b[0m raise IndexError('unsupported selection item for basic indexing; expected integer '\n\u001b[0;32m--> 277\u001b[0;31m 'or slice, got {!r}'.format(type(dim_sel)))\n\u001b[0m\u001b[1;32m 278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 279\u001b[0m \u001b[0mdim_indexers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim_indexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mIndexError\u001b[0m: unsupported selection item for basic indexing; expected integer or slice, got " - ] - } - ], - "source": [ - "za[['foo', 'baz']]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1D Benchmarking" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "800000000" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c = np.arange(100000000)\n", - "c.nbytes" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 480 ms, sys: 16 ms, total: 496 ms\n", - "Wall time: 141 ms\n" - ] - }, - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeint64
Shape(100000000,)
Chunk shape(97657,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes800000000 (762.9M)
No. bytes stored11854081 (11.3M)
Storage ratio67.5
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (100000000,)\n", - "Chunk shape : (97657,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 800000000 (762.9M)\n", - "No. bytes stored : 11854081 (11.3M)\n", - "Storage ratio : 67.5\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time zc = zarr.array(c)\n", - "zc.info" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "121 ms ± 1.49 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c.copy()" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "254 ms ± 942 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool dense selection" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9997476" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# relatively dense selection - 10%\n", - "ix_dense_bool = np.random.binomial(1, 0.1, size=c.shape[0]).astype(bool)\n", - "np.count_nonzero(ix_dense_bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "243 ms ± 5.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "433 ms ± 6.49 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "548 ms ± 5.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [], - "source": [ - "import tempfile\n", - "import cProfile\n", - "import pstats\n", - "\n", - "def profile(statement, sort='time', restrictions=(7,)):\n", - " with tempfile.NamedTemporaryFile() as f:\n", - " cProfile.run(statement, filename=f.name)\n", - " pstats.Stats(f.name).sort_stats(sort).print_stats(*restrictions)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:17:48 2017 /tmp/tmpruua2rs_\n", - "\n", - " 98386 function calls in 0.483 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 83 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1025 0.197 0.000 0.197 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.149 0.000 0.159 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.044 0.000 0.231 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1024 0.009 0.000 0.009 0.000 {built-in method numpy.core.multiarray.count_nonzero}\n", - " 1025 0.007 0.000 0.238 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.006 0.000 0.207 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/core.py:337()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Method ``nonzero`` is being called internally within numpy to convert bool to int selections, no way to avoid." - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:18:06 2017 /tmp/tmp7_bautep\n", - "\n", - " 52382 function calls in 0.592 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 2 0.219 0.110 0.219 0.110 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.096 0.000 0.101 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.094 0.047 0.094 0.047 ../zarr/indexing.py:630()\n", - " 1024 0.044 0.000 0.167 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.029 0.029 0.029 0.029 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 1 0.021 0.021 0.181 0.181 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "``.vindex[]`` is a bit slower, possibly because internally it converts to a coordinate array first." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int dense selection" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000000" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix_dense_int = np.random.choice(c.shape[0], size=c.shape[0]//10, replace=True)\n", - "ix_dense_int_sorted = ix_dense_int.copy()\n", - "ix_dense_int_sorted.sort()\n", - "len(ix_dense_int)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "62.2 ms ± 2.36 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "355 ms ± 3.53 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "351 ms ± 3.51 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "128 ms ± 137 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.71 s ± 5.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.68 s ± 3.87 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:09 2017 /tmp/tmpgmu5btr_\n", - "\n", - " 95338 function calls in 0.424 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 89 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.141 0.141 0.184 0.184 ../zarr/indexing.py:369(__init__)\n", - " 1024 0.099 0.000 0.106 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.046 0.000 0.175 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.027 0.000 0.027 0.000 ../zarr/indexing.py:424(__iter__)\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 1 0.010 0.010 0.010 0.010 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/function_base.py:1848(diff)\n", - " 1025 0.006 0.000 0.059 0.000 ../zarr/indexing.py:541(__iter__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_int_sorted]')" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:13 2017 /tmp/tmpay1gvnx8\n", - "\n", - " 52362 function calls in 0.398 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 85 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 2 0.107 0.054 0.107 0.054 ../zarr/indexing.py:630()\n", - " 1024 0.091 0.000 0.096 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.041 0.000 0.160 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.040 0.040 0.213 0.213 ../zarr/indexing.py:603(__init__)\n", - " 1 0.029 0.029 0.029 0.029 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 2048 0.011 0.000 0.011 0.000 ../zarr/indexing.py:695()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_int_sorted]')" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:20 2017 /tmp/tmpngsf6zpp\n", - "\n", - " 120946 function calls in 1.793 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 92 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.128 1.128 1.128 1.128 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 1024 0.139 0.000 0.285 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.132 0.132 1.422 1.422 ../zarr/indexing.py:369(__init__)\n", - " 1 0.120 0.120 0.120 0.120 {method 'take' of 'numpy.ndarray' objects}\n", - " 1024 0.116 0.000 0.123 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.034 0.000 0.034 0.000 ../zarr/indexing.py:424(__iter__)\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_int]')" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:22 2017 /tmp/tmpbskhj8de\n", - "\n", - " 50320 function calls in 1.730 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 86 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.116 1.116 1.116 1.116 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 1024 0.133 0.000 0.275 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2 0.121 0.060 0.121 0.060 ../zarr/indexing.py:654()\n", - " 1024 0.113 0.000 0.119 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.100 0.050 0.100 0.050 ../zarr/indexing.py:630()\n", - " 1 0.030 0.030 0.030 0.030 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.024 0.024 1.427 1.427 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_int]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When indices are not sorted, zarr needs to partially sort them so the occur in chunk order, so we only have to visit each chunk once. This sorting dominates the processing time and is unavoidable AFAIK." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool sparse selection" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9932" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# relatively sparse selection\n", - "ix_sparse_bool = np.random.binomial(1, 0.0001, size=c.shape[0]).astype(bool)\n", - "np.count_nonzero(ix_sparse_bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "15.7 ms ± 38.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "156 ms ± 2.1 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "133 ms ± 2.76 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:20:09 2017 /tmp/tmpb7nqc9ax\n", - "\n", - " 98386 function calls in 0.191 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 83 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.093 0.000 0.098 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.017 0.000 0.017 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.007 0.000 0.007 0.000 {built-in method numpy.core.multiarray.count_nonzero}\n", - " 1024 0.007 0.000 0.129 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.005 0.000 0.052 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.005 0.000 0.025 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_sparse_bool]')" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:20:09 2017 /tmp/tmphsko8nvh\n", - "\n", - " 52382 function calls in 0.160 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.093 0.000 0.098 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.017 0.008 0.017 0.008 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1025 0.008 0.000 0.014 0.000 ../zarr/indexing.py:674(__iter__)\n", - " 1024 0.006 0.000 0.127 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/indexing.py:695()\n", - " 2054 0.003 0.000 0.003 0.000 ../zarr/core.py:337()\n", - " 1024 0.002 0.000 0.005 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/core/arrayprint.py:381(wrapper)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_sparse_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int sparse selection" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000" - ] - }, - "execution_count": 81, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix_sparse_int = np.random.choice(c.shape[0], size=c.shape[0]//10000, replace=True)\n", - "ix_sparse_int_sorted = ix_sparse_int.copy()\n", - "ix_sparse_int_sorted.sort()\n", - "len(ix_sparse_int)" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "18.9 µs ± 392 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "20.3 µs ± 155 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "125 ms ± 296 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "109 ms ± 428 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "132 ms ± 489 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "108 ms ± 579 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:21:12 2017 /tmp/tmp0b0o2quo\n", - "\n", - " 120946 function calls in 0.196 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 92 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.105 0.000 0.111 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2048 0.006 0.000 0.013 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 1025 0.006 0.000 0.051 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.006 0.000 0.141 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/core.py:337()\n", - " 15373 0.004 0.000 0.010 0.000 {built-in method builtins.isinstance}\n", - " 1025 0.004 0.000 0.005 0.000 ../zarr/indexing.py:424(__iter__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_sparse_int]')" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:21:19 2017 /tmp/tmpdwju98kn\n", - "\n", - " 50320 function calls in 0.167 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 86 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.105 0.000 0.111 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.009 0.000 0.017 0.000 ../zarr/indexing.py:674(__iter__)\n", - " 1024 0.006 0.000 0.142 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/indexing.py:695()\n", - " 2054 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - " 1 0.003 0.003 0.162 0.162 ../zarr/core.py:591(_get_selection)\n", - " 1027 0.003 0.000 0.003 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_sparse_int]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For sparse selections, processing time is dominated by decompression, so we can't do any better." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### sparse bool selection as zarr array" - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typebool
Shape(100000000,)
Chunk shape(390625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes100000000 (95.4M)
No. bytes stored507131 (495.2K)
Storage ratio197.2
Chunks initialized256/256
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : bool\n", - "Shape : (100000000,)\n", - "Chunk shape : (390625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 100000000 (95.4M)\n", - "No. bytes stored : 507131 (495.2K)\n", - "Storage ratio : 197.2\n", - "Chunks initialized : 256/256" - ] - }, - "execution_count": 90, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zix_sparse_bool = zarr.array(ix_sparse_bool)\n", - "zix_sparse_bool.info" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "387 ms ± 5.47 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[zix_sparse_bool]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### slice with step" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "80.3 ms ± 377 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit np.array(c[::2])" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "168 ms ± 837 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::2]" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "136 ms ± 1.56 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::10]" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "104 ms ± 1.86 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::100]" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "100 ms ± 1.47 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::1000]" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:22:44 2017 /tmp/tmpg9dxqcpg\n", - "\n", - " 49193 function calls in 0.211 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 55 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.104 0.000 0.110 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.067 0.000 0.195 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.005 0.000 0.013 0.000 ../zarr/indexing.py:278(__iter__)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - " 2050 0.003 0.000 0.003 0.000 ../zarr/indexing.py:90(ceildiv)\n", - " 1025 0.003 0.000 0.006 0.000 ../zarr/indexing.py:109(__iter__)\n", - " 1024 0.003 0.000 0.003 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc[::2]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2D Benchmarking" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100000000,)" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100000, 1000)" - ] - }, - "execution_count": 100, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d = c.reshape(-1, 1000)\n", - "d.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeint64
Shape(100000, 1000)
Chunk shape(3125, 32)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes800000000 (762.9M)
No. bytes stored39228864 (37.4M)
Storage ratio20.4
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (100000, 1000)\n", - "Chunk shape : (3125, 32)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 800000000 (762.9M)\n", - "No. bytes stored : 39228864 (37.4M)\n", - "Storage ratio : 20.4\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 101, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zd = zarr.array(d)\n", - "zd.info" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool orthogonal selection" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [], - "source": [ - "ix0 = np.random.binomial(1, 0.5, size=d.shape[0]).astype(bool)\n", - "ix1 = np.random.binomial(1, 0.5, size=d.shape[1]).astype(bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "101 ms ± 577 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit d[np.ix_(ix0, ix1)]" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "373 ms ± 5.45 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int orthogonal selection" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "ix0 = np.random.choice(d.shape[0], size=int(d.shape[0] * .5), replace=True)\n", - "ix1 = np.random.choice(d.shape[1], size=int(d.shape[1] * .5), replace=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "174 ms ± 4.13 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit d[np.ix_(ix0, ix1)]" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "566 ms ± 12.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### coordinate (point) selection" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000000" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "n = int(d.size * .1)\n", - "ix0 = np.random.choice(d.shape[0], size=n, replace=True)\n", - "ix1 = np.random.choice(d.shape[1], size=n, replace=True)\n", - "n" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "243 ms ± 3.37 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit d[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2.03 s ± 17 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.vindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:24:31 2017 /tmp/tmp7c68z70p\n", - "\n", - " 62673 function calls in 2.065 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.112 1.112 1.112 1.112 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 3 0.244 0.081 0.244 0.081 ../zarr/indexing.py:654()\n", - " 3 0.193 0.064 0.193 0.064 ../zarr/indexing.py:630()\n", - " 1024 0.170 0.000 0.350 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1024 0.142 0.000 0.151 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1 0.044 0.044 0.044 0.044 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.043 0.043 1.676 1.676 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zd.vindex[ix0, ix1]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Points need to be partially sorted so all points in the same chunk are grouped and processed together. This requires ``argsort`` which dominates time." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## h5py comparison\n", - "\n", - "N.B., not really fair because using slower compressor, but for interest..." - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "import h5py\n", - "import tempfile" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [], - "source": [ - "h5f = h5py.File(tempfile.mktemp(), driver='core', backing_store=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hc = h5f.create_dataset('c', data=c, compression='gzip', compression_opts=1, chunks=zc.chunks, shuffle=True)\n", - "hc" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.16 s, sys: 172 ms, total: 1.33 s\n", - "Wall time: 1.32 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 0, 1, 2, ..., 99999997, 99999998, 99999999])" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time hc[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.11 s, sys: 0 ns, total: 1.11 s\n", - "Wall time: 1.11 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 1063, 28396, 37229, ..., 99955875, 99979354, 99995791])" - ] - }, - "execution_count": 81, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time hc[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [], - "source": [ - "# # this is pathological, takes minutes \n", - "# %time hc[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 38.3 s, sys: 136 ms, total: 38.4 s\n", - "Wall time: 38.1 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 0, 1000, 2000, ..., 99997000, 99998000, 99999000])" - ] - }, - "execution_count": 83, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# this is pretty slow\n", - "%time hc[::1000]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/blosc_microbench.ipynb b/notebooks/blosc_microbench.ipynb deleted file mode 100644 index 9361d8e95b..0000000000 --- a/notebooks/blosc_microbench.ipynb +++ /dev/null @@ -1,200 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.1'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10 loops, best of 3: 110 ms per loop\n", - "1 loop, best of 3: 235 ms per loop\n", - "Array((100000000,), int64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 11.2M; ratio: 67.8; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 loop, best of 3: 331 ms per loop\n", - "1 loop, best of 3: 246 ms per loop\n", - "Array((100000000,), float64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 724.8M; ratio: 1.1; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", - "data = np.random.normal(size=100000000)\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.2.dev0+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10 loops, best of 3: 92.7 ms per loop\n", - "1 loop, best of 3: 230 ms per loop\n", - "Array((100000000,), int64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 11.2M; ratio: 67.8; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 loop, best of 3: 338 ms per loop\n", - "1 loop, best of 3: 253 ms per loop\n", - "Array((100000000,), float64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 724.8M; ratio: 1.1; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", - "data = np.random.normal(size=100000000)\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/notebooks/dask_2d_subset.ipynb b/notebooks/dask_2d_subset.ipynb deleted file mode 100644 index 6e88b510d5..0000000000 --- a/notebooks/dask_2d_subset.ipynb +++ /dev/null @@ -1,869 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook has some profiling of Dask used to make a selection along both first and second axes of a large-ish multidimensional array. The use case is making selections of genotype data, e.g., as required for making a web-browser for genotype data as in www.malariagen.net/apps/ag1000g." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 2.1.1\n", - "dask 0.11.0\n" - ] - } - ], - "source": [ - "import zarr; print('zarr', zarr.__version__)\n", - "import dask; print('dask', dask.__version__)\n", - "import dask.array as da\n", - "import numpy as np" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Real data" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Group(/, 8)\n", - " arrays: 1; samples\n", - " groups: 7; 2L, 2R, 3L, 3R, UNKN, X, Y_unplaced\n", - " store: DirectoryStore" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# here's the real data\n", - "callset = zarr.open_group('/kwiat/2/coluzzi/ag1000g/data/phase1/release/AR3.1/variation/main/zarr2/zstd/ag1000g.phase1.ar3',\n", - " mode='r')\n", - "callset" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3R/calldata/genotype, (22632425, 765, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 32.2G; nbytes_stored: 1.0G; ratio: 31.8; initialized: 34540/34540\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: DirectoryStore" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# here's the array we're going to work with\n", - "g = callset['3R/calldata/genotype']\n", - "g" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 4 ms, sys: 0 ns, total: 4 ms\n", - "Wall time: 5.13 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# wrap as dask array with very simple chunking of first dim only\n", - "%time gd = da.from_array(g, chunks=(g.chunks[0], None, None))\n", - "gd" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "((22632425,), dtype('bool'), 13167162)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# load condition used to make selection on first axis\n", - "dim0_condition = callset['3R/variants/FILTER_PASS'][:]\n", - "dim0_condition.shape, dim0_condition.dtype, np.count_nonzero(dim0_condition)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "# invent a random selection for second axis\n", - "dim1_indices = sorted(np.random.choice(765, size=100, replace=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.3 s, sys: 256 ms, total: 15.5 s\n", - "Wall time: 15.5 s\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# setup the 2D selection - this is the slow bit\n", - "%time gd_sel = gd[dim0_condition][:, dim1_indices]\n", - "gd_sel" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.21 s, sys: 152 ms, total: 1.36 s\n", - "Wall time: 316 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " ..., \n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 1],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]]], dtype=int8)" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# now load a slice from this new selection - quick!\n", - "%time gd_sel[1000000:1100000].compute(optimize_graph=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 105406881 function calls (79072145 primitive calls) in 26.182 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "13167268/6 6.807 0.000 9.038 1.506 slicing.py:623(check_index)\n", - " 2 4.713 2.356 5.831 2.916 slicing.py:398(partition_by_size)\n", - "13167270/2 4.470 0.000 8.763 4.382 slicing.py:540(posify_index)\n", - " 52669338 4.118 0.000 4.119 0.000 {built-in method builtins.isinstance}\n", - " 2 2.406 1.203 8.763 4.382 slicing.py:563()\n", - " 1 0.875 0.875 0.875 0.875 slicing.py:44()\n", - " 13182474 0.600 0.000 0.600 0.000 {built-in method builtins.len}\n", - " 2 0.527 0.264 0.527 0.264 slicing.py:420(issorted)\n", - " 13189168 0.520 0.000 0.520 0.000 {method 'append' of 'list' objects}\n", - " 2 0.271 0.136 0.271 0.136 slicing.py:479()\n", - " 2 0.220 0.110 0.220 0.110 {built-in method builtins.sorted}\n", - " 1 0.162 0.162 0.162 0.162 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.113 0.056 26.071 13.035 core.py:1024(__getitem__)\n", - " 2 0.112 0.056 6.435 3.217 slicing.py:441(take_sorted)\n", - " 1 0.111 0.111 26.182 26.182 :1()\n", - " 2 0.060 0.030 24.843 12.422 slicing.py:142(slice_with_newaxes)\n", - " 106/3 0.039 0.000 1.077 0.359 slicing.py:15(sanitize_index)\n", - " 3 0.037 0.012 0.037 0.012 {built-in method _hashlib.openssl_md5}\n", - " 6726 0.012 0.000 0.017 0.000 slicing.py:567(insert_many)\n", - " 3364 0.004 0.000 0.021 0.000 slicing.py:156()\n", - " 20178 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 8 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 2 0.000 0.000 25.920 12.960 slicing.py:60(slice_array)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 106/4 0.000 0.000 0.037 0.009 utils.py:502(__call__)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 2 0.000 0.000 0.037 0.019 base.py:343(tokenize)\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 2 0.000 0.000 24.763 12.381 slicing.py:170(slice_wrap_lists)\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 6.962 3.481 slicing.py:487(take)\n", - " 1 0.000 0.000 26.182 26.182 {built-in method builtins.exec}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 1 0.000 0.000 0.037 0.037 base.py:314(normalize_array)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 2 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 4 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "# what's taking so long?\n", - "import cProfile\n", - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='time')" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 105406881 function calls (79072145 primitive calls) in 25.630 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 25.630 25.630 {built-in method builtins.exec}\n", - " 1 0.107 0.107 25.630 25.630 :1()\n", - " 2 0.102 0.051 25.523 12.761 core.py:1024(__getitem__)\n", - " 2 0.001 0.000 25.381 12.691 slicing.py:60(slice_array)\n", - " 2 0.049 0.024 24.214 12.107 slicing.py:142(slice_with_newaxes)\n", - " 2 0.000 0.000 24.147 12.073 slicing.py:170(slice_wrap_lists)\n", - "13167268/6 6.664 0.000 8.855 1.476 slicing.py:623(check_index)\n", - "13167270/2 4.354 0.000 8.466 4.233 slicing.py:540(posify_index)\n", - " 2 2.277 1.139 8.465 4.233 slicing.py:563()\n", - " 2 0.000 0.000 6.826 3.413 slicing.py:487(take)\n", - " 2 0.111 0.056 6.331 3.165 slicing.py:441(take_sorted)\n", - " 2 4.628 2.314 5.704 2.852 slicing.py:398(partition_by_size)\n", - " 52669338 4.026 0.000 4.026 0.000 {built-in method builtins.isinstance}\n", - " 106/3 0.071 0.001 1.167 0.389 slicing.py:15(sanitize_index)\n", - " 1 0.943 0.943 0.943 0.943 slicing.py:44()\n", - " 13182474 0.581 0.000 0.581 0.000 {built-in method builtins.len}\n", - " 13189168 0.497 0.000 0.497 0.000 {method 'append' of 'list' objects}\n", - " 2 0.495 0.248 0.495 0.248 slicing.py:420(issorted)\n", - " 2 0.281 0.141 0.281 0.141 slicing.py:479()\n", - " 2 0.234 0.117 0.234 0.117 {built-in method builtins.sorted}\n", - " 1 0.152 0.152 0.152 0.152 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.039 0.020 base.py:343(tokenize)\n", - " 106/4 0.000 0.000 0.039 0.010 utils.py:502(__call__)\n", - " 1 0.000 0.000 0.039 0.039 base.py:314(normalize_array)\n", - " 3 0.039 0.013 0.039 0.013 {built-in method _hashlib.openssl_md5}\n", - " 3364 0.003 0.000 0.019 0.000 slicing.py:156()\n", - " 6726 0.012 0.000 0.016 0.000 slicing.py:567(insert_many)\n", - " 20178 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 8 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 4 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 2 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='cumtime')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Synthetic data" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array((20000000, 200, 2), int8, chunks=(10000, 100, 2), order=C)\n", - " nbytes: 7.5G; nbytes_stored: 2.7G; ratio: 2.8; initialized: 4000/4000\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: dict" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# create a synthetic dataset for profiling\n", - "a = zarr.array(np.random.randint(-1, 4, size=(20000000, 200, 2), dtype='i1'),\n", - " chunks=(10000, 100, 2), compressor=zarr.Blosc(cname='zstd', clevel=1, shuffle=2))\n", - "a" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# create a synthetic selection for first axis\n", - "c = np.random.randint(0, 2, size=a.shape[0], dtype=bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# create a synthetic selection for second axis\n", - "s = sorted(np.random.choice(a.shape[1], size=100, replace=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 208 ms, sys: 0 ns, total: 208 ms\n", - "Wall time: 206 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d = da.from_array(a, chunks=(a.chunks[0], None, None))\n", - "d" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 12 s, sys: 200 ms, total: 12.2 s\n", - "Wall time: 12.2 s\n" - ] - } - ], - "source": [ - "%time ds = d[c][:, s]" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 80095589 function calls (60091843 primitive calls) in 19.467 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "10001773/6 4.872 0.000 6.456 1.076 slicing.py:623(check_index)\n", - " 2 3.517 1.758 4.357 2.179 slicing.py:398(partition_by_size)\n", - "10001775/2 3.354 0.000 6.484 3.242 slicing.py:540(posify_index)\n", - " 40007358 2.965 0.000 2.965 0.000 {built-in method builtins.isinstance}\n", - " 2 1.749 0.875 6.484 3.242 slicing.py:563()\n", - " 1 0.878 0.878 0.878 0.878 slicing.py:44()\n", - " 10019804 0.451 0.000 0.451 0.000 {built-in method builtins.len}\n", - " 10027774 0.392 0.000 0.392 0.000 {method 'append' of 'list' objects}\n", - " 2 0.363 0.181 0.363 0.181 slicing.py:420(issorted)\n", - " 2 0.270 0.135 4.786 2.393 slicing.py:441(take_sorted)\n", - " 1 0.207 0.207 0.207 0.207 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.158 0.079 0.158 0.079 {built-in method builtins.sorted}\n", - " 1 0.094 0.094 19.467 19.467 :1()\n", - " 2 0.079 0.040 19.373 9.686 core.py:1024(__getitem__)\n", - " 2 0.035 0.017 18.147 9.074 slicing.py:142(slice_with_newaxes)\n", - " 3 0.033 0.011 0.033 0.011 {built-in method _hashlib.openssl_md5}\n", - " 106/3 0.028 0.000 1.112 0.371 slicing.py:15(sanitize_index)\n", - " 8002 0.015 0.000 0.020 0.000 slicing.py:567(insert_many)\n", - " 4002 0.004 0.000 0.023 0.000 slicing.py:156()\n", - " 24006 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 8 0.001 0.000 0.001 0.000 {method 'update' of 'dict' objects}\n", - " 2 0.001 0.000 0.001 0.000 slicing.py:479()\n", - " 2 0.000 0.000 19.259 9.630 slicing.py:60(slice_array)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 106/4 0.000 0.000 0.034 0.008 utils.py:502(__call__)\n", - " 2 0.000 0.000 18.089 9.044 slicing.py:170(slice_wrap_lists)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 5.149 2.574 slicing.py:487(take)\n", - " 2 0.000 0.000 0.034 0.017 base.py:343(tokenize)\n", - " 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 1 0.000 0.000 19.467 19.467 {built-in method builtins.exec}\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 4 0.000 0.000 0.001 0.000 dicttoolz.py:19(merge)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 2 0.000 0.000 0.001 0.000 exceptions.py:15(merge)\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 4 0.000 0.000 0.001 0.000 functoolz.py:217(__call__)\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "cProfile.run('d[c][:, s]', sort='time')" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 452 ms, sys: 8 ms, total: 460 ms\n", - "Wall time: 148 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[[ 2, -1],\n", - " [ 2, 3],\n", - " [ 3, 0],\n", - " ..., \n", - " [ 1, 3],\n", - " [-1, -1],\n", - " [ 1, 1]],\n", - "\n", - " [[ 1, -1],\n", - " [ 2, 2],\n", - " [-1, 2],\n", - " ..., \n", - " [ 2, -1],\n", - " [ 1, 3],\n", - " [-1, -1]],\n", - "\n", - " [[ 1, -1],\n", - " [ 2, 0],\n", - " [ 0, 3],\n", - " ..., \n", - " [ 2, 2],\n", - " [ 3, 2],\n", - " [ 0, 2]],\n", - "\n", - " ..., \n", - " [[ 1, 2],\n", - " [ 3, -1],\n", - " [ 2, 1],\n", - " ..., \n", - " [ 1, 2],\n", - " [ 1, 0],\n", - " [ 2, 0]],\n", - "\n", - " [[ 1, 2],\n", - " [ 1, 0],\n", - " [ 2, 3],\n", - " ..., \n", - " [-1, 2],\n", - " [ 3, 3],\n", - " [ 1, -1]],\n", - "\n", - " [[-1, 3],\n", - " [ 2, 2],\n", - " [ 1, 1],\n", - " ..., \n", - " [ 3, 3],\n", - " [ 0, 0],\n", - " [ 0, 2]]], dtype=int8)" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time ds[1000000:1100000].compute(optimize_graph=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 80055494 function calls (60052157 primitive calls) in 19.425 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "10001670/3 5.032 0.000 6.671 2.224 slicing.py:623(check_index)\n", - " 1 3.459 3.459 4.272 4.272 slicing.py:398(partition_by_size)\n", - "10001671/1 3.287 0.000 6.378 6.378 slicing.py:540(posify_index)\n", - " 40006704 2.999 0.000 2.999 0.000 {built-in method builtins.isinstance}\n", - " 1 1.731 1.731 6.378 6.378 slicing.py:563()\n", - " 1 0.849 0.849 0.849 0.849 slicing.py:44()\n", - " 10011685 0.433 0.000 0.433 0.000 {built-in method builtins.len}\n", - " 10015670 0.381 0.000 0.381 0.000 {method 'append' of 'list' objects}\n", - " 1 0.355 0.355 0.355 0.355 slicing.py:420(issorted)\n", - " 1 0.196 0.196 0.196 0.196 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 1 0.193 0.193 0.193 0.193 slicing.py:479()\n", - " 1 0.157 0.157 0.157 0.157 {built-in method builtins.sorted}\n", - " 1 0.085 0.085 4.707 4.707 slicing.py:441(take_sorted)\n", - " 1 0.085 0.085 19.425 19.425 :1()\n", - " 1 0.079 0.079 19.341 19.341 core.py:1024(__getitem__)\n", - " 1 0.034 0.034 18.157 18.157 slicing.py:142(slice_with_newaxes)\n", - " 2 0.033 0.017 0.033 0.017 {built-in method _hashlib.openssl_md5}\n", - " 1 0.026 0.026 1.071 1.071 slicing.py:15(sanitize_index)\n", - " 4001 0.007 0.000 0.009 0.000 slicing.py:567(insert_many)\n", - " 2001 0.002 0.000 0.011 0.000 slicing.py:156()\n", - " 12003 0.001 0.000 0.001 0.000 {method 'pop' of 'list' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 1 0.000 0.000 19.228 19.228 slicing.py:60(slice_array)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n", - " 1 0.000 0.000 18.111 18.111 slicing.py:170(slice_wrap_lists)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 1 0.000 0.000 5.062 5.062 slicing.py:487(take)\n", - " 1 0.000 0.000 0.033 0.033 base.py:343(tokenize)\n", - " 1 0.000 0.000 19.425 19.425 {built-in method builtins.exec}\n", - " 2 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 3 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 2 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 1 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 2 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 4 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 1 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 1 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 4 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 2 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 2 0.000 0.000 0.033 0.017 utils.py:502(__call__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 1 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 1 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 2 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 1 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "# problem is in fact just the dim0 selection\n", - "cProfile.run('d[c]', sort='time')" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/notebooks/dask_copy.ipynb b/notebooks/dask_copy.ipynb deleted file mode 100644 index ba4391737a..0000000000 --- a/notebooks/dask_copy.ipynb +++ /dev/null @@ -1,1518 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Profile array copy via dask threaded scheduler" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook profiles a very simple array copy operation, using synthetic data." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.0.1.dev18+dirty\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "\n", - "(function(global) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " if (typeof (window._bokeh_onload_callbacks) === \"undefined\") {\n", - " window._bokeh_onload_callbacks = [];\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", - " delete window._bokeh_onload_callbacks\n", - " console.info(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(js_urls, callback) {\n", - " window._bokeh_onload_callbacks.push(callback);\n", - " if (window._bokeh_is_loading > 0) {\n", - " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " window._bokeh_is_loading = js_urls.length;\n", - " for (var i = 0; i < js_urls.length; i++) {\n", - " var url = js_urls[i];\n", - " var s = document.createElement('script');\n", - " s.src = url;\n", - " s.async = false;\n", - " s.onreadystatechange = s.onload = function() {\n", - " window._bokeh_is_loading--;\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", - " run_callbacks()\n", - " }\n", - " };\n", - " s.onerror = function() {\n", - " console.warn(\"failed to load library \" + url);\n", - " };\n", - " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", - " }\n", - " };\n", - "\n", - " var js_urls = ['https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-compiler-0.12.0.min.js'];\n", - "\n", - " var inline_js = [\n", - " function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - " \n", - " function(Bokeh) {\n", - " Bokeh.$(\"#d4821cb3-378c-411d-a941-d0708c0c532b\").text(\"BokehJS successfully loaded\");\n", - " },\n", - " function(Bokeh) {\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " for (var i = 0; i < inline_js.length; i++) {\n", - " inline_js[i](window.Bokeh);\n", - " }\n", - " }\n", - "\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(js_urls, function() {\n", - " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(this));" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", - "import bcolz\n", - "# don't let bcolz use multiple threads internally, we want to \n", - "# see whether dask can make good use of multiple CPUs\n", - "bcolz.set_nthreads(1)\n", - "import multiprocessing\n", - "import dask\n", - "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", - "from bokeh.io import output_notebook\n", - "output_notebook()" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import tempfile\n", - "import operator\n", - "from functools import reduce\n", - "from zarr.util import human_readable_size\n", - "\n", - "\n", - "def h5fmem(**kwargs):\n", - " \"\"\"Convenience function to create an in-memory HDF5 file.\"\"\"\n", - "\n", - " # need a file name even tho nothing is ever written\n", - " fn = tempfile.mktemp()\n", - "\n", - " # file creation args\n", - " kwargs['mode'] = 'w'\n", - " kwargs['driver'] = 'core'\n", - " kwargs['backing_store'] = False\n", - "\n", - " # open HDF5 file\n", - " h5f = h5py.File(fn, **kwargs)\n", - "\n", - " return h5f\n", - "\n", - "\n", - "def h5d_diagnostics(d):\n", - " \"\"\"Print some diagnostics on an HDF5 dataset.\"\"\"\n", - " \n", - " print(d)\n", - " nbytes = reduce(operator.mul, d.shape) * d.dtype.itemsize\n", - " cbytes = d._id.get_storage_size()\n", - " if cbytes > 0:\n", - " ratio = nbytes / cbytes\n", - " else:\n", - " ratio = np.inf\n", - " r = ' compression: %s' % d.compression\n", - " r += '; compression_opts: %s' % d.compression_opts\n", - " r += '; shuffle: %s' % d.shuffle\n", - " r += '\\n nbytes: %s' % human_readable_size(nbytes)\n", - " r += '; nbytes_stored: %s' % human_readable_size(cbytes)\n", - " r += '; ratio: %.1f' % ratio\n", - " r += '; chunks: %s' % str(d.chunks)\n", - " print(r)\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "def profile_dask_copy(src, dst, chunks, num_workers=multiprocessing.cpu_count(), dt=0.1, lock=True):\n", - " dsrc = da.from_array(src, chunks=chunks)\n", - " with Profiler() as prof, ResourceProfiler(dt=dt) as rprof:\n", - " da.store(dsrc, dst, num_workers=num_workers, lock=lock)\n", - " visualize([prof, rprof], min_border_top=60, min_border_bottom=60)\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## NumPy arrays" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1314, 2727, 2905, ..., 1958, 1325, 1971], dtype=uint16)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# a1 = np.arange(400000000, dtype='i4')\n", - "a1 = np.random.normal(2000, 1000, size=200000000).astype('u2')\n", - "a1" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'381.5M'" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "human_readable_size(a1.nbytes)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "a2 = np.empty_like(a1)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "chunks = 2**20, # 4M" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 56 ms, sys: 36 ms, total: 92 ms\n", - "Wall time: 91.7 ms\n" - ] - } - ], - "source": [ - "%time a2[:] = a1" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(a1, a2, chunks, lock=True, dt=.01)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(a1, a2, chunks, lock=False, dt=.01)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Zarr arrays (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((200000000,), uint16, chunks=(1048576,), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 381.5M; nbytes_stored: 318.2M; ratio: 1.2; initialized: 191/191\n", - " store: builtins.dict" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z1 = zarr.array(a1, chunks=chunks, compression='blosc', \n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", - "z1" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((200000000,), uint16, chunks=(1048576,), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 381.5M; nbytes_stored: 294; ratio: 1360544.2; initialized: 0/191\n", - " store: builtins.dict" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z2 = zarr.empty_like(z1)\n", - "z2" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(z1, z2, chunks, lock=True, dt=.02)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(z1, z2, chunks, lock=False, dt=0.02)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 251 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison, using blosc internal threads\n", - "%timeit -n3 -r5 z2[:] = z1" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " " - ] - } - ], - "source": [ - "%prun z2[:] = z1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Without the dask lock, we get better CPU utilisation. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## HDF5 datasets (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "h5f = h5fmem()\n", - "h5f" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " compression: lzf; compression_opts: None; shuffle: True\n", - " nbytes: 381.5M; nbytes_stored: 357.4M; ratio: 1.1; chunks: (1048576,)\n" - ] - } - ], - "source": [ - "h1 = h5f.create_dataset('h1', data=a1, chunks=chunks, compression='lzf', shuffle=True)\n", - "h5d_diagnostics(h1)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " compression: lzf; compression_opts: None; shuffle: True\n", - " nbytes: 762.9M; nbytes_stored: 0; ratio: inf; chunks: (1048576,)\n" - ] - } - ], - "source": [ - "h2 = h5f.create_dataset('h2', shape=h1.shape, chunks=h1.chunks, \n", - " compression=h1.compression, compression_opts=h1.compression_opts, \n", - " shuffle=h1.shuffle)\n", - "h5d_diagnostics(h2)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(h1, h2, chunks, lock=True, dt=0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(h1, h2, chunks, lock=False, dt=0.1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bcolz carrays (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "carray((200000000,), uint16)\n", - " nbytes := 381.47 MB; cbytes := 318.98 MB; ratio: 1.20\n", - " cparams := cparams(clevel=1, shuffle=2, cname='lz4', quantize=0)\n", - " chunklen := 1048576; chunksize: 2097152; blocksize: 16384\n", - "[1314 2727 2905 ..., 1958 1325 1971]" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c1 = bcolz.carray(a1, chunklen=chunks[0],\n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", - "c1" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "carray((200000000,), uint16)\n", - " nbytes := 381.47 MB; cbytes := 2.00 MB; ratio: 190.73\n", - " cparams := cparams(clevel=1, shuffle=2, cname='lz4', quantize=0)\n", - " chunklen := 1048576; chunksize: 2097152; blocksize: 4096\n", - "[0 0 0 ..., 0 0 0]" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c2 = bcolz.zeros(a1.shape, chunklen=chunks[0], dtype=a1.dtype, \n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", - "c2" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(c1, c2, chunks, lock=True, dt=0.05)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# not sure it's safe to use bcolz without a lock, but what the heck...\n", - "profile_dask_copy(c1, c2, chunks, lock=False, dt=0.05)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 649 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison\n", - "%timeit -n3 -r5 c2[:] = c1" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 557 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison\n", - "%timeit -n3 -r5 c1.copy()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/dask_count_alleles.ipynb b/notebooks/dask_count_alleles.ipynb deleted file mode 100644 index 8b9b7cec6e..0000000000 --- a/notebooks/dask_count_alleles.ipynb +++ /dev/null @@ -1,648 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Profile allele count from genotype data via dask.array" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.0.1.dev18+dirty\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "\n", - "(function(global) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " if (typeof (window._bokeh_onload_callbacks) === \"undefined\") {\n", - " window._bokeh_onload_callbacks = [];\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", - " delete window._bokeh_onload_callbacks\n", - " console.info(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(js_urls, callback) {\n", - " window._bokeh_onload_callbacks.push(callback);\n", - " if (window._bokeh_is_loading > 0) {\n", - " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " window._bokeh_is_loading = js_urls.length;\n", - " for (var i = 0; i < js_urls.length; i++) {\n", - " var url = js_urls[i];\n", - " var s = document.createElement('script');\n", - " s.src = url;\n", - " s.async = false;\n", - " s.onreadystatechange = s.onload = function() {\n", - " window._bokeh_is_loading--;\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", - " run_callbacks()\n", - " }\n", - " };\n", - " s.onerror = function() {\n", - " console.warn(\"failed to load library \" + url);\n", - " };\n", - " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", - " }\n", - " };\n", - "\n", - " var js_urls = ['https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-compiler-0.12.0.min.js'];\n", - "\n", - " var inline_js = [\n", - " function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - " \n", - " function(Bokeh) {\n", - " Bokeh.$(\"#b153ad5f-436a-4afb-945c-87790add89c8\").text(\"BokehJS successfully loaded\");\n", - " },\n", - " function(Bokeh) {\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " for (var i = 0; i < inline_js.length; i++) {\n", - " inline_js[i](window.Bokeh);\n", - " }\n", - " }\n", - "\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(js_urls, function() {\n", - " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(this));" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", - "import multiprocessing\n", - "import dask\n", - "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", - "from bokeh.io import output_notebook\n", - "output_notebook()\n", - "from functools import reduce\n", - "import operator\n", - "import allel" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", - "genotype" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((13167162, 765, 2), int8, chunks=(6553, 200, 2), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 18.8G; nbytes_stored: 683.2M; ratio: 28.1; initialized: 8040/8040\n", - " store: builtins.dict" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# copy into a zarr array\n", - "# N.B., chunks in HDF5 are too small really, use something bigger\n", - "chunks = (genotype.chunks[0], genotype.chunks[1] * 20, genotype.chunks[2])\n", - "genotype_zarr = zarr.array(genotype, chunks=chunks, compression='blosc',\n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", - "genotype_zarr" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We want to perform an allele count. Compare serial and parallel implementations, and compare working direct from HDF5 versus from Zarr." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1min 50s, sys: 512 ms, total: 1min 51s\n", - "Wall time: 1min 50s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))
nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;
compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);
data: bcolz.carray_ext.carray
0123
01523500
11527100
21527100
31527100
41527100
\n", - "

...

" - ], - "text/plain": [ - "AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))\n", - " nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;\n", - " compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);\n", - " data: bcolz.carray_ext.carray" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "# linear implementation from HDF5 on disk\n", - "allel.GenotypeChunkedArray(genotype).count_alleles()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 2min 27s, sys: 2.14 s, total: 2min 29s\n", - "Wall time: 1min 23s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))
nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;
compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);
data: bcolz.carray_ext.carray
0123
01523500
11527100
21527100
31527100
41527100
\n", - "

...

" - ], - "text/plain": [ - "AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))\n", - " nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;\n", - " compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);\n", - " data: bcolz.carray_ext.carray" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "# linear implementation from zarr in memory\n", - "# (although blosc can use multiple threads internally)\n", - "allel.GenotypeChunkedArray(genotype_zarr).count_alleles()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# multi-threaded implementation from HDF5 on disk\n", - "gd = allel.model.dask.GenotypeDaskArray.from_array(genotype, chunks=chunks)\n", - "ac = gd.count_alleles(max_allele=3)\n", - "with Profiler() as prof, ResourceProfiler(dt=1) as rprof:\n", - " ac.compute(num_workers=8)\n", - "visualize([prof, rprof], min_border_bottom=60, min_border_top=60);" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# multi-threaded implementation from zarr in memory\n", - "gdz = allel.model.dask.GenotypeDaskArray.from_array(genotype_zarr, chunks=chunks)\n", - "acz = gdz.count_alleles(max_allele=3)\n", - "with Profiler() as prof, ResourceProfiler(dt=1) as rprof:\n", - " acz.compute(num_workers=8)\n", - "visualize([prof, rprof], min_border_bottom=60, min_border_top=60);" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/genotype_benchmark_compressors.ipynb b/notebooks/genotype_benchmark_compressors.ipynb deleted file mode 100644 index b262e63fa0..0000000000 --- a/notebooks/genotype_benchmark_compressors.ipynb +++ /dev/null @@ -1,548 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.1.1.dev7+dirty\n", - "blosc ('1.10.0.dev', '$Date:: 2016-07-20 #$')\n" - ] - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import functools\n", - "import timeit\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "print('blosc', blosc.version())\n", - "import numpy as np\n", - "import h5py\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", - "genotype" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "n_variants = 500000" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(500000, 765, 2)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "genotype_sample = genotype[1000000:1000000+n_variants, ...]\n", - "genotype_sample.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "765000000" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nbytes = genotype_sample.nbytes\n", - "nbytes" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(685, 765, 2)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# 1M chunks of first dimension\n", - "chunks = (int(2**20 / (genotype_sample.shape[1] * genotype_sample.shape[2])), \n", - " genotype_sample.shape[1], \n", - " genotype_sample.shape[2])\n", - "chunks" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "8" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "blosc.get_nthreads()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((500000, 765, 2), int8, chunks=(685, 765, 2), order=C)\n", - " compression: blosc; compression_opts: {'cname': 'lz4', 'clevel': 1, 'shuffle': 2}\n", - " nbytes: 729.6M; nbytes_stored: 23.0M; ratio: 31.7; initialized: 730/730\n", - " store: builtins.dict" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='lz4', clevel=1, shuffle=2))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((500000, 765, 2), int8, chunks=(685, 765, 2), order=C)\n", - " compression: blosc; compression_opts: {'cname': 'zstd', 'clevel': 1, 'shuffle': 2}\n", - " nbytes: 729.6M; nbytes_stored: 12.0M; ratio: 60.7; initialized: 730/730\n", - " store: builtins.dict" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='zstd', clevel=1, shuffle=2))" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "compression_configs = (\n", - " (None, None),\n", - " ('zlib', 1),\n", - " ('bz2', 1),\n", - " ('lzma', dict(preset=1)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=2)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=2)),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def log(*msg):\n", - " print(*msg, file=sys.stdout)\n", - " sys.stdout.flush()" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "@functools.lru_cache(maxsize=None)\n", - "def compression_ratios():\n", - " x = list()\n", - " for compression, compression_opts in compression_configs:\n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " ratio = z.nbytes / z.nbytes_stored\n", - " x.append(ratio)\n", - " log(compression, compression_opts, ratio)\n", - " return x\n" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcFNW99/HPFwQJKLKYsCkDEjUuERk3FAKjqNEECdcg\ngrhcgxqveSIuSR6vG+CCmohPTKIm5iGokSTiDXpRQxSExmhERDYFxIVNUBTZrugjRub3/FGnsaan\nl+phZnpGfu/Xq19TXXWqzq9+XWKdPudUy8xwzjnnnHPOObfrmpQ6AOecc84555z7svAGlnPOOeec\nc87VEm9gOeecc84551wt8QaWc84555xzztUSb2A555xzzjnnXC3xBpZzzjnnnHPO1RJvYDnnnGv0\nJM2S9KtSx1EsSZWSzix1HM4552qP/HewnHOucZH0NeA64LvAfsAGYDHwGzObVsrYSkVSG+BfZvZx\nqWPJRtJEoL2ZDcpY/zVgs5n9qzSROeecq217lDoA55xzyUkqA/4JbAX+N1HDqglwMnAf0K1kweUg\nqVldNyDMbEtdHj+XXT03M/ugNuNxzjlXej5E0DnnGpf7gErgKDP7q5m9aWbLzewe4Ih0IUn7S3pM\n0v+E118ldYltHy3pVUnnS1opaZukCZKaSfqxpHckfSjpF/HKQ9nRkv4o6SNJ70m6OqNMpaTLQp3b\ngFvD+kMlPRnieV/SnyR1iO13uKQZkraGYy+Q1D9s20PSryStk/SppNWSxsX2rTJEUFIbSQ9K2iTp\nE0nTJR0a235BqOOkkIdtkmaGBmxO2c5NUhNJ/1fSilDXG5J+Gs81cAHw3bD/Dkn9Ysc7M1b28BDr\nJ5I2SpooqXW+mJxzzjUs3sByzrlGQlJb4NtEQwH/X+Z2M/ufUE7AVOCrQH+gAugMPJaxSzdgENFQ\nw38DhgJPAUcS9YiNBEZJ+l7GflcCS4BewI3AOEmDM8rcGI51OHCPpI7AbKIet6OBAUAr4L9j+/wJ\neDds7wmMAT4N20YB3wsxfh04G1hePUs7PQgcA5wR/n4C/F3SnrEyewLXAP8O9AbaAL/Nc8ys50b0\n/9K1wBDgG8C1wH9KujCUvxOYDMwAOgCdiHohq5DUEnga+J+Qg8HACcCEBDE555xrIHyIoHPONR5f\nBwS8XqDcyUQ3/weY2TsAks4B3pJ0kpnNDOWaAP9uZtuApZL+DvQDvmNmnwPLJb1A1BiKN4ReMrPb\nw/Jbko4FrgIej5X5i5n9If1G0lhgoZldG1v378BGSUeb2TygDPiFmb0ZiqyIHa8r8IaZvRDerwXm\nZDt5SQcSNay+lS4v6TxgDTACSMfVFLjMzN4KZe4kWWOmyrkFY2LLayQdBQwHJprZx5L+H9DSzDbk\nOe4IoCVwnpl9EmK6BJgl6QAzW5FnX+eccw2E92A551zjoYTlvgG8m25cAZjZSqLeoUNj5daExlXa\n+0SNmM8z1n0t4/gvZnl/aMa6VzLeHwX0D8PyPpL0EVGDx4AeocxdwARJz0q6VtLBsf0fAHqF4Xe/\nkfSd0FOXzTeAHcQaYKF379WMOLenG1fBu0Dz8MCMfDLPDUmXSnpZ0gfh3K4kahQW4xvA4nTjKvgn\n0ZDQzPw655xroLyB5ZxzjcebRA2SQ3bhGPFHx2Y+nMFyrKvJ/ysyn+bXBHiSaJ5Yz9jrwLAeMxtL\ndG6PEQ2NWxx6uTCzBUQ9XNcQNTQfBJ6pQVzx8/88x7ZC51vl3CSdDfwfop6xU4nO616geQ3iy8Uf\n+eucc42EN7Ccc66RMLPNRHN0/leYr1OFpH3C4jKgs6SusW0HEM3DWlILofTOeH98qDOf+cBhRL1m\nKzJeOxssZva2mf3GzAYSDde7KLbtYzObYmY/Ipo3NkDS17PUtYzo/2/Hp1eEB0V8k9o5/0x9gDlm\ndp+ZLQxD+TLj+oxoSGI+y4BvSmqVcWxROL/OOecaCG9gOedc4/IjohvueZKGSDpI0sGS/gNYBGBm\nM4iGw02SdJSko4GHgXlmlqqFGHpL+t+Svi7pYuBcouF9+dwD7ANMlnSspO6STpb0O0mtJLUIQ//6\nSyqTdBzQl9AgknSlpGGSvhEaVSOIHlW/NrOiMOxvKvA7SX0lfTOc/1bgzwXiTDoMM+4NoFzSaSEn\nNxDNZYtbBRwePq/2krI1tiYRPYzjofA0wX5ED934q8+/cs65xsMbWM4514iEuVTlwHTgdqJG1bNE\nT9i7IlZ0ENEPEM8M298lelJgbbiLaKjfAuAm4AYziz+hsNpwNjN7j6g3ZgcwDXgN+DXRUwK3h/Vt\ngYlED/H4K/ACkH4E/EfAT4GXgHmh/tPNLP2Uwcw6/x2YS/RwjjlETww8zcy2Fzi3QkPxsm3/HdFT\nAieFOrsSPTkw7vdEvVDzgA+IclHleOHJkN8GWhOd52NEORhZICbnnHMNiMx8WLdzzrlkJK0Efm1m\nhXqsnHPOud2S92A555xzzjnnXC3xBpZzzrli+LAH55xzLg8fIuicc84555xztcR7sJxzzjnnnHOu\nlngDyznnnHPOOedqiTewnHPOOeecc66W7FHqAJxzDYskn5jpnHPOOZeAmVX7gXrvwXLOVWNm/irw\nuuCCC0oeQ2N4eZ48T54nz1NDfXmePE+7+srFG1jOOVcD3bp1K3UIjYLnKRnPUzKep2Q8T8l4npLx\nPBXPG1jOOeecc845V0v8d7Ccc1X4HCznnHP5dOhQxvr1q0odxi775S9/yRVXXFHqMBo8z1NukrAs\nc7D8IRfOuSy8jVVYCqgocQyNQQrPUxIpPE9JpPA8JZGiLvP0/vvV7icbpSOPPLLUITQKnqfieQ+W\nc66KqAfL/11wzjmXi/JO8HfF69atG6tXry51GC6HsrIyVq1aVW19rh6sepuDJalM0qs5ts2SVF5f\nsWTU3VXSfEnTYutWliKWXCT1lzQxQbm8cUv6KPztJGlyWL5A0q9rcryEdY6WdFWh4xQjfkxJEyX1\nK1C+v6Qt4XOeL+n6BHXMktS1wPairllJQyQtlfRseP9nSQsljQrncWaB/ZOc6zmSFoXX85KOiG0b\nL2mJpP7FxO2cc865urV69eqSPxHPX7lfxTZ+6/shFw3x647BwDNmdnpsXUOMM0lMhcoYgJm9Z2ZD\nE+xXG3U2FM+ZWXl43VKiGEYCF5nZAEkdgaPN7Egzu7sW61gB9DOznsAtwP3pDWZ2NXAT8INarG83\nlip1AI1EqtQBNBKpUgfQSKRKHUAjkSp1AI1CKpUqdQjuS6q+G1jNJD0cvsWfLKlFZgFJwyUtDq/b\nw7om4dv7xeGb+VFhfQ9J00MvwDxJ3WsQUxvgg4x1G2LxnB/qXCDpwbBuoqS7Jb0g6a10z4OkVpJm\nhFgWSRoU1pdJWhb2Wy5pkqRTwv7LJR0dyrWUNEHSHEmvSDojhPEZsDXBuWwIxxkb4p0vaa2kCenT\nicUT703sGnpklku6MVseCtWZK1dxkg6QNE3Sy5JmSzpIUmtJq2JlWkpaI6lptvJZ6t9ClJ9Cih0w\nvhHYkevaC4ZKeknS65L6hPir9AhKekJSP0k3AH2BCZJ+DjwNdAmfUd8qgUrlklLhvKdJ6pD0XM1s\njpmlr5U5QJeMIuuJrnnnnHPOOVcH6vshFwcDF5rZnHDTfxlwV3qjpE7A7UAvopvJ6aGRshboYmZH\nhHKtwy6TgHFmNlVSc2rWYGwKVMZXmNlxoZ5DgWuB481ss6T4jWlHM+sj6RBgKjAF+BQYbGbbJLUn\nusGdGsr3AL5vZkslzQOGhf0HhTrOBK4DnjWzkZL2AeZKmmFmLwIvhpiOAn5oZpdknkg6bjMbDYwO\nx3gOSN/wx3ub4svHAIeF+F+W9KSZzU8fL5+EuUq7P8T+tqRjgftCb84CSf3NbDYwEPi7me2QVK08\nMCCj/ivTy5LGAi+b2ZNZ6j5e0kJgHfBTM1ta4LyGhGOWk/3aA2hqZsdJOh0YA5yS3j3L8W6WdBJw\nlZktkHQP8ISZlYfjjgx/9yD6vAaZ2UZJQ4FxwMgizjXtImBaxrpKomu+gDGx5Qp8Unk2FaUOoJGo\nKHUAjURFqQNoJCpKHUAjUVHqABqFioqKUofgGplUKpWo57O+G1hrzGxOWH4Y+DGxBhbRjf4sM9sE\nIGkS0I9oqFN3SXcDfwOekbQX0NnMpgKYWZJejCokCegZYsnmJOBRM9sc6tgS2/Z4WLdM0tfShwRu\nUzRPphLoHNu2MnZTvwSYEZZfBbqF5VOBMyT9NLxvDnQFlqcrNbNXgGqNqxweBu4ys4UFyk1Pn5uk\nKUQ9LfMT1pGWL1dIagWcADwa8g7QLPydDJwNzAaGAfcUKJ9VaFhm8wrQ1cw+CY2hx4FsvWHZrCDj\n2ottmxI7flnC4xXqSTsYOJzoywURfWnwbmahPOcaVSKdCFxI9FnGrQMOkrSnmW3PfYQxBcJ0zjnn\nnNu9VFRUVGmYjx07Nmu5Us/ByjZ/p9oNaLhZ70k0qPhS4Pe5ylY5kHRZbKhcx4xtTYCVwCHAU4mi\nryp+c5qOYwSwL9DLzHoRDT1skaV8Zex9JV80dEXUy9UrvLqb2XJqQNIYogZttaF6WST5XHZVE2Bz\nmAOVPr/Dw7apwGmS2gLlwMwC5YtiZtvM7JOwPI1oqGq7hPvmuvbgi89wB198hp9T9b+rasNgCxDw\nWuy8e2bMDyx8gOjBFvcT9YJtjm8zsxXAMmC1pMOKjM1VkSp1AI1EqtQBNBKpUgfQSKRKHUAjkSp1\nAI1CQ5+D1bFjNyTV2atjx26J4ujevTszZ87Muu3555/nkEMOqZXzzVdPIZ9++ilnnHEGbdq04eyz\nzwbg+uuv56tf/SqdO3dm9erVNGnShMrKygJHqh313cAqk5QednYO8I+M7XOBfpLaSWoKDAdmh+F2\nTc3sMeB6oNzMtgHvSPoegKTmkr4SP5iZ3RtuUsvNbH3Gtkoz6wbMI+o9yWYmcFb6Zjw0ALJJN7D2\nAT4ws8rQg1CWpUw+TwOX79xBqtEPDyiau3UyMCpzU45dTpHUJuRvMPBClmMuK1Bt3lyZ2UfASklD\nYsc8Imz7mOhzuBt40iI5yxcrNoeJMNRQsV7SGWFoaq59q117uYqGv6uAIxXZHzg2X2hZ1i0Hviqp\nd6h/jzD8MhFFTz78K3Cemb2dZfsRQHei3t8lSY/rnHPOufr1/vurib7zrptXdPxd07dvX5Yt++IW\ncVcaSbviv/7rv9iwYQObN2/mkUce4Z133uGuu+7i9ddf5913o4FAXwyIqnv13cB6HfiRpKVEE+1/\nG9ann263HriG6KuXBURzTJ4gmqifkrQA+GMoA3A+cLmkRUSNgp030kV4A8jamxGG9N1K1MhbAIyP\nxxsvGv5OAo4J8ZxL1FOQWSbb/mk3E/WuLFb0EIqbMgtIOirMTcrnSqAz0Xyq+aE3K1+9c4mGuy0k\nGuZXZXhgaGTklSdXcecCIxU9lOQ1YFBs2yNEPYB/ia0bkad8NYoe7jEwy6Yhkl4Lcf2SaBhieoho\nD2BTnsPmuvayXgNm9gJRI2tJqOuVzDI53qf3/xcwBLhD0ZyxBcDxRZzrDUTX872h93Zuxva2wCoz\nq5+vcL7UKkodQCNRUeoAGomKUgfQSFSUOoBGoqLUATQKPgfry2P16tUcdNBBOxtRq1evZt9996V9\n+4K3sHUj3zPfd4cX8FPg9lLH0ZBfwHeB/1XqOOrgvA4D7ix1HPV8zkOBPxcoY2D+8pe//OUvf+V4\nYa52Zctp3f//ONnn2K1bN7vtttvs0EMPtXbt2tkPfvAD2759u5mZpVIp22+//czM7LzzzrMmTZpY\ny5Ytbe+997Zf/OIX1Y714Ycf2sCBA61NmzbWrl0769evX5V67rzzTjviiCOsTZs2NmzYsJ31PPDA\nA9a3b98qx5Jkb7/9to0ePdqaN29uzZo1s7333tt+97vf2Ve+8hVr2rSp7b333nbhhRfaqlWrrEmT\nJrZjxw4zM9u6dauNHDnSOnXqZPvtt59df/31VllZWdTnE1tP5qu+H3LREE0BHpA0zYqc67K7MLOa\nzFFr8CwaIveTUsdRXySNB74F/GeC0nUdjnPOuUaqQ4eyUodQK1KplPdiJfSnP/2J6dOn07JlSwYO\nHMgtt9zCTTdFA63SvUYPPfQQ//jHP/jDH/7AiSeemPU448ePZ//992fjxo2YGXPmzKmy/dFHH+WZ\nZ55hzz335IQTTuCBBx7gkksuqVJPWvr9mDFjkMTbb7/NQw89BMDBBx/Meeedx5o1awCq/VDwBRdc\nQKdOnVixYgXbtm1j4MCBdO3alYsvvnhX0rTTbt/AsmieyrdKHYdzdc2iHxpOWrYuQ/lS8P8xJ+N5\nSsbzlIznKRnPk6ttP/7xj+ncuTMA1113HZdffvnOBlamfPcQzZo147333mPlypX06NGDPn36VNk+\natQoOnSIZvycccYZLFyY+0HYNb1Xef/995k2bRpbt25lzz33pEWLFlxxxRXcf//9tdbAqu85WM45\n96XgNy/JeJ6S8Twl43lKxvOUjOcpuf3222/ncllZ2c4HRxTrZz/7GT169ODUU0/l61//OnfccUeV\n7enGFUDLli3Ztm1bzQLOY82aNfzrX/+iU6dOtGvXjrZt23LppZfy4Ycf1lodu30PlnPOOeeccy63\nd955Z+fy6tWrd/ZmZSr0pL5WrVpx5513cuedd7J06VJOPPFEjj322JxDCuP7ffLJJzvfr1+/vsZP\nBdx///1p0aIFGzdurLMnC3oPlnPO1UBD//2UhsLzlIznKRnPUzKep2Q8T8ndc889rFu3jk2bNjFu\n3DiGDRuWtVzHjh1ZsWJFzuM89dRTvP129Csye++9N3vssQdNmzYtWH/Pnj1ZsmQJixcvZvv27Tl/\n4Def9JDCjh07cuqpp3LllVfy0UcfYWasWLGC5557ruhj5uINLOecc8455xqY6GEiqrNX0oeVSOKc\nc87ZOazvwAMP5Lrrrsta9pprruHmm2+mXbt23HXXXdW2v/nmm5x88snsvffe9OnThx/96Ef069dv\nZz25HHjggdx4440MGDCAgw46iG99q/jHJ8SP/9BDD/HZZ59x6KGH0q5dO8466yzWr1+fZ+8i6/LJ\n7M65OEnm/y4455xz9UeSP2CqAcv1+YT11VqG3oPlnHPOOeecc7XEH3LhnKumriZ9Oufc7q5Dlw6s\nX1t7Q5Gy8ce0J+N5cnXFG1jOuerGlDqARmAl0L3UQTQCnqdkPE/JfAny9P6Y90sdgnOujvkcLOdc\nFZLMG1jOOVdHxviPubvqfA5Ww9Zg52BJKpP0ao5tsySV11csGXV3lTRf0rTYupWliCUXSf0lTUxQ\nLm/ckj4KfztJmhyWL5D065ocL2GdoyVdVeg4xYgfU9JESf0KlO8vaUv4nOdLuj5BHbMkdS2wvahr\nVtIQSUslPRve/1nSQkmjwnmcWWD/gucayv1K0pvh2EfG1o+XtERS/2Lids4555xzydX3Qy4aYtN8\nMPCMmZ0eW9cQ40wSU6EyBmBm75nZ0AT71UadDcVzZlYeXreUKIaRwEVmNkBSR+BoMzvSzO6urQok\nnQ70MLMDgR8Cv01vM7OrgZuAH9RWfbu1BvU1TAPmeUrG85SM5ykR/32nZDxPrq7UdwOrmaSHw7f4\nkyW1yCwgabikxeF1e1jXJHx7v1jSIkmjwvoekqaHb+rnSarJyOw2wAcZ6zbE4jk/1LlA0oNh3URJ\nd0t6QdJb6Z4HSa0kzQixLJI0KKwvk7Qs7Ldc0iRJp4T9l0s6OpRrKWmCpDmSXpF0RgjjM2BrgnPZ\nEI4zNsQ7X9JaSRPSpxOLJ96b2DX0yCyXdGO2PBSqM1eu4iQdIGmapJclzZZ0kKTWklbFyrSUtEZS\n02zls9S/hSg/hRT71IaNwI5c114wVNJLkl6X1CfEX6VHUNITkvpJugHoC0yQ9HPgaaBL+Iz6VglU\nKpeUCuc9TVKHIs71e8BDAGb2ErBPbH+A9UTXvHPOOeecqwP1/ZCLg4ELzWxOuOm/DNj5K2SSOgG3\nA72Ibianh0bKWqCLmR0RyrUOu0wCxpnZVEnNqVmDsSlQGV9hZseFeg4FrgWON7PNkuI3ph3NrI+k\nQ4CpwBTgU2CwmW2T1B6YE7YB9AC+b2ZLJc0DhoX9B4U6zgSuA541s5GS9gHmSpphZi8CL4aYjgJ+\naGaXZJ5IOm4zGw2MDsd4Dkjf8Md7m+LLxwCHhfhflvSkmc1PHy+fhLlKuz/E/rakY4H7Qm/OAkn9\nzWw2MBD4u5ntkFStPDAgo/4r08uSxgIvm9mTWeo+XtJCYB3wUzNbWuC8hoRjlpP92gNoambHhV6j\nMcAp6d2zHO9mSScBV5nZAkn3AE+YWXk47sjwdw+iz2uQmW2UNBQYB4xMeK5dgHdi79eFdelZ1ZVE\n13x+s2LL3Wj0k8rrhOckGc9TMp6nZDxPifiT8ZLxPLlipVKpRD2f9d2DtcbM5oTlh4m+0Y87Bphl\nZpvMrJKoAdUPWAF0D71G3wY+krQX0NnMpgKY2Wdm9mkxwUgS0JOoAZfNScCjZrY51LEltu3xsG4Z\n8LX0IYHbJC0CZgCdJaW3rYzd1C8J2wFeJbqFBTgVuEbSAiAFNAeqzAMys1eyNa5yeBi4y8wWFig3\n3cy2hPxNofrnkkS+XCGpFXAC8Gg4v98B6Z6VycDZYXkY8EiB8lmZ2egcjatXgK5mdiTwG8Jnl1C1\nay+2bUrs+Ml+Dr1wT9rBwOFEXy4sIGp0d84slOdcC1kHHCRpz7ylToy9/IbGOeecq3cd9+uIpDp7\nddyvY6lPcacTTzyRP/zhDzXe/8ILL6Rdu3b07t0bgPvuu4+OHTvSunVrNm3aRJMmTVixYsUux1lR\nUcGYMWN2vnKp7x6szG/2s83fqXYDamZbJPUEvg1cCpwFXJGtbJUDSZcBF4d6vmNm62PbmhDdPG8H\nniriHNK2Z4l5BLAv0MvMKhU9AKJFlvKVsfeVfPE5iKiX680axFOFpDFEDdpqQ/WySPK57KomwOZ0\nj02GqcCtktoC5cBMYK885YtiZttiy9Mk3SupnZltSrBvtmvvorA5/Rnu4IvP8HOqfnFRbRhsAQJe\nM7M+Re6Xtg7YP/Z+v7AOADNbIWkZsFrSADNbUsN63JfgcdH1wvOUjOcpGc9TIv77Tsk09Dy9v+79\nOv3ZlC/LTwY8//zzPPvss7z77ru0aNGCzz//nKuvvpq5c+dy+OGHA/X/+5713YNVJik97Owc4B8Z\n2+cC/SS1k9QUGA7MDsPtmprZY8D1QHm4aX5H0vcAJDWX9JX4wczsXjPrFR5ssD5jW6WZdQPm8UXv\nSaaZwFmS2oU62uYol/7U9gE+CI2rE6naq5Hkk30auHznDrEnwBVD0dytk4FRmZty7HKKpDYhf4OB\nF7Icc1mBavPmysw+AlZKGhI75hFh28dEn8PdwJMWyVm+WPE5SGGoodKNK0Vz5jrl2bfatZeraPi7\nCjhSkf2BY/OFlmXdcuCrknqH+vcIwy+TmgqcH/btDWwxs53/goYcdifq/fXGlXPOOecatVWrVtGt\nWzdatIi+016/fj3bt2/nkEMO2Vmmvh+BX98NrNeBH0laSjTRPv2Es/TT7dYD1xANj1tANMfkCaI5\nJKkwZOqPoQxEN5KXhyF5L1BgCFkObwDtsm0IQ/puJWrkLQDGx+ONFw1/JwHHhHjOBZZlKZNt/7Sb\niR4EsljRQyhuyiwg6agwNymfK4mGlb2s6CEKYwrUO5douNtComF+8zPqbF+gvny5ijsXGKnooSSv\nAYNi2x4h6gH8S2zdiDzlq1H0cI+BWTYNkfRaiOuXRMMQ00NEewD5erJyXXtZrwEze4GokbUk1PVK\nZpkc79P7/wsYAtyhaM7YAuD4pOdqZn8japi+RTSs8rKMIm2BVWEIrtsV/i16Mp6nZDxPyXieEmnI\nvTINiecpme7duzN+/Hh69uxJ27ZtGT58OJ999sUzt37/+99z4IEHsu+++zJ48GDee++9rMfZvn07\n5513Hvvuuy9t27bluOOOY8OGL56ntmrVKvr27Uvr1q057bTT2LQpuj2bPXs2+++/f5Vjde/enZkz\nZ/KHP/yBiy++mBdffJHWrVszYsQIvvGNbwDQtm1bTj755GpxfPbZZ/zkJz+hrKyMTp06cdlll7F9\n+/Zq5XbFbv9Dw5J+CrQ3s2sKFt5NSfou0N3MflPqWGqTpMOIHrryk1LHUl8UPTTj38xseJ4y/kPD\nzjlXV8b4Dw276pTlh2wl1ekQwaTXYvfu3enQoQP//d//zZ577skJJ5zAFVdcwSWXXMLMmTM5++yz\nmTFjBoceeihXX301ixYtYvbs2dWOc//99/PUU08xefJkmjdvzsKFCznwwAPZa6+9OPHEE1m7di1/\n//vf2W+//TjttNM4/vjjGTduHLNnz+a8885jzZo1VWKaMGECJ510Eg8++CATJkzgueeeA2D16tUc\ncMABfP755zuHBjZp0oS33nqLAw44gCuvvJKVK1fy4IMPsscee3DOOedw+OGHc+utt+bMQbbPJ7a+\n2oik+p6D1RBNAR6QNC3jt7BcYGY1maPW4IUhcrtT42o88C3gPwsWHlPX0Tjn3O6pQ5eaDLYpTkOf\nW9RQeJ6SGzVqFB06RNfuGWecwcKF0fPT/vSnPzFy5Eh69uwJwG233Ubbtm1Zs2YNXbtWeU4bzZo1\nY+PGjbzxxht885vfpFevXlW2X3jhhfTo0QOAoUOH8sQTT+xSzGaWde7V73//e1599VX22WcfAK65\n5hpGjBiRt4FVrN2+gWVmbxPddDr3pRZ+aDhp2boM5UvB/8ecjOcpGc9TMp4n50oj3bgCaNmy5c5h\ngO+++y5HHXXUzm2tWrWiffv2rFu3rloD6/zzz2ft2rUMGzaMrVu3MmLECMaNG0fTptGvx3Ts2LFK\nHdu2baO2bdiwgU8++aRKzJWVlbV+31Pfc7Ccc+5LwW/ykvE8JeN5SsbzlIznKRnP067r3Lkzq1ev\n3vn+448/ZuPGjXTp0qVa2aZNm3LDDTewZMkS/vnPf/Lkk0/y0EMPFayjVatWfPLJJzvf79ixo8rc\nrWLsu+87K3VnAAAgAElEQVS+tGzZkiVLlrBp0yY2bdrEli1b2Lp1a42Ol4s3sJxzzjnnnHNFGz58\nOBMnTmTx4sVs376da6+9lt69e1frvYKoB/q1116jsrKSvfbai2bNmu3svcrnoIMO4tNPP2XatGl8\n/vnn3HLLLVUespFNrh4pSVx88cVcccUVOxtp69at45lnnklwtsnt9kMEnXOuJnyoUjKep2Q8T8l4\nnpLxPCXT0PPUoUuHOv2tqqTzAfP9htSAAQO4+eabOfPMM9myZQsnnHACf/nLX7KWXb9+PZdeeinr\n1q1jr732YtiwYZx77rkF62jdujX33nsvI0eOpLKykp/97Gfst99+RcUcf3/HHXcwduxYevfuvbO3\n7T/+4z849dRT8x6zGLv9UwSdc1VJMv93obCG/j/mhsLzlIznKRnPUzKep2QaUp5yPaXONQzFPkXQ\nG1jOuSq8geWcc87VL29gNWzFNrB8DpZzzjnnnHPO1RKfg+WcqybfWGjnnGvsOnQoY/36VaUOo840\npKFvDZnnydUVb2A557LwYQqFpYCKEsfQGKTwPCWRwvOURIrayNP77/uXSM65uuNzsJxzVUgyb2A5\n577cfL6La1h8DlbD1mDnYEkqk/Rqjm2zJJXXVywZdXeVNF/StNi6laWIJRdJ/SVNTFAub9ySPgp/\nO0maHJYvkPTrmhwvYZ2jJV1V6DjFiB9T0kRJ/QqUHyRpkaQFkuZJOilBHbMkVf8Rh6rbi7pmJQ2R\ntFTSs+H9nyUtlDQqnMeZBfZPcq7nhHNdJOl5SUfEto2XtERS/2Lids4551zdKisrQ5K/GuirrKys\nqM+zvh9y0RCb5oOBZ8zs9Ni6hhhnkpgKlTEAM3vPzIYm2K826mwIZphZTzPrBVwI3F+iOEYCF5nZ\nAEkdgaPN7Egzu7sW61gB9DOznsAtxM7VzK4GbgJ+UIv17cZSpQ6gkUiVOoBGIlXqABqJVKkDaBRS\nqVSpQ2gUGlKeVq1ahZk1yNesWbNKHkOpX6tWrSrq86zvBlYzSQ+Hb/EnS2qRWUDScEmLw+v2sK5J\n+PZ+cfhmflRY30PSdEW9APMkda9BTG2ADzLWbYjFc76+6P14MKybKOluSS9Iekuh50FSK0kzQiyL\nJA0K68skLQv7LZc0SdIpYf/lko4O5VpKmiBpjqRXJJ0RwvgM2JrgXDaE44wN8c6XtFbShPTpxOKJ\n9yZ2VdQjs1zSjdnyUKjOXLmKk3SApGmSXpY0W9JBklpLWhUr01LSGklNs5XPUv8WovzkZGafxN7u\nBXyY4Lw2AjtyXXvBUEkvSXpdUp8Qf5UeQUlPSOon6QagLzBB0s+Bp4Eu4TPqm5GnckmpcN7TJKV/\nCTDJuc4xs/S1MgfoklFkPdE175xzzjnn6kJ9tfyAMqAS6B3eTwCuCsuzgHKgE7AaaEfU+HsWGBS2\nPRM7Vuvwdw4wKCw3B1rUIK6xwBU5th0KvA60De/bhL8TgUfC8iHAm2G5KbBXWG4fW19GdGN8aHg/\nD5gQlgcBU8LyrcA5YXkfYDnwlYyYjgLuT3hu+wCLgCPD+/+JxbM4LF8ArCO66W4BvAqU1yCPuXI1\nOvY5zwB6hOVjgWfD8mNA/7A8NH1+ecrvPGaWz3JgjvgGA8uAzcCxRZxXrmtvFvCLsHw6MD2Wz1/F\nyj9B1KOU3qdX5mcQu6bOJHrwzAtA+1g+JhRzrrEyP8m8VoBvAU8W2M9gdOw1y8D85S9/+etL9MKc\nc65Ys2bNstGjR+98hX9LyHzV91ME15jZnLD8MPBj4K7Y9mOAWWa2CUDSJKAf0VCn7pLuBv4GPCNp\nL6CzmU0FMLO83+xnI0lAzxBLNicBj5rZ5lDHlti2x8O6ZZK+lj4kcJuieTKVQOfYtpVmtjQsLyFq\nPEDUoOkWlk8FzpD00/C+OdCVqKFFqO8V4JKEp/gwcJeZLSxQbnr63CRNIeppmZ+wjrR8uUJSK+AE\n4NGQd4Bm4e9k4GxgNjAMuKdA+azMbHSebY8Dj4feoj8CByc8rxVkXHuxbVPC31eIGkxJFHp01cHA\n4cD0cN5NgHczC+U7VwBJJxINh+ybsWkdcJCkPc1se+4jjCkQpnPOOefc7qWioqLKo/3Hjh2btVyp\n52BlvocsN6DhZr0n0eDrS4Hf5ypb5UDSZbGhch0ztjUBVhL1QD2VKPqq4jen6ThGAPsS9VL0Ihp6\n2CJL+crY+0q+eFy+gO+bWa/w6m5my6kBSWOIGrTVhuplkeRz2VVNgM1mVh47v8PDtqnAaZLaEvUY\nzSxQvsbM7HlgD0ntE5bPde3BF5/hDr74DD+n6n9X1YbBFiDgtdh597Sq8wMLHyB6sMX9RL27m+Pb\nzGwFUU/eakmHFRmbqyJV6gAaiVSpA2gkUqUOoJFIlTqARqEhzS1qyDxPyXieilffDawySceF5XOA\nf2Rsnwv0k9ROUlNgODA73Aw3NbPHgOuJhrBtA96R9D0ASc0lfSV+MDO7N9yklpvZ+oxtlWbWjWi4\n3tk54p0JnCWpXaijbY5y6QbWPsAHZlYZehDKspTJ52ng8p07SEcm2Kd6MNHcrZOBUZmbcuxyiqQ2\nIX+DiYaoZR5zWYFq8+bKzD4CVkoaEjvmEWHbx0Sfw91Ew9csX/liSeoRWy4PdW4M72dI6pRn32rX\nXq6i4e8q4EhF9ica2pjz8FnWLQe+Kql3qH8PSYfmOUZmvF2BvwLnmdnbWbYfAXQn6v1dkvS4zjnn\nnHMumfpuYL0O/EjSUqI5P78N6w0gNIKuIfqKagHwspk9QTRRPyVpAdHwrmvCfucDl0taRNQoSD8M\noBhvEM35qiYM6buVqJG3ABgfjzdeNPydBBwT4jmXqKcgs0y2/dNuJnoQyOLwEIqbMgtIOkpSoafg\nXQl0Bl4OvXdjCtQ7l2i420KiYX5Vhgcm6e3Jk6u4c4GRih5K8hrR/LO0R4h6AP8SWzciT/lqFD3c\nY2CWTd+X9Jqk+USNuGGhvIAewKY8h8117WW9BszsBaJG1hLgl0TDB8m3T8b+/wKGAHdIWkj038Hx\nRZzrDUTX872h93Zuxva2wCozq8yyrytKRakDaCQqSh1AI1FR6gAaiYpSB9AoxIcwudw8T8l4noq3\n2//QcJjv1N7MrilYeDcl6btAdzP7TaljqU1hiNyFZvaTUsdSXyQNBf7NzIbnKWN1M0rUOecaCv9R\nV+fcrlOpf2i4AZsC9FHsh4ZdVWb21JetcQVgZkt2s8bVeKInC/7fUsfy5ZAqdQCNRKrUATQSqVIH\n0EikSh1Ao+BzZpLxPCXjeSpefT9FsMEJ81S+Veo4nKtrFv3QcEJJpgw651zj1KFD0ge/Oudc8Xb7\nIYLOuaokmf+74JxzzjmXnw8RdM4555xzzrk65g0s55yrAR+TnoznKRnPUzKep2Q8T8l4npLxPBXP\nG1jOOeecc845V0t8DpZzrgqfg+Wcc845V5jPwXLOOeecc865OuYNLOecqwEfk56M5ykZz1Mynqdk\nPE/JeJ6S8TwVb7f/HSznXHWS/w6Wc7u7Dl06sH7t+lKH4ZxzjY7PwXLOVSHJGFPqKJxzJTcG/B7B\nOedy8zlYzjnnnHPOOVfH6q2BJalM0qs5ts2SVF5fsWTU3VXSfEnTYutWliKWXCT1lzQxQbm8cUv6\nKPztJGlyWL5A0q9rcryEdY6WdFWh4xQjfkxJEyX1K1B+kKRFkhZImifppAR1zJLUtcD2oq5ZSUMk\nLZX0bHj/Z0kLJY0K53Fmgf0Lnmso9ytJb4ZjHxlbP17SEkn9i4nb5dCg/pVowDxPyXieEvG5IMl4\nnpLxPCXjeSpefc/BaohjDQYDz5jZNbF1DTHOJDEVKmMAZvYeMDTBfrVRZ0Mww8ymAkj6JvAY8PUS\nxDESuMjM/impI3C0mR0Y4irYgE5C0ulADzM7UNJxwG+B3gBmdrWkucAPgNm1UZ9zzjnnnKuqvocI\nNpP0cPgWf7KkFpkFJA2XtDi8bg/rmoRv7xeHnohRYX0PSdPDN/XzJHWvQUxtgA8y1m2IxXN+rPfj\nwbBuoqS7Jb0g6a10z4OkVpJmhFgWSRoU1pdJWhb2Wy5pkqRTwv7LJR0dyrWUNEHSHEmvSDojhPEZ\nsDXBuWwIxxkb4p0vaa2kCenTicUT703sGnpklku6MVseCtWZK1dxkg6QNE3Sy5JmSzpIUmtJq2Jl\nWkpaI6lptvJZ6t9ClJ+czOyT2Nu9gA8TnNdGYEeuay8YKuklSa9L6hPir9IjKOkJSf0k3QD0BSZI\n+jnwNNAlfEZ9M/JULikVznuapA5JzxX4HvBQOO+XgH1i+wOsJ7rm3a6qyb82uyPPUzKep0QqKipK\nHUKj4HlKxvOUjOepePXdg3UwcKGZzQk3/ZcBd6U3SuoE3A70IrqZnB4aKWuBLmZ2RCjXOuwyCRhn\nZlMlNadmDcamQGV8hZkdF+o5FLgWON7MNkuK35h2NLM+kg4BpgJTgE+BwWa2TVJ7YE7YBtAD+L6Z\nLZU0DxgW9h8U6jgTuA541sxGStoHmCtphpm9CLwYYjoK+KGZXZJ5Ium4zWw0MDoc4zkgfcMf722K\nLx8DHBbif1nSk2Y2P328fBLmKu3+EPvbko4F7jOzAaFB1t/MZgMDgb+b2Q5J1coDAzLqvzK9LGks\n8LKZPZlZsaTBwG1AR+DbCc5rSNivnOzXHkBTMztOUa/RGOCU9O5ZjnezoqGJV5nZAkn3AE+YWXk4\n7sjwdw+iz2uQmW2UNBQYB4xMeK5dgHdi79eFde+H95VE13x+s2LL3fCbP+ecc87t9lKpVKIhk/Xd\ng7XGzOaE5YeJvtGPOwaYZWabzKySqAHVD1gBdA+9Rt8GPpK0F9A5PfTLzD4zs0+LCUaSgJ5EDbhs\nTgIeNbPNoY4tsW2Ph3XLgK+lDwncJmkRMAPoLCm9baWZLQ3LS8J2gFeJbmEBTgWukbQASAHNgSrz\ngMzslWyNqxweBu4ys4UFyk03sy0hf1Oo/rkkkS9XSGoFnAA8Gs7vd0C6Z2UycHZYHgY8UqB8VmY2\nOlvjKmx73MwOAc4A/ljEeVW79mLbpoS/rwBlCY9X6PnnBwOHE325sICo0d05s1C+cy1gHXCQpD3z\nljox9vLGVXY+ZyYZz1MynqdEfC5IMp6nZDxPyXievlBRUcGYMWN2vnIp9RysbPN3qt2AmtkWST2J\neh4uBc4CrshWtsqBpMuAi0M93zGz9bFtTYhunrcDTxVxDmnbs8Q8AtgX6GVmlYoeANEiS/nK2PtK\nvvgcRNTL9WYN4qlC0hiiBm21oXpZJPlcdlUTYHO6xybDVOBWSW2BcmAm0VC+XOVrzMyel7SHpPZm\ntjFB+WzX3kVhc/oz3MEXn+HnVP3iotow2AIEvGZmfYrcL20dsH/s/X5hHQBmtkLSMmC1pAFmtqSG\n9TjnnHPOuSzquwerTNHEe4BzgH9kbJ8L9JPUTlJTYDgwOwy3a2pmjwHXA+Vmtg14R9L3ACQ1l/SV\n+MHM7F4z62Vm5fHGVdhWaWbdgHl80XuSaSZwlqR2oY62OcqlG1j7AB+ExtWJVO3VSPLLrU8Dl+/c\nIfYEuGIomrt1MjAqc1OOXU6R1CbkbzDwQpZjLitQbd5cmdlHwEpJQ2LHPCJs+5joc7gbeNIiOcsX\nS1KP2HJ5qHNjeD8jDE3NtW+1ay9X0fB3FXCkIvsDx+YLLcu65cBXJfUO9e8Rhl8mNRU4P+zbG9hi\nZunhgekcdifq/fXG1a7wnr1kPE/JeJ4S8bkgyXiekvE8JeN5Kl59N7BeB34kaSnRRPvfhvXpp9ut\nB64hGh63gGiOyRNEc0hSYcjUH0MZiG4kLw9D8l6gwBCyHN4A2mXbEIb03UrUyFsAjI/HGy8a/k4C\njgnxnAssy1Im2/5pNxM9CGSxoodQ3JRZQNJRYW5SPlcSDSt7OTxEYUyBeucSDXdbSDTMb35Gne0L\n1JcvV3HnAiMVPZTkNWBQbNsjRD2Af4mtG5GnfDWKHu4xMMum70t6TdJ8okbcsFBeRHPjNuU5bK5r\nL+s1YGYvEDWylgC/JBo+SL59Mvb/FzAEuEPSQqL/Do5Peq5m9jeihulbRMMqL8so0hZYFYbgOuec\nc865Wqbd/VfaJf0UaJ/xmHYXI+m7QHcz+02pY6lNkg4jeujKT0odS30JD834NzMbnqeMMab+Ymq0\nVuK9Dkl4npJpiHkaAw3tHiGVSvm36Ql4npLxPCXjecpNEmZWbURSfc/BaoimAA9ImmZmp5c6mIbI\nzGoyR63BC0PkdqfG1XjgW8B/Fiw8pq6jcc41dB261GRQiHPOud2+B8s5V5Uk838XnHPOOefyy9WD\nVd9zsJxzzjnnnHPuS8sbWM45VwP+uyDJeJ6S8Twl43lKxvOUjOcpGc9T8byB5ZxzzjnnnHO1xOdg\nOeeq8DlYzjnnnHOF+Rws55xzzjnnnKtj3sByzrka8DHpyXiekvE8JeN5SsbzlIznKRnPU/H8d7Cc\nc9VI1Xq7nXOuQejQoYz161eVOgznnMvJ52A556qQZOD/LjjnGirh9y7OuYbA52A555xzzjnnXB2r\ntwaWpDJJr+bYNktSeX3FklF3V0nzJU2LrVtZilhykdRf0sQE5fLGLemj8LeTpMlh+QJJv67J8RLW\nOVrSVYWOU4z4MSVNlNSvQPmDJf1T0qdJYwnXZNcC24u6ZiUNkbRU0rPh/Z8lLZQ0KpzHmQX2T3Ku\n50haFF7PSzoitm28pCWS+hcTt8slVeoAGolUqQNoJFKlDqCRSJU6gEbB58wk43lKxvNUvPqeg9UQ\n+/QHA8+Y2TWxdQ0xziQxFSpjAGb2HjA0wX61UWdDsBH4MdFnXUojgYvM7J+SOgJHm9mBEDWeaqmO\nFUA/M9sq6TTgfqA3gJldLWku8ANgdi3V55xzzjnnYup7iGAzSQ+Hb/EnS2qRWUDScEmLw+v2sK5J\n+PZ+cfhmflRY30PS9NALME9S9xrE1Ab4IGPdhlg854c6F0h6MKybKOluSS9Ieivd8yCplaQZIZZF\nkgaF9WWSloX9lkuaJOmUsP9ySUeHci0lTZA0R9Irks4IYXwGbE1wLhvCccaGeOdLWitpQvp0YvHE\nexO7hh6Z5ZJuzJaHQnXmylWcpAMkTZP0sqTZkg6S1FrSqliZlpLWSGqarXyW+rcQ5ScnM/vQzF4B\nPk9wPmkbgR25rr1gqKSXJL0uqU+Iv0qPoKQnJPWTdAPQF5gg6efA00CX8Bn1zchTuaRUOO9pkjoU\nca5zzCx9rcwBumQUWU90zbtdVlHqABqJilIH0EhUlDqARqKi1AE0ChUVFaUOoVHwPCXjeSpeffdg\nHQxcaGZzwk3/ZcBd6Y2SOgG3A72Ibianh0bKWqCLmR0RyrUOu0wCxpnZVEnNqVmDsSlQGV9hZseF\neg4FrgWON7PNkuI3ph3NrI+kQ4CpwBTgU2CwmW2T1J7oBndqKN8D+L6ZLZU0DxgW9h8U6jgTuA54\n1sxGStoHmCtphpm9CLwYYjoK+KGZXZJ5Ium4zWw0MDoc4zkgfcMf722KLx8DHBbif1nSk2Y2P328\nfBLmKu3+EPvbko4F7jOzAaFB1t/MZgMDgb+b2Q5J1coDAzLqvzK9LGks8LKZPVko7gTnNSQcs5zs\n1x5AUzM7TtLpwBjglPTuWY53s6STgKvMbIGke4AnzKw8HHdk+LsH0ec1yMw2ShoKjANG1uBcLwKm\nZayrJLrmCxgTW67Ab2qcc845t7tLpVKJhkzWdwNrjZnNCcsPEw3buiu2/RhglpltApA0CegH3AJ0\nl3Q38DfgGUl7AZ3NbCqAmeX9Zj8bSQJ6hliyOQl41Mw2hzq2xLY9HtYtk/S19CGB2xTNk6kEOse2\nrTSzpWF5CTAjLL8KdAvLpwJnSPppeN8c6AosT1caemKqNa5yeBi4y8wWFig3PX1ukqYQ9bTMT1hH\nWr5cIakVcALwaMg7QLPwdzJwNtGwtWHAPQXKZxUalrVtBRnXXmzblPD3FaAs4fEKPf/8YOBwoi8X\nRPSlwbuZhQqdq6QTgQuJPsu4dcBBkvY0s+25jzCmQJgumgtSUeIYGoMUnqckUniekkiVOoBGIZVK\nea9DAp6nZDxPX6ioqKiSi7Fjx2YtV+o5WNnm71S7ATWzLZJ6At8GLgXOAq7IVrbKgaTLgItDPd8x\ns/WxbU2Ibp63A08VcQ5p8ZvTdBwjgH2BXmZWqegBEC2ylK+Mva/ki89BRL1cb9YgniokjSFq0FYb\nqpdFks9lVzUBNqd7bDJMBW6V1BYoB2YCe+UpX29yXHsXhc3pz3AHX3yGn1O1J7XaMNgCBLxmZn1q\nFjEoerDF/cBp6QZvmpmtkLQMWC1pgJktqWk9zjnnnHOuuvqeg1UmKT3s7BzgHxnb5wL9JLWT1BQY\nDswOw+2amtljwPVAuZltA96R9D0ASc0lfSV+MDO718x6mVl5vHEVtlWaWTdgHlHvSTYzgbMktQt1\ntM1RLt3A2gf4IDSuTqRqr0aSX259Grh85w7SkQn2qR5MNHfrZGBU5qYcu5wiqU3I32DghSzHXFag\n2ry5MrOPgJWShsSOeUTY9jHR53A38KRFcpbfRVVyoGjOXKechbNcewWOuwo4UpH9gWOTxhIsB74q\nqXeof48w/DIRRU8+/Ctwnpm9nWX7EUB3ot5fb1ztkopSB9BIVJQ6gEaiotQBNBIVpQ6gUfDehmQ8\nT8l4nopX3w2s14EfSVpKNNH+t2F9+ul264FriMYALCCaY/IE0UT9lKQFwB9DGYDzgcslLSJqFKQf\nBlCMN4B22TaEIX23EjXyFgDj4/HGi4a/k4BjQjznAsuylMm2f9rNRA8CWazoIRQ3ZRaQdFSYm5TP\nlUBnovlU80NvVr565xINd1tINMyvyvDA0MjIK0+u4s4FRip6KMlrwKDYtkeIegD/Els3Ik/5ahQ9\n3GNglvUdJL1DlJfrFD1EY68wBK8HsCnPYXNde1mvATN7gaiRtQT4JdHwQfLtk7H/v4AhwB2SFhL9\nd3B80nMFbiC6nu8Nc9vmZmxvC6wys8rquzrnnHPOuV2l3f3X0MN8p/YZj2l3MZK+C3Q3s9+UOpba\nJOkwooeu/KTUsdSX8NCMfzOz4XnKWON4+n6ppfBv05NI4XlKIoXnKYkUcCK7+71LIT5nJhnPUzKe\np9wkYWbVRiTV9xyshmgK8ICkaWZ2eqmDaYjMrCZz1Bq8MERud2pcjQe+BfxngtJ1HY5zztVIhw5J\nnynknHOlsdv3YDnnqpJk/u+Cc84551x+uXqw6nsOlnPOOeecc859aXkDyznnaiDJDw06z1NSnqdk\nPE/JeJ6S8Twl43kqnjewnHPOOeecc66W+Bws51wVPgfLOeecc64wn4PlnHPOOeecc3XMG1jOOVcD\nPiY9Gc9TMp6nZDxPyXiekvE8JeN5Kp7/DpZzrhrJfwfLuYakQ5cOrF+7vtRhOOecS8DnYDnnqpBk\njCl1FM65KsaA///aOecaFp+D5ZxzzjnnnHN1rN4aWJLKJL2aY9ssSeX1FUtG3V0lzZc0LbZuZSli\nyUVSf0kTE5TLG7ekj8LfTpImh+ULJP26JsdLWOdoSVcVOk4x4seUNFFSvwLlD5b0T0mfJo0lXJNd\nC2wv6pqVNETSUknPhvd/lrRQ0qhwHmcW2L/guYZyv5L0Zjj2kbH14yUtkdS/mLhdDg3qX4kGzPOU\njOcpEZ8LkoznKRnPUzKep+LV9xyshji+YTDwjJldE1vXEONMElOhMgZgZu8BQxPsVxt1NgQbgR8T\nfdalNBK4yMz+KakjcLSZHQhR46k2KpB0OtDDzA6UdBzwW6A3gJldLWku8ANgdm3U55xzzjnnqqrv\nIYLNJD0cvsWfLKlFZgFJwyUtDq/bw7om4dv7xZIWSRoV1veQND18Uz9PUvcaxNQG+CBj3YZYPOeH\nOhdIejCsmyjpbkkvSHor3fMgqZWkGSGWRZIGhfVlkpaF/ZZLmiTplLD/cklHh3ItJU2QNEfSK5LO\nCGF8BmxNcC4bwnHGhnjnS1oraUL6dGLxxHsTu4YemeWSbsyWh0J15spVnKQDJE2T9LKk2ZIOktRa\n0qpYmZaS1khqmq18lvq3EOUnJzP70MxeAT5PcD5pG4Edua69YKiklyS9LqlPiL9Kj6CkJyT1k3QD\n0BeYIOnnwNNAl/AZ9c3IU7mkVDjvaZI6JD1X4HvAQ+G8XwL2ie0PsJ7omne7qib/2uyOPE/JeJ4S\nqaioKHUIjYLnKRnPUzKep+LVdw/WwcCFZjYn3PRfBtyV3iipE3A70IvoZnJ6aKSsBbqY2RGhXOuw\nyyRgnJlNldScmjUYmwKV8RVmdlyo51DgWuB4M9ssKX5j2tHM+kg6BJgKTAE+BQab2TZJ7YE5YRtA\nD+D7ZrZU0jxgWNh/UKjjTOA64FkzGylpH2CupBlm9iLwYojpKOCHZnZJ5omk4zaz0cDocIzngPQN\nf7y3Kb58DHBYiP9lSU+a2fz08fJJmKu0+0Psb0s6FrjPzAaEBll/M5sNDAT+bmY7JFUrDwzIqP/K\n9LKkscDLZvZkobgTnNeQcMxysl97AE3N7DhFvUZjgFPSu2c53s2STgKuMrMFku4BnjCz8nDckeHv\nHkSf1yAz2yhpKDAOGJnwXLsA78Terwvr3g/vK4mu+fxmxZa74Td/zjnnnNvtpVKpREMm67uBtcbM\n5oTlh4mGbd0V234MMMvMNgFImgT0A24Buku6G/gb8IykvYDOZjYVwMwKfbNfjSQBPUMs2ZwEPGpm\nm0MdW2LbHg/rlkn6WvqQwG2K5slUAp1j21aa2dKwvASYEZZfJbqFBTiV/8/e/UdJVd353n9/QBlF\nUcEf/DBja1wa442jYvwVvdr6RExMNF6UJGoGV+JosnCpmJi1fDJ50uAvjI5JdObJGI1DouIa5F6S\nQY0iYpejKILQgAqiTtCIT1Bv/MksRq/yff7Y34LT1XW6ThXQRdHf11q1+tQ5+5z9Pd9zGmr33vsU\nnDQBvhkAACAASURBVC7ph/5+ELAPsLJcqffE9Ghc5bgb+JmZLalRbk753CTNJPW0LC5YR1lvuULS\nTsAXgBmed4Dt/ee9wDdIw9a+Cfy/NcpX5Q3Lze2PVNx7mW0z/ecioK3g8Wo9//wzwOdIf1wQ6Y8G\n/19loU0419eBAyX9lZl9mFvqpAaP3p+sIhqeRUSeiok8FVIqleKv6QVEnoqJPBUTedqovb29Wy4m\nT55ctVyz52BVm7/T4wOomb0r6VDgVOB7wDhgYrWy3Q4kTQAu9HpOM7M1mW0DSB+ePwQeqOMcyrIf\nTstxnAfsARxuZuuVHgCxQ5Xy6zPv17PxOojUy/VSA/F0I2kSqUHbY6heFUWuy6YaALxT7rGpMAu4\nVtJQYDTwKLBzL+X7TM6993e+uXwNP2HjNfyY7j2pPYbB1iDgOTM7rrGIeR3468z7T/k6AMzsj5JW\nAK9K+r/M7PkG6wkhhBBCCFX09RysNqWJ9wDnAo9XbF8AnCBpmKSBwDnAYz7cbqCZ/Q74MTDazNYC\nr0n6GoCkQZJ2zB7MzH5pZoeb2ehs48q3rTezfYFnSL0n1TwKjJM0zOsYmlOu3MDaFXjTG1cn0b1X\no8g3t84GLt2wQ+YJcPVQmrv1ReCyyk05u5wiaTfP35nAvCrHXFGj2l5zZWYfAKsknZ055t/4tv8k\nXYebgfstyS2/ibrlQGnO3MjcwlXuvRrHfQU4TMlfA0cVjcWtBPaUdIzXv50PvyxqFjDe9z0GeNfM\nysMDyzncj9T7G42rTRG9DcVEnoqJPBUSf0UvJvJUTOSpmMhT/fq6gfUCcLGk5aSJ9rf6+vLT7dYA\nVwIloIs0x+Q+0hySkqQu4C4vA+mD5KWSlpIaBdnJ/EW9CAyrtsGH9F1LauR1ATdl480W9Z/TgCM9\nnm8BK6qUqbZ/2dWkB4EsU3oIxVWVBSQd4XOTenM5MIo0n2qx92b1Vu8C0nC3JaRhft2GB3ojo1e9\n5CrrW8AFSg8leQ44I7NtOqkH8F8z687rpXwPSg/3+GqV9cMlvUbKy98rPURjZx+Ctz/wdi+Hzbv3\nqt4DZjaP1Mh6HvgFafggve1Tsf//Ac4GfippCen34Nii52pmfyA1TF8GfkWa55g1FHjFzNZX7htC\nCCGEEDad+vs3w/t8p90rHtMeMiR9BdjPzP6p2bFsTpL+G+mhK1c0O5a+4g/N+B9mdk4vZYxJfRdT\ny4o5M8VEnoqpladJ0N//v4aYC1JU5KmYyFMxkad8kjCzHiOS+noO1tZoJvAbSQ+a2ZebHczWyMwa\nmaO21fMhcv2pcXUT8N+B/7tm4UlbOpoQQj2G793IAI0QQgjN0O97sEII3Umy+HchhBBCCKF3eT1Y\nfT0HK4QQQgghhBC2WdHACiGEBhT5osEQeSoq8lRM5KmYyFMxkadiIk/1iwZWCCGEEEIIIWwmMQcr\nhNBNzMEKIYQQQqgt5mCFEEIIIYQQwhYWDawQQmhAjEkvJvJUTOSpmMhTMZGnYiJPxUSe6hffgxVC\n6EHq0dsdQgghhD40fHgba9a80uwwQgNiDlYIoRtJBvHvQgghhNBcIj6nb91iDlYIIYQQQgghbGFb\ntIElqU3SsznbOiWN3pL155G0j6TFkh7MrFvVjFjySDpR0tQC5eqOW9JlknbI2Xa+pFt8uUPS+BrH\nOl9SR40yH9QbYy3lY/o91lmgfKekFyR1+bXfo0b5XvPv2++rM+ZBkuZ4/eMkHS/pOX9/UN7vSmb/\nmucqaUdJ90taIelZSddlth3o9U2vJ+6Qp9TsAFpEqdkBtIhSswNoEaVmB9AiSs0OoEWUmh1AS4g5\nWPXrix6srbFv80zgYTP7cmbd1hhnkZgaiXsiMLiB/RqNYUvk1nKWe3OOmR1uZqPN7H/XWUcj2yuN\nBszrnwGcB1xnZqOBdQWPV6TMjWb2WeBw4HhJp5IqftHMPgccImm/OmMPIYQQQggF9EUDa3tJd0ta\nLuneaj0nks6RtMxf1/u6AZKm+rqlki7z9ft7L8ASSc80+EFxN+DNinVvZeIZ73V2Sfqtr5sq6WZJ\n8yS9LGmsr99J0iMey1JJZ/j6Nu9FmCpppaRpkk7x/VdK+ryXGyzpDknzJS2SdLqH8RHwXoFzecuP\nMznTO7PajznYezO6PI/jJF0CjAI6Jc31fb/tMc0Hjsscey3pg39v1nk5JO0laaZfmy5Jx5RTmsnt\nFZIWeJkOXzdF0oRMmQ5J388rX+ET4O0CeYL67vcN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0d4\n79mewF3AkX6ci4CvA1dn9/V9Bki6QdLTft4XFj1XM1tnZo/58sfAYuBTFcXeIP0OhE3S3uwAWkR7\nswNoEe3NDqBFtDc7gBbR3uwAWkR7swNoCe3t7c0OofWY2RZ7AW3AeuAYf38H8H1f7iT9RX8k8Cow\njPQBeC5whm97OHOsXfznfOAMXx4E7NBAXJOBiTnbDgZeAIb6+93851Rgui9/FnjJlwcCO/vy7pn1\nbaQP6Qf7+2eAO3z5DGCmL18LnOvLuwIrgR0rYjoCuK3gue0KLCX1XowFfpXZNsR//jFzfiMy+d8O\neAK4pcHr/a/Apb6sTH3v+89TyvH49vuA44HDgFLmOM8De+eV9/cfVKl/JHB/TmydwLOkBseP6zyv\nWcCxvjzY79MTgXe8TgFPAl/I5HdY5to96ssnArMyx50KjM3cL8t8+ULgR5l7fCHQVvRcM2V2A/4D\n2Ldi/Vzg873sZ9CReXUaWLziFa94xSte8erTFxa2Lp2dndbR0bHh5deIyldf9GD9yczm+/LdpA/U\nWUcCnWb2tpmtB6YBJ5A+pO7nvUanAh9I2hkYZWazSGf0kZn9Vz3BSBJwKLA6p8jJwAwze8freDez\n7fe+bgWwV/mQwBRJS4FHgFGSyttWmdlyX37et0P6oL+vL48BrpTURRoMPAjYJxuQmS0ys4sKnuLd\nwE1m1uX1nOI9RMebWXkulNjYq3Q0G/P/MbAp83NOBv7ZY7ZMfWVjPJ7FpIbOZ4ADzGwJsKekEZL+\nBnjbzF7PK59XuZn92cy+mrP5XDM7BPjvwH+X9K06zmse8HPv/Rvq9ynAAq/TgCVsvKab+ozzMcB4\nvyeeJjV+u513jXNF0kDgHuAXZvZKxebVpN+BXkzKvNqLR96vlJodQIsoNTuAFlFqdgAtotTsAFpE\nqdkBtIhSswNoCTEHa6P29nYmTZq04ZWnL74Hy2q8hyofSM3sXUmHAqcC3wPGkeYO9frh1YeaXej1\nnGZmazLbBpAabh8CD9RxDmUfVon5PGAP4HAzW6/00IkdqpRfn3m/no25F3CWmb3UQDzdSJpEatDe\nCWBmLyk9SOQ04BpJj5jZNdV23dS6XbVrW1nPFDO7vcq2GaRrPIKNjbzeyteqq3thsz/7z/+UdA9w\nFKkxWmTfn0q6H/gKME/SGN+Uvb6fsPGafszG4YhVHyZSg4BLzGxOA/uW3QasNLN/rLLtV8BsSUeZ\n2Xc3oY4QQgghhFChL3qw2iQd7cvnAo9XbF8AnCBpmP/V/RzgMUm7AwPN7HfAj4HRZrYWeE3S12DD\nU9l2zB7MzH5pGx9ksKZi23oz25c0XO8bOfE+CozLzKEZmlOu3CjZFXjTG1cnkYZ6VZbpzWzg0g07\nSIcV2KdnMGnu1heByzLrRgLrzOwe4EbSsEuA94FdfPlpUv6HStqe1MipdvyLs/OkcswFJnj5AZKG\nlHf3n7OB75TnMEka5XOTAO4FvgmcRWps5ZXfo+KYNUka6PcTfo5fBZ7z92cq86S9nP0/bWbPm9kN\npOF6B9WochVpaCB+PvWaDUyQtJ3Xf0DlfV4j3mtIQ2ovzylyBXBBNK42VXuzA2gR7c0OoEW0NzuA\nFtHe7ABaRHuzA2gR7c0OoCXEHKz69UUD6wXgYknLSXNCbvX1BuCNoCtJ/bRdwEIzu480B6fkw6Tu\n8jIA44FLfUjePGB4AzG9SBp21YMP6buW1MjrAm7Kxpst6j+nkR5csBT4FrCiSplq+5ddTXoQyDKl\nx3RfVVnAH5RwWy/nA3A56eEVC/0hCpOAQ4AFfh4/Acq9V7cDD0ma6/mfTJrb9jiwvMeRk4OAv9SI\nYSJwkqRlpEbswb6+fK3nkIatPeVlZgA7+7blwBBgtZm90Uv5IdljZkka6T1Nlf6K1GOzhDTUcLXn\nAGB/aj9MZKLSI8+XkubVPVilTDaeq4BbJC0g9Wblybsnfk26Dov9nriVit7mvHOVtDfwI+DgzIM5\nvlNRbCjwci9xhRBCCCGEBilNH+lfJP0Q2N3MrqxZOAAgaRbpgQy9NRhajqQ7gcvNrFbjcZvgcxCX\nAWeb2cqcMlbnCMx+qkT89bOIEpGnIkpEnoooEXkqokTkqYgSW3eexNbwOb1UKkUvVg5JmFmPUVV9\n0YO1NZoJHKfMFw2H3pnZGdta4wrAzMb3o8bVgaRe4i5SL24IIYQQQtjM+mUPVgghX+rBCiGEEEIz\nDR/expo1rzQ7jNCLvB6svniKYAihxcQfXkIIIYQQGtNfhwiGEMImie8FKSbyVEzkqZjIUzGRp2Ii\nT8VEnuoXDawQQgghhBBC2ExiDlYIoRtJFv8uhBBCCCH0Lp4iGEIIIYQQQghbWDSwQgihATEmvZjI\nUzGRp2IiT8VEnoqJPBUTeapfNLBCCCGEEEIIYTOJOVghhG7ie7BCCCFUM3zv4axZvabZYYSw1cib\ngxUNrBBCN5KMSc2OIoQQwlZnUnxPYghZ8ZCLEELYnFY1O4AWEXkqJvJUTOSpmMhTITG3qJjIU/22\naANLUpukZ3O2dUoavSXrzyNpH0mLJT2YWbdV/XMk6URJUwuUqztuSZdJ2iFn2/mSbvHlDknjaxzr\nfEkdNcp8UG+MtZSP6fdYZ4HynZJekNTl136PGuV7zb9vv6/OmAdJmuP1j5N0vKTn/P1Beb8rmf2L\nnutoScskvSjpF5n1B3p90+uJO4QQQgghFNcXPVhbY1/ymcDDZvblzLqtMc4iMTUS90RgcAP7NRrD\nlsit5Sz35hwzO9zMRpvZ/66zjka2VxoNmNc/AzgPuM7MRgPrCh6vSJl/Bi4wswOBAyWdSqr4RTP7\nHHCIpP3qjD1UigwWE3kqJvJUTOSpmMhTIe3t7c0OoSVEnurXFw2s7SXdLWm5pHur9ZxIOsf/4r5M\n0vW+boCkqb5uqaTLfP3+3guwRNIzDX5Q3A14s2LdW5l4xnudXZJ+6+umSrpZ0jxJL0sa6+t3kvSI\nx7JU0hm+vk3SCt9vpaRpkk7x/VdK+ryXGyzpDknzJS2SdLqH8RHwXoFzecuPMznTO7PajzlY0v2+\nfpn3mlwCjAI6Jc31fb/tMc0Hjsscey3pg39v1nk5JO0laaZfmy5Jx5RTmsntFZIWeJkOXzdF0oRM\nmQ5J388rX+ET4O0CeYL67vcN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0d479mewF3AkX6ci4Cv\nA1dn9/V9Bki6QdLTft4XFj1XSSOAIWa20FfdSfqDQtYbpN+BEEIIIYSwmW3XB3V8Bvi2mc2XdAcw\nAfhZeaOkkcD1wOHAu8Acb6SsBvY2s7/xcrv4LtNIf/WfJWkQjTUSBwLrsyvM7Giv52DgR8CxZvaO\npOwH0RFmdpykzwKzgJnAfwFnmtlaSbsD830bwP7AWWa2XNIzwDd9/zO8jrHA3wNzzewCSbsCCyQ9\nYmZPAU95TEcA3zWziypPpBy3mXUAHX6Mfwf+CfgS8LqZfdWPM8TMPpB0OdDu5zcCmETK//tACVjs\nx7ypViLN7N7M21uAkpmNlSRg53Ixr/8U4AAzO8q3z5J0PDAd+AXwSy//dWBMXnkzewJvtJnZauBs\nP/5I4Pby+VbxG0n/B5hpZtfUOK8N+Qd+AEwws6ckDSZdc4DDgIOBNcA8SV8wsyfp2ctkZvaWpL8D\nfmBm5Ub4scB9ZjZTUlum/AXAu2Z2tN/j8yQ9bGavFjjXvUm/O2WrfV3WetLvQL7sQMR9ib+GVrOK\nyEsRkadiIk/FRJ6KiTwVUiqVonemgMjTRqVSqdCctL5oYP3JzOb78t3AJWQaWMCRQKeZvQ0gaRpw\nAnANsJ+km4E/AA9L2hkYZWazAMzso3qD8Q/qh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0gf\nXEdltq0ys+W+/DzwiC8/S/rYCjAGOF3SD/39IGAfYGW5UjNbBPRoXOW4G7jJzLokrQX+QdIU4AFv\nmJRjLvcqHU33/E8HDihYV6WTgb/1mA2onHs1BjhF0mKvfydSA2qqpD29sbcX8LaZvS5pYrXywBNU\nYWZ/BvIaV+ea2Z+992mmpG+ZWd49UGke8HO/N2d6bAALvE4kLSFd0yfJ9Ng1aAxpGN84f78L6bxf\nLReoca61rCb9DjyTW+KkBo8cQgghhLCNam9v79bYnDx5ctVyfdHA6vHX/CplenwgNbN3JR0KnAp8\nDxhHmjvU64dXH2p2oddzmpmtyWwbAPwR+BB4oI5zKPuwSsznAXsAh5vZeqWHTuxQpfz6zPv1bMy9\nSL1cLzUQTzeSJpEatHcCmNlLSg8SOQ24xnvGqvXcbGqDoKzW/CABU8zs9irbZpCu8QhSj1at8nXN\nfyo3hMzsPyXdAxxFfiO7ct+fSrof+AqpN2mMb8pe30/YeE0/ZmPPatWHidQg4BIzm9PAvq8Df515\n/ylfl/UrYLako8zsuw3UESD+OlxU5KmYyFMxkadiIk+FRK9MMZGn+vXFHKw2SUf78rnA4xXbFwAn\nSBomaSBwDvCYD7cbaGa/A34MjDaztcBrkr4GG57KtmP2YGb2y8yDDNZUbFtvZvuS/nL/jZx4HwXG\nZebQDM0pV26U7Aq86Y2rk4C2KmV6Mxu4dMMO0mEF9ukZTJq79UXgssy6kcA6M7sHuJH0kAVIQwHL\nQy6fJuV/qKTtSY2case/ODtPKsdc0hDQ8jyiIeXd/eds4DvlOUySRvncJIB7gW8CZ5EaW3nl96g4\nZk2SBvr9hJ/jV4Hn/P2Zkq6rsf+nzex5M7sBWAgcVKPKVcARvnxW0TgzZgMTJG3n9R9QeZ/n8Xv+\nPUnlYZXjgX+rKHYF6SEY0bgKIYQQQtjM+qKB9QJwsaTlpIn1t/p6gw0fCK8kzf3pAhaa2X2keSMl\nSV2khwNc6fuNBy6VtJQ0dGt4AzG9CAyrtsGH9F1LauR1AeV5SHk9cdNIDy5YCnwLWFGlTLX9y64m\nPQhkmdJjuq+qLKD0oITbejkfgMtJD69YqPQQhUnAIaQ5XV3AT0jDLgFuBx6SNNfzP5k0d+xxYHmP\nIycHAX+pEcNE4CRJy0iN2IN9fflazwHuAZ7yMjPweVqe9yHAajN7o5fyQ7LHzJI00nuaKv0Vqcdm\nCWl+2WrPAaR5crUeJjJR0rN+jT8CHqxSJhvPVcAtkhaQerPy5N0TvyZdh8V+T9xKRW9zL+cKcDFw\nB+k+f8nMHqrYPhR4uZe4QhFb1Rc7bMUiT8VEnoqJPBUTeSokvt+pmMhT/dQfv5Hb5zvtbmZX1iwc\nAJA0CxhrZr01GFqOpDuBy82sVuNxm+C9WsuAs81sZU4ZY1KfhtWaYhJ5MZGnYiJPxUSeitlSeZoE\n29Lnxnh4QzGRp3ySMLMeo6r6awNrf+A3wNqK78IKYZsl6UDSUMxlwPmW88svqf/9oxBCCKGm4XsP\nZ83qNbULhtBPRAMrhFCIpLy2VwghhBBCcHkNrL6YgxVCCNucGJNeTOSpmMhTMZGnYiJPxUSeiok8\n1S8aWCGEEEIIIYSwmcQQwRBCNzFEMIQQQgihthgiGEIIIYQQQghbWDSwQgihATEmvZjIUzGRp2Ii\nT8VEnoqJPBUTeapfNLBCCCGEEEIIYTOJOVghhG7ie7BCCCH0d8OHt7FmzSvNDiNs5eJ7sEIIhaQG\nVvy7EEIIoT8T8Rk51BIPuQghhM2q1OwAWkSp2QG0iFKzA2gRpWYH0CJKzQ6gRZSaHUBLiDlY9dui\nDSxJbZKezdnWKWn0lqw/j6R9JC2W9GBm3apmxJJH0omSphYoV3fcki6TtEPOtvMl3eLLHZLG1zjW\n+ZI6apT5oN4Yaykf0++xzgLlH5TUJek5Sb+WtF2N8r3m37ffV2fMgyTN8XtvnKTjPZ7Fkg7K+13J\n7F/zXCXtKOl+SSskPSvpusy2A72+6fXEHUIIIYQQiuuLHqytsX/1TOBhM/tyZt3WGGeRmBqJeyIw\nuIH9Go1hS+TWcpbzjDOzw83sc8BuwDfqrKOR7ZVGA2Zmo81sBnAecJ2ZjQbWFTxekTI3mtlngcOB\n4yWdSqr4RT//QyTtV2fsoYf2ZgfQItqbHUCLaG92AC2ivdkBtIj2ZgfQItqbHUBLaG9vb3YILacv\nGljbS7pb0nJJ91brOZF0jqRl/rre1w2QNNXXLZV0ma/f33sBlkh6psEPirsBb1aseysTz3ivs0vS\nb33dVEk3S5on6WVJY339TpIe8ViWSjrD17d5L8JUSSslTZN0iu+/UtLnvdxgSXdImi9pkaTTPYyP\ngPcKnMtbfpzJHu9iSav9mIO9N6PL8zhO0iXAKKBT0lzf99se03zguMyx15I++PdmnZdD0l6SZvq1\n6ZJ0TDmlmdxeIWmBl+nwdVMkTciU6ZD0/bzyFT4B3q6VJDMrx7g9MAj4S41dNuTfe6vKuV0kaScv\nM0TSDL/Od2XiXyVpmC8fodRbuydwF3CkH+ci4OvA1dl9fZ8Bkm6Q9LSf94VFz9XM1pnZY778MbAY\n+FRFsTdIvwMhhBBCCGFzM7Mt9gLagPXAMf7+DuD7vtxJ+ov+SOBVYBipwTcXOMO3PZw51i7+cz5w\nhi8PAnZoIK7JwMScbQcDLwBD/f1u/nMqMN2XPwu85MsDgZ19effM+jbSh/SD/f0zwB2+fAYw05ev\nBc715V2BlcCOFTEdAdxW8Nx2BZaSei/GAr/KbBviP/+YOb8RmfxvBzwB3NLg9f5X4FJfVqa+9/3n\nKeV4fPt9wPHAYUApc5zngb3zyvv7D6rUPxK4v5f4HiI1rKbXeV6zgGN9ebDfpycC73idAp4EvpDJ\n77DMtXvUl08EZmWOOxUYm7lflvnyhcCPMvf4QqCtnnMt37vAfwD7VqyfC3y+l/0MOjKvTgOLV49X\n5CXyFHmKPG2tr8jTpucJC0lnZ2ezQ9hqdHZ2WkdHx4aX3ydUvnqdh7KZ/MnM5vvy3cAlwM8y248E\nOs3sbQBJ04ATgGuA/STdDPwBeFjSzsAoM5tFOqOP6g1GkoBDPZZqTgZmmNk7Xse7mW2/93UrJO1V\nPiQwRdIJpMbkqMy2VWa23JefBx7x5WeBfX15DHC6pB/6+0HAPqSGFl7fIuCigqd4N3CTmXVJWgv8\ng6QpwANm9kQm5nKv0tF0z/904ICCdVU6Gfhbj9mAyrlXY4BTJC32+ncCDjCzqZL2lDQC2At428xe\nlzSxWnlSI7AHM/sz8NW84MzsS5IGAfdKGm9mdxY8r3nAz/3enOmxASzwOpG0hHRNnyTTY9egMaRh\nfOP8/S6k8341cy69nqukgcA9wC/M7JWKzatJvwPP5Icwqf6oQwghhBC2Ye3t7d2GTE6ePLlqub5o\nYFmN91DlA6mZvSvpUOBU4HvAONLcoV4/vPpQswu9ntPMbE1m2wBS78KHwAN1nEPZh1ViPg/YAzjc\nzNYrPXRihyrl12fer2dj7gWcZWYvNRBPN5ImkRq0dwKY2UtKDxI5DbhG0iNmdk21XTe1blft2lbW\nM8XMbq+ybQbpGo8AphcoX6uu6gGafSTpfwFHAYUaWGb2U0n3A18B5kka45uy1/cTNl7Tj9k4/Lbq\nw0RqEHCJmc1pYN+y24CVZvaPVbb9Cpgt6Sgz++4m1NHPtTc7gBbR3uwAWkR7swNoEe3NDqBFtDc7\ngBbR3uwAWkLMwapfX8zBapN0tC+fCzxesX0BcIKkYf5X93OAxyTtDgw0s98BPwZGW5pH85qkr8GG\np7LtmD2Ymf3S0sMMRmcbV75tvZntS/rLfd5DDh4FxmXm0AzNKVdulOwKvOmNq5NIQ70qy/RmNnDp\nhh2kwwrs0zOYNHfri8BlmXUjgXVmdg9wI2nYJcD7pF4RgKdJ+R/q85PGUYWki7PzpHLMBSZ4+QGS\nhpR395+zge+U5zBJGuVzkwDuBb4JnEVqbOWV36PimDUpzZMb4cvbkRpKS/z9mco8aS9n/0+b2fNm\ndgNpuN5BNapcRRoaiJ9PvWYDEzxWJB1QeZ/XiPca0pDay3OKXAFcEI2rEEIIIYTNry8aWC8AF0ta\nTpoTcquvNwBvBF1J+jKCLmChmd1HmoNTktRFejjAlb7feOBSSUtJQ7eGNxDTi6Q5Rz34kL5rSY28\nLuCmbLzZov5zGunBBUuBbwErqpSptn/Z1aQHgSxTekz3VZUF/EEJt/VyPgCXkx5esdAfojAJOARY\n4OfxE9KwS4DbgYckzfX8TybNbXscWN7jyMlB1H4wxETgJEnLSI3Yg319+VrPIQ1be8rLzAB29m3L\ngSHAajN7o5fyQ7LHzJI00nuaKu0EzPJhfIuA14B/8W37U/thIhOVHnm+lDSv7sEqZbLxXAXcImkB\nqTcrT9498WvSdVjs98StVPQ2552rpL2BHwEHZx7M8Z2KYkOBl3uJKxRSanYALaLU7ABaRKnZAbSI\nUrMDaBGlZgfQIkrNDqAlxPdg1U9pqkz/4vOddjezK2sWDgBImkV6IENvDYaWI+lO4HIzq9V43Cb4\nHMRlwNlmtjKnjDU4ArOfKRHDS4ooEXkqokTkqYgSkaciSkSeiiiRnyfRHz8jV1MqlWKYYA5JmFmP\nUVX9tYG1P/AbYK11/y6sELZZkg4kDcVcBpxvOb/8qYEVQggh9F/Dh7exZs0rzQ4jbOWigRVCKERS\nXtsrhBBCCCG4vAZWX8zBCiGEbU6MSS8m8lRM5KmYyFMxkadiIk/FRJ7qFw2sEEIIIYQQQthMbNT0\nHgAAIABJREFUYohgCKGbGCIYQgghhFBbDBEMIYQQQgghhC0sGlghhNCAGJNeTOSpmMhTMZGnYiJP\nxUSeiok81S8aWCGEEEIIIYSwmcQcrBBCN/E9WCGE0DeG7z2cNavXNDuMEEKD4nuwQgiFSDImNTuK\nEELoByZBfA4LoXXFQy5CCGFzWtXsAFpE5KmYyFMxkadCYs5MMZGnYiJP9duiDSxJbZKezdnWKWn0\nlqw/j6R9JC2W9GBm3Vb1z7akEyVNLVCu7rglXSZph5xt50u6xZc7JI2vcazzJXXUKPNBvTHWUj6m\n32OdBco/KKlL0nOSfi1puxrle82/b7+vzpgHSZrj9944Scd7PIslHZT3u5LZv+i5jpa0TNKLkn6R\nWX+g1ze9nrhDCCGEEEJxfdGDtTX2fZ8JPGxmX86s2xrjLBJTI3FPBAY3sF+jMWyJ3FrOcp5xZna4\nmX0O2A34Rp11NLK90mjAzGy0mc0AzgOuM7PRwLqCxytS5p+BC8zsQOBASaeSKn7Rz/8QSfvVGXuo\nFBksJvJUTOSpmMhTIe3t7c0OoSVEnoqJPNWvLxpY20u6W9JySfdW6zmRdI7/xX2ZpOt93QBJU33d\nUkmX+fr9vRdgiaRnGvyguBvwZsW6tzLxjPc6uyT91tdNlXSzpHmSXpY01tfvJOkRj2WppDN8fZuk\nFb7fSknTJJ3i+6+U9HkvN1jSHZLmS1ok6XQP4yPgvQLn8pYfZ7LHu1jSaj/mYEn3+/pl3mtyCTAK\n6JQ01/f9tsc0Hzguc+y1pA/+vVnn5ZC0l6SZfm26JB1TTmkmt1dIWuBlOnzdFEkTMmU6JH0/r3yF\nT4C3ayXJzMoxbg8MAv5SY5cN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0co9dbuCdwFHOnHuQj4\nOnB1dl/fZ4CkGyQ97ed9YdFzlTQCGGJmC33VnaQ/KGS9QfodCCGEEEIIm1mvw6Q2k88A3zaz+ZLu\nACYAPytvlDQSuB44HHgXmOONlNXA3mb2N15uF99lGumv/rMkDaKxRuJAYH12hZkd7fUcDPwIONbM\n3pGU/SA6wsyOk/RZYBYwE/gv4EwzWytpd2C+bwPYHzjLzJZLegb4pu9/htcxFvh7YK6ZXSBpV2CB\npEfM7CngKY/pCOC7ZnZR5YmU4zazDqDDj/HvwD8BXwJeN7Ov+nGGmNkHki4H2v38RgCTSPl/HygB\ni/2YN9VKpJndm3l7C1Ays7GSBOxcLub1nwIcYGZH+fZZko4HpgO/AH7p5b8OjMkrb2ZP4I02M1sN\nnO3HHwncXj7fSpIeAo4EHjGzh2qc14b8Az8AJpjZU5IGk645wGHAwcAaYJ6kL5jZk/TsZTIze0vS\n3wE/MLNyI/xY4D4zmympLVP+AuBdMzva7/F5kh42s1cLnOvepN+dstW+Lms96XcgX3Yg4r7EX42r\nWUXkpYjIUzGRp2IiT4WUSqXodSgg8lRM5GmjUqlUaE5aXzSw/mRm8335buASMg0s0gfeTjN7G0DS\nNOAE4BpgP0k3A38AHpa0MzDKzGYBmNlH9QbjH9QP9ViqORmYYWbveB3vZrb93tetkLRX+ZDAFEkn\nkD64jspsW2Vmy335eeARX36W9LEVYAxwuqQf+vtBwD7AynKlZrYI6NG4ynE3cJOZdUlaC/yDpCnA\nA94wKcdc7lU6mu75nw4cULCuSicDf+sxG1A592oMcIqkxV7/TqQG1FRJe3pjby/gbTN7XdLEauWB\nJ6jCzP4MVG1c+fYveYPlXknjzezOguc1D/i535szPTaABV4nkpaQrumTZHrsGjSGNIxvnL/fhXTe\nr2bOpddzrWE16XfgmdwSJzV45BBCCCGEbVR7e3u3xubkyZOrluuLBlaPv+ZXKdPjA6mZvSvpUOBU\n4HvAONLcoV4/vPpQswu9ntPMbE1m2wDgj8CHwAN1nEPZh1ViPg/YAzjczNYrPXRihyrl12fer2dj\n7kXq5XqpgXi6kTSJ1KC9E8DMXlJ6kMhpwDXeM3ZNtV03tW5Xa36QgClmdnuVbTNI13gEqUerVvmG\n5nWZ2UeS/hdwFGn4XJF9firpfuArpN6kMb4pe30/YeM1/ZiNPatVHyZSg4BLzGxOA/u+Dvx15v2n\nfF3Wr4DZko4ys+82UEeA+Ct6UZGnYiJPxUSeConehmIiT8VEnurXF3Ow2iQd7cvnAo9XbF8AnCBp\nmKSBwDnAYz7cbqCZ/Q74MTDa59G8JulrsOGpbDtmD2Zmv/SHGYzONq5823oz25f0l/u8hxw8CozL\nzKEZmlOu3CjZFXjTG1cnAW1VyvRmNnDphh2kwwrs0zOYNHfri8BlmXUjgXVmdg9wI+khC5CGApaH\nXD5Nyv9Qn580jiokXZydJ5VjLmkIaHke0ZDy7v5zNvCd8hwmSaN8bhLAvcA3gbNIja288ntUHLMm\npXlyI3x5O1JDaYm/P1PSdTX2/7SZPW9mNwALgYNqVLkKOMKXzyoaZ8ZsYILHiqQDKu/zPH7Pvyep\nPKxyPPBvFcWuID0EIxpXIYQQQgibWV80sF4ALpa0nDSx/lZfb7DhA+GVpLk/XcBCM7uPNG+kJKmL\n9HCAK32/8cClkpaShm4NbyCmF4Fh1Tb4kL5rSY28LqA8DymvJ24a6cEFS4FvASuqlKm2f9nVpAeB\nLFN6TPdVlQWUHpRwWy/nA3A56eEVC5UeojAJOIQ0p6sL+Alp2CXA7cBDkuZ6/ieT5o49DizvceTk\nIGo/GGIicJKkZaRG7MG+vnyt5wD3AE95mRn4PC3P+xBgtZm90Uv5IdljZkka6T1NlXYizd9aAiwC\nXgP+xbftT+2HiUyU9Kxf44+AB6uUycZzFXCLpAWk3qw8effEr0nXYbHfE7dS0dvcy7kCXAzcQbrP\nX6oy32wo8HIvcYUitqovdtiKRZ6KiTwVE3kqJL63qJjIUzGRp/qpP36DuM932t3MrqxZOAAgaRYw\n1sx6azC0HEl3ApebWa3G4zbBe7WWAWeb2cqcMsakPg2rNcVk+2IiT8VEnorZ1vI0CbbE57B4KEEx\nkadiIk/5JGFmPUZV9dcG1v7Ab4C1Fd+FFcI2S9KBpKGYy4DzLeeXX1L/+0chhBCaYPjew1mzek3t\ngiGErVI0sEIIhUjKa3uFEEIIIQSX18DqizlYIYSwzYkx6cVEnoqJPBUTeSom8lRM5KmYyFP9ooEV\nQgghhBBCCJtJDBEMIXQTQwRDCCGEEGqLIYIhhBBCCCGEsIVFAyuEEBoQY9KLiTwVE3kqJvJUTOSp\nmMhTMZGn+kUDK4QQQgghhBA2k5iDFULoJr4HK4QQQn8zfHgba9a80uwwQouJ78EKIRSSGljx70II\nIYT+RMRn4lCveMhFCCFsVqVmB9AiSs0OoEWUmh1Aiyg1O4AWUWp2AC2i1OwAWkLMwarfFm1gSWqT\n9GzOtk5Jo7dk/Xkk7SNpsaQHM+tWNSOWPJJOlDS1QLm645Z0maQdcradL+kWX+6QNL7Gsc6X1FGj\nzAf1xlhL+Zh+j3UWKH+NpD9Jer/g8XvNv2+/r3jEIGmQpDl+742TdLyk5/z9QXm/K5n9a56rpB0l\n3S9phaRnJV2X2Xag1ze9nrhDCCGEEEJxfdGDtTX2t54JPGxmX86s2xrjLBJTI3FPBAY3sF+jMWyJ\n3FrOcp5ZwJGbUEcj2yuNBszMRpvZDOA84DozGw2sK3i8ImVuNLPPAocDx0s6lVTxi2b2OeAQSfvV\nGXvoob3ZAbSI9mYH0CLamx1Ai2hvdgAtor3ZAbSI9mYH0BLa29ubHULL6YsG1vaS7pa0XNK91XpO\nJJ0jaZm/rvd1AyRN9XVLJV3m6/f3XoAlkp5p8IPibsCbFeveysQz3uvskvRbXzdV0s2S5kl6WdJY\nX7+TpEc8lqWSzvD1bd6LMFXSSknTJJ3i+6+U9HkvN1jSHZLmS1ok6XQP4yPgvQLn8pYfZ7LHu1jS\naj/mYO/N6PI8jpN0CTAK6JQ01/f9tsc0Hzguc+y1pA/+vVnn5ZC0l6SZfm26JB1TTmkmt1dIWuBl\nOnzdFEkTMmU6JH0/r3yFT4C3ayXJzBaY2Ru1ymVsyL/3VpVzu0jSTl5miKQZfp3vysS/StIwXz5C\nqbd2T+Au4Eg/zkXA14Grs/v6PgMk3SDpaT/vC4ueq5mtM7PHfPljYDHwqYpib5B+B0IIIYQQwuZm\nZlvsBbQB64Fj/P0dwPd9uZP0F/2RwKvAMFKDby5whm97OHOsXfznfOAMXx4E7NBAXJOBiTnbDgZe\nAIb6+93851Rgui9/FnjJlwcCO/vy7pn1baQP6Qf7+2eAO3z5DGCmL18LnOvLuwIrgR0rYjoCuK3g\nue0KLCX1XowFfpXZNsR//jFzfiMy+d8OeAK4pcHr/a/Apb6sTH3v+89TyvH49vuA44HDgFLmOM8D\ne+eV9/cfVKl/JHB/jRjfb+C8ZgHH+vJgv09PBN7xOgU8CXwhk99hmWv3qC+fCMzKHHcqMDZzvyzz\n5QuBH2Xu8YVAWwPnuhvwH8C+FevnAp/vZT+Djsyr08Di1eMVeYk8RZ4iT1vrK/JUf56wUF1nZ2ez\nQ9hqdHZ2WkdHx4aX3zdUvrZjy/uTmc335buBS4CfZbYfCXSa2dsAkqYBJwDXAPtJuhn4A/CwpJ2B\nUWY2i3RGH9UbjCQBh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0iNyVGZbavMbLkvPw884svP\nAvv68hjgdEk/9PeDgH1IDS28vkXARQVP8W7gJjPrkrQW+AdJU4AHzOyJTMzlXqWj6Z7/6cABBeuq\ndDLwtx6zAZVzr8YAp0ha7PXvBBxgZlMl7SlpBLAX8LaZvS5pYrXypEZgD2b2Z+CrDcbem3nAz/3e\nnOmxASzwOpG0hHRNnyTTY9egMaRhfOP8/S6k8361XKDWuUoaCNwD/MLMXqnYvJr0O/BMfgiT6o86\nhBBCCGEb1t7e3m3I5OTJk6uW64sGltV4D1U+kJrZu5IOBU4FvgeMI80d6vXDqw81u9DrOc3M1mS2\nDSD1LnwIPFDHOZR9WCXm84A9gMPNbL3SQyd2qFJ+feb9ejbmXsBZZvZSA/F0I2kSqUF7J4CZvaT0\nIJHTgGskPWJm11TbdVPrdtWubWU9U8zs9irbZpCu8QhgeoHyterabMzsp5LuB74CzJM0xjdlr+8n\nbLymH7Nx+G3Vh4nUIOASM5vTSLzuNmClmf1jlW2/AmZLOsrMvrsJdfRz7c0OoEW0NzuAFtHe7ABa\nRHuzA2gR7c0OoEW0NzuAlhBzsOrXF3Ow2iQd7cvnAo9XbF8AnCBpmP/V/RzgMUm7AwPN7HfAj4HR\nZrYWeE3S12DDU9l2zB7MzH5pZodbepDAmopt681sX9Jf7r+RE++jwLjMHJqhOeXKjZJdgTe9cXUS\naahXZZnezAYu3bCDdFiBfXoGk+ZufRG4LLNuJLDOzO4BbiQNuwR4n9QrAvA0Kf9DJW1PauRUO/7F\n2XlSOeYCE7z8AElDyrv7z9nAd8pzmCSN8rlJAPcC3wTOIjW28srvUXHMenXbT9KZyjxpr+oO0qfN\n7Hkzu4E0XO+gGnWsIg0NhHQ+9ZoNTJC0ndd/QOV9XiPea0hDai/PKXIFcEE0rkIIIYQQNr++aGC9\nAFwsaTlpTsitvt4AvBF0JenLCLqAhWZ2H2kOTklSF+nhAFf6fuOBSyUtJQ3dGt5ATC+S5hz14EP6\nriU18rqAm7LxZov6z2mkBxcsBb4FrKhSptr+ZVeTHgSyTOkx3VdVFvAHJdzWy/kAXE56eMVCf4jC\nJOAQYIGfx09Iwy4BbgcekjTX8z+ZNLftcWB5jyMnBwF/qRHDROAkSctIjdiDfX35Ws8hDVt7ysvM\nAHb2bcuBIcBq84dR5JQfkj1mlqSR3tPUg6SfSnoN2FHpce0/8U37U/thIhOVHnm+lDSv7sEqZbLx\nXAXcImkBqTcrT9498WvSdVjs98StVPQ2552rpL2BHwEHZx7M8Z2KYkOBl3uJKxRSanYALaLU7ABa\nRKnZAbSIUrMDaBGlZgfQIkrNDqAlxPdg1U9pqkz/4vOddjezK2sWDgBImkV6IENvDYaWI+lO4HIz\nq9V43Cb4HMRlwNlmtjKnjPXhCMwWViKGlxRRIvJURInIUxElIk9FlIg8FVFiY55Ef/xMXESpVIph\ngjkkYWY9RlX11wbW/sBvgLXW/buwQthmSTqQNBRzGXC+5fzyRwMrhBBC/xMNrFC/aGCFEApJDawQ\nQgih/xg+vI01a15pdhihxeQ1sPriKYIhhBYTf3ipLYZMFBN5KibyVEzkqZjIUzGRp2IiT/Xri4dc\nhBBCCCGEEEK/EEMEQwjdSMqbnhVCCCGEEFzeEMHowQohhBBCCCGEzSQaWCGE0ID4XpBiIk/FRJ6K\niTwVE3kqJvJUTOSpftHACiGEEEIIIYTNJOZghRC6iTlYIYQQQgi1xWPaQwiFST3+rQghhH5l+N7D\nWbN6TbPDCCG0oOjBCiF0I8mY1OwoWsAqYL9mB9ECIk/FRJ6K6cs8TWrd7wSM7y0qJvJUTOQpXzxF\nMIQQQgghhBC2sC3awJLUJunZnG2dkkZvyfrzSNpH0mJJD2bWrWpGLHkknShpaoFydcct6TJJO+Rs\nO1/SLb7cIWl8jWOdL6mjRpkP6o2xlvIx/R7rLFD+Gkl/kvR+weP3mn/ffl/xiEHSIElz/N4bJ+l4\nSc/5+4Pyflcy+xc919GSlkl6UdIvMusP9Pqm1xN3yBG9DcVEnoqJPBUTeSokehuKiTwVE3mqX1/0\nYG2N/etnAg+b2Zcz67bGOIvE1EjcE4HBDezXaAxbIreWs5xnFnDkJtTRyPZKowEzs9FmNgM4D7jO\nzEYD6woer0iZfwYuMLMDgQMlnUqq+EUz+xxwiKT4mBJCCCGEsAX0RQNre0l3S1ou6d5qPSeSzvG/\nuC+TdL2vGyBpqq9bKukyX7+/9wIskfRMgx8UdwPerFj3Viae8V5nl6Tf+rqpkm6WNE/Sy5LG+vqd\nJD3isSyVdIavb5O0wvdbKWmapFN8/5WSPu/lBku6Q9J8SYskne5hfAS8V+Bc3vLjTPZ4F0ta7ccc\nLOl+X7/Me00uAUYBnZLm+r7f9pjmA8dljr2W9MG/N+u8HJL2kjTTr02XpGPKKc3k9gpJC7xMh6+b\nImlCpkyHpO/nla/wCfB2rSSZ2QIze6NWuYwN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0co9dbu\nCdwFHOnHuQj4OnB1dl/fZ4CkGyQ97ed9YdFzlTQCGGJmC33VnaQ/KGS9QfodCJtiq+rz3opFnoqJ\nPBUTeSokvreomMhTMZGn+vXFUwQ/A3zbzOZLugOYAPysvFHSSOB64HDgXWCON1JWA3ub2d94uV18\nl2mkv/rPkjSIxhqJA4H12RVmdrTXczDwI+BYM3tHUvaD6AgzO07SZ0k9IjOB/wLONLO1knYH5vs2\ngP2Bs8xsuaRngG/6/md4HWOBvwfmmtkFknYFFkh6xMyeAp7ymI4AvmtmF1WeSDluM+sAOvwY/w78\nE/Al4HUz+6ofZ4iZfSDpcqDdz28EMImU//eBErDYj3lTrUSa2b2Zt7cAJTMbK0nAzuViXv8pwAFm\ndpRvnyXpeGA68Avgl17+68CYvPJm9gTeaDOz1cDZfvyRwO3l890U2fwDPwAmmNlTkgaTrjnAYcDB\nwBpgnqQvmNmT9OxlMjN7S9LfAT8ws3Ij/FjgPjObKaktU/4C4F0zO9rv8XmSHjazVwuc696k352y\n1b4uaz3pdyBfdiDivsSwnBBCCCH0e6VSqVCDsy8aWH8ys/m+fDdwCZkGFmnYVqeZvQ0gaRpwAnAN\nsJ+km4E/AA9L2hkYZWazAMzso3qD8Q/qh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0gfXEdl\ntq0ys+W+/DzwiC8/S/rYCjAGOF3SD/39IGAfYGW5UjNbBPRoXOW4G7jJzLokrQX+QdIU4AFvmJRj\nLvcqHU33/E8HDihYV6WTgb/1mA2onHs1BjhF0mKvfydSA2qqpD29sbcX8LaZvS5pYrXywBNUYWZ/\nBja5cVXFPODnfm/O9NgAFnidSFpCuqZPkumxa9AY0jC+cf5+F9J5v1ousInnupr0O/BMbomTGjxy\nfxKNzmIiT8VEnoqJPBUSc2aKiTwVE3naqL29vVs+Jk+eXLVcXzSwevw1v0qZHh9IzexdSYcCpwLf\nA8aR5g71+uHVh5pd6PWcZmZrMtsGAH8EPgQeqOMcyj6sEvN5wB7A4Wa2XumhEztUKb8+8349G3Mv\nUi/XSw3E042kSaQG7Z0AZvaS0oNETgOu8Z6xa6rtuql1u1rzgwRMMbPbq2ybQbrGI0g9WrXK99mc\nOTP7qaT7ga+QepPG+Kbs9f2Ejdf0Yzb2rFZ9mEgNAi4xszkN7Ps68NeZ95/ydVm/AmZLOsrMvttA\nHSGEEEIIIUdfzMFqk3S0L58LPF6xfQFwgqRhkgYC5wCP+XC7gWb2O+DHwGgzWwu8JulrsOGpbDtm\nD2ZmvzSzw/1BAmsqtq03s31Jf7n/Rk68jwLjMnNohuaUKzdKdgXe9MbVSUBblTK9mQ1cumEH6bAC\n+/QMJs3d+iJwWWbdSGCdmd0D3Eh6yAKkoYDlIZdPk/I/VNL2pEZOteNfnJ0nlWMuaQhoeR7RkPLu\n/nM28J3yHCZJo3xuEsC9wDeBs0iNrbzye1Qcs17d9pN0pqTret1B+rSZPW9mNwALgYNq1LEKOMKX\nz2ogxtnABEnbef0HVN7nefyef09SeVjleODfKopdQXoIRjSuNkXMBSkm8lRM5KmYyFMhMWemmMhT\nMZGn+vVFA+sF4GJJy0kT62/19QYbPhBeSZr70wUsNLP7SPNGSpK6SA8HuNL3Gw9cKmkpaejW8AZi\nehEYVm2DD+m7ltTI6wLK85DyeuKmkR5csBT4FrCiSplq+5ddTXoQyDKlx3RfVVnAH5RwWy/nA3A5\n6eEVC/0hCpOAQ0hzurqAn5CGXQLcDjwkaa7nfzJp7tjjwPIeR04OAv5SI4aJwEmSlpEasQf7+vK1\nngPcAzzlZWbg87Q870OA1eWHUeSUH5I9Zpakkd7T1IOkn0p6DdhR6XHtP/FN+1P7YSITJT3r1/gj\n4MEqZbLxXAXcImkBqTcrT9498WvSdVjs98StVPQ293auwMXAHaT7/CUze6hi+1Dg5V7iCiGEEEII\nDVKrfkv5pvD5Trub2ZU1CwcAJM0CxppZbw2GliPpTuByM6vVeNwmeK/WMuBsM1uZU8aY1KdhhRDC\n1mcS9MfPSCGE4iRhZj1GVfXXBtb+wG+AtRXfhRXCNkvSgaShmMuA8y3nl19S//tHIYQQKgzfezhr\nVq+pXTCE0G9FAyuEUIikvLZXyCiVSvFkpQIiT8VEnoqJPBUTeSom8lRM5ClfXgOrL+ZghRBCCCGE\nEEK/ED1YIYRuogcrhBBCCKG26MEKIYQQQgghhC0sGlghhNCA+F6QYiJPxUSeiok8FRN5KibyVEzk\nqX7RwAohhBBCCCGEzSTmYIUQuok5WCGEEEIIteXNwdquGcGEELZu6fuIQwihfxk+vI01a15pdhgh\nhBYXQwRDCFVYvGq+OreCGFrhFXmKPLVOnt5441W2BTFnppjIUzGRp/pFAyuEEEIIIYQQNpPN0sCS\n1Cbp2ZxtnZJGb4566iVpH0mLJT2YWbeqGbHkkXSipKkFyq3yn7m5rig/RNJrkm7JHkPSsDpiq5kr\nv7779LL9fEn/WLTOgnGdXz4vSR2Sxtcof6SkLn8tlfSNAnVMlXRCje1j64z7eEnP+T35V5JulPSs\npJ/6eXy/xv5FzvWLkp7x81wo6aTMth9IeqHI+Yci2psdQItob3YALaK92QG0iPZmB9AS2tvbmx1C\nS4g8FRN5qt/mnINlm/FYm8uZwMNmdmVm3dYYZ5GYLGc5z9XAYw3Usynlt/RxGvUscISZrZc0AnhO\n0v80s0/6OI7zgOvM7B4ASRcCQ83MJHVspjreAr5qZmsk/TdgNvApADO7SdITwI3A9M1UXwghhBBC\nyNicQwS3l3S3pOWS7pW0Q2UBSedIWuav633dAO8NWOZ/db/M1+8vaY6kJf4X+f0aiGk34M2KdW9l\n4hnvdXZJ+q2vmyrpZknzJL1c7qWQtJOkRzK9A2f4+jZJK3y/lZKmSTrF918p6fNebrCkOyTNl7RI\n0ukexkfAewXO5a3KFZJuz/TMvCnp//H1RwB7AQ9X7gJc6vUvlXRg5tz+xa/BEkn/I6/OKv4CfOLH\n+ZIfe4mkOVXi3UPS/5T0tL+OVbJK0i6Zci9K2rNa+Sr1rwXW9Ragmf2Xma33tzsC7xVoXL1LujZI\nut57npZIuiFT5sQq98mJku7LnMs/+n12AfB14GpJd0n6N2BnYJGkcRV5+rSkB70H6rHydQI+KHCu\nS81sjS8/D+wgaftMkTXArjXOPRRSanYALaLU7ABaRKnZAbSIUrMDaAkxZ6aYyFMxkaf6bc4erM/w\n/7N372F2VHW+/98fMokRAgkhkEA0F+IZJA6JSUBEEVoBHWYGRBQU5aIPjyAXQS4eGUZJIojMIJwT\n9ScOwsQIkesAAyJDQLujB4hAroSEIBAFZBJACJIZBKG/vz9q7VC9u/betTsddnfyeT3Pfrp21aqq\n7/pWdbJXr7VqwxciYoGkK4GTgUsrGyXtDFwETCH78HpXaqQ8DYyOiEmpXOWD9lyyv/bfKmkQPWsM\nDgA68ysiYu90nonAucA+EfGipGG5YqMi4oOSdgduBW4C/gwcFhHrJe0ALEjbACYAn4yIFZIeBD6T\n9j80neNw4J+AX0TE8ZKGAvdLujsi7gPuSzFNA06MiBOqK1KJu2rdF9N+Y4A7gNmSBHyHrLfkoIKc\nPBsR0ySdBJwNnAB8A1iXuwZDa52zIIZPpX1GAJcD+0bEk1X5rJgFXBoR90p6J3BnREwmIQquAAAg\nAElEQVSUdAvwCWCOpPcBv4uI5yTNrS4PTKw6/yWVZUknZqvi8uoTp+P+GzAe+GyJep2R9htOdt3f\nnd5vlytWdJ9AQY9dRFwpaV/gtoi4KR3rTxExNS3ne7AuJ7sPHk9xXwYcEBH536eadc2V+RSwKCL+\nklvdSanf+xm55TY8LMfMzMy2dB0dHaUanL3ZwHoyIhak5auBL5NrYAF7Ae0R8QJA+vC8H3ABMF7S\nLODnwDxJQ4BdIuJWgIh4rdlgUkNjcoqlyEeAGyLixXSOdbltt6R1KyXtVDkk8G1l83I6gV1y21ZH\nxIq0/DBwd1p+CBiXlj8KHCLpq+n9IGAMsKpy0ohYSNbgaaaeg4EbgFMj4mlJpwC3R8QzWQqoft72\nzennQrJGDcCBwIZ5ORFRpket2vuB+RHxZDrGuoIyBwK7p2sDMETS1sD1wHnAHOAzvDl8rVb5QhHx\nr3W23Q/8jaTdgDsltUfEn0rU6yXgFUlXALcDP8ttK7pPNoqkbYAPADfk6j2wuly9uqbjvAf4Nt0b\n2c8DO0oaVuMaJTPKB73Famt1AP1EW6sD6CfaWh1AP9HW6gD6Bc+ZKcd5Ksd5elNbW1uXfMycObOw\n3Kacg1U076bbl+tExDpJk4GPAV8CjgC+UlS2y4Gkk4EvpvP8XWVYVNq2FfAE8CrZh+JmvVoQ8+eA\nEcCUNJdnNTC4oHxn7n2+t0BkvVy/7UE89VwG3BgR7en9PsC+KT/bkg3dfDkizq2K9Q16/3vQGn15\nkoC9q3pUAO5TNiR0BNm8uW/WK6+N+I6miFgl6XHgf5E1MhuVfyP1Ih1Adm+empah+D55na69rd2G\nyjawFfBipWerJyS9g6w37ZiI+F1+W0S8Iula4AlJn46IbkM5zczMzKznenMO1lhJlSFlnwV+XbX9\nfmA/ScMlDQCOAuan4XYDIuJm4OvA1IhYDzwl6eMAkgZJenv+YBHxg4iYEhFT842rtK0zIsYBD5Lr\nmanyS+CINAQMSdvXKFf54DyUbHhdp7Ins40tKFPPncBpG3aQ3ltin7pSb9WQiLi4si4ijo6IcRGx\nK9kQwJ/kGle13AWckjtut+F9yuaf7VznGAuAD0kam8oX5XMecHrumJNz224m6/FcketZqVe+NEnj\n0j1Hiu9dwG/T+zlK8+Rq7LsNMCwi/hM4E5hUq2j6+XtgoqSBKY8H1Cif32eDiHgZWJ2G91ViqHXO\noniHkvWyfS3Xo5zfPozsd2K0G1cbq6PVAfQTHa0OoJ/oaHUA/URHqwPoFzxnphznqRznqXm92cB6\nBDhF0gqyh0v8MK0PgNQIOofsX8fFwAMRcRswGuiQtBi4KpUBOJbsgQxLgXuAkT2I6VGg8LHkaUjf\nt8gaeYuBylyeWj1xc4G9UjxHAysLyhTtX3E+WW/SMmWPWf9mdQFJ0yTVnFNT4CxgD2UPuVgkqdHw\nwlqxXQAMV/bI8MVUjcFIQ9UmAC/UPHDE82TDG29Ox7i2oNjpwJ7KHrCxHDgxt+16sl7Ca0uW70bS\niTVysC+wVNKidJ4TcsMDJwHP1DnstsDP0nX/FXBGWl94n0TE0+kcy1NdFlWXqfO+4mjgeGUP1VgO\nHFpdoE5dTyW7Vufl7osRue1DgbURUfdhGWZmZmbWM4po9RO0N50032mHqse0W5PSfJ4vRMTZrY6l\nN0naFrgiIraY74VKwx1nRUTRExkrZaL1T9Y3M2sFsTl/LjKz3iWJiOg2Imlzb2BNAH4MrI+Ig1sc\njllLSTqLrJfw4oi4pk65zfcfBTOzOkaOHMuaNb9rdRhm1k9skQ0sM2uepPC/C411dHT4yUolOE/l\nOE/lOE/lOE/lOE/lOE+11Wpg9eYcLDMzMzMzsy2ae7DMrAv3YJmZmZk15h4sMzMzMzOzTcwNLDOz\nHvD3gpTjPJXjPJXjPJXjPJXjPJXjPDXPDSwzMzMzM7Ne4jlYZtaF52CZmZmZNVZrDtZftSIYM+vb\npG7/VphZHzNy9EjWPL2m1WGYmVkV92CZWReSghmtjqIfWA2Mb3UQ/YDzVE5P8jQDtrT/w/19POU4\nT+U4T+U4T7X5KYJmZmZmZmabWK80sCSNlfRQjW3tkqb2xnmaJWmMpEWS7sitW92KWGqRtL+k2SXK\nrU4/a+a6qvy2kp6S9N38MSQNbyK2hrlK13dMne3HSfpe2XOWjOu4Sr0kTZd0bIPye0lanF5LJX26\nxDlmS9qvwfbDm4x7X0nL0z35NkkXS3pI0j+nepzZYP+GdU3l/lHSbyWtlPTR3PqzJD1Spv5Wgntl\nynGeynGeSvFf0ctxnspxnspxnprXmz1YfXGcwmHAvIg4OLeuL8ZZJqaosVzL+cD8HpxnY8pv6uP0\n1EPAtIiYAnwM+P8kDWhBHJ8DLoyIqRHxKvBFYFJEfK23TiBpd+BIYHfgYOAHShOqIuIS4DjglN46\nn5mZmZl11ZsNrIGSrpa0QtL1kgZXF5B0lKRl6XVRWrdV6g1YlnoXTk/rJ0i6S9ISSQ9K6snf94YB\nz1atey4Xz7HpnIslzUnrZkuaJekeSY9VeikkbSPp7hTLUkmHpvVjU0/BbEmrJM2VdFDaf5WkPVO5\nrSVdKWmBpIWSDklhvAa8VKIuz1WvkPSjXM/Ms5K+kdZPA3YC5lXvApyWzr9U0l/n6vZv6RoskfSJ\nWucs8EfgjXScv03HXiLproJ4R0i6UdJv0msfZVZL2i5X7lFJOxaVLzj/euCVegFGxJ8jojO9fTvw\nUkS80aBe68iuDZIuSj1PSyT9S67M/gX3yf6SbsvV5XvpPjuerOFzvqSrJP0HMARYKOmIqjztKukO\nSQ9Iml+5TsDLjeoKfBy4NiJej4jfAb8F3pfbvgYY2uAYVkaf6gvvw5yncpynUvx9POU4T+U4T+U4\nT83rzacI7gZ8ISIWSLoSOBm4tLJR0s7ARcAUsg+vd6VGytPA6IiYlMpVPmjPJftr/62SBtGzxuAA\noDO/IiL2TueZCJwL7BMRL0oalis2KiI+mHoDbgVuAv4MHBYR6yXtACxI2wAmAJ+MiBWSHgQ+k/Y/\nNJ3jcOCfgF9ExPGShgL3S7o7Iu4D7ksxTQNOjIgTqitSibtq3RfTfmOAO4DZqbfiO2S9JQcV5OTZ\niJgm6STgbOAE4BvAutw1GFrrnAUxfCrtMwK4HNg3Ip6symfFLODSiLhX0juBOyNioqRbgE8AcyS9\nD/hdRDwnaW51eWBi1fkvqSxLOjFbFZdXnzgd99/IBuJ8tkS9zkj7DSe77u9O77fLFSu6T6Cgxy4i\nrpS0L3BbRNyUjvWniJialqfnil9Odh88nuK+DDggIvK/T7XqOpp0PyV/SOsqOinze9+eWx6Hhy+Z\nmZnZFq+jo6NUg7M3G1hPRsSCtHw18GVyDSxgL6A9Il4ASB+e9wMuAMZLmgX8HJgnaQiwS0TcChAR\nrzUbTGpoTE6xFPkIcENEvJjOsS637Za0bqWknSqHBL6tbF5OJ7BLbtvqiFiRlh8G7k7LD5F9PAX4\nKHCIpK+m94OAMcCqykkjYiFZg6eZeg4GbgBOjYinJZ0C3B4Rz2QpoPrJJjennwvJGjUABwIb5uVE\nRJketWrvB+ZHxJPpGOsKyhwI7J6uDcAQSVsD1wPnAXOAzwDXNShfKCL+tc62+4G/kbQbcKek9oj4\nU4l6vQS8IukK4HbgZ7ltRffJRpG0DfAB4IZcvQdWl6tX1waeB3aUNKzGNcp8uIdH35K40VmO81SO\n81SK54KU4zyV4zyV4zy9qa2trUs+Zs6cWViuNxtY1X+1L5p30+0xhhGxTtJksrkxXwKOAL5SVLbL\ngaSTyeawBPB3EbEmt20r4AngVbIPxc16tSDmzwEjgCkR0ansARCDC8p35t7newtE1sv12x7EU89l\nwI0RUelz2AfYN+VnW7Khmy9HxLlVsb5B738PWqMvTxKwd0T8pWr9fcqGhI4gmzf3zXrltRHf0RQR\nqyQ9DvwvskZmo/JvpF6kA8juzVPTMhTfJ6/Ttbe121DZBrYCXqz0bPXAH4B35t6/I60DICJekXQt\n8ISkT0dEt6GcZmZmZtZzvTkHa6ykypCyzwK/rtp+P7CfpOHKHjBwFDA/DbcbEBE3A18HpkbEeuAp\nSR8HkDRI0tvzB4uIH0TElPTAgDVV2zojYhzwILmemSq/BI5IQ8CQtH2NcpUPzkPJhtd1SvowMLag\nTD13Aqdt2EF6b4l96kq9VUMi4uLKuog4OiLGRcSuZEMAf5JrXNVyF7kHHxQN71M2/2znOsdYAHxI\n0thUviif84DTc8ecnNt2M1mP54pcz0q98qVJGpfuOVJ87yKbm4SkOUrz5Grsuw0wLCL+EzgTmFSr\naPr5e2CipIEpjwfUKJ/fZ4OIeBlYLelTuRhqnbPIrcBn0u/MeLK63p871jCy34nRblxtJM+ZKcd5\nKsd5KsVzQcpxnspxnspxnprXmw2sR4BTJK0ge7jED9P6AEiNoHOADmAx8EBE3EY2P6RD0mLgqlQG\n4FiyBzIsBe4BRvYgpkeBwseSpyF93yJr5C0GKnN5avXEzQX2SvEcDawsKFO0f8X5ZL1Jy5Q9Zv2b\n1QUkTZPUbf5QHWcBeyh7yMUiSY2GF9aK7QJguLJHhi8G2qriEtk8sxdqHjjiebLhjTenY1xbUOx0\nYE9lD9hYDpyY23Y9WS/htSXLdyPpxBo52BdYKmlROs8JueGBk4Bn6hx2W+Bn6br/CjgjrS+8TyLi\n6XSO5akui6rL1HlfcTRwvLKHaiwHDq0uUKuu6b6+HlhBNuT25Oj6TaRDgbUR0ehhGWZmZmbWA9qc\nvwU+zXfaISLOaVjYapL0HrIHmJzd6lh6k6RtgSsiYov5Xqg03HFWRBQ9kbFSJpjx1sVkZj00Azbn\n/8PNzPo6SUREtxFJm3sDawLwY2B91XdhmW1xJJ1F1kt4cURcU6fc5vuPgtlmZOTokax5ek3jgmZm\ntknUamD15hDBPiciHo+ID7lxZZY90j7NWazZuMqV9avBq729veUx9IeX87Tp8rQlNq48F6Qc56kc\n56kc56l5m3UDy8zMzMzM7K20WQ8RNLPmSQr/u2BmZmZW3xY5RNDMzMzMzOyt5AaWmVkPeEx6Oc5T\nOc5TOc5TOc5TOc5TOc5T89zAMjMzMzMz6yWeg2VmXXgOlpmZmVljteZg/VUrgjGzvk3q9m+FmVmh\nkSPHsmbN71odhplZn+EhgmZWIPxq+GrvAzH0h5fztLnnae3a3/NW8VyQcpyncpyncpyn5rmBZWZm\nZmZm1kt6pYElaaykh2psa5c0tTfO0yxJYyQtknRHbt3qVsRSi6T9Jc0uUW51+lkz11Xlt5X0lKTv\n5o8haXgTsTXMVbq+Y+psP07S98qes2Rcx1XqJWm6pGMblB8u6ZeSXs7no8E+syXt12D74U3Gva+k\n5emefJukiyU9JOmfUz3ObLB/mboeKOlBSUslPSDpw7ltZ0l6RNKnm4nbamlrdQD9RFurA+gn2lod\nQL/Q1tbW6hD6BeepHOepHOepeb3ZgxW9eKzechgwLyIOzq3ri3GWiSlqLNdyPjC/B+fZmPKb+jg9\n9Wfg68BZLY7jc8CFETE1Il4FvghMioiv9eI5ngP+ISImA58HrqpsiIhLgOOAU3rxfGZmZmaW05sN\nrIGSrpa0QtL1kgZXF5B0lKRl6XVRWrdV6g1Ylv7qfnpaP0HSXZKWpL/Ij+9BTMOAZ6vWPZeL59h0\nzsWS5qR1syXNknSPpMcqvRSStpF0d6534NC0fqyklWm/VZLmSjoo7b9K0p6p3NaSrpS0QNJCSYek\nMF4DXipRl+eqV0j6UYp9saRnJX0jrZ8G7ATMq94FOC2df6mkv87V7d/SNVgi6RO1zlngj8Ab6Th/\nm469RNJdBfGOkHSjpN+k1z7KrJa0Xa7co5J2LCpfcP71wCv1AoyI/4mIe4FXS9SnYh3ZtUHSRann\naYmkf8mV2b/gPtlf0m25unwv3WfHA0cC50u6StJ/AEOAhZKOqMrTrpLuSD1Q8yvXCXi5RF2XRsSa\ntPwwMFjSwFyRNcDQJvJgNXW0OoB+oqPVAfQTHa0OoF/wXJBynKdynKdynKfm9eZTBHcDvhARCyRd\nCZwMXFrZKGln4CJgCtmH17tSI+VpYHRETErlKh+055L9tf9WSYPoWWNwANCZXxERe6fzTATOBfaJ\niBclDcsVGxURH5S0O3ArcBNZL8hhEbFe0g7AgrQNYALwyYhYIelB4DNp/0PTOQ4H/gn4RUQcL2ko\ncL+kuyPiPuC+FNM04MSIOKG6IpW4q9Z9Me03BrgDmC1JwHfIeksOKsjJsxExTdJJwNnACcA3gHW5\nazC01jkLYvhU2mcEcDmwb0Q8WZXPilnApRFxr6R3AndGxERJtwCfAOZIeh/wu4h4TtLc6vLAxKrz\nX1JZlnRitioubxR3iXqdkY45nOy6vzu93y5XrOg+gYIeu4i4UtK+wG0RcVM61p8iYmpanp4rfjnZ\nffB4ysdlwAERkf99alhXSZ8CFkXEX3KrOyn1ez8jt9yGhy+ZmZnZlq6jo6NUg7M3G1hPRsSCtHw1\n8GVyDSxgL6A9Il4ASB+e9wMuAMZLmgX8HJgnaQiwS0TcChARrzUbTGpoTE6xFPkIcENEvJjOsS63\n7Za0bqWknSqHBL6tbF5OJ7BLbtvqiFiRlh8G7k7LDwHj0vJHgUMkfTW9HwSMAVZVThoRC8kaPM3U\nczBwA3BqRDwt6RTg9oh4JksB1c/bvjn9XEjWqAE4ENgwLyciyvSoVXs/MD8inkzHWFdQ5kBg93Rt\nAIZI2hq4HjgPmAN8BriuQflCEfGvPYi7kZeAVyRdAdwO/Cy3reg+2SiStgE+ANyQq/fA6nKN6irp\nPcC36d7Ifh7YUdKwGtcomVE+6C1WW6sD6CfaWh1AP9HW6gD6Bc8FKcd5Ksd5Ksd5elNbW1uXfMyc\nObOwXG82sKr/al8076bbl+tExDpJk4GPAV8CjgC+UlS2y4Gkk8nmsATwd5VhUWnbVsATZEPCbm+i\nDhX5oWSVOD4HjACmRESnsgdADC4o35l7n+8tEFkv1297EE89lwE3RkR7er8PsG/Kz7ZkQzdfjohz\nq2J9g97/HrRGX54kYO+qHhWA+5QNCR1BNm/um/XK6y38jqaIeCP1Ih1Adm+empah+D55na69rd2G\nyjawFfBipWerJyS9g6w37ZiI+F1+W0S8Iula4AlJn46IbkM5zczMzKznenMO1lhJlSFlnwV+XbX9\nfmA/ZU90GwAcBcxPw+0GRMTNZA8imBoR64GnJH0cQNIgSW/PHywifhARU9IDA9ZUbeuMiHHAg+R6\nZqr8EjgiDQFD0vY1ylU+OA8lG17XqezJbGMLytRzJ3Dahh2k95bYp67UWzUkIi6urIuIoyNiXETs\nSjYE8Ce5xlUtd5F78EHR8D5l8892rnOMBcCHJI1N5YvyOQ84PXfMybltN5P1eK7I9azUK99TXa6V\npDlK8+QKC2c9SsMi4j+BM4FJDY77e2CipIEpjwfUKN8tFoCIeBlYnYb3VWKodc6ieIeS9bJ9Ldej\nnN8+jOx3YrQbVxuro9UB9BMdrQ6gn+hodQD9gueClOM8leM8leM8Na83G1iPAKdIWkH2cIkfpvUB\nkBpB55D9L7IYeCAibgNGAx2SFpM98eyctN+xZA9kWArcA4zsQUyPAoWPJU9D+r5F1shbDFTm8tTq\niZsL7JXiORpYWVCmaP+K88l6k5Ype8z6N6sLSJomqZn5Q2cBeyh7yMUiSY2GF9aK7QJguLJHhi+m\naqxKGqo2AXih5oEjnicb3nhzOsa1BcVOB/ZU9oCN5cCJuW3Xk/USXluyfDeSTqyVg9TjeAlwnKQn\nJb07bZoEPFPnsNsCP0vX/VfAGWl94X0SEU+nuixPdVlUXabO+4qjgeOVPVRjOXBoQX1q1fVUsmt1\nXu6+GJHbPhRYGxF1H5ZhZmZmZj2jiFY/QXvTSfOddoiIcxoWtprSfJ4vRMTZrY6lN0naFrgiIraY\n74VKwx1nRUTRExkrZaL1T9Y3s/5DbM6fJczMapFERHQbkbS5N7AmAD8G1ld9F5bZFkfSWWS9hBdH\nxDV1yrmBZWZNcAPLzLZMtRpYvTlEsM+JiMcj4kNuXJllj7RPcxZrNq7eJL/88suvUq+RI/NTkjct\nzwUpx3kqx3kqx3lqXm8/Rc7MNgP+a3RjHR0dfnRtCc5TOc6TmdnmY7MeImhmzZMU/nfBzMzMrL4t\ncoigmZmZmZnZW8kNLDOzHvCY9HKcp3Kcp3Kcp3Kcp3Kcp3Kcp+a5gWVmZmZmZtZLPAfLzLrwHCwz\nMzOzxjwHy8zMzMzMbBNzA8vMupHkl1996jXqHaNa/WuxSXmOQznOUznOUznOUznOU/P8PVhm1t2M\nVgfQD6wGxrc6iH6gl/K0dsbajT+ImZnZW8BzsMysC0nhBpb1OTP8BdhmZta3SJtwDpaksZIeqrGt\nXdLU3jhPsySNkbRI0h25datbEUstkvaXNLtEudXpZ81cV5XfVtJTkr6bP4ak4U3E1jBX6fqOqbP9\nOEnfK3vOknEdV6mXpOmSjm1QfrikX0p6OZ+PBvvMlrRfg+2HNxn3vpKWp3vybZIulvSQpH9O9Tiz\nwf4N65rK/aOk30paKemjufVnSXpE0qebidvMzMzMyuvNOVh98U+LhwHzIuLg3Lq+GGeZmKLGci3n\nA/N7cJ6NKb+pj9NTfwa+DpzV4jg+B1wYEVMj4lXgi8CkiPhab51A0u7AkcDuwMHADyQJICIuAY4D\nTumt823R+tSfavow56kUz3Eox3kqx3kqx3kqx3lqXm82sAZKulrSCknXSxpcXUDSUZKWpddFad1W\nqTdgmaSlkk5P6ydIukvSEkkPSurJKP5hwLNV657LxXNsOudiSXPSutmSZkm6R9JjlV4KSdtIujvF\nslTSoWn92NRTMFvSKklzJR2U9l8lac9UbmtJV0paIGmhpENSGK8BL5Woy3PVKyT9KMW+WNKzkr6R\n1k8DdgLmVe8CnJbOv1TSX+fq9m/pGiyR9Ila5yzwR+CNdJy/TcdeIumugnhHSLpR0m/Sax9lVkva\nLlfuUUk7FpUvOP964JV6AUbE/0TEvcCrJepTsY7s2iDpotTztETSv+TK7F9wn+wv6bZcXb6X7rPj\nyRo+50u6StJ/AEOAhZKOqMrTrpLukPSApPmV6wS83KiuwMeBayPi9Yj4HfBb4H257WuAoU3kwczM\nzMya0JsPudgN+EJELJB0JXAycGllo6SdgYuAKWQfXu9KjZSngdERMSmVq3zQnkv21/5bJQ2iZ43B\nAUBnfkVE7J3OMxE4F9gnIl6UNCxXbFREfDD1BtwK3ETWC3JYRKyXtAOwIG0DmAB8MiJWSHoQ+Eza\n/9B0jsOBfwJ+ERHHSxoK3C/p7oi4D7gvxTQNODEiTqiuSCXuqnVfTPuNAe4AZqfeiu+Q9ZYcVJCT\nZyNimqSTgLOBE4BvAOty12BorXMWxPCptM8I4HJg34h4siqfFbOASyPiXknvBO6MiImSbgE+AcyR\n9D7gdxHxnKS51eWBiVXnv6SyLOnEbFVc3ijuEvU6Ix1zONl1f3d6v12uWNF9AgU9dhFxpaR9gdsi\n4qZ0rD9FxNS0PD1X/HKy++DxlI/LgAMiIv/7VKuuo0n3U/KHtK6ikzK/9+255XH4YQ5FnJNynKdS\n2traWh1Cv+A8leM8leM8leM8vamjo6NUj15vNrCejIgFaflq4MvkGljAXkB7RLwAkD487wdcAIyX\nNAv4OTBP0hBgl4i4FSAiXms2mNTQmJxiKfIR4IaIeDGdY11u2y1p3UpJO1UOCXxb2bycTmCX3LbV\nEbEiLT8M3J2WHyL7eArwUeAQSV9N7wcBY4BVlZNGxEKyBk8z9RwM3ACcGhFPSzoFuD0inslSQPXE\nu5vTz4VkjRqAA4EN83IiokyPWrX3A/Mj4sl0jHUFZQ4Edk/XBmCIpK2B64HzgDnAZ4DrGpQvFBH/\n2oO4G3kJeEXSFcDtwM9y24ruk40iaRvgA8ANuXoPrC63EXV9HthR0rAa1yjz4R4e3czMzGwz1dbW\n1qXBOXPmzMJym3IOVtG8m25P2Ugf8iYDHcCXgB/VKtvlQNLJaWjcIkmjqrZtRTbyf3eyD8XNyg8l\nq8TxOWAEMCUippANPRxcUL4z9z7fWyCyXq4p6TU+Ilax8S4DboyISp/DPsCpkp4g68k6RtKFBXV7\ng95/TH/da5a2753LwZg0fO8+YELqBTsM+Pd65Xs55roi4g2yIXY3Av8A/Gduc9F98jpdf6+6DZVt\nYCvgxTRPq1Lvv2li/z8A78y9f0daB0BEvAJcCzwhqaiH08ry3KJynKdSPMehHOepHOepHOepHOep\neb3ZwBorqTKk7LPAr6u23w/sp+yJbgOAo4D5abjdgIi4mexBBFMjYj3wlKSPA0gaJOnt+YNFxA/S\nh8+pEbGmaltnRIwDHiTXM1Pll8ARaQgYkravUa7ywXko2fC6TkkfBsYWlKnnTuC0DTtI7y2xT12p\nt2pIRFxcWRcRR0fEuIjYlWwI4E8i4twGh7qL3IMPiob3KZt/tnOdYywAPiRpbCpflM95wOm5Y07O\nbbuZrMdzRa5npV75nupyrSTNUZonV1g461EaFhH/CZwJTGpw3N8DEyUNTHk8oGwsABHxMrBa0qdy\nMdQ6Z5Fbgc+k35nxwLvIfvcqxxpG9jsxOiK6zZMzMzMzs43Tmw2sR4BTJK0ge7jED9P6AEiNoHPI\neqoWAw9ExG1k80M6JC0GrkplAI4leyDDUuAeYGQPYnoUKHwseRrS9y2yRt5ioDKXp1ZP3FxgrxTP\n0cDKgjJF+1ecT/YgkGXKHrP+zeoCkqZJamb+0FnAHrmevEbDC2vFdgEwXNkjwxcDbVVxiWye2Qs1\nDxzxPNnwxpvTMa4tKHY6sKeyB2wsB07MbbuerJfw2pLlu5F0Yq0cKHvk/CXAcSw5BAgAACAASURB\nVJKelPTutGkS8Eydw24L/Cxd918BZ6T1hfdJRDyd6rI81WVRdZk67yuOBo5X9lCN5cChBfUprGu6\nr68HVpANuT05un550FBgberJso3huUXlOE+leI5DOc5TOc5TOc5TOc5T8zbrLxpO8512iIhzGha2\nmiS9h+wBJme3OpbeJGlb4IqI2GK+Fyo9NGNWRBQ9kbFSxl80bH3PDH/RsJmZ9S2q8UXDm3sDawLw\nY2B91XdhmW1xJJ1F1kt4cURcU6fc5vuPgvVbI0ePZM3TaxoX7Kc6Ojr8V+ISnKdynKdynKdynKfa\najWwevshB31KRDwOfKjVcZj1BemR9pc0LIh7CsrwfzjlOE9mZral2ax7sMyseZLC/y6YmZmZ1Ver\nB6s3H3JhZmZmZma2RXMDy8ysB/y9IOU4T+U4T+U4T+U4T+U4T+U4T81zA8vMzMzMzKyXeA6WmXXh\nOVhmZmZmjXkOlpmZmZmZ2SbmBpaZdSPJL7+2+NeoUePest85z3Eox3kqx3kqx3kqx3lq3mb9PVhm\n1lMeIthYB9DW4hj6gw76a57Wru026sPMzKwhz8Eysy4khRtYZgDyl26bmVlNUh+agyVprKSHamxr\nlzT1rY4pnXuMpEWS7sitW92KWGqRtL+k2SXKrU4/a+a6qvy2kp6S9N3cunZJYxrsN1vSfg3iva3R\n+ZuRP6ak4yRNL7HPP0t6SNIySUeWKD9d0rENtp/ZZNy7SVosaaGk8ZJOk7RC0lWpHt9rsH/Dukqa\nLOneVNcl+bpKOkrSI5LOaCZuMzMzMyuvlXOw+uKfBQ8D5kXEwbl1fTHOMjFFjeVazgfm9yycpmLZ\nFMese3xJfwe8F5gEvB84W9KQTRBTI4cBN0TEtIhYDZwEHBgRx6TtzV7XIv8NHBMRewAHA/9X0nYA\nEXENsD/gBlav6Gh1AP1ER6sD6Bc8x6Ec56kc56kc56kc56l5rWxgDZR0dfoL/vWSBlcXSH9xX5Ze\nF6V1W6Vek2WSlko6Pa2fIOmu9Ff7ByWN70FMw4Bnq9Y9l4vn2HTOxZLmpHWzJc2SdI+kxyQdntZv\nI+nuFMtSSYem9WMlrUz7rZI0V9JBaf9VkvZM5baWdKWkBanH45AUxmvASyXq8lz1Ckk/SrEvlvSs\npG+k9dOAnYB5Vbv8EXijwXnWpZiQtFeqx5IU9zZV5y+sk6T7JO2eK9cuaWqdHOS9AqxvEONE4FeR\n+R9gGfC3DfZ5OR2b1NP0cKrXT3Nl3pNifUzSl1PZLj2Gks5KvV0HA18BTpL0C0mXAbsCd1Tu4dw+\nIyTdKOk36bVP2bpGxGMR8Xha/i+y+3nH3Pa1wNAGdTczMzOznoqIt/wFjAU6gfen91cCZ6bldmAq\nsDPwe2A4WUPwF8Chadu83LG2Sz8XAIem5UHA4B7ENRP4So1tE4FHgO3T+2Hp52zgurS8O/DbtDwA\nGJKWd8itH0vWIJmY3j8IXJmWDwVuSsvfAj6blocCq4C3V8U0Dbi8RK6XVa0bAzwMvANQyvkuwHHA\nd3t4TQcCjwNT0/sh6brtD9xar07A6cCMtH4UsLJB+Q3HrIrhkMpxqtYfBPw67TsixXlGE3X7AzCw\n6n6bDvw/sgfF7AA8n655l3wDZwHn5fY5M7ftidz9tCH3wFzgA2n5ncCKsnWtKvM+4OGC9S832C9g\neu7VHhB++bUFvggzM7OK9vb2mD59+oZX+n+C6lcrnyL4ZEQsSMtXA18GLs1t3wtoj4gXACTNBfYD\nLgDGS5oF/ByYl4Z77RIRtwJExGvNBiNJwOQUS5GPkA3vejGdY11u2y1p3UpJO1UOCXxb2fykTmCX\n3LbVEbEiLT8M3J2WHwLGpeWPAodI+mp6P4isYbSqctKIWAic0GQ9BwM3AKdGxNOSTgFuj4hnshTQ\n08dm7QY8ExGLUmzr0/nyZWrV6Qay3rMZwJHAjQ3KF4qI24Bu870i4i5JewH3kvXo3Evjnrm8pcBP\nJd1CutbJ7RHxOvBHSWuBkU0cE7JcF+X7QGB3vZm8IZK2jqz3Dahd1w0HlnYGfgIcU7D5BUkTIvV0\nFZvRMHgzMzOzLUlbWxttbW0b3s+cObOwXF+ag1X9Hgo+fKaGzWSygf1fAn5Uq2yXA0knp6FxiySN\nqtq2FbCarAfq9lLRd/VqQcyfI+stmRIRU8g+2A8uKN+Ze9/Jm4/OF/DJiJiSXuMjYhUb7zLgxoho\nT+/3AU6V9ATwHeAYSRf28NiNGmeFdYqIZ4DnJe0BfBq4LrdPr+QgIi5Mx/gY2X3/aBO7/z3wfbLe\n0wfS/QLdr+NfAa+T9WRVdBv6WoKAvXP1HpNvXDXcWdoW+BnwjxHxQEGRWcASSZ/vQWy2QUerA+gn\nOlodQL/gOQ7lOE/lOE/lOE/lOE/Na2UDa6ykvdPyZ8mGcOXdD+wnabikAcBRwHxJOwADIuJm4Otk\nQ9LWA09J+jiApEGS3p4/WET8IH1YnRoRa6q2dUbEOLLhep+uEe8vgSMkDU/n2L5GuUojYyjwbER0\nSvow2dCx6jL13AmctmEH6b0l9qkr9VYNiYiLK+si4uiIGBcRuwJnAz+JiHML9p1TmR9WwypgVJrP\nhaQh6brl1avTdcD/JhuCt7xE+dKUzdurXLdJwB6k+WaSLqzcNzX2FTAmIuYD5wDbkQ1/rGUtsKOk\n7SW9DfiHHoQ8j2zYZCWGyWV3lDSQrJdtTvodKXIu8K6I+HEPYjMzMzOzOlrZwHoEOEXSCrKHS/ww\nrQ+A1Ag6h+zPn4uBB9KwqNFAh6TFwFWpDMCxwGmSlgL30PxwLch6NYYXbUhD+r5F1shbDFySjzdf\nNP2cC+yV4jkaWFlQpmj/ivPJHgSyLD004ZvVBSRNk3R5nfpUOwvYI9eT18zwwknAM7U2RsRfyBqn\n35e0hKyR8LaqYvXq9O907726oE75biQdImlGwaaBwK8lLSe7z46OiM60bQ9gTcE+FQOAq9N1XAjM\niog/FZSr3LevpzgfIGsgriwo22WfAqcDeyp7OMpy4MTqAnXqeiSwL/D53HWeVFVmUGQPu7CN0tbq\nAPqJtlYH0C/kh5xYbc5TOc5TOc5TOc5T8/xFwzlprs8OEXFOw8JbkDTk7IqIqNW7129JuiO6PpZ/\ns5bmAS6NiJ3rlInabT+zLYm/aNjMzGpTX/qi4T7sJuCDyn3RsEFEvLw5Nq4AtrDG1VFkPYv/UqK0\nX35t8a+RI/Mjuzctz3Eox3kqx3kqx3kqx3lqXiufItjnpKeqfajVcZhtCpF90fA1Jctu4mj6v46O\nDg+bKMF5MjOzLY2HCJpZF5LC/y6YmZmZ1echgmZmZmZmZpuYG1hmZj3gMenlOE/lOE/lOE/lOE/l\nOE/lOE/NcwPLzMzMzMysl3gOlpl14TlYZmZmZo15DpaZmZmZmdkm5gaWmXUjyS+/Nslr1DtGtfr2\n7pM8x6Ec56kc56kc56kc56l5/h4sM+tuRqsD6AdWA+NbHUQ/UJWntTPWtiwUMzOzt4LnYJlZF5LC\nDSzbZGb4i6zNzGzzIPWhOViSxkp6qMa2dklT3+qY0rnHSFok6Y7cutWtiKUWSftLml2i3Or0s2au\nq8pvK+kpSd/NrWuXNKbBfrMl7dcg3tsanb8Z+WNKOk7S9BL7/LOkhyQtk3RkifLTJR3bYPuZTca9\nm6TFkhZKGi/pNEkrJF2V6vG9BvuXretxkh6VtCpfB0lHSXpE0hnNxG1mZmZm5bVyDlZf/BPmYcC8\niDg4t64vxlkmpqixXMv5wPyehdNULJvimHWPL+nvgPcCk4D3A2dLGrIJYmrkMOCGiJgWEauBk4AD\nI+KYtL3Z69qNpO2B84C9gL2B6ZKGAkTENcD+gBtYvaFP/emlD3OeSvEch3Kcp3Kcp3Kcp3Kcp+a1\nsoE1UNLV6S/410saXF0g/cV9WXpdlNZtlXpNlklaKun0tH6CpLskLZH0oKSezI4YBjxbte65XDzH\npnMuljQnrZstaZakeyQ9JunwtH4bSXenWJZKOjStHytpZdpvlaS5kg5K+6+StGcqt7WkKyUtSD0e\nh6QwXgNeKlGX56pXSPpRin2xpGclfSOtnwbsBMyr2uWPwBsNzrMuxYSkvVI9lqS4t6k6f2GdJN0n\nafdcuXZJU+vkIO8VYH2DGCcCv4rM/wDLgL9tsM/L6diknqaHU71+mivznhTrY5K+nMp26TGUdFbq\n7ToY+ApwkqRfSLoM2BW4o3IP5/YZIelGSb9Jr32aqOvHyP5I8FJErCO7phvqGhFrgaENjmFmZmZm\nPdTKh1zsBnwhIhZIuhI4Gbi0slHSzsBFwBSyD/F3pUbK08DoiJiUym2XdpkLXBgRt0oaRM8ajwOA\nzvyKiNg7nWcicC6wT0S8KGlYrtioiPhgaiTcCtwE/Bk4LCLWS9oBWJC2AUwAPhkRKyQ9CHwm7X9o\nOsfhwD8Bv4iI41MPxP2S7o6I+4D7UkzTgBMj4oTqilTirlr3xbTfGOAOYLYkAd8BPgccVFX+U40S\nFhFnpGMOBK4FjoiIRamH6JWq4oV1Svt9GpghaVTK5yJJ36pRPn/+6yvLqQE2LSJmVJ13KXCepEuB\nbYAPAw83qNelubdfA8ZFxF9y9xtk93AbWYNllaQfVHbvfri4Q9IPgZcrx5b0MaAt3U/H5crPAi6N\niHslvRO4E5hYsq6jgady7/+Q1uU1/t1ozy2Pww9zKOKclOM8ldLW1tbqEPoF56kc56kc56kc5+lN\nHR0dpXr0WtnAejIiFqTlq4Evk2tgkQ1xao+IFwAkzQX2Ay4AxkuaBfwcmJc+zO8SEbcCRMRrzQaT\nGhqTUyxFPkI2vOvFdI51uW23pHUrJe1UOSTwbWXzkzqBXXLbVkfEirT8MFBpNDxE9nEW4KPAIZK+\nmt4PAsYAqyonjYiFQLfGVYN6DgZuAE6NiKclnQLcHhHPZCmg20S9knYDnomIRSm29el8+TK16nQD\nWU/LDOBI4MYG5QtFxG1At/leEXGXpL2Ae8l6KO+lcc9c3lLgp5JuIV3r5PaIeB34o6S1wMgmjglZ\nrovyfSCwu95M3hBJW6feN6B2XUt6QdKEiHi8ZokP9/DIZmZmZpuptra2Lg3OmTNnFpbrS3OwiuaW\ndPvwmRo2k4EO4EvAj2qV7XIg6eQ0NG5R6iXJb9uKbKbA7sDtpaLv6tWCmD8HjACmRMQUsg/2gwvK\nd+bed/Jmo1dkvVxT0mt8RKxi410G3BgRlT6KfYBTJT1B1pN1jKQLe3jsRo2zwjpFxDPA85L2IOvJ\nui63T6/kICIuTMf4GNl9/2gTu/898H1gKvBAul+g+3X8K+B1sp7Qim5DX0sQsHeu3mPyjasG/kDX\nRug70rq8WcASSZ/vQWxW4blF5ThPpXiOQznOUznOUznOUznOU/Na2cAaK6kyjO2zwK+rtt8P7Cdp\nuKQBwFHA/DTcbkBE3Ax8HZiaekuekvRxAEmDJL09f7CI+EH6sDo1ItZUbeuMiHHAg2Qf8Iv8EjhC\n0vB0ju1rlKs0MoYCz0ZEp6QPA2MLytRzJ3Dahh2k95bYp67UWzUkIi6urIuIoyNiXETsCpwN/CQi\nzi3Yd47S/LAaVgGj0rBFJA1J1y2vXp2uA/43sF1ELC9RvjRl8/Yq120SsAdpvpmkCyv3TY19BYyJ\niPnAOcB2QL0HZKwFdpS0vaS3Af/Qg5DnARvmZUma3MS+dwIHSRqa7tGD0rq8c4F3RcSPexCbmZmZ\nmdXRygbWI8ApklaQPVzih2l9AKRG0DlkPVWLgQfSsKjRQIekxcBVqQzAscBpkpYC99D8cC3IejWG\nF21IQ/q+RdbIWwxcko83XzT9nAvsleI5GlhZUKZo/4rzyR4Esiw9NOGb1QUkTZN0eZ36VDsL2CPX\nk9fM8MJJwDO1NkbEX8gap9+XtISskfC2qmL16vTvdO+9uqBO+W4kHSJpRsGmgcCvJS0nu8+OjojK\nXLs9gDUF+1QMAK5O13EhMCsi/lRQrnLfvp7ifICsYbOyoGyXfQqcDuyp7OEoy4ETqwvUqmsawno+\n2R8LfgPMrBrOCjAoPezCNobnFpXjPJXiOQ7lOE/lOE/lOE/lOE/N8xcN56S5PjtExDkNC29BJG0L\nXBERtXr3+i1Jd1Q9ln+zluYBLo2IneuU8RcN26Yzw180bGZmmwfV+KJhN7ByJE0Afgys35I+dNuW\nQdJRZE9EnBMR/6dOOf+jYJvMyNEjWfN0vU7jLVNHR4f/SlyC81SO81SO81SO81RbrQZWK58i2Oek\np6p9qNVxmG0K6YuGrylZdhNH0//5P5xynCczM9vSuAfLzLqQFP53wczMzKy+Wj1YrXzIhZmZmZmZ\n2WbFDSwzsx7w94KU4zyV4zyV4zyV4zyV4zyV4zw1zw0sMzMzMzOzXuI5WGbWhedgmZmZmTXmOVhm\nZmZmZmabmBtYZtaNJL/88mszeo0aNa7V/6z0Cs8FKcd5Ksd5Ksd5ap6/B8vMCniIYGMdQFuLY+gP\nOnCeyuhgU+Zp7dpuI1jMzGwT8RwsM+tCUriBZba5kb9A3Mysl0l9aA6WpLGSHqqxrV3S1Lc6pnTu\nMZIWSbojt251K2KpRdL+kmaXKLc6/ayZ66ry20p6StJ3c+vaJY1psN9sSfs1iPe2RudvRv6Yko6T\nNL3EPm+ka7tY0i0lyk+XdGyD7Wc2Gfdu6fwLJY2XdJqkFZKuSvX4XoP9G9ZV0mRJ90p6SNISSUfm\nth0l6RFJZzQTt5mZmZmV18o5WH3xT2mHAfMi4uDcur4YZ5mYosZyLecD83sWTlOxbIpjljn+f0fE\n1IiYEhGHbYJ4yjgMuCEipkXEauAk4MCIOCZtb/a6Fvlv4JiI2AM4GPi/krYDiIhrgP0BN7B6RUer\nA+gnOlodQD/R0eoA+gXPBSnHeSrHeSrHeWpeKxtYAyVdnf6Cf72kwdUF0l/cl6XXRWndVqnXZJmk\npZJOT+snSLor/dX+QUnjexDTMODZqnXP5eI5Np1zsaQ5ad1sSbMk3SPpMUmHp/XbSLo7xbJU0qFp\n/VhJK9N+qyTNlXRQ2n+VpD1Tua0lXSlpQerxOCSF8RrwUom6PFe9QtKPUuyLJT0r6Rtp/TRgJ2Be\n1S5/BN5ocJ51KSYk7ZXqsSTFvU3V+QvrJOk+SbvnyrVLmlonB3mvAOsbxAjQ7ASEl9OxST1ND6d6\n/TRX5j0p1sckfTmV7dJjKOms1Nt1MPAV4CRJv5B0GbArcEflHs7tM0LSjZJ+k177lK1rRDwWEY+n\n5f8iu593zG1fCwxtMhdmZmZmVlZEvOUvYCzQCbw/vb8SODMttwNTgZ2B3wPDyRqCvwAOTdvm5Y61\nXfq5ADg0LQ8CBvcgrpnAV2psmwg8Amyf3g9LP2cD16Xl3YHfpuUBwJC0vENu/ViyBsnE9P5B4Mq0\nfChwU1r+FvDZtDwUWAW8vSqmacDlJXK9rGrdGOBh4B1kDY92YBfgOOC7PbymA4HHganp/ZB03fYH\nbq1XJ+B0YEZaPwpY2aD8hmNWxXBI5TgF215Lub4X+HiTdfsDMLDqfpsO/D+yB8XsADyfrnmXfANn\nAefl9jkzt+2J3P20IffAXOADafmdwIpm6por8z7g4YL1LzfYL2B67tUeEH755Ve/fhFmZrZx2tvb\nY/r06Rte6d9Wql+tfIrgkxGxIC1fDXwZuDS3fS+gPSJeAJA0F9gPuAAYL2kW8HNgnqQhwC4RcStA\nRLzWbDCSBExOsRT5CNnwrhfTOdbltt2S1q2UtFPlkMC3lc1P6gR2yW1bHREr0vLDwN1p+SFgXFr+\nKHCIpK+m94PIGkarKieNiIXACU3WczBwA3BqRDwt6RTg9oh4JktB0z09FbsBz0TEohTb+nS+fJla\ndbqBrPdsBnAkcGOD8oUi4jag1nyvsRHxX6ln85eSlkU2TK+MpcBPlc3dys/fuj0iXgf+KGktMLLk\n8SpEcb4PBHbXm8kbImnriPifSoEGdUXSzsBPgGMKNr8gaUKknq5iMxoGb2ZmZrYlaWtro62tbcP7\nmTNnFpbrS3Owqt9DwYfP1LCZTDZg/UvAj2qV7XIg6eQ0NG6RpFFV27YCVpP1QN1eKvquXi2I+XPA\nCGBKREwhG6o1uKB8Z+59J28+Ol/AJyObMzQlIsZHxCo23mXAjRHRnt7vA5wq6QngO8Axki7s4bEb\nNc4K6xQRzwDPS9oD+DRwXW6fXslBZMPlSI2qDmBKE7v/PfB9st7TB9L9At2v418Br5P1ZFV0G/pa\ngoC9c/Uek29cNdxZ2hb4GfCPEfFAQZFZwBJJn+9BbLZBR6sD6Cc6Wh1AP9HR6gD6Bc8FKcd5Ksd5\nKsd5al4rG1hjJe2dlj8L/Lpq+/3AfpKGSxoAHAXMl7QDMCAibga+TjYkbT3wlKSPA0gaJOnt+YNF\nxA/Sh9WpEbGmaltnRIwjG0L26Rrx/hI4QtLwdI7ta5SrNDKGAs9GRKekD5MNHasuU8+dwGkbdpDe\nW2KfulJv1ZCIuLiyLiKOjohxEbErcDbwk4g4t2DfOZX5YTWsAkal+VxIGpKuW169Ol0H/G+yIXjL\nS5QvTdIwSYPS8gjgg8CK9P7Cyn1TY18BYyJiPnAOsB3Z8Mda1gI7Stpe0tuAf+hByPPIhk1WYphc\ndkdJA8l62eak35Ei5wLviogf9yA2MzMzM6ujlQ2sR4BTJK0ge7jED9P6AEiNoHPI/qy3GHggDYsa\nDXRIWgxclcoAHAucJmkpcA/ND9cCeJRszlc3aUjft8gaeYuBS/Lx5oumn3OBvVI8RwMrC8oU7V9x\nPtmDQJalhyZ8s7qApGmSLq9Tn2pnAXvkevKaGV44CXim1saI+AtZ4/T7kpaQNRLeVlWsXp3+ne69\nVxfUKd+NpEMkzSjYtDvwYLpuvwAujIhH0rY9gDUF+1QMAK5O13EhMCsi/lRQrnLfvp7ifICsgbiy\noGyXfQqcDuyp7OEoy4ETqwvUqeuRwL7A53PXeVJVmUGRPezCNkpbqwPoJ9paHUA/0dbqAPqF/NAc\nq815Ksd5Ksd5ap6/aDgnzfXZISLOaVh4C5KGnF0REbV69/otSXdE18fyb9bSPMClEbFznTJRu+1n\nZv2Tv2jYzKy3qS990XAfdhPwQeW+aNggIl7eHBtXAFtY4+oosp7Ff2l1LJuHjlYH0E90tDqAfqKj\n1QH0C54LUo7zVI7zVI7z1LxWPkWwz0lPVftQq+Mw2xQi+6Lha8qV7unDJM2sLxo5cmzjQmZm1is8\nRNDMupAU/nfBzMzMrD4PETQzMzMzM9vE3MAyM+sBj0kvx3kqx3kqx3kqx3kqx3kqx3lqnhtYZmZm\nZmZmvcRzsMysC8/BMjMzM2vMc7DMzMzMzMw2MTewzMx6wGPSy3GeynGeynGeynGeynGeynGemufv\nwTKzbiR/D5b1npGjR7Lm6TWtDsPMzOwt4TlYZtaFpGBGq6OwzcoM8P81Zma2ufEcLDMzMzMzs02s\nJQ0sSWMlPVRjW7ukqW91TOncYyQtknRHbt3qVsRSi6T9Jc0uUW51+lkz11Xlt5X0lKTv5ta1SxrT\nYL/ZkvZrEO9tjc7fjPwxJR0naXqJfd5I13axpFtKlJ8u6dgG289sMu7d0vkXShov6TRJKyRdlerx\nvQb7l63rcZIelbQqXwdJR0l6RNIZzcRtNfSpfxn6MOepFM9xKMd5Ksd5Ksd5Ksd5al4r52D1xfEi\nhwHzIuKc3Lq+GGeZmKLGci3nA/N7Fk5TsWyKY5Y5/n9HREsa7jmHATdExIUAkk4CDoiIZyQdR/PX\ntRtJ2wPnAVMBAQsl/UdEvBQR10j6JfAA8H82piJmZmZmVqyVQwQHSro6/QX/ekmDqwukv7gvS6+L\n0rqtUq/JMklLJZ2e1k+QdJekJZIelDS+BzENA56tWvdcLp5j0zkXS5qT1s2WNEvSPZIek3R4Wr+N\npLtTLEslHZrWj5W08v9n7+7Dra7q/P8/X5CIiiJqgTpf0JguJyfvQEYdHTyaN6l5MyYVYdpMl9lo\natlopI0eU/Om9MrJ1Ex/yAiaWt6ghiLGIRIJhcNBBckKy5tBLaVB8/68f3+s94bP2WffrH06uDny\nflzXuc7ea63P5/Ne77057HXWWp/jxy2TNFXSgX78Mkm7e7uNJV0vaZ7PeBzuYbwF/CWjLy+VF0j6\nscfeLulFSf/l5aOBDwEzyg75M/Buneus9JiQNMb7scjj3qTs+hX7JOlhSR8ttJslaVSNHBS9Drxa\nJ0ZIg41GrPJz4zNNT3i/biq0+UeP9beSTvG2XWYMJX3dZ7sOAb4K/IekByVdDXwYmF56DxeO2UrS\nTyX92r/2aqCvB5N+SfAXM1tJek0/Uao0sxeAwQ3mIlTSk58w66PIU5aWlpZmh9AnRJ7yRJ7yRJ7y\nRJ4a18wZrB2AfzOzeZKuB04CLi9VStoauBjYjfQh/gEfpDwLbGtmO3u7zfyQqcB3zGyapAH0bPDY\nH+gsFpjZHn6dHYGzgL3M7BVJmxeaDTOzvX2QMA24HXgDOMrMXpW0JTDP6wBGAp8ysyWSHgU+68cf\n4dc4GjgbeNDMvihpMDBf0kwzexh42GMaDZxoZl8q70gp7rKyE/y44cB0YJIkAd8DJgAHlrU/pl7C\nzOxrfs4NgJ8A48xsoaRB+ACloGKf/LjPAK2Shnk+F0q6sEr74vVvLT32AdhoM2utEOqGnuu3gEvM\n7K46/bq88PQbwHZm9nbh/QbpPdxCGrAsk3RV6fDup7Ppkq4BVpXOLelgoMXfT8cX2l8BXG5mcyX9\nP+B+YMfMvm4LPFN4/pyXFdX/tzGr8Hg74kNyCCGEENZ7bW1tWUsmmzmD9Uczm+ePpwD7lNWPAWaZ\n2ctm1kkaQI0Ffg9s77NGBwOr/MP8NmY2DcDM3jKzNxoJxgcau5AGcJXsT1re9YpfY2Wh7k4vW0qa\nCYI0Y3KRpA5gJrCNpFLdcjNb4o+f8HqAx0gfZwEOAiZKagfagAFAl/1QzdMG8wAAIABJREFUZrag\n0uCqTj8HArcBXzGzZ0kD23vN7PlC3D2xA/C8mS302F71162oWp9uA0qDuU8DP63TviIzu7vK4Apg\nhJntThpIfr/BGc4O4CZJE+g6o3evmb1jZn8GXgCGNnBOSLmulO8DgCu939OAQZI2Ljao09d6XpY0\nsmaL/QpfMbiqLPYW5Yk8ZYk9DnkiT3kiT3kiT3kiT2u0tLTQ2tq6+quadWkPVqW9Jd0+fJrZSkm7\nkJZCfRkYR1p6VXNgIOkk4AS/zqFmtqJQ1480cHsTuLeBPpS8WSHmCcBWwG5m1ql004mBFdp3Fp53\nsuY1EWmW66kexFPL1cBPzaw0R7EXsI/nZ1PS0s1VZnZWD85db3BWtU+S/iRpJ9JM1omFqm7tfZar\nIWb2v/59uaQ20sxo7ke/w0iD+yOAsyV9zMvLX8cPAO+QZkJLui19zSBgDzN7uwfHPkeaVSv5O7rO\nR0GaIVsk6RQzu6EH1wghhBBCCFU0cwZrhKTSMrbPAXPK6ucDYyVtIak/MB6Y7cvt+pvZHcC3gFFm\n9irwjKQjASQNkLRR8WRmdpWZ7WZmo4qDK6/rNLPtgEdJH/Ar+QUwTtIWfo0hVdqVBhmDgRd9cLUf\nMKJCm1ruB05dfYC0a8YxNUk6GRhkZt8tlZnZsWa2nZl9GPhP4H8qDa4kTZbvD6tiGTDMly0iaZC/\nbkW1+nQLcCawmZk9ntE+m6TNfdkokrYC9gaW+PPvlN43VY4VMNzMZgMTgc2AQTUu9wLwQUlDJG0I\nfLIHIc8AVu/L8l8o5LofOFDSYH+PHuhlRWcBfx+Dq79RzOzliTxliT0OeSJPeSJPeSJPeSJPjWvm\nAOtJ4GRJS0g3l7jGyw3AB0ETSUvD2oFHzOxu0n6SNl8+daO3ATgOONWX5D1E48u1AH4DbFGpwpf0\nXUga5LUDlxXjLTb171OBMR7PscDSCm0qHV9yPmk2abHfNOHb5Q0kjZZ0bY3+lPs6sJPSTS4WSmpk\neeHOwPPVKn225TOkpW2LSIOEDcua1erTz/z4WwplF9Ro342kwyW1Vqj6KPCov24PkvbqPel1OwEr\nKhxT0h+Y4q/jAuAKM/u/Cu1K79t3PM5HSAObpRXadjmmgtOA3ZVujvI4XWf0gOp99SWs55N+WfBr\n4Lyy5awAA/xmFyGEEEIIoZfJbF28C3lzSDoD2LLsNu3rPUmbAteZWbXZvT5L0nQzO6TZcbxXfB9g\nh5ltXaON0frexdRnLSdmZ3IsByZD/F9TW1tbW/yWOEPkKU/kKU/kKU/kqTpJmFm3lWnN3IO1Lrod\nuGF9+9Bdj5mtovrSyT5tfXqdJY0n3RHx0rqNW9d2NGF9MnTbniwoCCGEEPqmmMEKIXQhyeLnQggh\nhBBCbdVmsJq5ByuEEEIIIYQQ3ldigBVCCD0QfxckT+QpT+QpT+QpT+QpT+QpT+SpcTHACiGEEEII\nIYReEnuwQghdxB6sEEIIIYT6Yg9WCCGEEEIIIaxlMcAKIYQeiDXpeSJPeSJPeSJPeSJPeSJPeSJP\njYu/gxVC6EbqNtsdQliHDB06ghUrnm52GCGEECqIPVghhC4kGcTPhRDWbSL+/w4hhOaKPVghhBBC\nCCGEsJY1ZYAlaYSkx6rUzZI06r2Oya89XNJCSdMLZcubEUs1kvaVNCmj3XL/XjXXZe03lfSMpP8u\nlM2SNLzOcZMkja0T7931rt+I4jklHS/p3Ixjpkt6RdK0zGucK+m4OvWn50cNknaQ1C5pgaTtJZ0q\naYmkG70fP6hzfN2+StpF0lxJj0laJOnThbrxkp6U9LVG4g7VtDU7gD6irdkB9BFtzQ6gT4i9IHki\nT3kiT3kiT41r5gzWuri24ShghpkdUihbF+PMicmqPK7mfGB2z8JpKJa1cc6c818KHLsW4mjEUcBt\nZjbazJYD/wEcYGaf9/pGX9dKXgM+b2Y7AYcA35e0GYCZ3QzsC8QAK4QQQghhLWnmAGsDSVP8N/i3\nShpY3sB/477Yvy72sn4+a7JYUoek07x8pKQH/Lf2j0ravgcxbQ68WFb2UiGe4/ya7ZIme9kkSVdI\nekjSbyUd7eWbSJrpsXRIOsLLR0ha6sctkzRV0oF+/DJJu3u7jSVdL2mez3gc7mG8Bfwloy8vlRdI\n+rHH3i7pRUn/5eWjgQ8BM8oO+TPwbp3rrPSYkDTG+7HI496k7PoV+yTpYUkfLbSbJWlUjRwUvQ68\nWidGzGxWTruCVX5ufKbpCe/XTYU2/+ix/lbSKd62y4yhpK/7bNchwFeB/5D0oKSrgQ8D00vv4cIx\nW0n6qaRf+9deuX01s9+a2e/88f+S3s8fLNS/AAxuIA+hqpZmB9BHtDQ7gD6ipdkB9AktLS3NDqFP\niDzliTzliTw1rpl3EdwB+DczmyfpeuAk4PJSpaStgYuB3Ugf4h/wQcqzwLZmtrO328wPmQp8x8ym\nSRpAzwaP/YHOYoGZ7eHX2RE4C9jLzF6RtHmh2TAz29sHCdOA24E3gKPM7FVJWwLzvA5gJPApM1si\n6VHgs378EX6No4GzgQfN7IuSBgPzJc00s4eBhz2m0cCJZval8o6U4i4rO8GPGw5MByZJEvA9YAJw\nYFn7Y+olzMy+5ufcAPgJMM7MFkoahA9QCir2yY/7DNAqaZjnc6GkC6u0L17/1tJjH4CNNrPWenFn\n9OvywtNvANuZ2duF9xuk93ALacCyTNJVpcO7n86mS7oGWFU6t6SDgRZ/Px1faH8FcLmZzZX0/4D7\ngR0b7aukfwI2KA24CjL+bRRP20J8+AshhBDC+q6trS1ryWQzZ7D+aGbz/PEUYJ+y+jHALDN72cw6\nSQOoscDvge191uhgYJV/mN/GzKYBmNlbZvZGI8H4QGMX0gCukv1Jy7te8WusLNTd6WVLSTNBAAIu\nktQBzAS2kVSqW25mS/zxE14P8BiwnT8+CJgoqZ20OH8A0GU/lJktqDS4qtPPgcBtwFfM7FnSwPZe\nM3u+EHdP7AA8b2YLPbZX/XUrqtan24DSYO7TwE/rtK/IzO7ujcFVBR3ATZIm0HVG714ze8fM/gy8\nAAxt8Lyicr4PAK70fk8DBknauNigXl/9FxT/A3yhQvXLkkbWDq218NVSu+l6q63ZAfQRbc0OoI9o\na3YAfULsBckTecoTecoTeVqjpaWF1tbW1V/VNHMGq9tv+Su06fbh08xWStoFOBj4MjCOtPSq5sBA\n0knACX6dQ81sRaGuH2ng9iZwbwN9KHmzQswTgK2A3cysU+mmEwMrtO8sPO9kzWsi0izXUz2Ip5ar\ngZ/6kjmAvYB9PD+bkpZurjKzs3pw7nqDs6p9kvQnSTuRZrJOLFR1a++zXO+lw0iD+yOAsyV9zMvL\nX8cPAO+QZkJLui19zSBgDzN7uwfHImlT4B7gm2b2SIUmVwCLJJ1iZjf05BohhBBCCKGyZs5gjZBU\nWsb2OWBOWf18YKykLST1B8YDs325XX8zuwP4FjDKzF4FnpF0JICkAZI2Kp7MzK4ys93MbFRxcOV1\nnWa2HfAo6QN+Jb8Axknawq8xpEq70iBjMPCiD672A0ZUaFPL/cCpqw+Qds04piZJJwODzOy7pTIz\nO9bMtjOzDwP/CfxPpcGVpMny/WFVLAOG+bJFJA3y162oVp9uAc4ENjOzxzPa90S3GSNJ3ym9byoe\nkGY2h5vZbGAisBkwqMY1XgA+KGmIpA2BT/YgzhnA6n1Z/guFLL5U805gsv8bqeQs4O9jcPW3aml2\nAH1ES7MD6CNamh1AnxB7QfJEnvJEnvJEnhrXzAHWk8DJkpaQbi5xjZcbgA+CJpLWTbQDj5jZ3cC2\nQJsvn7rR2wAcB5zqS/IeovHlWgC/AbaoVOFL+i4kDfLagcuK8Rab+vepwBiP51hgaYU2lY4vOZ80\nm7TYb5rw7fIGkkZLurZGf8p9HdhJ6SYXCyU1srxwZ+D5apU+2/IZ0tK2RaRBwoZlzWr16Wd+/C2F\nsgtqtO9G0uGSWqvU/dLPvb+kP0oq7TfbCVhR6RjXH5jir+MC4Aoz+78K7Urv23c8zkdIA8SlFdp2\nOaaC04DdlW6O8jhdZ/RK/anW10+Tltt+ofA671zWZoDf7CKEEEIIIfQyxV+CX0PSGcCWZjaxbuP1\niC85u87Mqs3u9VmSppfdlv99zfcBdpjZ1jXa2Lr51wnWNW3ErEOONiJPOdpoLE9iffz/u62tLX6b\nniHylCfylCfyVJ0kzKzbyrRm7sFaF90O3LC+feiux8xWUX3pZJ+2Pr3OksaT7oh4aUbrtR1OCOFv\nMHToiPqNQgghNEXMYIUQupBk8XMhhBBCCKG2ajNYzdyDFUIIIYQQQgjvKzHACiGEHoi/C5In8pQn\n8pQn8pQn8pQn8pQn8tS4GGCFEEIIIYQQQi+JPVghhC5iD1YIIYQQQn2xByuEEEIIIYQQ1rIYYIUQ\nQg/EmvQ8kac8kac8kac8kac8kac8kafGxd/BCiF0I8XfwQp/m6HbDmXFsyuaHUYIIYTwnos9WCGE\nLiQZrc2OIvR5rRD/v4QQQng/iz1YIYQQQgghhLCWNWWAJWmEpMeq1M2SNOq9jsmvPVzSQknTC2XL\nmxFLNZL2lTQpo91y/14112XtN5X0jKT/LpTNkjS8znGTJI2tE+/d9a7fiOI5JR0v6dyMY6ZLekXS\ntMxrnCvpuDr1p+dHDZJ2kNQuaYGk7SWdKmmJpBu9Hz+oc3xuX4+X9BtJy4p9kDRe0pOSvtZI3KGK\ndeonw7or1u7niTzliTzliTzliTzliTw1rpkzWOvi2pGjgBlmdkihbF2MMycmq/K4mvOB2T0Lp6FY\n1sY5c85/KXDsWoijEUcBt5nZaDNbDvwHcICZfd7rG31du5E0BDgHGAPsAZwraTCAmd0M7AvEACuE\nEEIIYS1p5gBrA0lT/Df4t0oaWN7Af+O+2L8u9rJ+PmuyWFKHpNO8fKSkByQtkvSopO17ENPmwItl\nZS8V4jnOr9kuabKXTZJ0haSHJP1W0tFevomkmR5Lh6QjvHyEpKV+3DJJUyUd6Mcvk7S7t9tY0vWS\n5vmMx+EexlvAXzL68lJ5gaQfe+ztkl6U9F9ePhr4EDCj7JA/A+/Wuc5KjwlJY7wfizzuTcquX7FP\nkh6W9NFCu1mSRtXIQdHrwKt1YsTMZuW0K1jl58Znmp7wft1UaPOPHutvJZ3ibbvMGEr6us92HQJ8\nFfgPSQ9Kuhr4MDC99B4uHLOVpJ9K+rV/7dVAXw8m/ZLgL2a2kvSafqKQhxeAwQ3kIVTTk58w66GW\nlpZmh9AnRJ7yRJ7yRJ7yRJ7yRJ4a18y7CO4A/JuZzZN0PXAScHmpUtLWwMXAbqQP8Q/4IOVZYFsz\n29nbbeaHTAW+Y2bTJA2gZ4PH/kBnscDM9vDr7AicBexlZq9I2rzQbJiZ7e2DhGnA7cAbwFFm9qqk\nLYF5XgcwEviUmS2R9CjwWT/+CL/G0cDZwINm9kWfgZgvaaaZPQw87DGNBk40sy+Vd6QUd1nZCX7c\ncGA6MEmSgO8BE4ADy9ofUy9hZvY1P+cGwE+AcWa2UNIgfIBSULFPftxngFZJwzyfCyVdWKV98fq3\nlh77AGy0mbXWizujX5cXnn4D2M7M3i683yC9h1tIA5Zlkq4qHd79dDZd0jXAqtK5JR0MtPj76fhC\n+yuAy81srqT/B9wP7JjZ122BZwrPn/Oyovr/NmYVHm9HDCZCCCGEsN5ra2vLWjLZzBmsP5rZPH88\nBdinrH4MMMvMXjazTtIAaizwe2B7nzU6GFjlH+a3MbNpAGb2lpm90UgwPtDYhTSAq2R/0vKuV/wa\nKwt1d3rZUtJMEICAiyR1ADOBbSSV6pab2RJ//ITXAzxG+jgLcBAwUVI70AYMALrshzKzBZUGV3X6\nORC4DfiKmT1LGtjea2bPF+LuiR2A581socf2qr9uRdX6dBtQGsx9GvhpnfYVmdndvTG4qqADuEnS\nBLrO6N1rZu+Y2Z+BF4ChDZ5XVM73AcCV3u9pwCBJGxcb/I19fVnSyJot9it8xeCqstiDlSXW7ueJ\nPOWJPOWJPOWJPOWJPK3R0tJCa2vr6q9qmjmD1e23/BXadPvwaWYrJe1CWgr1ZWAcaelVzYGBpJOA\nE/w6h5rZikJdP9LA7U3g3gb6UPJmhZgnAFsBu5lZp9JNJwZWaN9ZeN7JmtdEpFmup3oQTy1XAz/1\nJXMAewH7eH42JS3dXGVmZ/Xg3PUGZ1X7JOlPknYizWSdWKjq1t5nud5Lh5EG90cAZ0v6mJeXv44f\nAN4hzYSWdFv6mkHAHmb2dg+OfY40q1byd3Sdj4I0Q7ZI0ilmdkMPrhFCCCGEEKpo5gzWCEmlZWyf\nA+aU1c8HxkraQlJ/YDww25fb9TezO4BvAaPM7FXgGUlHAkgaIGmj4snM7Coz283MRhUHV17XaWbb\nAY+SPuBX8gtgnKQt/BpDqrQrDTIGAy/64Go/YESFNrXcD5y6+gBp14xjapJ0MjDIzL5bKjOzY81s\nOzP7MPCfwP9UGlxJmlzaH1bFMmCYL1tE0iB/3Ypq9ekW4ExgMzN7PKN9T3SbMZL0ndL7puIBaWZz\nuJnNBiYCmwGDalzjBeCDkoZI2hD4ZA/inAGs3pflv1DIdT9woKTB/h490MuKzgL+PgZXf6OY2csS\na/fzRJ7yRJ7yRJ7yRJ7yRJ4a18wB1pPAyZKWkG4ucY2XG4APgiaSloa1A4+Y2d2k/SRtvnzqRm8D\ncBxwqi/Je4jGl2sB/AbYolKFL+m7kDTIawcuK8ZbbOrfpwJjPJ5jgaUV2lQ6vuR80mzSYr9pwrfL\nG0gaLenaGv0p93VgJ6WbXCyU1Mjywp2B56tV+mzLZ0hL2xaRBgkbljWr1aef+fG3FMouqNG+G0mH\nS2qtUvdLP/f+kv4oqbTfbCdgRaVjXH9gir+OC4ArzOz/KrQrvW/f8TgfIQ1sllZo2+WYCk4Ddle6\nOcrjdJ3RK/WnYl99Cev5pF8W/Bo4r2w5K8AAv9lFCCGEEELoZTJbF+9C3hySzgC2NLOJdRuvRyRt\nClxnZtVm9/osSdPLbsv/vub7ADvMbOsabYzW9y6mPms5MYtVSyuYGW1tbfHbzwyRpzyRpzyRpzyR\npzyRp+okYWbdVqY1cw/Wuuh24Ib17UN3PWa2iupLJ/u09el1ljSedEfES+s2bl3b0YT3u6Hb9mQR\nQQghhND3xQxWCKELSRY/F0IIIYQQaqs2g9XMPVghhBBCCCGE8L4SA6wQQuiB+LsgeSJPeSJPeSJP\neSJPeSJPeSJPjYsBVgghhBBCCCH0ktiDFULoIvZghRBCCCHUF3uwQgghhBBCCGEtiwFWCCH0QKxJ\nzxN5yhN5yhN5yhN5yhN5yhN5alz8HawQQjdSt9nuEHrN0KEjWLHi6WaHEUIIIawVsQcrhNCFJIP4\nuRDWJhH/94QQQujrYg9WCCGEEEIIIaxlWQMsSSMkPValbpakUb0bVh5JwyUtlDS9ULa8GbFUI2lf\nSZMy2i0vtL+7WhtJW/RiXBWvU6ivGbe/L2bVadPr74/iOXNeb0k7S5orqUPSXZIGZRxT87ySVuVH\nvPqY70p6TNIlkraSNE/SAkn75Ly2mX29VNJSSYsk/UzSZoW6X0qaL+lDjcYeKmlrdgB9RFuzA+gT\nYo9DnshTnshTnshTnshT4xqZwVoX13McBcwws0MKZetinDkxWZXHjZ6nEfXO12jczZBz/euAM81s\nF+AO4MxeOG9P+n0CsLOZfQM4AFhsZqPN7FeZ58tpMwP4RzPbFXgK+Obqg83GAguAwxqOPIQQQggh\nZGlkgLWBpCmSlki6VdLA8gaSxkta7F8Xe1k/SZO8rEPSaV4+UtID/pv2RyVt34P4NwdeLCt7qRDP\ncX7NdkmTvWySpCskPSTpt5KO9vJNJM30WDokHeHlI3xGYJKkZZKmSjrQj18maXdvt7Gk6wuzEod7\nGG8Bf8noy0uFx4Ml3SPpSUlXFcpXr/GUdLrPhiwu5HRjP67dy8d5+RiPd5HHt0nxwpLu9ZnAdkkr\nJX0+M+53gZf9HP0KMzSLJJ1c3tjzNtdzfIvHe7CkWwttVs+sSTqovH2dvFXzER/EAMwEPpVxzEse\nwzBJsz0/iyXtvSZUXeB9nSvpg144qfSe8uer/PtdwCBggaQzgUuAo/y8A+n62k6Q9Guvu1pafceJ\nun01s5lm1ulP5wF/V9ZkBenfTfibtTQ7gD6ipdkB9AktLS3NDqFPiDzliTzliTzliTw1rpG7CO4A\n/JuZzZN0PXAScHmpUtLWwMXAbsBK4AEfpDwLbGtmO3u70pKlqcB3zGyapAH0bD9Yf6CzWGBme/h1\ndgTOAvYys1ckFT9UDjOzvSV9FJgG3A68ARxlZq9K2pL04XSatx8JfMrMlkh6FPisH3+EX+No4Gzg\nQTP7oqTBwHxJM83sYeBhj2k0cKKZfam8I6W43Rjgo8AfgfslHW1mt5cqlZbHHe/t+gO/ltTmcT5n\nZp/0dptK2gD4CTDOzBYqLY97vezahxXO+/8Bd5rZqlLc1ZjZs8Ax/vRLwAjSDI2V5RvP6beAj5vZ\n6z7IOB24CPiRpI3M7HXgM8BN3v7sCu0vqJY3SfcCXzSzFWWhPiHpCDObBnya7oOOSn0rnfdzwH1m\ndpEPdEqDvE2AuWb2LUmXkGanvlPpVH6+IyX9n5mVlja+AIw2s1P9eakP/+A5+Gcze1fSD4EJwJTM\nvhb9O+m1L+okvWfqaC08biE+JIcQQghhfdfW1pa1ZLKRQc0fzWyeP54C7FNWPwaYZWYv+2/QpwJj\ngd8D2/us0cHAKv+Qv41/4MXM3jKzNxqIBf+wuwtpAFfJ/sBtZvaKX2Nloe5OL1sKlPajCLhIUgdp\nlmMbrdmrstzMlvjjJ7we4DFgO398EDBRUjtp08EAYHgxIDNbUGlwVcF8M/uDpdts3Uz3XO8D3GFm\nb5jZa6QB4r94PAdKukjSPj5I2gF43swWegyvFmY4VpO0FXAjMN6Pa9QBwI885vJ8A+wJ7Ag85Dk6\nDhhuZu8C9wGHS+pPWr42rVr7WgGY2WFVBhz/Dpws6RHSwOitBvr1CPBvks4hDR5f8/I3zezn/ngB\na94H5XLvd15a/vdxYBTwiPd7f+DD3RpX72u6qHQ28LaZ3VRW9Rywc/1wWgtfLfWbr5famh1AH9HW\n7AD6hNjjkCfylCfylCfylCfytEZLSwutra2rv6ppZAarfP9Hpf0g3T5MmtlKSbsABwNfBsYBX63U\ntsuJpJNIswIGHFr8MCmpH2ng9iZwbwN9KHmzQswTgK2A3cysU+mGAgMrtO8sPO9kTQ5FmuV6qgfx\nlMvJdfeDzJ7yWahDgfMlPUgaTNbLdT/SQK7VB51rg0j75SZUqLsF+ArwCvCImb3mA+hq7RtiZr8h\nvf+Q9BEa2INkZnMkjfVjbpB0mZlNAd4uNHuXNe+Dd/BfXHgfNmgwXAGTzezsBo9bcwLpC6T3wP4V\nqm8HzpG0xMx27Ok1QgghhBBCZY3MYI2QVFw2Naesfj4wVtIWPhMxHpjtS736m9kdpCVio8zsVeAZ\nSUcCSBogaaPiyczsKjPbzcxGlf+m3sw6zWw74FHScqpKfgGMk9+ZTdKQKu1Kg4/BwIs+uNqPtNyt\nvE0t9wOnrj5A2jXjmGr2UNr71Y/Uv/JczyHt3xmotJ/qX4E5vkzzdZ+1+B5pJmQZMMyXJyJpkL8+\nRZcAHWZ2W6VglPZwTa4T8wPAiaVzV8j3PGBvSSO9fmMf7ADM9lhPYM2StlrtG1LYH9WP9B68xp9v\nI2lmnWOHk94X15NullG6I2K198TTwO7++Ei6DrBqvY9KdQ8CxxRiHuIxZJH0CeAM4Agze7NCk+OA\n6TG46g0tzQ6gj2hpdgB9QuxxyBN5yhN5yhN5yhN5alwjA6wnScuslpA2yV/j5aUlYSuAiaT1IO2k\nmYi7gW2BNl/udKO3gfRB71RfkvcQMLQH8f8GqHhra1/SdyFpkNcOXFaMt9jUv08Fxng8xwJLK7Sp\ndHzJ+aQbgSxWuqX9t8sbSBot6doa/SmZD1xJWo74OzO7s3htM2sHbiAtX3sYuNbMOoCdSHu/2oFz\ngAvM7G3SIO1KSYtId5nbsOx6XwcOUrrJxUJJnyyrHw78tU7M1wHPAIv9+uPLYv4T8AXgZs/xXNLy\nRXzJ4j3AJ/x7zfZUeQ2UbtYxrELVeEnLgCWkPWo3ePnWdJ2JqqQF6JC0kLR/6/u1YgB+DOzrOdgT\neK1QV2smspSnpaRB4Azv9wygW59q9PUHpJtpPOCv5VVl9UNIdxcMIYQQQghrgXzLTJ8k6QxgSzOb\nWLdx6DG/icONZvZ4s2PpTUp3OvyDmd3T7FjeK37TjMVm9qMabaz5d9/vC9qI2ZkcbXTPk+jL//es\nDW1tbfFb4gyRpzyRpzyRpzyRp+okYWbdVig1sgdrXXQ7aV/M9LK/hRV6kf/dpvcdM/ths2N4L0ma\nTdo3WOluhyGEEEIIoRf06RmsEELvSzNYIaw9Q4eOYMWKp5sdRgghhPA3eb/OYIUQ1oL4xUsIIYQQ\nQs/05I/7hhDCei/+LkieyFOeyFOeyFOeyFOeyFOeyFPjYoAVQgghhBBCCL0k9mCFELqQZPFzIYQQ\nQgihtmp7sGIGK4QQQgghhBB6SQywQgihB2JNep7IU57IU57IU57IU57IU57IU+NigBVCCCGEEEII\nvST2YIUQuoi/g9V3DN12KCueXdHsMEIIIYT1UrU9WDHACiF0IclobXYUIUtr/M2yEEIIoVniJhch\nhNCbljc7gL4h1u7niTzliTzliTzliTzliTw1LmuAJWmEpMeq1M2SNKp3w8ojabikhZKmF8rWqY89\nkvaVNCmj3fJC+7urtZG0RS/GVfE6hfqacfv7YladNr3+/iieM+f1lrSzpLmSOiTdJWlQxjE1zytp\nVX7Eq4/5rqTHJF0iaStJ8yQtkLRPzmub2dchkmZIWibpfkmDC3UTog2GAAAgAElEQVS/lDRf0oca\njT2EEEIIIeRpZAZrXVyHchQww8wOKZSti3HmxGRVHjd6nkbUO1+jcTdDzvWvA840s12AO4Aze+G8\nPen3CcDOZvYN4ABgsZmNNrNfZZ4vp81EYKaZ7QD8Avjm6oPNxgILgMMajjx0t32zA+gbWlpamh1C\nnxB5yhN5yhN5yhN5yhN5alwjA6wNJE2RtETSrZIGljeQNF7SYv+62Mv6SZrkZR2STvPykZIekLRI\n0qOSevJxZXPgxbKylwrxHOfXbJc02csmSbpC0kOSfivpaC/fRNJMj6VD0hFePkLSUj9umaSpkg70\n45dJ2t3bbSzp+sKsxOEexlvAXzL68lLh8WBJ90h6UtJVhfLVazwlne6zIYsLOd3Yj2v38nFePsbj\nXeTxbVK8sKR7fSawXdJKSZ/PjPtd4GU/R7/CDM0iSSeXN/a8zfUc3+LxHizp1kKb1TNrkg4qb18n\nb9V8xAcxADOBT2Uc85LHMEzSbM/PYkl7rwlVF3hf50r6oBdOKr2n/Pkq/34XMAhYIOlM4BLgKD/v\nQLq+thMk/drrrpZUqsvp65HAZH88mfRLiKIVpH83IYQQQghhLWhkgLUDcKWZ7QisAk4qVkraGrgY\naAF2Bcb4IGVXYFsz29lnEErLzqYCPzCzXYF/Bv63B/H3BzqLBWa2h8ezI3AW0GJmuwGnFZoNM7O9\ngcNJH3QB3gCOMrPdgf2BywrtRwLf9VmBHYDP+vFn+DUAzgYeNLM9/fjvSdrIzB42s695TKMlXVup\nI6W43RjgZOCjwN8XP7D7eUYBx3u7vYATJO0CfAJ4zsx2M7OdgfskbQD8BDjFc30A8HrZtQ8zs1HA\nF4GngTuLcVdjZs+a2TH+9EvACNIMza6k17cY85bAt4CPe44XAKeTBjz/JGkjb/oZ4CZvf3aF9lXz\n5gPFYRVCfaI0YAY+DfxdrX6VnfdzwH2en12ARV6+CTDX+zqHNDtV8VR+viOBv5rZKDO7FDgH+Ik/\nf6PQh3/wHPyzX7MTmNBAXz9kZi94+xVA+XLATtK/m9pmFb7WqUW365DIS5ZYu58n8pQn8pQn8pQn\n8pQn8rRGW1sbra2tq7+q+UAD5/yjmc3zx1OAU4DLC/VjgFlmVprRmAqMBS4Atpd0BfBzYIbSHpht\nzGwagJm91UAc+PlF+sA7pUqT/YHbzOwVv8bKQt2dXrZUa/ajCLhI0ljSh9BtCnXLzWyJP36CNCgA\neAzYzh8fBBwu6Qx/PgAYDiwrXdTMFpAGIvXMN7M/eD9vBvYBbi/U7wPcUfpgLul24F+A+0kDu4uA\ne83sV5I+BjxvZgs9hlf9mC4XlLQVcCNwjJk1vL+INHC72vyWZmX5BtgT2BF4yF+7DUgDlHcl3UfK\n3c9Iy9fOIA3Uu7WvFYCZVVv69u/ADyT9FzCNNDuX6xHgeh+o3mVmHV7+ppn93B8vIPW/km53lqmi\ntPzv48Ao4BHv90DghW6Nq/e12nlLniPltrb9Ms8eQgghhLCeaGlp6bJk8rzzzqvYrpEBVvkHtUr7\nQbp9mDSzlT67cjDwZWAc8NVKbbucSDqJNCtgwKH+2/hSXT/g98CbwL0N9KHkzQoxTwC2AnYzs06l\nGwoMrNC+s/C8kzU5FPApM3uqB/GUy8l194PMnvLZrUOB8yU9SBpM1st1P+BmoNXMlvYg3hwi7Zeb\nUKHuFuArwCvAI2b2mg8uqrVviJn9hvT+Q9JHaGAPkpnN8UH3YcANki4zsynA24Vm77LmffAOPjNc\nGBg2QsBkMzu7weNKXpA01Mxe8Bmu8iW0twPnSFris9Ghp2IPVpZYu58n8pQn8pQn8pQn8pQn8tS4\nRpYIjpBUXDY1p6x+PjBW0haS+gPjgdm+1Ku/md1BWiI2ymdRnpF0JICkAYUlYgCY2VW+1G1UcXDl\ndZ1mth3wKGk5VSW/AMbJ78wmaUiVdqXBx2DgRR9c7Uda7lbeppb7gVNXHyDtmnFMNXso7f3qR+pf\nea7nkPbvDPT9VP8KzPFlmq+b2U3A90gzIcuAYZJGe1yD/PUpugToMLPbKgWjtIdrcqW6ggeAE0vn\nrpDvecDekkZ6/cY+2AGY7bGeQFrOWK99Qwr7o/qR3oPX+PNtJM2sc+xw0vvietLNMkp3RKz2nnga\n2N0fH0nXAVat91Gp7kHgmELMQzyGXNOAL/jj44G7yuqPA6bH4CqEEEIIYe1oZID1JHCypCWkTfLX\neHlpSdgK0h3M2oB20kzE3cC2QJukdtIStIl+3HHAqZI6gIeAoT2I/zdAxVtb+5K+C0mDvHbW7Kmq\nNjs0lbRvrAM4FlhaoU2l40vOJ90IZLHSLe2/Xd6g1h6sMvOBK0nLEX9nZncWr21m7cANpOVrDwPX\n+tK1nYD53t9zgAvM7G3SIO1KSYuAGcCGZdf7OnCQ0k0uFkr6ZFn9cOCvdWK+DngGWOzXH18W859I\nH/xv9hzPJe1nw8w6gXtIe8juqdeeKq9BjX1J4yUtA5aQ9qjd4OVb03UmqpIWoEPSQtL+re/XigH4\nMbCv52BP4LVCXa2ZyFKelpIGgTO83zOAbn2q0ddLgAO9vx8n7YssGgL0xixriD1YWWLtfp7IU57I\nU57IU57IU57IU+PkW2b6JN/vtKWZTazbOPSYpEuAG83s8WbH0puU7nT4BzO7p9mxvFck/ZB0e/gf\n1WhjtL53MfVZy2n+MsFWWNd/hre1tcXykgyRpzyRpzyRpzyRpzyRp+okYWbdVij19QHWSNJMzqtl\nfwsrhFBG0mzSvsFjzey5Gu367g+F9czQbYey4tkV9RuGEEIIode9LwdYIYTeJ8ni50IIIYQQQm3V\nBliN7MEKIYTgYk16nshTnshTnshTnshTnshTnshT42KAFUIIIYQQQgi9JJYIhhC6iCWCIYQQQgj1\nxRLBEEIIIYQQQljLYoAVQgg9EGvS80Se8kSe8kSe8kSe8kSe8kSeGhcDrBBCCCGEEELoJbEHK4TQ\nRfwdrBDeX4YOHcGKFU83O4wQQnjfib+DFULIkgZY8XMhhPcPEf/XhxBC74ubXIQQQq9qa3YAfURb\nswPoI9qaHUCfEHtB8kSe8kSe8kSeGpc1wJI0QtJjVepmSRrVu2HlkTRc0kJJ0wtly5sRSzWS9pU0\nKaPd8kL7u6u1kbRFL8ZV8TqF+ppx+/tiVp02vf7+KJ4z5/WWdK6kZ/29slDSJzKOqXleSavyI159\nzHclPSbpEklbSZonaYGkfXJe28y+XippqaRFkn4mabNC3S8lzZf0oUZjDyGEEEIIeRqZwVoX1xcc\nBcwws0MKZetinDkxWZXHjZ6nEfXO12jczZB7/cvNbJR/3dcL5+1Jv08AdjazbwAHAIvNbLSZ/Srz\nfDltZgD/aGa7Ak8B31x9sNlYYAFwWMORhwpamh1AH9HS7AD6iJZmB9AntLS0NDuEPiHylCfylCfy\n1LhGBlgbSJoiaYmkWyUNLG8gabykxf51sZf1kzTJyzokneblIyU94L9pf1TS9j2If3PgxbKylwrx\nHOfXbJc02csmSbpC0kOSfivpaC/fRNJMj6VD0hFePsJnBCZJWiZpqqQD/fhlknb3dhtLur4wK3G4\nh/EW8JeMvrxUeDxY0j2SnpR0VaF89RpPSaf7bMjiQk439uPavXycl4/xeBd5fJsULyzpXp/ZaZe0\nUtLnM+N+F3jZz9GvMEOzSNLJ5Y09b3M9x7d4vAdLurXQZvXMmqSDytvXyVst3dbH1vGSxzBM0mzP\nz2JJe68JVRd4X+dK+qAXTiq9p/z5Kv9+FzAIWCDpTOAS4Cg/70C6vrYTJP3a666WVKqr21czm2lm\nnf50HvB3ZU1WkP7dhBBCCCGEtcHM6n4BI4BOYE9/fj1wuj+eBYwCtgb+AGxBGrg9CBzhdTMK59rM\nv88DjvDHA4CBObGUxXUe8NUqdTsCTwJD/Pnm/n0ScIs//ijwlD/uDwzyx1sWykeQBhs7+vNHgev9\n8RHA7f74QuBz/ngwsAzYqCym0cC1dfq0L/BXv65IMxJHe91yz+8ooAMYCGwCPA7sAhwN/Khwrk2B\nDYDfAaO8bJC/PvsC08quPQpYBGzag9fiy8CtrLlxSinfpffHlsDsUk6AM4Fved6fLpRfBYyv1r54\nzgox3AsMq1B+ruduEXAdMLiBfp0OfNMfC9jEH3cCh/rjS4CzCu+vowvH/1+Vx8cD/114Xnpt/wGY\nBvT38h8Cx+b2tazNtNJ7slD2X8B/1jnO4NzC1ywDi69uX5GXyFNfyRP2fjFr1qxmh9AnRJ7yRJ7y\nRJ7WmDVrlp177rmrv/znK+VfHyDfH81snj+eApwCXF6oHwPMMrPSjMZUYCxwAbC9pCuAnwMzJA0C\ntjGzaaTI3mogDvz8Ig0qplRpsj9wm5m94tdYWai708uWFvajCLhI0ljSh+dtCnXLzWyJP34CmOmP\nHwO288cHAYdLOsOfDwCGkwZa+PUWAF/K6N58M/uD9/NmYB/g9kL9PsAdZvaGt7kd+BfgfuB7ki4C\n7jWzX0n6GPC8mS30GF71Y7pcUNJWwI3AMWbW8P4i0pK3q83M/Dory+r3JA16H/LXbgNgrpm9K+k+\nUu5+Rlq+dgZpvUy39rUCMLNqS9+uAr5tZibpAtL79ouZ/XoEuF7SBsBdZtbh5W+a2c/98QJS/yvJ\nnTkz//5x0oD0Ee/3QOCFbo2r9zVdVDobeNvMbiqreo6stUit9ZuEEEIIIaxHWlpauiyZPO+88yq2\na2SAZXWeQ4UPk2a2UtIuwMGkWY5xwFcrte1yIukk0p4VI80UrCjU9QN+D7xJ+k1+o96sEPMEYCtg\nNzPrVLqhwMAK7TsLzztZk0MBnzKzp3oQT7mcXHc/yOwppZs/HAqcL+lB0mCyXq77ATcDrWa2tAfx\n5hBpJnNChbpbgK8ArwCPmNlrPrio1r4hZlZcWvdjoOrNPSocO8cH3YcBN0i6zMymAG8Xmr3LmvfB\nO/jS28LAsBECJpvZ2Q0et+YE0hdI74H9K1TfDpwjaYmZ7djTawSIPTO5WpodQB/R0uwA+oTYC5In\n8pQn8pQn8tS4RvZgjZC0hz/+HDCnrH4+MFbSFpL6k5Z5zZa0JWm50x2kJWGjfBblGUlHAkgaIGmj\n4snM7Coz283STQlWlNV1mtl2pOV6n6kS7y+AcfI7s0kaUqVdafAxGHjRB1f7kZbolbep5X7g1NUH\nSLtmHFPNHkp7v/qR+lee6zmk/TsDfT/VvwJzJG0NvO6zFt8jzYQsA4ZJGu1xDfLXp+gSoMPMbqsU\njO/hmlwn5geAE0vnrpDvecDekkZ6/caSPuJ1sz3WE4CfZLRviKRhhadHk5ZUImkbSTMrH7X62OGk\n98X1pOWFpTsiVntPPA3s7o+PpOsAq9b7qFT3IHBMYU/XEI8hi9IdEs8gLb99s0KT44DpMbgKIYQQ\nQlg7GhlgPQmcLGkJaZP8NV5eWhK2AphI+mMe7aSZiLuBbYE2Se2kJWgT/bjjgFMldQAPAUN7EP9v\nSPtWuvElfReSBnntwGXFeItN/ftUYIzHcyywtEKbSseXnE+6EchipVvaf7u8gaTRkq6t0Z+S+cCV\npOWIvzOzO4vXNrN24AbS8rWHSfu6OoCdgPne33OAC8zsbdIg7UpJi0h7ujYsu97XgYOUbnKxUNIn\ny+qHk/aF1XId8Ayw2K8/vizmPwFfAG72HM8FdvC6TuAe4BP+vWZ7qrwGSjfrGFah6lJ/XRaR9p59\nzcu3putMVCUtQIekhcCnge/XioE0Q7av52BP4LVCXa2ZyFKelpJ+ETHD+z0D6NanGn39AWmf3QP+\nWl5VVj+EdHfB8Ddra3YAfURbswPoI9qaHUCfEH+PJ0/kKU/kKU/kqXGlGxL0Sb7faUszm1i3cegx\nSZcAN5rZ482OpTcp3enwD2Z2T7Njea9I+iHp9vA/qtHGMlelrufaiGVdOdqIPOVoY+3lSfTl/+uL\n2traYrlShshTnshTnshTdZIws24rlPr6AGskaSbnVev6t7BCCGUkzSbtGzzWzJ6r0a7v/lAIIXQz\ndOgIVqx4utlhhBDC+877coAVQuh9kix+LoQQQggh1FZtgNXIHqwQQggu1qTniTzliTzliTzliTzl\niTzliTw1LgZYIYQQQgghhNBLYolgCKGLWCIYQgghhFBfLBEMIYQQQgghhLUsBlghhNADsSY9T+Qp\nT+QpT+QpT+QpT+QpT+SpcTHACiGEEEIIIYReEnuwQghdxN/BCs0wdNuhrHh2RbPDCCGEELLF38EK\nIWSRZLQ2O4qw3mmF+P8ohBBCXxI3uQghhN60vNkB9BGRpyyxxyFP5ClP5ClP5ClP5KlxWQMsSSMk\nPValbpakUb0bVh5JwyUtlDS9ULZO/XcuaV9JkzLaLS+0v7taG0lb9GJcFa9TqK8Zt78vZtVp0+vv\nj+I5c15vSedKetbfKwslfSLjmJrnlbQqP+LVx3xX0mOSLpG0laR5khZI2ifntc3s6xBJMyQtk3S/\npMGFul9Kmi/pQ43GHkIIIYQQ8jQyg7Uurt04CphhZocUytbFOHNisiqPGz1PI+qdr9G4myH3+peb\n2Sj/uq8XztuTfp8A7Gxm3wAOABab2Wgz+1Xm+XLaTARmmtkOwC+Ab64+2GwssAA4rOHIQ3fbNzuA\nPiLylKWlpaXZIfQJkac8kac8kac8kafGNTLA2kDSFElLJN0qaWB5A0njJS32r4u9rJ+kSV7WIek0\nLx8p6QFJiyQ9Kqkn/w1vDrxYVvZSIZ7j/JrtkiZ72SRJV0h6SNJvJR3t5ZtImumxdEg6wstHSFrq\nxy2TNFXSgX78Mkm7e7uNJV1fmJU43MN4C/hLRl9eKjweLOkeSU9KuqpQvnqNp6TTfTZkcSGnG/tx\n7V4+zsvHeLyLPL5NiheWdK/P7LRLWinp85lxvwu87OfoV5ihWSTp5PLGnre5nuNbPN6DJd1aaLN6\nZk3SQeXt6+Stlm7rY+t4yWMYJmm252expL3XhKoLvK9zJX3QCyeV3lP+fJV/vwsYBCyQdCZwCXCU\nn3cgXV/bCZJ+7XVXSyrV5fT1SGCyP55M+iVE0QrSv5sQQgghhLAWNDLA2gG40sx2BFYBJxUrJW0N\nXAy0ALsCY3yQsiuwrZntbGa7AKVlZ1OBH5jZrsA/A//bg/j7A53FAjPbw+PZETgLaDGz3YDTCs2G\nmdnewOGkD7oAbwBHmdnuwP7AZYX2I4Hv+qzADsBn/fgz/BoAZwMPmtmefvz3JG1kZg+b2dc8ptGS\nrq3UkVLcbgxwMvBR4O+LH9j9PKOA473dXsAJknYBPgE8Z2a7mdnOwH2SNgB+ApziuT4AeL3s2oeZ\n2Sjgi8DTwJ3FuKsxs2fN7Bh/+iVgBGmGZlfS61uMeUvgW8DHPccLgNOBmcA/SdrIm34GuMnbn12h\nfdW8+UBxWJVwv+KDoetUWDZXo2+l834OuM/zswuwyMs3AeZ6X+eQZqcqnsrPdyTwV59BuxQ4B/iJ\nP3+j0Id/8Bz8s1+zE5jQQF8/ZGYvePsVQPlywE7Sv5vaZhW+1qlFt+uQyEueyFOW2OOQJ/KUJ/KU\nJ/KUJ/K0RltbG62trau/qvlAA+f8o5nN88dTgFOAywv1Y4BZZlaa0ZgKjAUuALaXdAXwc2CGpEHA\nNmY2DcDM3mogDvz8In3gnVKlyf7AbWb2il9jZaHuTi9bqjX7UQRcJGks6UPoNoW65Wa2xB8/QRoU\nADwGbOePDwIOl3SGPx8ADAeWlS5qZgtIA5F65pvZH7yfNwP7ALcX6vcB7ih9MJd0O/AvwP2kgd1F\nwL1m9itJHwOeN7OFHsOrfkyXC0raCrgROMbMGt5fRBq4XW1+G7CyfAPsCewIPOSv3QakAcq7ku4j\n5e5npOVrZ5AG6t3a1wrAzKotfbsK+LaZmaQLSO/bL2b26xHgeh+o3mVmHV7+ppn93B8vIPW/ktyZ\ns9Lyv48Do4BHvN8DgRe6Na7e12rnLXmOlNva9ss8ewghhBDCeqKlpaXLksnzzjuvYrtGBljlH9Qq\n7Qfp9mHSzFb67MrBwJeBccBXK7XtciLpJNKsgAGH+m/jS3X9gN8DbwL3NtCHkjcrxDwB2ArYzcw6\nlW4oMLBC+87C807W5FDAp8zsqR7EUy4n190PMnvKZ7cOBc6X9CBpMFkv1/2Am4FWM1vag3hziLRf\nbkKFuluArwCvAI+Y2Ws+uKjWviFmVlxa92Og6s09Khw7xwfdhwE3SLrMzKYAbxeavcua98E7+Mxw\nYWDYCAGTzezsBo8reUHSUDN7wWe4ypfQ3g6cI2mJz0aHnoq9RXkiT1lij0OeyFOeyFOeyFOeyFPj\nGlkiOEJScdnUnLL6+cBYSVtI6g+MB2b7Uq/+ZnYHaYnYKJ9FeUbSkQCSBhSWiAFgZlf5UrdRxcGV\n13Wa2XbAo6TlVJX8AhgnvzObpCFV2pUGH4OBF31wtR9puVt5m1ruB05dfYC0a8Yx1eyhtPerH6l/\n5bmeQ9q/M9D3U/0rMMeXab5uZjcB3yPNhCwDhkka7XEN8ten6BKgw8xuqxSM0h6uyZXqCh4ATiyd\nu0K+5wF7Sxrp9RtL+ojXzfZYTyAtZ6zXviFlS+mOBh738m0kzax81Opjh5PeF9cD13mcUP098TSw\nuz8+kq4DrFrvo1Ldg8AxhT1dQzyGXNOAL/jj44G7yuqPA6bH4CqEEEIIYe1oZID1JHCypCWkTfLX\neHlpSdgK0h3M2oB20kzE3cC2QJukdtIStIl+3HHAqZI6gIeAoT2I/zdAxVtb+5K+C0mDvHbW7Kmq\nNjs0lbRvrAM4FlhaoU2l40vOJ90IZLHSLe2/Xd6g1h6sMvOBK0nLEX9nZncWr21m7cANpOVrDwPX\n+tK1nYD53t9zgAvM7G3SIO1KSYuAGcCGZdf7OnCQ0k0uFkr6ZFn9cOCvdWK+DngGWOzXH18W859I\nH/xv9hzPJe1nw8w6gXtIe8juqdeeKq9BjX1Jl/rrsgjYFyjtLduarjNRlbQAHZIWAp8Gvl8rBtIM\n2b6egz2B1wp1tWYiS3laSvpFxAzv9wygW59q9PUS4EBJy0jLDS8uqx8C9MYsa4i9RXkiT1lij0Oe\nyFOeyFOeyFOeyFPj5Ftm+iTf77SlmU2s2zj0mKRLgBvN7PFmx9KblO50+Aczu6fZsbxXJP2QdHv4\nH9VoY7S+dzH1WcuJ5W85cvPUCn35/6O/VVtbWyzDyRB5yhN5yhN5yhN5qk4SZtZthVJfH2CNJM3k\nvFr2t7BCCGUkzSbtGzzWzJ6r0a7v/lAIfdbQbYey4tkV9RuGEEII64j35QArhND7JFn8XAghhBBC\nqK3aAKuRPVghhBBcrEnPE3nKE3nKE3nKE3nKE3nKE3lqXAywQgghhBBCCKGXxBLBEEIXsUQwhBBC\nCKG+WCIYQgghhBBCCGtZDLBCCKEHYk16nshTnshTnshTnshTnshTnshT42KAFUIIIYQQQgi9JPZg\nhRC6iL+DFULzDB06ghUrnm52GCGEEDLE38EKIWRJA6z4uRBCc4j4fzmEEPqGuMlFCCH0qrZmB9BH\ntDU7gD6irdkB9AmxFyRP5ClP5ClP5KlxWQMsSSMkPValbpakUb0bVh5JwyUtlPT/s3fncXJVdd7H\nP98EkG0IEDUBhMBklEdUCAEUBZMGd9kxLAEGhgEdXzqigDgoahIWBQXGCKLymAloMgiMKLuEpTsT\ngjEhZAGyyBIWcQI8QjQ4ipj+PX/cU8nt6lpONR26m3zfr1e/+tY55977u7+qdOrUOefW7aWyFX0R\nSz2SxkqamtFuRan9zfXaSNq2F+OqeZ5SfcO40+uivUmbXn99lI+Z83xLGifpIUlrcmNpdlxJq/Oi\n7bLPtyU9KOkiSW+UNEfSfEn75zy3mdf6LUlLJS2U9DNJW5Xq/lvSXElvbjV2MzMzM8vTyghWf5yz\ncDgwIyI+Virrj3HmxBR1tls9TiuaHa/VuPtCzvkfBI4AZvbicXty3Z8Edo+IfwM+CCyOiL0i4t7M\n4+W0mQG8IyJGAY8AX167c8QYYD5wUMuRWw1tfR3AANHW1wEMEG19HcCA0NbW1tchDAjOUx7nKY/z\n1LpWOlgbS5omaYmk6yRtWt1A0nhJi9PPhalskKSpqWyRpM+n8pGS7kyftN8vaZcexL818FxV2fOl\neE5M51wg6epUNlXSZEmzJT0q6chUvoWku1IsiyQdmspHpBGBqZKWS5ou6UNp/+WS9k7tNpc0pTQq\ncUgK46/AHzKu5fnS9hBJt0haJumKUvnaOZ6SzkijIYtLOd087bcglR+VyvdJ8S5M8W1RPrGkW9NI\n4AJJqyT9Y2bca4AX0jEGlUZoFkr6bHXjlLf7Uo6vTfF+RNJ1pTZrR9Ykfbi6fZO81RQRyyPikXL+\nMjyfYhguaWbKz2JJ+60LVeena71P0ptS4dTKayo9Xp1+3whsCcyX9CXgIuDwdNxN6frcHi/p16nu\n+5IqdTnXeldEdKaHc4C3VDVZSfHvxszMzMzWh4ho+gOMADqBfdPjKcAZabsdGA1sBzwJbEvRcbsb\nODTVzSgda6v0ew5waNreBNg0J5aquCYBX6hTtxuwDNgmPd46/Z4KXJu23w48krYHA1um7aGl8hEU\nnY3d0uP7gSlp+1DghrR9AXBc2h4CLAc2q4ppL+DKJtc0FvjfdF5RjEgcmepWpPyOBhYBmwJbAA8B\newBHAj8sHevvgI2Bx4DRqWzL9PyMBW6qOvdoYCHwdz14Lj4NXMe6G6dU8l15fQylGEHaLJV/Cfhq\nyvsTpfIrgPH12pePWSOGW4HhDWKsuV+T6zoD+HLaFrBF2u4EPp62LwK+Unp9HVna/491tk8Cvlt6\nXHlu/w9wEzA4lX8POKHVa01tbqq8JktlXwO+2GS/gAmln/aA8E+3H+fFeVofeSKsvvb29r4OYUBw\nnvI4T3mcp3Xa29tjwoQJa3/S32yqfzYi31MRMSdtTwM+BwsvocoAACAASURBVFxaqt8HaI+IyojG\ndGAMcD6wi6TJwG3ADElbAttHxE0Ukf21hThIxxdFp2JanSYHAtdHxIvpHKtKdb9IZUtL61EEfFPS\nGIo3z9uX6lZExJK0/TBwV9p+ENg5bX8YOETSWenxJsBOFB0t0vnmA5/KuLy5EfFkus5rgP2BG0r1\n+wM/j4i/pDY3AO8H7gAulvRN4NaIuFfSO4HfRcQDKYaX0j5dTijpjcBPgHER0fL6Ioopb9+PiEjn\nWVVVvy9Fp3d2eu42Bu6LiDWSfkmRu59RTF87i2K+TLf2jQKIiPUx9W0eMEXSxsCNEbEolb8cEbel\n7fkU119L7qhZpN8foOiQzkvXvSnwbLfGTa5V0jnAKxHxn1VVz5A1F2li8yZmZmZmG5C2trYuUyYn\nTZpUs10rHaxo8hhqvJmMiFWS9gA+QjHKcRTwhVptuxxI+gzFmpWgGClYWaobBDwOvEzxSX6rXq4R\n8/HAG4E9I6JTxQ0FNq3RvrP0uJN1ORTwiSimor1aObnuvlPEIypu4vBx4DxJd1N0JpvlehBwDTAx\nIpb2IN4cohjJPL5G3bXAvwIvAvMi4k+pc1Gv/WsmImalTvdBwFWSLomIacArpWZrWPc6+Btp6m2p\nY9gKAVdHxDk9jVnSP1G8Bg6sUX0D8HVJSyJit56ew8BrZnK19XUAA0RbXwcwIHgtSB7nKY/zlMd5\nal0ra7BGSHpP2j4OmFVVPxcYI2lbSYMppnnNlDSUYrrTzymmhI1OoyhPSzoMQNImkjYrHywiroiI\nPSNidLlzleo6I2Jniul6x9SJ9x7gKKU7s0napk67SudjCPBc6lwdQDFFr7pNI3cAp63dQRqVsU89\n71Gx9msQxfVV53oWxfqdTdN6qiOAWZK2A/6cRi0uphgJWQ4Ml7RXimvL9PyUXQQsiojrawWT1nBd\n3STmO4F/qRy7Rr7nAPtJGpnqN5f01lQ3M8X6SeCnGe1fjfJap+0l3dWwsbQTxetiCvCjFGeX41R5\nAtg7bR9G1w5Wo9dRpe5uYFxpTdc2KYYskj5KMQJ4aES8XKPJicDt7lyZmZmZrR+tdLCWAZ+VtIRi\nkfwPUnllSthK4GyKL/NYQDEScTOwA9AhaQHFFLSz034nAqdJWgTMBob1IP7fUKxb6SZN6buAopO3\nALikHG+5afo9HdgnxXMCsLRGm1r7V5xHcSOQxSpuaX9udQNJe0m6ssH1VMwFLqeYjvhYRPyifO6I\nWABcRTF97VcU67oWAe8C5qbr/TpwfkS8QtFJu1zSQoo1XW+oOt+ZwIdV3OTiAUkHV9XvRLEurJEf\nAU8Di9P5x1fF/P+AfwKuSTm+D9g11XUCtwAfTb8btqfOc6DiZh3Da5QfLulpimmKt2jdbf23o+tI\nVC1twCJJDwBHA99pFAPwf4GxKQf7An8q1TUaiazkaSnFBxEz0nXPAGpdU81rBS6jWGd3Z3our6iq\n34bi7oL2qnX0dQADREdfBzBAdPR1AAOCv48nj/OUx3nK4zy1rnJDggEprXcaGhFnN21sPSbpIuAn\nEfFQX8fSm1Tc6fDJiLilr2N5rUj6HsXt4X/YoE1kzkrdwHXgaV05OnCecnRQ5EkM5P+X17eOjg5P\nV8rgPOVxnvI4T/VJIiK6zVAa6B2skRQjOS9F1+/CMrMqkmZSrBs8ISKeadDOHSyzPuMOlpnZQPG6\n7GCZWe8rOlhm1heGDRvBypVP9HUYZmaWoV4Hq5U1WGa2gaj1nQ7+6frT3t7e5zEMhB/nqbU8uXPV\nmNeC5HGe8jhPeZyn1rmDZWZmZmZm1ks8RdDMupAU/rtgZmZm1pinCJqZmZmZma1n7mCZmfWA56Tn\ncZ7yOE95nKc8zlMe5ymP89Q6d7DMzMzMzMx6iddgmVkXXoNlZmZm1ly9NVgb9UUwZta/Sd3+Vmyw\nhu0wjJW/XdnXYZiZmdkA4REsM+tCUjCxr6PoRyYW3wtWraOjg7a2ttc8nIHGecrjPOVxnvI4T3mc\npzzOU32+i6CZmZmZmdl6ltXBkjRC0oN16tolje7dsPJI2knSA5JuL5Wt6ItY6pE0VtLUjHYrSu1v\nrtdG0ra9GFfN85TqG8adXhftTdr0+uujfMyc51vSOEkPSVqTG0uz40panRdtl32+LelBSRdJeqOk\nOZLmS9o/57nNvNZtJM2QtFzSHZKGlOr+W9JcSW9uNXbrzp/m5XGe8jhPeZynPM5THucpj/PUulZG\nsPrjXMLDgRkR8bFSWX+MMyemqLPd6nFa0ex4rcbdF3LO/yBwBDCzF4/bk+v+JLB7RPwb8EFgcUTs\nFRH3Zh4vp83ZwF0RsStwD/DltTtHjAHmAwe1HLmZmZmZZWmlg7WxpGmSlki6TtKm1Q0kjZe0OP1c\nmMoGSZqayhZJ+nwqHynpTkkLJd0vaZcexL818FxV2fOleE5M51wg6epUNlXSZEmzJT0q6chUvoWk\nu1IsiyQdmspHSFqa9lsuabqkD6X9l0vaO7XbXNKU0qjEISmMvwJ/yLiW50vbQyTdImmZpCtK5Wvn\neEo6I42GLC7ldPO034JUflQq3yfFuzDFt0X5xJJuTSOBCyStkvSPmXGvAV5IxxhUGqFZKOmz1Y1T\n3u5LOb42xfsRSdeV2qwdWZP04er2TfJWU0Qsj4hHyvnL8HyKYbikmSk/iyXtty5UnZ+u9T5Jb0qF\nUyuvqfR4dfp9I7AlMF/Sl4CLgMPTcTel63N7vKRfp7rvS2vvONH0WoHDgKvT9tUUH0KUraT4d2Ov\nkr8XJI/zlMd5yuM85XGe8jhPeZyn1rVyF8FdgZMjYo6kKcBngEsrlZK2Ay4E9gRWAXemTspvgR0i\nYvfUbqu0y3TgGxFxk6RN6Nl6sMFAZ7kgIt6TzrMb8BXgvRHxoqTym8rhEbGfpLcDNwE3AH8BDo+I\nlyQNBeakOoCRwCciYomk+4Fj0/6HpnMcCZwD3B0Rp6RpWXMl3RURvwJ+lWLaC/iXiPhU9YVU4k72\nAd4OPAXcIenIiLihUqlimttJqd1g4NeSOlKcz0TEwand30naGPgpcFREPCBpS+DPVec+qHTc/wB+\nERGrK3HXExG/Bcalh58CRlCM0ERVvkk5/SrwgYj4c+pknAF8E/ihpM0i4s/AMcB/pvbn1Gh/fr28\nSboVOCUiXvUt30rHPQ74ZUR8M3V0Kp28LYD7IuKrki6iGJ36Rq1DpeMdJumPEVGZ2vgssFdEnJYe\nV67h/6QcvC8i1kj6HnA8MC3zWt8cEc+mc65U9+mAnRSvmcbKEz93Bnry8YeZmZnZ60hHR0dWh7OV\nDtZTETEnbU8DPkepg0XxZr89IiojGtOBMRRviHeRNBm4DZiR3uRvHxE3AUTEX1uIg3R8AXukWGo5\nELg+Il5M51hVqvtFKltaegMq4JuSxlC8Cd2+VLciIpak7YeBu9L2gxRvPwE+DBwi6az0eBNgJ2B5\n5aQRMZ+iI9LM3Ih4Ml3nNcD+FJ3Aiv2Bn0fEX1KbG4D3A3cAF0v6JnBrRNwr6Z3A7yLigRTDS2mf\nLieU9EbgJ8C41Llq1QeB71e+QKkq3wD7ArsBs9NztzFFB2WNpF9S5O5nFNPXzgLaarVvFEClo9jL\n5gFTUkf1xohYlMpfjojb0vZ8iuuvJXfUrDL97wPAaGBeuu5NgWe7Nc6/1upphc9Q5LaxAzKPvgHz\nnPQ8zlMe5ymP85THecrjPOVxntZpa2vrko9JkybVbNdKB6v6jVqt9SDd3kxGxCpJewAfAT4NHAV8\noVbbLgeSPkMxKhDAx8uf1EsaBDwOvAzc2sI1VLxcI+bjgTcCe0ZEp4obCmxao31n6XEn63IoilGu\nR3oQT7WcXHffKeKRNAr1ceA8SXdTdCab5XoQcA0wMSKW9iDeHKJYL3d8jbprgX8FXgTmRcSfUuei\nXvvXTETMSp3ug4CrJF0SEdOAV0rN1rDudfA30mhsqWPYCgFXR8Q5PQz5WUnDIuJZScPpPoX2BuDr\nkpZExG49PIeZmZmZ1dHKtLwRksrTpmZV1c8FxkjaVtJgYDwwM031GhwRP6eYIjY6jaI8LekwAEmb\nSNqsfLCIuCIi9oyI0dXToCKiMyJ2Bu6nmE5Vyz3AUUp3ZpO0TZ12lc7HEOC51Lk6gGK6W3WbRu4A\nTlu7gzQqY5963qNi7dcgiuurzvUsivU7m6pYT3UEMCtN0/xzRPwncDHFSMhyYHianoikLdPzU3YR\nsCgirq8VjIo1XFfXqiu5E/iXyrFr5HsOsJ+kkal+c0lvTXUzU6yfpJjO2Kz9q1Fe67S9pLsaNpZ2\nonhdTAF+lOLscpwqTwB7p+3D6NrBavQ6qtTdDYwrrenaJsWQ6ybgn9L2ScCNVfUnAre7c/XqeU56\nHucpj/OUx3nK4zzlcZ7yOE+ta6WDtQz4rKQlFIvkf5DKK1PCVlLcwawDWEAxEnEzsAPQIWkBxRS0\ns9N+JwKnSVoEzAaG9SD+3wA1b22dpvRdQNHJWwBcUo633DT9ng7sk+I5AVhao02t/SvOo7gRyGIV\nt7Q/t7qBpL0kXdngeirmApdTTEd8LCJ+UT53RCwArqKYvvYr4Mo0de1dFGu/FgBfB86PiFcoOmmX\nS1oIzADeUHW+M4EPq7jJxQOSDq6q3wn43yYx/wh4Gliczj++Kub/R/HG/5qU4/so1vUREZ3ALcBH\n0++G7anzHKi4WcfwGuWHS3qaYpriLVp3W//t6DoSVUsbsEjSA8DRwHcaxQD8X2BsysG+wJ9KdY1G\nIit5WkrxQcSMdN0zgFrXVPNaKTrLH5K0nGK64YVV9dsAvTHKamZmZmY1KC2ZGZDSeqehEXF208bW\nY+kmDj+JiIf6OpbepOJOh09GxC19HctrJd00Y3FE/LBBm2DiaxdTvzcRBvLfSTMzM1s/JBER3WYo\nDfQO1kiKkZyXqr4Ly8yqSJpJsW7whIh4pkG7gftHYT0YtsMwVv72Vd+Y0szMzF5n6nWwenJr9H4j\nIh6LiPe7c2XWXESMjYgDGnWuSm39k37qda48Jz2P85THecrjPOVxnvI4T3mcp9YN6A6WmZmZmZlZ\nfzKgpwiaWe+TFP67YGZmZtbY63KKoJmZmZmZWX/iDpaZWQ94Tnoe5ymP85THecrjPOVxnvI4T61z\nB8vMzMzMzKyXeA2WmXXhNVhmZmZmzdVbg7VRXwRjZv2b1O1vhfUzw4aNYOXKJ/o6DDMzM6viKYJm\nVkP4p+lPe5+e/9lnn6z/9PUjnrufx3nK4zzlcZ7yOE95nKfWNexgSRoh6cE6de2SRq+fsBqTtJOk\nByTdXipb0Rex1CNprKSpGe36VdxlObE1ayNpgqQzei+qrseUNFXSmCbtt5Z0g6RFkuZI2i3jHO2S\ndmpS39LrX9I4SUsk3Z0eXyNpoaTPp+s4ssn+Odd6XLrORZLulbR7qe4SSQ9LGttK3GZmZmaWL2cE\nqz8uxjgcmBERHyuV9cc4c2Lqj3FXDPT4K74CLIiIPYCTgO/2URynAKdGxAckDQf2johRETG5F8/x\nODAmXev5wJWViog4EzgX+OdePN8GrK2vAxgQ2tra+jqEAcF5yuM85XGe8jhPeZyn1uV0sDaWNC19\n8n6dpE2rG0gaL2lx+rkwlQ1Kn7gvTp+mfz6Vj5R0Z/rk/n5Ju/Qg7q2B56rKni/Fc2I65wJJV6ey\nqZImS5ot6dHKaIGkLSTdlWJZJOnQVD5C0tK033JJ0yV9KO2/XNLeqd3mkqakkZH5kg5JYfwV+EPG\ntTyfjjNc0sw0MrdY0n6pfLWk81O+7pP0plR+cOmcM0rlEyT9OLVdLunUVD42Hf8WScskXaHCyZL+\nvZS7UyVdUp3TZvHXy3uZpL+XdLukeSmWt0naStITpTabS3pK0uBa7WucfxVFrhvZDbgHICKWAztX\n8tXA74E19V7HydGSfp3yWXm+TpJ0Wel6bpY0RtLXgP2BKZK+BdwB7JCe7/2r8jRaUke67tslDcu9\n1oiYExGV190cYIeqJisp/v2YmZmZ2foQEXV/gBFAJ7BvejwFOCNttwOjge2AJ4FtKTpsdwOHproZ\npWNtlX7PAQ5N25sAmzaKoU5ck4Av1KnbDVgGbJMeb51+TwWuTdtvBx5J24OBLdP20FL5CIo3s7ul\nx/cDU9L2ocANafsC4Li0PQRYDmxWFdNewJVNrukM4MtpW8AWabsT+Hjavgj4SuVcpX1PAb6dticA\nC1JuhwJPAcOBscD/pusSMAM4EtgCeBQYnPafDbyjB89JvbxPKL1m7gJGpu13A3en7Z8DY9P20ZVc\nNWi/9pg1XhcH1yi/ALikdJy/AntmXle913F7KecfA+5M2ycB3y21v5liRKmyz56l19fiUrup6fnY\nKD0HQ0v5mJJ7rVVtvlj9ugPeD9zSZL+A8E/Tn/Y+Pj8xELS3t/d1CAOC85THecrjPOVxnvI4T/Wl\n/4u7vZfKuYvgUxExJ21PAz4HXFqq3wdoj4gXACRNB8ZQTE/aRdJk4DZghqQtge0j4iaKiJqNPHQj\nScAeKZZaDgSuj4gX0zlWlep+kcqWSnpz5ZDAN1WsbekEti/VrYiIJWn7YYo3/AAPAjun7Q8Dh0g6\nKz3eBNiJoqNFOt984FNNLm0exejGxsCNEbEolb8cEbel7fnAB9P2jpKuo+jgbgysKB3rxpTb30u6\nh6JT8QdgbkQ8CcX6H2D/iLhBxZqggyUtAzaKiIebxFpLo7wjaQvgfcD16TkkxQ1wHXAMMBM4Fvhe\nk/Y1RcSEOlUXApMlPUDx3C0A1mRe1+NUvY5LdTek3/MpOkw5mt2eb1fgncCd6boHAb+rbtTgWouT\nSAcAJ1OMmpU9A7xN0hsi4uX6R5hY2m7D0+HMzMxsQ9fR0ZF104+cDlY0eQw13jRGxCpJewAfAT4N\nHAV8oVbbLgeSPgN8Mp3n4xGxslQ3iOIN78vArRmxVyu/oazEcTzwRoqRhU4VN23YtEb7ztLjTtbl\nTsAnIuKRHsSzVkTMSp28g4CrJF0SEdOAV0rN1pTOexlwcUTcquKmBeU33OXnSNR+zsrtplCsU1pG\nMZKyPgwCXoyIWjeGuAm4QNI2FCNG9wBbNmjfkohYTWndUXqOH8/ct9br+NRUXXk9lJ+Xv9F16m23\nKbVNCHgoIvZrcb91ByhubHEl8NFKh7ciIh6XtBR4UtIH6nemJ/b09BuQtr4OYEDw3P08zlMe5ymP\n85THecrjPK3T1tbWJR+TJk2q2S5nDdYISe9J28cBs6rq5wJjJG0raTAwHpgpaSjFtLOfA18FRkfE\nS8DTkg4DkLSJpM3KB4uIKyJiz4gYXe5cpbrOiNiZYrreMXXivQc4StK26Rzb1GlX6WANAZ5LnasD\n6DoSkfNlQHcAp63dQRqVsU/3YIo71j0XEVOAH1F0NBrFsBXrRjZOqqo7LOV2KMXUwHmpfB8Va8sG\nUeTvXoCImAvsSPHcXVMnvqVNLqFh3lMnZ4WkcaVj7p7q/kTxnE6mmL4Wjdq3StKQNDKIpE8CM9Nr\nERXr77ZrsG+313G9pun3E8CotL5tR4rRw7qHr1G2HHiTpH3T+TdSxl0PS/HuBPwM+MeIeKxG/e7A\nLhQjyT0ZqTQzMzOzBnI6WMuAz0paQrE4/gepvFisUXSCzgY6KKZezYuImykW13dIWgD8JLUBOBE4\nTdIiirUmlQX8rfgNxZqvbtKUvgsoOnkLgMoNG+qNxE2n6HgsAk4AltZoU2v/ivMobgSyWMUt7c+t\nbiBpL0lXdt+1izZgUZrGdjTwnSbnnQT8l6R5dL8ZxWKK5+M+4NxSR/V+4HKK6Y6PpU5DxXXA7Fh3\ng4Ry/EObxN4o72UnAKeouGHHQxRr2SqupRhN/Gmp7PgG7buRNEnSwTWq3g48lDqJHwEqN1wRMBJ4\nocFh672Oa76eImI2RSfrYYrncH51mzqPK/u/AowDLpK0kOLf1HtbuNavUfzbuELFzUbmVtVvAzwR\nEZ019rWWdPR1AAOCvz8lj/OUx3nK4zzlcZ7yOE+tU7E+a2BJ652GRsTZTRtvYCRNAFZHxKVV5WOB\nMyOiZidF0s3ApRHRXqPuIGCXiLh8fcTcVyS9Azg5Ir7Y17G8ViQdDRwREeMbtIn6/Xpbp4O+nSYo\nBsLf746ODk8vyeA85XGe8jhPeZynPM5TfZKIiG4zkgZqB2skcBXwUnT9LqwNXqsdLElDKKZ5LoiI\nY1+7SO21puL2+++nuFvl3Q3auYM1IAyMDpaZmdnr1euqg2Vm60/RwbL+btiwEaxc+URfh2FmZrbB\nqtfBylmDZWYbmFrf6eCfrj/t7e19ev6B0rny3P08zlMe5ymP85THecrjPLXOHSwzMzMzM7Ne4imC\nZtaFpPDfBTMzM7PGPEXQzMzMzMxsPXMHy8ysBzwnPY/zlMd5yuM85XGe8jhPeZyn1rmDZWZmZmZm\n1ku8BsvMuvAaLDMzM7Pm6q3B2qgvgjGz/k3q9rfC+sCwHYax8rcr+zoMMzMza4FHsMysC0nBxL6O\nYgBYAeyyns8xsfhOsoGso6ODtra2vg6j33Oe8jhPeZynPM5THuepvh7dRVDSCEkP1qlrlzS6twJs\nhaSdJD0g6fZS2Yq+iKUeSWMlTc1o16/iLsuJrVkbSRMkndF7UXU9pqSpksY0ab+1pBskLZI0R9Ju\nGedol7RTk/qWXv+SxklaIunu9PgaSQslfT5dx5FN9m96randdyU9ko49qlR+iaSHJY1tJW4zMzMz\ny5dzk4v++PHp4cCMiPhYqaw/xpkTU3+Mu2Kgx1/xFWBBROwBnAR8t4/iOAU4NSI+IGk4sHdEjIqI\nyb11AkkfA0ZGxFuBfwF+UKmLiDOBc4F/7q3zbdDW9+jV64Q/9czjPOVxnvI4T3mcpzzOU+tyOlgb\nS5qWPnm/TtKm1Q0kjZe0OP1cmMoGpU/cF6eRg8+n8pGS7kyfrt8vqSdvU7YGnqsqe74Uz4npnAsk\nXZ3KpkqaLGm2pEcrowWStpB0V4plkaRDU/kISUvTfsslTZf0obT/ckl7p3abS5qSRkbmSzokhfFX\n4A8Z1/J8Os5wSTPTyNxiSful8tWSzk/5uk/Sm1L5waVzziiVT5D049R2uaRTU/nYdPxbJC2TdIUK\nJ0v691LuTpV0SXVOm8VfL+9lkv5e0u2S5qVY3iZpK0lPlNpsLukpSYNrta9x/lUUuW5kN+AegIhY\nDuxcyVcDvwfW1HsdJ0dL+nXKZ+X5OknSZaXruVnSGElfA/YHpkj6FnAHsEN6vvevytNoSR3pum+X\nNKyFaz0M+HG61l8DQ0r7A6yk+PdjZmZmZutBTgdrV+DyiNgNWA18plwpaTvgQqANGAXskzopo4Ad\nImL3NHJQmS43HbgsIkYB7wP+pwdxDwY6ywUR8Z4Uz24UIxZtEbEnUH5DPDwi9gMOAS5KZX8BDo+I\nvYEDgUtK7UcC346IXVMejk37n5XOAXAOcHdE7Jv2v1jSZhHxq4g4PcW0l6Qra11IJW7gOOCXETEa\n2ANYmMq3AO5L+ZoFfDKVz4qIfSNiL+Ba4Eulw76L4vl4H/B1FaMlAPsAnwXeDvwDcARwHXCIpMGp\nzcnAf1TFVldm3iuuBP41IvahyOH3I+KPwAKtm7Z2cMrDmlrta5z/9IiYk2KYJOngGuddBFQ61O8G\ndgLe0uS6xkXEM9R/HQMMTtd/OnRZtdRtVC8izgPuB46LiC8BhwKPRsToiLi30k7SRsBlwCfSdU8F\nvtHCte4APF16/Ewqq+ik+Pdjr1a/ndzbv/j7U/I4T3mcpzzOUx7nKY/z1Lqcuwg+VXlTB0wDPgdc\nWqrfB2iPiBcAJE0HxgDnA7tImgzcBsyQtCWwfUTcBBARzT6N70aSKDog0+o0ORC4PiJeTOdYVar7\nRSpbKunNlUMC31SxtqUT2L5UtyIilqTth4G70vaDwM5p+8MUHZSz0uNNKN7AL6+cNCLmA59qcmnz\nKEY3NgZujIhFqfzliLgtbc8HPpi2d5R0HbAdsDFd3+7dmHL7e0n3AO+mGE2bGxFPQrH+B9g/Im5Q\nsSboYEnLgI0i4uEmsdbSKO9I2oKiw3d9eg5JcUPRyTsGmAkcC3yvSfuaImJCnaoLgcmSHqB47hYA\nazKv63GqXseluhvS7/nAiMzjNbs9367AO4E703UPAn5X3ajBtTbzDPA2SW+IiJfrtmovbe+Mp8OZ\nmZnZBq+joyOrw5nTwar+NL7WmptubxojYpWkPYCPAJ8GjgK+UKttlwNJn6EYpQng4xGxslQ3iOIN\n78vArRmxVyu/oazEcTzwRmDPiOhUcdOGTWu07yw97mRd7kQx2vBID+JZKyJmpU7eQcBVki6JiGnA\nK6Vma0rnvQy4OCJuTaM/5Tfc5edI1F8nVSmfQjH6tIyuIzS9aRDwYhqhq3YTcIGkbYDRFNP5tmzQ\nviURsZrSuqP0HD+euW+t1/Gpqbryeig/L3+j68hwtym1TQh4KI2U9sQzwI6lx29JZQBExOOSlgJP\nSvpA3c70AT08+4bEnc4snrufx3nK4zzlcZ7yOE95nKd12trauuRj0qRJNdvlTBEcIak8jW1WVf1c\nYIykbdM0s/HATElDKaZQ/Rz4KjA6Il4CnpZ0GICkTSRtVj5YRFwREXumqVMrq+o6I2JniqlWx9SJ\n9x7gKEnbpnNsU6ddpYM1BHguda4OoOtIRM6XAd0BnLZ2h9Jd21qh4o51z0XEFOBHFB2NRjFsxbqR\njZOq6g5LuR0KjKUYHYNi+uaI1FE9BrgXICLmUrwpHw9cUye+pU0uoWHeUydnhaRxpWPunur+RPGc\nTgZuiULd9q2SNCSNDCLpk8DM9FpExfq77Rrs2+11XK9p+v0EMEqFHSlGD+sevkbZcuBNkvZN599I\nGXc9LLkJODHtuy+wKiKeLV3P7hRdg+17OFJpZmZmZg3kdLCWAZ+VtIRicXzlrmQBkDpBZwMdFFOv\n5kXEzRTrPjokLQB+ktpA8ebvNEmLgNlAeQF+rt8A9uLOlAAAIABJREFU29aqSFP6LqDo5C1g3Zqq\neiNx0yk6HouAE4ClNdrU2r/iPIobgSxWcUv7c6sbNFqDVdIGLErT2I4GvtPkvJOA/5I0j+43o1hM\n8XzcB5xb6qjeD1xOMd3xsdRpqLgOmB0R3W7MkToZDTXIe9kJwCkqbtjxEMU6pIprKUYTf1oqO75B\n+24arEt6O/BQ6iR+hLQ+LE3BGwm80OCw9V7HNV9PETGbopP1MMVzOL+6TZ3Hlf1fAcYBF0laSPFv\n6r2515qmk66Q9CjwQ6rWTALbAE9ERGf1vtYir8HK4rn7eZynPM5THucpj/OUx3lq3YD8ouG03mlo\nRJzdtPEGRtIEYHVEXFpVPhY4MyJqdlIk3QxcGhHtNeoOAnaJiMvXR8x9RdI7gJMj4ot9HctrRdLR\nwBERMb5BG3/RcA5/0XAWf0FlHucpj/OUx3nK4zzlcZ7qU50vGh6oHayRwFXAS1XfhbXBa7WDJWkI\nxTTPBRFx7GsXqb3WVNx+//3AlyPi7gbt3MHqLyYO/A6WmZnZ69XrqoNlZuuPJP9R6CeG7TCMlb9d\n2byhmZmZvebqdbBy1mCZ2QYmIvzT5Ke9vX29n+P10Lny3P08zlMe5ymP85THecrjPLXOHSwzMzMz\nM7Ne4imCZtaFpPDfBTMzM7PGPEXQzMzMzMxsPXMHy8ysBzwnPY/zlMd5yuM85XGe8jhPeZyn1rmD\nZWZmZmZm1ku8BsvMuvAaLDMzM7PmvAbLzLJJqvkzfPjOfR2amZmZWb/mDpaZ1RA1f5599sk+jao/\n8Zz0PM5THucpj/OUx3nK4zzlcZ5a17CDJWmEpAfr1LVLGr1+wmpM0k6SHpB0e6lsRV/EUo+ksZKm\nZrTrV3GX5cTWrI2kCZLO6L2ouh5T0lRJYzL2+a6kRyQtlDQqo327pJ2a1Lf0+pc0TtISSXenx9ek\neD6fruPIJvs3vVZJx0lalH7ulbR7qe4SSQ9LGttK3GZmZmaWb6OMNv1xMcbhwIyIOLtU1h/jzImp\nP8ZdMdDjB0DSx4CREfFWSe8BfgDs2wehnAKcGhH3SRoO7B0Rb00xNu2MZ3ocGBMRf5D0UeBK0rVG\nxJmS5gL/DMzspfNtsNra2vo6hAHBecrjPOVxnvI4T3mcpzzOU+typghuLGla+uT9OkmbVjeQNF7S\n4vRzYSoblD5xX5w+Tf98Kh8p6c70yf39knbpQdxbA89VlT1fiufEdM4Fkq5OZVMlTZY0W9KjldEC\nSVtIuivFskjSoal8hKSlab/lkqZL+lDaf7mkvVO7zSVNkTRH0nxJh6Qw/gr8IeNank/HGS5pZhqZ\nWyxpv1S+WtL5KV/3SXpTKj+4dM4ZpfIJkn6c2i6XdGoqH5uOf4ukZZKuUOFkSf9eyt2pki6pzmmz\n+OvlvUzS30u6XdK8FMvbJG0l6YlSm80lPSVpcK32Nc6/iiLXjRwG/BggIn4NDJE0rMk+vwfW1Hsd\nJ0dL+nXKZ+X5OknSZaXruVnSGElfA/YHpkj6FnAHsEN6vvevytNoSR3pum8vxdr0WiNiTkRUXndz\ngB2qmqyk+PdjZmZmZutBTgdrV+DyiNgNWA18plwpaTvgQqANGAXskzopo4AdImL3iNgDqHxCPx24\nLCJGAe8D/qcHcQ8GOssFEfGeFM9uwFeAtojYEyi/IR4eEfsBhwAXpbK/AIdHxN7AgcAlpfYjgW9H\nxK4pD8em/c9K5wA4B7g7IvZN+18sabOI+FVEnJ5i2kvSlbUupBI3cBzwy4gYDewBLEzlWwD3pXzN\nAj6ZymdFxL4RsRdwLfCl0mHfRfF8vA/4uorREoB9gM8Cbwf+ATgCuA44RNLg1OZk4D+qYqsrM+8V\nVwL/GhH7UOTw+xHxR2CB1k1bOzjlYU2t9jXOf3pEzEkxTJJ0cI3z7gA8XXr8DN07HtXHHRcRz1D/\ndQwwOF3/6cDE8u41jncecD9wXER8CTgUeDQiRkfEvZV2kjYCLgM+ka57KvCNFq617FTg9qqyTop/\nP/YqeU56Hucpj/OUx3nK4zzlcZ7yOE+ty5ki+FTlTR0wDfgccGmpfh+gPSJeAJA0HRgDnA/sImky\ncBswQ9KWwPYRcRNARDQbeehGkig6INPqNDkQuD4iXkznWFWq+0UqWyrpzZVDAt9UsbalE9i+VLci\nIpak7YeBu9L2g8DOafvDFB2Us9LjTYCdgOWVk0bEfOBTTS5tHsXoxsbAjRGxKJW/HBG3pe35wAfT\n9o6SrgO2AzYGymuhbky5/b2ke4B3U4ymzY2IJ6FY/wPsHxE3qFgTdLCkZcBGEfFwk1hraZR3JG1B\n0eG7Pj2HpLih6OQdQzFt7Vjge03a1xQRE3oQdzOPU/U6LtXdkH7PB0ZkHq/brTyr7Aq8E7gzXfcg\n4HfVjZpdq6QDKDrL+1dVPQO8TdIbIuLl+keYWNpuSz9mZmZmG66Ojo6sDmdP1mDVWnPT7U1jRKyS\ntAfwEeDTwFHAF2q17XIg6TMUozQBfDwiVpbqBlG84X0ZuDUj9mrlN5SVOI4H3gjsGRGdKm7asGmN\n9p2lx52sy50oRhse6UE8a0XErNTJOwi4StIlETENeKXUbE3pvJcBF0fErWn0p/yGu/wcifrrpCrl\nUyhGn5bRdYSmNw0CXkwjdNVuAi6QtA0wGrgH2LJB+1Y9A+xYevyWVNZUndfxqam68nooPy9/o+vI\ncLcptU0IeCiNlPaIihtbXAl8tNLhrYiIxyUtBZ6U9IH6nemJPT39BsNz0vM4T3mcpzzOUx7nKY/z\nlMd5Wqetra1LPiZNmlSzXc4UwREqbgwAxTS2WVX1c4ExkrZN08zGAzMlDaWYQvVz4KvA6Ih4CXha\n0mEAkjaRtFn5YBFxRUTsmaZOrayq64yInSmmWh1TJ957gKMkbZvOsU2ddpUO1hDgudS5OoCuIxHN\nRhugWEtz2todMu5QVzOY4o51z0XEFOBHFB2NRjFsxbqRjZOq6g5LuR0KjKUYHYNi+uaI1FE9BrgX\nICLmUnRAxgPX1IlvaZNLaJj3iFgNrJA0rnTM3VPdnyie08nALVGo274HbgJOTMfYF1gVEc+mx3el\naa411Xod12uafj8BjFJhR4rRw7qHr1G2HHhTihNJG6Xpl1nS6+hnwD9GxGM16ncHdqEYSe7JSKWZ\nmZmZNZDTwVoGfFbSEorF8T9I5QGQOkFnAx3AAmBeRNxMscalQ9IC4CepDRRvdE+TtAiYDTS72UAt\nvwG2rVWRpvRdQNHJW8C6NVX1RuKmU3Q8FgEnAEtrtKm1f8V5FDcCWazilvbnVjdotAarpA1YJOkB\n4GjgO03OOwn4L0nz6H4zisUUz8d9wLmljur9wOUU0x0fS52GiuuA2aUbJJTjH9ok9kZ5LzsBOEXF\nDTseoliHVHEtxWjiT0tlxzdo3029dUlpiuUKSY8CPyStI0xT8EYCLzQ4bL3Xcc3XU0TMpuhkPUzx\nHM6vblPncWX/V4BxwEWSFlL8m3pv7rUCX6P4t3GFipuNzK2q3wZ4IiI6u+9qrfCc9DzOUx7nKY/z\nlMd5yuM85XGeWqeIfn+X7W7SeqehVbdpN4q7CAKrI+LSqvKxwJkRUbOTIulm4NKIaK9RdxCwS0Rc\nvj5i7iuS3gGcHBFf7OtYXiuSjgaOiIjxDdpE/X69GIh/M9aHjo4OT5vI4DzlcZ7yOE95nKc8zlMe\n56k+SUREtxlJA7WDNRK4CngpIj7Wx+H0K612sCQNoZjmuSAijn3tIrXXmorb778f+HJE3N2gnTtY\nZmZmZk28rjpYZrb+uINlZmZm1ly9DlbOGiwz2+Co5s+wYbl3o3/985z0PM5THucpj/OUx3nK4zzl\ncZ5al3ObdjPbwHiUyszMzKxnPEXQzLqQFP67YGZmZtaYpwiamZmZmZmtZ+5gmZn1gOek53Ge8jhP\neZynPM5THucpj/PUOnewzMzMzMzMeonXYJlZF16DZWZmZtac12CZmZmZmZmtZ+5gmVk3kmr+DH/L\n8L4Ord/wnPQ8zlMe5ymP85THecrjPOVxnlrn78Eys+4m1i5+duKzr2kYZmZmZgNNwxEsSSMkPVin\nrl3S6PUTVmOSdpL0gKTbS2Ur+iKWeiSNlTQ1o12/irssJ7ZmbSRNkHRG70XV9ZiSpkoak7HPdyU9\nImmhpFEZ7dsl7dSkvqXXv6RxkpZIujs9vibF8/l0HUc22f9VXaukSyQ9LGlsK3FbbW1tbX0dwoDg\nPOVxnvI4T3mcpzzOUx7nqXU5UwT742r3w4EZEfGxUll/jDMnpv4Yd8VAjx8ASR8DRkbEW4F/AX7Q\nR6GcApwaER+QNBzYOyJGRcTk3jpBo2uNiDOBc4F/7q3zmZmZmVlXOR2sjSVNS5+8Xydp0+oGksZL\nWpx+Lkxlg9In7oslLZL0+VQ+UtKd6dP1+yXt0oO4twaeqyp7vhTPiemcCyRdncqmSposabakRyuj\nBZK2kHRXimWRpENT+QhJS9N+yyVNl/ShtP9ySXundptLmiJpjqT5kg5JYfwV+EPGtTyfjjNc0sw0\nMrdY0n6pfLWk81O+7pP0plR+cOmcM0rlEyT9OLVdLunUVD42Hf8WScskXaHCyZL+vZS7UyVdUp3T\nZvHXy3uZpL+XdLukeSmWt0naStITpTabS3pK0uBa7WucfxVFrhs5DPgxQET8GhgiaViTfX4PrKn3\nOk6OlvTrlM/K83WSpMtK13OzpDGSvgbsD0yR9C3gDmCH9HzvX5Wn0ZI60nXfXoq1N651JcW/H3uV\nPCc9j/OUx3nK4zzlcZ7yOE95nKfW5XSwdgUuj4jdgNXAZ8qVkrYDLgTagFHAPqmTMgrYISJ2j4g9\ngMp0uenAZRExCngf8D89iHsw0FkuiIj3pHh2A74CtEXEnkD5DfHwiNgPOAS4KJX9BTg8IvYGDgQu\nKbUfCXw7InZNeTg27X9WOgfAOcDdEbFv2v9iSZtFxK8i4vQU016Srqx1IZW4geOAX0bEaGAPYGEq\n3wK4L+VrFvDJVD4rIvaNiL2Aa4EvlQ77Lorn433A11WMlgDsA3wWeDvwD8ARwHXAIZIGpzYnA/9R\nFVtdmXmvuBL414jYhyKH34+IPwILtG7a2sEpD2tqta9x/tMjYk6KYZKkg2ucdwfg6dLjZ1JZo+sa\nFxHPUP91DDA4Xf/pdF211G1ULyLOA+4HjouILwGHAo9GxOiIuLfSTtJGwGXAJ9J1TwW+0YvX2knx\n78fMzMzM1oOcm1w8VXlTB0wDPgdcWqrfB2iPiBcAJE0HxgDnA7tImgzcBsyQtCWwfUTcBBARzT6N\n70aSKDog0+o0ORC4PiJeTOdYVar7RSpbKunNlUMC31SxtqUT2L5UtyIilqTth4G70vaDwM5p+8MU\nHZSz0uNNgJ2A5ZWTRsR84FNNLm0exejGxsCNEbEolb8cEbel7fnAB9P2jpKuA7YDNgbKa6FuTLn9\nvaR7gHdTjKbNjYgnoVj/A+wfETeoWBN0sKRlwEYR8XCTWGtplHckbUHR4bs+PYekuKHo5B0DzASO\nBb7XpH1NETGhB3E38zhVr+NS3Q3p93xgRObxun1XQpVdgXcCd6brHgT8rrrRq7jWZ4C3SXpDRLxc\nt1V7aXtnoCfjzK9znpOex3nK4zzlcZ7yOE95nKc8ztM6HR0dWSN6OR2s6k/ja6256famMSJWSdoD\n+AjwaeAo4Au12nY5kPQZilGaAD4eEStLdYMo3vC+DNyaEXu18hvKShzHA28E9oyIThU3bdi0RvvO\n0uNO1uVOFKMNj/QgnrUiYlbq5B0EXCXpkoiYBrxSaramdN7LgIsj4tY0+lN+w11+jkT9dVKV8ikU\no0/L6DpC05sGAS+mEbpqNwEXSNoGGA3cA2zZoH2rngF2LD1+Syprqs7r+NRUXXk9lJ+Xv9F1ZLjb\nlNomBDyURkp7ouG1RsTjkpYCT0r6QN3O9AE9PLuZmZnZ61RbW1uXDuekSZNqtsuZIjhCUnka26yq\n+rnAGEnbpmlm44GZkoZSTKH6OfBVYHREvAQ8LekwAEmbSNqsfLCIuCIi9kxTp1ZW1XVGxM4UU62O\nqRPvPcBRkrZN59imTrtKB2sI8FzqXB1A15GIZqMNUKylOW3tDhl3qKsZTHHHuuciYgrwI4qORqMY\ntmLdyMZJVXWHpdwOBcZSjI5BMX1zROqoHgPcCxARcynelI8HrqkT39Iml9Aw7xGxGlghaVzpmLun\nuj9RPKeTgVuiULd9D9wEnJiOsS+wKiKeTY/vStNca6r1Oq7XNP1+Ahilwo4Uo4d1D1+jbDnwphQn\nkjZK0y9z1b3WVLY7xXjU9j0cqbTEc9LzOE95nKc8zlMe5ymP85THeWpdTgdrGfBZSUsoFsdX7koW\nAKkTdDbQASwA5kXEzRTrPjokLQB+ktpA8ebvNEmLgNlAs5sN1PIbYNtaFWlK3wUUnbwFrFtTVW8k\nbjpFx2MRcAKwtEabWvtXnEdxI5DFKm5pf251g0ZrsEragEWSHgCOBr7T5LyTgP+SNI/uN6NYTPF8\n3AecW+qo3g9cTjHd8bHUaai4DpgdEd1uzJE6GQ01yHvZCcApKm7Y8RDFOqSKaylGE39aKju+Qftu\n6q1LSlMsV0h6FPghaR1hmoI3EnihwWHrvY5rvp4iYjZFJ+thiudwfnWbOo8r+78CjAMukrSQ4t/U\ne1/ttZZsAzwREZ3V+5qZmZnZq6eIfn+X7W7SeqehEXF208YbGEkTgNURcWlV+VjgzIio2UmRdDNw\naUS016g7CNglIi5fHzH3FUnvAE6OiC/2dSyvFUlHA0dExPgGbaLeFw0zEQbi3wwzMzOz3iaJiOg2\nIylnBKs/ugHYT6UvGraekTRE0nLgT7U6VwARcevrrXMFEBEPb2Cdq0uAL1JMQTUzMzOz9WBAjmCZ\n2fojqe4fhWE7DGPlb1fWq96gdHR0+M5KGZynPM5THucpj/OUx3nK4zzVV28EK+cugma2gfEHL2Zm\nZmY94xEsM+tCUvjvgpmZmVljr7c1WGZmZmZmZv2OO1hmZj3g7wXJ4zzlcZ7yOE95nKc8zlMe56l1\n7mCZmZmZmZn1Eq/BMrMuvAbLzMzMrDmvwTIzMzMzM1vPfJt2M+tG6vZhjJmZbQA2pO879Pc75XGe\nWucOlpl1N7GvAxgAVgC79HUQA4DzlMd5yuM85XkVeXp24rO9GorZhshTBK3PSVrd1zFUSBot6UFJ\nU0plK/oolj0kfaz0+CRJEzL2u13Si5JuqiofL2mZpNPXR7wbHL/Jy+M85XGe8jhPeZynLB6VyeM8\ntc4dLOsP+tMdFU4AvhcRp5TKWopPUm/9uxoFfLyqLCeWb1FcR9cdI64BxgLuYJmZmZmtJ+5gWb8h\naZKkBZIekPRbSVMkjZC0VNJUScslTZf0IUmz0+O90777SLpP0nxJ90p6aw/D2Bp4rqrs+XSOsZJm\nSroljQRdUYp9taSLJS0A9k0jYR2S5qURpWGp3WmSHpa0UNJ/prLN07XOSfEfImlj4Fzg6JSPo4D/\nBV5qdgER0V6vXUQ8CwxpOSvWXZ+Maw5AzlMe5ymP85THecri73fK4zy1zmuwrN+IiAnABElDgP8G\nLktVI4FPRMQSSfcDx0bEfpIOBc4BjgCWAvtHRKekDwDfBMb1IIzBQGdVXO8pPdwHeDvwFHCHpCMj\n4gZgC+BXEfFFSRsBM4FDI+L3ko4GvgGcAvwbsHNEvCJpq3TMc4C7I+KUdO1zgbuArwN7RcRp1UFK\nOiTVTezBNfqDFTMzM7P1xB0s64+mAZdExEJJI4AVEbEk1T1M0fkAeBAYkba3Bn6cRq6CHry2U8fo\nHazr2NUyNyKeTO2vAfYHbgDWpN8AuwLvBO5UcTu+QcDvUt0i4D8l/QL4RSr7MHCIpLPS402AnRrF\nGhE3AzfnX10XL0gaGRGP1W3RXtreGc/nr8U5yeM85XGe8jhPeZynLF5blMd5WqejoyNrRM8dLOtX\nJE0EnoqIH5eKXy5td5Yed7LuNXwecE9EHJk6ZeUuQuXY5wMHARERo6vq3kIxcvRoRNzfIMTqNVCV\nx38ufTuvgIciYr8a+x8EjAEOBc6R9K7U/hMR8UhVTPs2iOPVmAwslPS5iLiqZosD1tOZzczMzAao\ntra2Lh3OSZMm1WznqULWHwjWTnv7IPD5WvVNDAGeSdsn12oQEV+NiD2rO1ep7rfADkUYamtwnnen\ndWGDgGOAWTViXA68qdJBkrSRpN1S3U4RMRM4G9iKYmrhHcDaaYCSRqXN1alNT4j6efsK8A91O1eW\nx2sc8jhPeZynPM5THucpi9cW5XGeWucOlvUHlZGf04HtgXnpxg4Tq+qrt8u+BVwoaT49fF2nEahH\ngW0bNLsfuJxiquJjEVGZ5rc2roh4hWL910WSFgILgPemKYjTJC0C5gOTI+KPFKNvG0taLOlBiptb\nQDEKt1vpJhdrpRthTKwVoKT/Bq4FDpT0lKQPVTXZJN3swszMzMx6mdbNajIzSd8DHoyIH9SoGwuc\nGRGHvvaR9Q5JbwYWRcR2DdqEv2jYzGwDNRH83tAsjyQiotuMIY9gmXX1Y+Dk8hcNv15IGg/MoBjt\nMzMzM7P1wCNYZtaFJP9RMDPbQA3bYRgrf7uyr8N4TXR0dPgOeRmcp/rqjWD5LoJm1o0/eGnO/+Hk\ncZ7yOE95nKc8zpNZ3/IIlpl1ISn8d8HMzMysMa/BMjMzMzMzW8/cwTIz6wF/L0ge5ymP85THecrj\nPOVxnvI4T61zB8vMzMzMzKyXeA2WmXXhNVhmZmZmzXkNlpmZmZmZ2Xrm27SbWTdStw9jzMzMzAac\nvvhuN08RNLMuJAUT+zqKAWAFsEtfBzEAOE95nKc8zlMe5ymP85RnoOdp4vr7fk9PETTrZySNkPRg\nC+2/JWmppIWSfiZpqxbPt6uk+yT9RdIZrUdsXQzk/2xeS85THucpj/OUx3nK4zzlcZ5a5g6WWd9q\n5SOVGcA7ImIU8Ajw5RbP9Xvgc8C3W9zPzMzMzDK5g2XWtzaWNE3SEv3/9u49yJK6POP490FQEQUV\nw1Jh5RZBRQV2uYUC4mAUbwHRRAyKAhpjFZRsAhgTk8IlmggYJQQvkYgrohhBUYHyhsAoKyLI7nJH\no1wES1YtAVeNROTNH6cHembOzPYZZvfMst9P1anp/p3uc9556uzsvNO/7k7OTfL4JLslWZ5kWZLr\nkvweoKq+XlUPNvtdCcwf5I2q6udVdQ3wwCx/D+un24ZdwDrCnLoxp27MqRtz6sacujGngdlgScP1\nTOADVbUTsAo4qqquqaoFVbUQ+Ar9jzi9EfjyWqxTkiRJHXgVQWm4flRVVzbLn6Q3he/9AEleAywA\nDmjvkOQfgd9V1TlrrKrLWsvb4vzrfsykG3Pqxpy6MaduzKkbc+rGnB4yOjrK6OjoarezwZKGa+I5\nWAWQ5LnACcB+7bv+JjkCeBnwgn4vluTdwMuBao6Azcz+M95TkiTpUWlkZISRkZGH1k888cS+2zlF\nUBqubZLs1Sy/FliaZDPgHOANVfWLsQ2TvAR4G3BQVd3f78Wq6p9a0wun442uHinnpHdjTt2YUzfm\n1I05dWNO3ZjTwDyCJQ3XLcDRSZYANwAfBg4Btgb+K707/o4djTodeCxwcXMj4Cur6qiub5RkHvBd\n4EnAg0kWATtV1a9m8xuSJElan3mjYUnjeKNhSZL0qLHYGw1LkiRJ0jrLI1iSxkniDwVJkvSoMG+r\nedx9191r5LWnOoLlOViSJvEPL6s3Ojo67kpC6s+cujGnbsypG3Pqxpy6MafBeQRL0jhJyp8LkiRJ\n0/McLEmSJElaw2ywJGkGutzJXebUlTl1Y07dmFM35tSNOQ3OBkuSJEmSZonnYEkax3OwJEmSVs9z\nsCRJkiRpDbPBkjRJEh8+fMzBx5bztxz2j4eh8lyQbsypG3PqxpwG532wJE22eNgFrANuA7YbdhHr\nAHPqpmNOKxevXOOlSJIeGc/BkuaYJLcBu1XVL5Israp9kzwfOL6qDnwEr3sm8GfAyqraeZrtygZL\nmqMWeyNwSZorEs/BktYVD/32VFX79hufoSXAix/ha0iSJGkaNljSkCR5S5LlSZYluTXJJWNPtbZZ\n1dplsyQXJbklyYcGfb+qWgrc8wjL1pjbhl3AOsKcujGnTjwXpBtz6sacujGnwdlgSUNSVR+pqgXA\nnsCdwPv6bdZa3gM4Gng28Iwkr1rzVUqSJGkQXuRCGr7/AC6tqi+tZrurquoOgCSfBvYFzl8jFV3W\nWt4WL1LQj5l0Y07dmFMnIyMjwy5hnWBO3ZhTN+b0sNHR0U5H9GywpCFKcgTw9Ko6qsPmE8/BGree\nZE/gI834CVV10YwL23/Ge0qSJD0qjYyMjGs4TzzxxL7bOUVQGpIkuwHHAYdNt1lrea8k2yTZAHgN\nsLS9YVVdVVULqmrhNM1VJrymZspzZroxp27MqRPPBenGnLoxp27MaXA2WNLwHA08BbisudDFGc14\n+8hUe/kq4APAjcAPq+rzg7xZknOAK4Adk/woyZEzL12SJEn9eB8sSeN4HyxpDlvsfbAkaa7wPliS\nJEmStIbZYEnSTHjOTDfm1I05deK5IN2YUzfm1I05Dc6rCEqabPGwC5DUz7yt5g27BEnSangOlqRx\nkpQ/FyRJkqbnOViSJEmStIbZYEnSDDgnvRtz6sacujGnbsypG3PqxpwGZ4MlSZIkSbPEc7AkjeM5\nWJIkSavnOViSJEmStIbZYEnSDDgnvRtz6sacujGnbsypG3PqxpwGZ4MlaZIkfR9bzt9y2KVJkiTN\naZ6DJWmcJDXljYYXgz8zJEmSPAdL6iTJg0ne21o/LskJQ6plm6aeo1tjpyd5wzDqkSRJ0urZYEnj\n3Q+8KslTh11I46fAoiQbDrsQjeec9G7MqRtz6sacujGnbsypG3ManA2WNN4DwBnAsROfaI4oXZJk\nRZKLk8xvxpckOS3Jt5L8IMmrWvscn+SqZp8OkRXqAAAKwklEQVR3zqCenwGXAEf0qWfXJN9uXvtz\nSTZrxi9LclKS7yS5Jck+zfgGSU5pxlckefMM6pEkSdI0bLCk8Qr4IPC6JE+a8NzpwJKq2hU4p1kf\ns2VV7QMcCJwMkORFwA5VtSewANg9yb4zqOdk4PgkE+f4ngW8rannBqDdwD2mqvYC/hYeOqPqTcC9\nzfiewF8n2WbAetQYGRkZdgnrBHPqxpy6MaduzKkbc+rGnAbntCNpgqr6VZKzgEXA/7ae2ht4ZbN8\nNk0j1fhCs+/NSbZoxg4AXpRkGRBgE2AHYOmA9dye5ErgdWNjSTYFNquqsdc6Czi3tdv5zddrgLEm\n6gDgeUle3axv2tRzx6Q3vay1vC2w3SAVS5IkPfqMjo52mjLpESypv9PoHfHZpDU23eXz7m8tp/X1\nPVW1sKoWVNWOVbWkvVOSg5MsT7IsycJpXv89wNsnjE26ak2fen7Pw39ICfDWppYFVfVHVfX1vnvv\n33rYXPXlnPRuzKkbc+rGnLoxp27MqRtzetjIyAiLFy9+6DEVGyxpvABU1T30jgi9qfXcFcChzfJh\nwOXTvQbwVeCNSTYBSPKHSf6gvWFVfaFpdhZW1bJp6vkecBNwULP+S+AXY+dXAa8HvtGhnqPGLpiR\nZIckG0+xjyRJkmbA+2BJLUl+WVWbNstbALcCJ1fVu5JsDSwBNqd38Ykjq+quJB8DLqqq8/u8xluB\nsYtJrAIOq6rbOtayDXBhVe3crO8MLAPeWFWfSLIL8J/Axk2dR1bVfUkuBY6vqmVJNgeurqrtm3O4\n3k3vPLHQu0LhwVW1asL7eh8sSZKk1ZjqPlg2WJLGscGSJElaPW80LEmzyDnp3ZhTN+bUjTl1Y07d\nmFM35jQ4GyxJky3u/5i31bwhFTT3rFixYtglrBPMqRtz6sacujGnbsypG3ManJdplzSJ0wBX7957\n7x12CesEc+rGnLoxp27MqRtz6sacBucRLEmSJEmaJTZYkjQDt99++7BLWCeYUzfm1I05dWNO3ZhT\nN+Y0OK8iKGmcJP5QkCRJ6sDLtEuSJEnSGuQUQUmSJEmaJTZYkiRJkjRLbLAkSZIkaZbYYEkCIMlL\nktyS5PtJ3j7seuaSJGcmWZnkutbYU5J8Lcn3knw1yWbDrHHYksxPcmmSG5Ncn+SYZtycWpI8Lsl3\nkixvsvrXZtyc+kiyQZJlSS5o1s2pjyS3J7m2+Vxd1YyZ1QRJNktyXpKbm39/e5nTeEl2bD5Hy5qv\n9yU5xpwGY4MliSQbAB8AXgw8Bzg0ybOGW9WcsoReNm1/D3y9qp4JXAr8w1qvam55ADi2qp4D7A0c\n3XyGzKmlqu4H9q+qBcDOwAuS7IM5TWURcFNr3Zz6exAYqaoFVbVnM2ZWk50GfKmqng3sAtyCOY1T\nVd9vPkcLgd2AXwOfx5wGYoMlCWBP4H+q6o6q+h3w38ArhlzTnFFVS4F7Jgy/AjirWT4LOHitFjXH\nVNXdVbWiWf4VcDMwH3OapKp+0yw+jt7/w/dgTpMkmQ+8DPhoa9ic+guTf6czq5YkmwL7VdUSgKp6\noKruw5ym80Lgh1V1J+Y0EBssSQBbAXe21u9qxjS1LapqJfSaC2CLIdczZyTZFtgVuBKYZ07jNdPe\nlgN3A6NVdRPm1M+pwNuA9v1kzKm/Ai5OcnWSv2rGzGq87YCfJ1nSTH87I8kTMKfpvAY4p1k2pwHY\nYEnS7PCmgkCSJwKfBRY1R7Im5rLe51RVDzZTBOcD+yUZwZzGSfJyYGVzVHTSTTxb1uucWvZppnS9\njN703P3wMzXRhsBC4INNVr+mN+3NnPpIshFwEHBeM2ROA7DBkgTwY2Dr1vr8ZkxTW5lkHkCSLYGf\nDrmeoUuyIb3m6uyq+mIzbE5TqKpfAl8CdsecJtoHOCjJrcCn6Z2rdjZwtzlNVlU/ab7+DPgCvWnf\nfqbGuwu4s6q+26x/jl7DZU79vRS4pqp+3qyb0wBssCQBXA08I8k2SR4L/CVwwZBrmmvC+L+kXwAc\n0SwfDnxx4g7roY8BN1XVaa0xc2pJ8rSxq28l2Rh4EbAccxqnqt5RVVtX1fb0fh5dWlWvBy7EnMZJ\n8oTmyDFJNgEOAK7Hz9Q4zfS2O5Ps2Az9KXAj5jSVQ+n9cWOMOQ0gVR7hk9S7TDu9KyxtAJxZVScN\nuaQ5I8k5wAiwObASeCe9vxKfBzwduAM4pKruHVaNw9ZcCe+b9H6xq+bxDuAq4FzMCYAkz6N3gvjY\nRQnOrqp/S/JUzKmvJM8Hjquqg8xpsiTb0bvKW9GbBvepqjrJrCZLsgu9i6ZsBNwKHAk8BnMapzk3\n7Q5g+6pa1Yz5eRqADZYkSZIkzRKnCEqSJEnSLLHBkiRJkqRZYoMlSZIkSbPEBkuSJEmSZokNliRJ\nkiTNEhssSZIkSZolNliSJGmdkuSiJJuuxffbJclLW+sHJvm7tfX+ktYt3gdLkiStFUkeU1W/H3Yd\n/UxXW5LDgd2r6q1ruSxJ6yCPYEmStJ5J8oYk1yZZnuSsZmybJJckWZHk4iTzm/ElST6U5NtJfpBk\nJMnHk9yU5GOt11yV5P1Jbmj237wZvyzJqUmuBo5J8rQkn03yneaxd7Pd85t6liW5JskmSbZM8o1m\n7Lok+zTb3pbkqc3ysUmub55f1PpebkpyRlPPV5I8rk8OS5J8OMmVwMlJ9khyRfP+S5PskGQj4J+B\nQ5o6Xp3k8CSnT5ebpPWXDZYkSeuRJDsB7wBGqmoBsKh56nRgSVXtCpzTrI95clXtDRwLXACcUlU7\nATsn2bnZZhPgqqp6LvBN4J2t/Teqqj2q6lTgNOD9VbUX8BfAmc02xwFHVdVCYD/gt8Brga80Y7sA\nK5ptq/leFgKHA3sAewNvTrJLs80zgNObeu4D/nyKSLaqqj+uquOBm4F9q2q3pv73VNXvgBOAz1TV\nwqo6r13DanKTtB7acNgFSJKkteoFwHlVdQ9AVd3bjO8NvLJZPhs4ubXPhc3X64GfVNVNzfqNwLbA\ndcCDwLnN+CeBz7X2/0xr+YXAs5OkWX9ikicA3wJOTfIp4Pyq+nFz1OvM5ijSF6vq2gnfy77A56vq\ntwBJzqfXnF0I3FZV1zfbXdPU2c95reUnA59IsgO9BqrL70kTczulwz6SHsU8giVJkuDhIzL93N98\nfbC1PLY+VRPSfr1ft5YD7FVVC5rH1lX1m6o6GXgTsDHwrSQ7VtXlwJ8APwY+nuSwAb6fdp2/n6bO\ndm3vAi6tqucBBwKP7/A+E3Pz5HZpPWeDJUnS+uVS4NWtc5ie0oxfARzaLB8GXD7F/plifAN6U/4A\nXgcsnWK7r/HwtETGpvQl2b6qbqyqU4CrgWcl2Rr4aVWdCXwUWDihhsuBg5M8Pskm9I4kXT5hm0Fs\nSq+ZAziyNb6qea6frrlJWk/YYEmStB5ppvf9C/CNJMuB9zVPHQMcmWQFvQZprAma7gjNxKNUeya5\nHhihd2GIfvsvAnZvLrJxA/CWZvxvmotVXAv8H/Dl5nWuTbIMOAT49/ZrVtVy4OP0GrJvA2e0phF2\nOZI0cZv3AicluYbxvyNdBuw0dpGLCftMlZuk9ZSXaZckSY9YklVV9aRh1yFJw+YRLEmSNBv8i60k\n4REsSZIkSZo1HsGSJEmSpFligyVJkiRJs8QGS5IkSZJmiQ2WJEmSJM0SGyxJkiRJmiX/DxZWyxUh\naxJjAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "ratios = compression_ratios() \n", - "labels = ['%s - %s' % (c, o)\n", - " for c, o in compression_configs]\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression ratio', fontsize=14, y=1.01)\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "ax.set_xlim(0, max(ratios)+3)\n", - "ax.set_ylim(0, len(ratios))\n", - "ax.set_xlabel('compression ratio')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();\n" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "@functools.lru_cache(maxsize=None)\n", - "def compression_decompression_times(repeat=3, number=1):\n", - " c = list()\n", - " d = list()\n", - " for compression, compression_opts in compression_configs:\n", - " \n", - " def compress():\n", - " zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", - " t = timeit.Timer(stmt=compress, globals=locals())\n", - " compress_times = t.repeat(repeat=repeat, number=number)\n", - " c.append(compress_times)\n", - " \n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", - " def decompress():\n", - " z[:]\n", - " \n", - " t = timeit.Timer(stmt=decompress, globals=locals())\n", - " decompress_times = t.repeat(repeat=repeat, number=number)\n", - " d.append(decompress_times)\n", - " \n", - " log(compression, compression_opts, compress_times, decompress_times)\n", - " \n", - " return c, d\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcVcWZ//HPlwaCoAhowqYsEjEuAWk3FAIoanSiSAwi\niMsoxjjmF3FLhnGjiRE1I05MoiZmEDUaI86QDOCggtI4GgnKqoCorIriwqKiEZV+fn+cunD69l2b\nPrdbeN6v13n1vafqVNV57hFP3ao6V2aGc84555xzzrmd16i+G+Ccc84555xzuwrvYDnnnHPOOedc\nHfEOlnPOOeecc87VEe9gOeecc84551wd8Q6Wc84555xzztUR72A555xzzjnnXB3xDpZzzrmvFEmz\nJP26vttRXyRdLWlVfbfDOedcZt7Bcs65BkzSNyTdKekNSZ9JelPS45JOre+21aPvA/9W342oZ/4j\nls4510A1ru8GOOecy0xSZ+BvwIfAvwKLib4YOxG4B+hSb43LQlITM/siyTrMbHOS5TvnnHM7w0ew\nnHOu4boHqAKOMLP/NrPXzWy5md0F9EhlkrS/pL9I+ihs/y2pYyx9jKSXJZ0vaZWkLZImSGoi6Sdh\nVOwDSf8erzzkHSPpj5I+lvSOpKvT8lRJuizUuQW4Oew/RNK00J53Jf1JUtvYcYdJminpw1D2Akn9\nQ1pjSb+WtC6M2q2RNC52bLUpgpJaSXpA0kZJn0qaIemQWPoFoY4TQhy2SHomdGCzkvQjScsl/UPS\n+5KmS2oU0iZKmirpOknrQ/n3SfpaWhk/C6OPn0paJGlEWnoHSX8Obd8YYvbNDGW8E2J5P7BnrnY7\n55yrX97Bcs65BkhSa+C7wG/N7B/p6Wb2UcgnYArwdaA/MADoAPwl7ZAuwCDge0RT7IYCjwOHE42I\njQRGSToj7bgrgSVAL+BGYJykwWl5bgxlHQbcJakdMJtoxO1IYCDQAvif2DF/At4O6T2BCuCzkDYK\nOCO08ZvA2cDymlHa7gHgKOD08PdT4Im0zs7XgNHAPwO9gVbA77IVKOkI4LfAGKA7cALwRFq2/kQd\n3ROAM4GTgdtiZdwMXAj8C3AwcAvwu9T0Tkl7ALOAT4DvhHa9DcyU1CzkGQrcBNwAlAOvAVfliIVz\nzrn6Zma++eabb741sI2oo1AFnJEn30nAF8D+sX1dgW3ACeH9GKKb+D1jeR4D3gUax/bNAn4de78K\neDKtvj8Az8beVwG/SsszFpiRtq91yHtkeP8hcF6Wc7oz/fi09O3tBA4M5faJpbcENgMXhfcXhHh8\nM5bnHOAfOer4PrAJaJElfSKwEdgjtm8E8A9gD6A5UUevT9px/wFMC68vApanpZcBHwBDwvvngd+l\n5ZkBrKzva9Q333zzzbfMm49gOedcw6QC830LeNvM3kztMLNVRCMhh8TyrTWzLbH37wKvmdmXafu+\nkVb+CxneH5K2b17a+yOA/mHa3MeSPgbWEj2YoVvIcwcwQdLTkq6VdFDs+PuBXpJek/RbSf8URuoy\n+RZR52lOaodFo3svp7Vzq5m9EXv/NtBUUqss5c4A1gCrJT0UplemT81bbNVHF18AmoZzPARoRjSS\nFo/DpcABIX85cEBa+mai0bVUnA6On1usHueccw2UP+TCOecapteJOiQHU31qXTHiT5pLf/CEZdlX\nVot6Pkl73wiYBlxNzY7iuwBmNlbSQ8CpwCnAGEk/MrP7zWxBWB/1XaLphQ8AC4lG64oRP/8vs6Rl\n/KLRzLZIKgf6hXpHE02PPNLM1ueoM3W+qXJPA95My/NFLM8CoimQ6XHamKMO55xzDZiPYDnnXANk\nZpuAJ4H/J6l5erqkvcPLZUAHSZ1iaQcQrcNaUgdN6Z32/thQZy7zgUOJRs1Wpm3bO2NmtsLMfmtm\npwETgItjaZ+Y2WQz+zHRurGB6Q9/CJYR/b/s2NQOSS2Bb7OT529mVWZWaWbXEa0Ta0HUYUr5dlhH\nlXIssBVYASwNr7tkiEGqwzWfaI3Zhgx5Uk9KXEbmz8A551wD5R0s55xruH5MNLLxkqQhkrpLOkjS\nvwCLAMxsJtF0uIclHSHpSOAh4CUzq6yDNvSW9K+Svinph8C5RNP7crkL2BuYJOloSV0lnSjp95Ja\nSGoWpv71l9RZ0jFAX0KHSNKVkoZJ+lboVI0gWrP1VnpFYdrfFOD3kvpK+nY4/w+BR/K0M+s0TEnf\nk3S5pMND53UE0dP7lsayNQbuU/TExJOIHmJxr5n9I0zHvB24XdKFkrpJ6hmeTJjqSD5MNKL3P5L6\nSeoS/t4uKTVF8E7gAkkXh8/g34Cj85yXc865euRTBJ1zroEys1Vhmtq1wK1AR2AD8ApwRSzrIODX\nwDPh/Qzg8jpqxh1ET8q7HtgC3GBm8ScU1vjBWzN7R1Ifog7HdKK1SGuBp4hGdUT00IuJQPtwTlOB\nn4YiPg6vvxnKXwCcamappwym1/nPwK+IplI2A54DTjGzrXnOLdeP9W4GBhM9va850ajUSDP7WyzP\nbKJO4SyiB1v8F9HvlaXicIOk9URTJe8GPiKa6vjLkP4PSf2IPttJRJ3St0N5m0KeSZK6Ar8I7ZgC\njA/n7JxzrgGSmf8YvHPOuZokrQJ+Y2b5Rqx2O5ImAvuY2aD6botzzrmGxacIOuecc84551wd8Q6W\nc865bHyKg3POOVcknyLonHPOOeecc3XER7Ccc84555xzro54B8s555xzzjnn6oh3sJxzzjnnnHOu\njvjvYDnnqpHkCzOdc8455wpgZjV+tN5HsJxzNZiZb3W8XXDBBfXehl1189h6XL9Km8fVY/tV2zyu\n2bdsvIPlnHMl0KVLl/puwi7LY5sMj2syPK7J8dgmw+NaPO9gOeecc84551wd8Q6Wc64GSXWytWvX\npb5PpcFo1apVfTdhl+WxTYbHNRke1+R4bJPhcS2eP+TCOZdB3Tzn4t13a6z73G0dfvjh9d2EXZbH\nNhke12R4XJPjsU2Gx7V4yrVAyzm3+4meIlhX/y4o5yJQ55xzzkXrnNasWVPfzXBZdO7cmdWrV9fY\nLwmrz6cISuos6eUsabMklZeqLWl1d5I0X9L02L5V9dGWbCT1lzSxgHw52y3p4/C3vaRJ4fUFkn5T\nm/IKrHOMpKvylVOMeJmSJkrqlyd/f0mbw+c8X9L1BdQxS1KnPOlFXbOShkhaKunp8P4RSQsljQrn\ncWae4ws513MkLQrbc5J6xNLGS1oiqX8x7XbOOedcstasWVPvT8TzLftWbOe31GuwGuJX2YOBp8zs\n1Ni+htjOQtqUL48BmNk7Zja0gOPqos6G4lkzKw/bL+qpDSOBi81soKR2wJFmdriZ3VmHdawE+plZ\nT+AXwL2pBDO7Gvg5cFEd1ucKVFlZWd9N2GV5bJPhcU2GxzU5HlvXUJS6g9VE0kPhW/xJkpqlZ5A0\nXNLisN0a9jUK394vDt/Mjwr7u0maEUYBXpLUtRZtagW8l7bv/Vh7zg91LpD0QNg3UdKdkp6X9EZq\n5EFSC0kzQ1sWSRoU9neWtCwct1zSw5JOCscvl3RkyNdc0gRJcyTNk3R6aMbnwIcFnMv7oZyxob3z\nJb0laULqdGLtiY8mdgojMssl3ZgpDvnqzBarOEkHSJou6UVJsyV1l9RS0upYnuaS1koqy5Q/Q/2b\nieKTT7GLgTYA27Jde8FQSX+X9KqkPqH91UYEJU2V1E/SDUBfYIKkXwJPAh3DZ9S3WkOlckmV4byn\nS2pb6Lma2RwzS10rc4COaVnWE13zzjnnnHMuAaV+yMVBwIVmNifc9F8G3JFKlNQeuBXoRXQzOSN0\nUt4COppZj5CvZTjkYWCcmU2R1JTadRjLgKr4DjM7JtRzCHAtcKyZbZIUvzFtZ2Z9JB0MTAEmA58B\ng81si6R9iG5wp4T83YAfmNlSSS8Bw8Lxg0IdZwLXAU+b2UhJewNzJc00sxeAF0KbjgB+ZGaXpJ9I\nqt1mNgYYE8p4Fkjd8MdHm+KvjwIODe1/UdI0M5ufKi+XAmOVcm9o+wpJRwP3hNGcBZL6m9ls4DTg\nCTPbJqlGfmBgWv1Xpl5LGgu8aGbTMtR9rKSFwDrgp2a2NM95DQlllpP52gMoM7NjJJ0KVAAnpQ7P\nUN5Nkk4ArjKzBZLuAqaaWXkod2T425jo8xpkZhskDQXGASOLONeUi4HpafuqiK75PCpirweEze2M\nAQMG1HcTdlke22R4XJPhcU2Ox9YlrbKysqCR0lJ3sNaa2Zzw+iHgJ8Q6WEQ3+rPMbCOApIeBfkRT\nnbpKuhP4X+ApSXsCHcxsCoCZFTKKUY0kAT1DWzI5AXjMzDaFOjbH0v4a9i2T9I1UkcAtitbJVAEd\nYmmrYjf1S4CZ4fXLQJfw+mTgdEk/De+bAp2A5alKzWweUKNzlcVDwB1mtjBPvhmpc5M0mWikZX6B\ndaTkihWSWgDHAY+FuAM0CX8nAWcDs4FhwF158mcUOpaZzAM6mdmnoTP0VyDTaFgmK0m79mJpk2Pl\ndy6wvHwjaQcBhxF9uSCiLw3eTs+U41yjSqTjgQuJPsu4dUB3SV8zs63ZS6jI00znnHPOud3LgAED\nqnXkx44dmzFffa/ByrR+p8YNaLhZ7wlUApcCf8iWt1pB0mWxqXLt0tIaAauAg4HHC2p9dfGb01Q7\nRgD7Ar3MrBfR1MNmGfJXxd5XsaOjK6JRrl5h62pmy6kFSRVEHdoaU/UyKORz2VmNgE1hDVTq/A4L\naVOAUyS1BsqBZ/LkL4qZbTGzT8Pr6URTVdsUeGy2aw92fIbb2PEZfkn1/65qTIPNQ8ArsfPumbY+\nMH8B0YMt7iUaBdsUTzOzlcAyYI2kQ4tsm9sJvjYgOR7bZHhck+FxTc6uFtt27brU2e9SZtoK/a3K\nrl278swzz2RMe+655zj44IPr5Hxz1ZPPZ599xumnn06rVq04++yzAbj++uv5+te/TocOHVizZg2N\nGjWiqqoqT0l1o9QdrM6SUtPOzgH+Ly19LtBPUhtJZcBwYHaYbldmZn8BrgfKzWwL8KakMwAkNZW0\nR7wwM7s73KSWm9n6tLQqM+sCvEQ0epLJM8BZqZvx0AHIJNXB2ht4z8yqwghC5wx5cnkSuHz7AVKt\nfnhA0dqtE4FR6UlZDjlJUqsQv8HA8xnKXJan2pyxMrOPgVWShsTK7BHSPiH6HO4Eplkka/5ixdYw\nEaYaKjZKOjNMTc12bI1rL1vW8Hc1cLgi+wNH52pahn3Lga9L6h3qbxymXxZE0ZMP/xs4z8xWZEjv\nAXQlGv1dUmi5zjnnnCutd99dQ/SddzJbVP7O6du3L8uW7bhF3JlO0s74r//6L95//302bdrEo48+\nyptvvskdd9zBq6++yttvRxOBdkyISl6pO1ivAj+WtJRoof3vwv7U0+3WA6OJRgsWEK0xmUq0UL9S\n0gLgjyEPwPnA5ZIWEXUKtt9IF+E1IONoRpjSdzNRJ28BMD7e3njW8Pdh4KjQnnOJRgrS82Q6PuUm\notGVxYoeQvHz9AySjghrk3K5EuhAtJ5qfhjNylXvXKLpbguJpvlVmx4YOhk55YhV3LnASEUPJXkF\nGBRLe5RoBPDPsX0jcuSvQdHDPU7LkDRE0iuhXb8imoaYmiLaDdiYo9hs117Ga8DMnifqZC0Jdc1L\nz5Plfer4L4AhwG2K1owtAI4t4lxvILqe7w6jt3PT0lsDq82sNF/huO18bUByPLbJ8Lgmw+OaHI/t\n7mvNmjV07959eydqzZo17LvvvuyzT95b2GTkeub77rABPwVure92NOQN+B7w/+q7HQmc16HA7fXd\njhKf81DgkTx5DKyONsw555xzuWX6/2Xd/v+49v+P7tKli91yyy12yCGHWJs2beyiiy6yrVu3mplZ\nZWWl7bfffmZmdt5551mjRo2sefPmttdee9m///u/1yjrgw8+sNNOO81atWplbdq0sX79+lWr5/bb\nb7cePXpYq1atbNiwYdvruf/++61v377VypJkK1assDFjxljTpk2tSZMmttdee9nvf/9722OPPays\nrMz22msvu/DCC2316tXWqFEj27Ztm5mZffjhhzZy5Ehr37697bfffnb99ddbVVVVUZ9PbH+Ne6lS\nj2A1RJOBPor90LCrzsweN7Pf1nc76pqZLTGza+q7HaUiaTxwDfCfBeSuk61t20Kf/bHr29XWBjQk\nHttkeFyT4XFNjsc2OX/605+YMWMGK1asYPny5fziFzt+UjQ1avTggw/SqVMnpk2bxkcffcQ119S8\nxRo/fjz7778/GzZs4L333mPcuHHV0h977DGeeuopVq1axaJFi7j//vtr1JP+vqKigmuvvZZhw4bx\n0UcfcckllzB9+nQ6dOjARx99xH333VejHRdccAFNmzZl5cqVLFiwgBkzZvCf/1nA7VGBdvsOlpmt\nMLPvWJEPEnDuq8bMrjazo83s6QLy1sm2fv3qEpyZc84555L0k5/8hA4dOtCqVSuuu+46Hnnkkax5\no4GdzJo0acI777zDqlWrKCsro0+fPtXSR40aRdu2bWnVqhWnn346CxdmfxB2rnpyeffdd5k+fTr/\n8R//QbNmzdh333254oorcp5TsXb7DpZzzpWCrw1Ijsc2GR7XZHhck+OxTc5+++23/XXnzp23Pzii\nWD/72c/o1q0bJ598Mt/85je57bbbqqW3bbvjcQrNmzdny5YttWtwDmvXruWLL76gffv2tGnThtat\nW3PppZfywQcf1Fkdpf4dLOecc84559xXyJtvvrn99Zo1a+jQoUPGfPme1NeiRQtuv/12br/9dpYu\nXcrxxx/P0UcfzfHHH5/3uE8//XT7+/Xr19f6qYD7778/zZo1Y8OGDYk9WdBHsJxzrgR8bUByPLbJ\n8Lgmw+OaHI9tcu666y7WrVvHxo0bGTduHMOGDcuYr127dqxcuTJrOY8//jgrVkS/IrPXXnvRuHFj\nysrK8tbfs2dPlixZwuLFi9m6dWvWH/jNJTWlsF27dpx88slceeWVfPzxx5gZK1eu5Nlnny26zGy8\ng+Wcc84551wDEz0oqm4eOrUzD6KSxDnnnLN9Wt+BBx7IddddlzHv6NGjuemmm2jTpg133HFHjfTX\nX3+dE088kb322os+ffrw4x//mH79+m2vJ5sDDzyQG2+8kYEDB9K9e3e+853vFNT29PNIefDBB/n8\n88855JBDaNOmDWeddRbr16/PcXSRddV2gZhzbtckyfzfBeecc650JNX6oQ0uedk+n7C/Rs/QR7Cc\nc84555xzro54B8s5V4Okndra7deuvk+hwfG1Acnx2CbD45oMj2tyPLauofCnCDrnaqrYucPfrXi3\nTprhnHPOOfdV42uwnHPVSLKd7WBRUfsfAHTOOed2N74Gq2FrsGuwJHWW9HKWtFmSykvVlrS6O0ma\nL2l6bN+q+mhLNpL6S5pYQL6c7Zb0cfjbXtKk8PoCSb+pTXkF1jlG0lX5yilGvExJEyX1y5O/v6TN\n4XOeL+n6AuqYJalTnvSirllJQyQtlfR0eP+IpIWSRoXzODPP8XnPNeT7taTXQ9mHx/aPl7REUv9i\n2u2cc8455wpX6jVYDbFrPhh4ysxOje1riO0spE358hiAmb1jZkMLOK4u6mwonjWz8rD9op7aMBK4\n2MwGSmoHHGlmh5vZnXVVgaRTgW5mdiDwI+B3qTQzuxr4OXBRXdXnCudrA5LjsU2GxzUZHtfkeGxd\nQ1HqDlYTSQ+Fb/EnSWqWnkHScEmLw3Zr2NcofHu/WNIiSaPC/m6SZoRv6l+S1LUWbWoFvJe27/1Y\ne84PdS6Q9EDYN1HSnZKel/RGauRBUgtJM0NbFkkaFPZ3lrQsHLdc0sOSTgrHL5d0ZMjXXNIESXMk\nzZN0emjG58CHBZzL+6GcsaG98yW9JWlC6nRi7YmPJnYKIzLLJd2YKQ756swWqzhJB0iaLulFSbMl\ndZfUUtLqWJ7mktZKKsuUP0P9m4nik0+xP9W9AdiW7doLhkr6u6RXJfUJ7a82IihpqqR+km4A+gIT\nJP0SeBLoGD6jvtUaKpVLqgznPV1S2yLO9QzgQQAz+zuwd+x4gPVE17xzzjnnnEtAqR9ycRBwoZnN\nCTf9lwHbf4VMUnvgVqAX0c3kjNBJeQvoaGY9Qr6W4ZCHgXFmNkVSU2rXYSwDquI7zOyYUM8hwLXA\nsWa2SVL8xrSdmfWRdDAwBZgMfAYMNrMtkvYB5oQ0gG7AD8xsqaSXgGHh+EGhjjOB64CnzWykpL2B\nuZJmmtkLwAuhTUcAPzKzS9JPJNVuMxsDjAllPAukbvjjo03x10cBh4b2vyhpmpnNT5WXS4GxSrk3\ntH2FpKOBe8JozgJJ/c1sNnAa8ISZbZNUIz8wMK3+K1OvJY0FXjSzaRnqPlbSQmAd8FMzW5rnvIaE\nMsvJfO0BlJnZMWHUqAI4KXV4hvJuknQCcJWZLZB0FzDVzMpDuSPD38ZEn9cgM9sgaSgwDhhZ4Ll2\nBN6MvV8X9qWeOlFFdM3nNiv2ugtQm68uXDUDBgyo7ybssjy2yfC4JsPjmhyPrUtaZWVlQSOlpR7B\nWmtmc8Lrh4i+0Y87CphlZhvNrIqoA9UPWAl0DaNG3wU+lrQn0MHMpgCY2edm9lkxjZEkoCdRBy6T\nE4DHzGxTqGNzLO2vYd8y4BupIoFbJC0CZgIdJKXSVsVu6peEdICXiW5hAU4GRktaAFQCTYFq64DM\nbF6mzlUWDwF3mNnCPPlmmNnmEL/J1PxcCpErVkhqARwHPBbO7/dAamRlEnB2eD0MeDRP/ozMbEyW\nztU8oJOZHQ78lvDZFajGtRdLmxwrv7CfQ88/knYQcBjRlwsLiDrdHdIz5TjXfNYB3SV9LWeu42Ob\nd66cc865kmu3X7ud/tmUXFtD+kmV448/nvvuu6/Wx1944YW0adOG3r17A3DPPffQrl07WrZsycaN\nG2nUqBErV67c6XYOGDCAioqK7Vs2pR7BSv9mP9P6nRo3oGa2WVJP4LvApcBZwBWZ8lYrSLoM+GGo\n55/MbH0srRHRzfNW4PEiziFla4Y2jwD2BXqZWZWiB0A0y5C/Kva+ih2fg4hGuV6vRXuqkVRB1KGt\nMVUvg0I+l53VCNiUGrFJMwW4WVJroBx4BtgzR/6imNmW2Ovpku6W1MbMNhZwbKZr7+KQnPoMt7Hj\nM/yS6l9c1JgGm4eAV8ysT5HHpawD9o+93y/sA8DMVkpaBqyRNNDMltSyHlekyspK/3Y1IR7bZHhc\nk+FxTc6uFtt317270z+bkrP8XeQnVZ577jmefvpp3n77bZo1a8aXX37J1Vdfzdy5cznssMOA6Gl/\npVTqEazOklLTzs4B/i8tfS7QT1IbSWXAcGB2mG5XZmZ/Aa4HysNN85uSzgCQ1FTSHvHCzOxuM+sV\nHmywPi2tysy6AC+xY/Qk3TPAWZLahDpaZ8mX+tT2Bt4LnavjqT6qUcgn+yRw+fYDYk+AK4aitVsn\nAqPSk7IccpKkViF+g4HnM5S5LE+1OWNlZh8DqyQNiZXZI6R9QvQ53AlMs0jW/MWKr0EKUw2V6lwp\nWjPXPsexNa69bFnD39XA4YrsDxydq2kZ9i0Hvi6pd6i/cZh+WagpwPnh2N7AZjPb/i9oiGFXotFf\n71w555xz7itt9erVdOnShWbNou+0169fz9atWzn44IO35yn1I/BL3cF6FfixpKVEC+1TTzhLPd1u\nPTCaaHrcAqI1JlOJ1pBUhilTfwx5ILqRvDxMyXuePFPIsngNaJMpIUzpu5mok7cAGB9vbzxr+Psw\ncFRoz7nAsgx5Mh2fchPRg0AWK3oIxc/TM0g6IqxNyuVKomllLyp6iEJFnnrnEk13W0g0zW9+Wp37\n5KkvV6zizgVGKnooySvAoFjao0QjgH+O7RuRI38Nih7ucVqGpCGSXgnt+hXRNMTUFNFuQK6RrGzX\nXsZrwMyeJ+pkLQl1zUvPk+V96vgvgCHAbYrWjC0Aji30XM3sf4k6pm8QTau8LC1La2B1mILrSmhX\n+la1ofHYJsPjmgyPa3I8tsno2rUr48ePp2fPnrRu3Zrhw4fz+ec7nrn1hz/8gQMPPJB9992XwYMH\n884772QsZ+vWrZx33nnsu+++tG7dmmOOOYb339/xPLXVq1fTt29fWrZsySmnnMLGjdHt2ezZs9l/\n//2rldW1a1eeeeYZ7rvvPn74wx/ywgsv0LJlS0aMGMG3vvUtAFq3bs2JJ55Yox2ff/4511xzDZ07\nd6Z9+/ZcdtllbN26tUa+nbHb/9CwpJ8C+5jZ6LyZd1OSvgd0NbPf1ndb6pKkQ4keunJNfbelVBQ9\nNOP7ZjY8Rx7/oWHnnHOuhJThh2wlJTpFsND/V3ft2pW2bdvyP//zP3zta1/juOOO44orruCSSy7h\nmWee4eyzz2bmzJkccsghXH311SxatIjZs2fXKOfee+/l8ccfZ9KkSTRt2pSFCxdy4IEHsueee3L8\n8cfz1ltv8cQTT7DffvtxyimncOyxxzJu3Dhmz57Neeedx9q1a6u1acKECZxwwgk88MADTJgwgWef\nfRaANWvWcMABB/Dll19unxrYqFEj3njjDQ444ACuvPJKVq1axQMPPEDjxo0555xzOOyww7j55puz\nxiDT5xPbX2NGUqnXYDVEk4H7JU1P+y0sF5hZbdaoNXhhitzu1LkaD3wH+Le8mSt2rq62HWszmLxr\n29XWBjQkHttkeFyT4XFNjsc2OaNGjaJt2+j/7aeffjoLF0bPT/vTn/7EyJEj6dmzJwC33HILrVu3\nZu3atXTqVO05bTRp0oQNGzbw2muv8e1vf5tevXpVS7/wwgvp1q0bAEOHDmXq1Kk71WYzy7j26g9/\n+AMvv/wye++9NwCjR49mxIgROTtYxdrtO1hmtoLoptO5XVr4oeFC8ybZFOecc859haQ6VwDNmzff\nPg3w7bff5ogjjtie1qJFC/bZZx/WrVtXo4N1/vnn89ZbbzFs2DA+/PBDRowYwbhx4ygri349pl27\ndtXq2LJlC3Xt/fff59NPP63W5qqqqjq/7yn1GiznnNst+beqyfHYJsPjmgyPa3I8tqXXoUMH1qxZ\ns/39J5/ThHwuAAAgAElEQVR8woYNG+jYsWONvGVlZdxwww0sWbKEv/3tb0ybNo0HH3wwbx0tWrTg\n008/3f5+27Zt1dZuFWPfffelefPmLFmyhI0bN7Jx40Y2b97Mhx9+WKvysvEOlnPOOeecc65ow4cP\nZ+LEiSxevJitW7dy7bXX0rt37xqjVxBN4XzllVeoqqpizz33pEmTJttHr3Lp3r07n332GdOnT+fL\nL7/kF7/4RbWHbGSSbURKEj/84Q+54oortnfS1q1bx1NPPVXA2RZut58i6JxzpeBrA5LjsU2GxzUZ\nHtfk7GqxbduxbaK/VVXoeulcvyE1cOBAbrrpJs4880w2b97Mcccdx5///OeMedevX8+ll17KunXr\n2HPPPRk2bBjnnntu3jpatmzJ3XffzciRI6mqquJnP/sZ++23X1Ftjr+/7bbbGDt2LL17994+2vYv\n//IvnHzyyTnLLMZu/xRB51x1ksz/Xah7u9r/+BsSj20yPK7J8Lgm56sc22xPqXMNQ7FPEfQOlnOu\nGu9gOeecc6XlHayGrdgOlq/Bcs4555xzzrk64h0s51wNknaJrV27LvUdyu0qKyvruwm7LI9tMjyu\nyfC4Jsdj6xoKf8iFcy6DXWOawrvvZl8065xzzjmXBF+D5ZyrRpLtKh0s8DntzjnnGj5fg9WwNdg1\nWJI6S3o5S9osSeWlakta3Z0kzZc0PbZvVX20JRtJ/SVNLCBfznZL+jj8bS9pUnh9gaTf1Ka8Ausc\nI+mqfOUUI16mpImS+uXJP0jSIkkLJL0k6YQC6pglqeaPOFRPL+qalTRE0lJJT4f3j0haKGlUOI8z\n8xxfyLmeE851kaTnJPWIpY2XtERS/2La7Zxzzrlkde7cud6n1fuWfevcuXNRn2ep12A1xK75YOAp\nMzs1tq8htrOQNuXLYwBm9o6ZDS3guLqosyGYaWY9zawXcCFwbz21YyRwsZkNlNQOONLMDjezO+uw\njpVAPzPrCfyC2Lma2dXAz4GL6rA+VyBfG5Acj20yPK7J8Lgm56sc29WrV2NmDXKbNWtWvbehvrfV\nq1cX9XmWuoPVRNJD4Vv8SZKapWeQNFzS4rDdGvY1Ct/eLw7fzI8K+7tJmqFoFOAlSV1r0aZWwHtp\n+96Pted87Rj9eCDsmyjpTknPS3pDYeRBUgtJM0NbFkkaFPZ3lrQsHLdc0sOSTgrHL5d0ZMjXXNIE\nSXMkzZN0emjG58CHBZzL+6GcsaG98yW9JWlC6nRi7YmPJnZSNCKzXNKNmeKQr85ssYqTdICk6ZJe\nlDRbUndJLSWtjuVpLmmtpLJM+TPUv5koPlmZ2aext3sCHxRwXhuAbdmuvWCopL9LelVSn9D+aiOC\nkqZK6ifpBqAvMEHSL4EngY7hM+qbFqdySZXhvKdLSv0SYCHnOsfMUtfKHKBjWpb1RNe8c84555xL\nQKkfcnEQcKGZzQk3/ZcBd6QSJbUHbgV6Ed1MzgidlLeAjmbWI+RrGQ55GBhnZlMkNaV2HcYyoCq+\nw8yOCfUcAlwLHGtmmyTFb0zbmVkfSQcDU4DJwGfAYDPbImkfohvcKSF/N+AHZrZU0kvAsHD8oFDH\nmcB1wNNmNlLS3sBcSTPN7AXghdCmI4Afmdkl6SeSareZjQHGhDKeBVI3/PHRpvjro4BDQ/tflDTN\nzOanysulwFil3BvavkLS0cA9Fo3mLJDU38xmA6cBT5jZNkk18gMD0+q/MvVa0ljgRTObll6xpMHA\nLUA74LsFnNeQcFw5ma89gDIzO0bSqUAFcFLq8Azl3aRoauJVZrZA0l3AVDMrD+WODH8bE31eg8xs\ng6ShwDhgZKHnGnMxMD1tXxXRNZ9HRez1gLC5nfFV/fHLrwKPbTI8rsnwuCbHY5sMj+sOlZWVBY2U\nlrqDtdbM5oTXDwE/IdbBIrrRn2VmGwEkPQz0I5rq1FXSncD/Ak9J2hPoYGZTAMws5zf7mUgS0DO0\nJZMTgMfMbFOoY3Ms7a9h3zJJ30gVCdyiaJ1MFdAhlrbKzJaG10uAmeH1y0CX8Ppk4HRJPw3vmwKd\ngOWpSs1sHlCjc5XFQ8AdZrYwT74ZqXOTNJlopGV+gXWk5IoVkloAxwGPhbgDNAl/JwFnA7OBYcBd\nefJnFDqW2dL+Cvw1jBb9kaizX4iVpF17sbTJ4e88oNDJufkea3cQcBjRlwsi+tLg7fRMuc4VQNLx\nRNMh+6YlrQO6S/qamW3NXkJFnmY655xzzu1eBgwYUK3DOXbs2Iz56nsNVqb1OzVuQMPNek+gErgU\n+EO2vNUKki6LTZVrl5bWCFgFHAw8XlDrq4vfnKbaMQLYF+hl0Xqf94BmGfJXxd5XsaOjK6JRrl5h\n62pmy6kFSRVEHdoaU/UyKORz2VmNgE1mVh47v8NC2hTgFEmtgXLgmTz5a83MngMahxHGQvJnu/Zg\nx2e4jR2f4ZdU/++qxjTYPAS8EjvvnlZ9fWD+AqIHW9xLNAq2KZ5mZiuBZcAaSYcW2Ta3E77KawMa\nOo9tMjyuyfC4JsdjmwyPa/FK3cHqLCk17ewc4P/S0ucC/SS1kVQGDAdmh5vhMjP7C3A9UG5mW4A3\nJZ0BIKmppD3ihZnZ3eEmtdzM1qelVZlZF+AlotGTTJ4BzpLUJtTROku+VAdrb+A9M6sKIwidM+TJ\n5Ung8u0HSIcXcEzNxkRrt04ERqUnZTnkJEmtQvwGA89nKHNZnmpzxsrMPgZWSRoSK7NHSPuE6HO4\nE5hmkaz5iyWpW+x1eahzQ3g/M0xNzXZsjWsvW9bwdzVwuCL7A0fnalqGfcuBr0vqHepvHKZfFkTR\nkw//GzjPzFZkSO8BdCUa/V1SaLnOOeecc64wpe5gvQr8WNJSooX2vwv7U0+3Ww+MJhotWEC0xmQq\n0UL9SkkLiKZ3jQ7HnQ9cLmkRUacg9TCAYrwGtMmUEKb03UzUyVsAjI+3N541/H0YOCq051yikYL0\nPJmOT7mJ6EEgixU9hOLn6RkkHRHWJuVyJdCBaD3V/DCalaveuUTT3RYSTfOrNj2wkNGeHLGKOxcY\nqeihJK8Ag2JpjxKNAP45tm9Ejvw1KHq4x2kZkn4g6RVJ84k6ccNCfhGtjduYo9hs117Ga8DMnifq\nZC0BfkU0fZBcx6Qd/wUwBLhN0kKi/w6OLeJcbyC6nu8Oo7dz09JbA6vNrKrmoS5JPoc9OR7bZHhc\nk+FxTY7HNhke1+Lt9j80HNY77WNmo/Nm3k1J+h7Q1cx+W99tqUthityFZnZNfbelVMJDM75vZsNz\n5PEfGnbOOeecy0P1/UPDDdhkoI9iPzTsqjOzx3e1zhWAmS3ZzTpX44FrgP+s77bsjnwOe3I8tsnw\nuCbD45ocj20yPK7FK/VTBBucsE7lO/XdDueSZtEPDReokCWDDV/btsX98rpzzjnn3M7a7acIOueq\nk2T+74JzzjnnXG4+RdA555xzzjnnEuYdLOecKwGfw54cj20yPK7J8Lgmx2ObDI9r8byD5Zxzzjnn\nnHN1xNdgOeeq8TVYzjnnnHP5+Ros55xzzjnnnEuYd7Ccc64EfA57cjy2yfC4JsPjmhyPbTI8rsXb\n7X8HyzlXk/TV/B2sth3bsv6t9fXdDOecc87txnwNlnOuGklGRX23opYqwP9Nc84551wp+Bos55xz\nzjnnnEtYyTpYkjpLejlL2ixJ5aVqS1rdnSTNlzQ9tm9VfbQlG0n9JU0sIF/Odkv6OPxtL2lSeH2B\npN/UprwC6xwj6ap85RQjXqakiZL65ck/SNIiSQskvSTphALqmCWpU570oq5ZSUMkLZX0dHj/iKSF\nkkaF8zgzz/F5zzXk+7Wk10PZh8f2j5e0RFL/Ytrt6obPYU+OxzYZHtdkeFyT47FNhse1eKVeg9UQ\n5+4MBp4ys9GxfQ2xnYW0KV8eAzCzd4ChBRxXF3U2BDPNbAqApG8DfwG+WQ/tGAlcbGZ/k9QOONLM\nDgztytuBLoSkU4FuZnagpGOA3wG9AczsaklzgYuA2XVRn3POOeecq67UUwSbSHoofIs/SVKz9AyS\nhktaHLZbw75G4dv7xWEkYlTY303SjPBN/UuSutaiTa2A99L2vR9rz/mx0Y8Hwr6Jku6U9LykN1Ij\nD5JaSJoZ2rJI0qCwv7OkZeG45ZIelnRSOH65pCNDvuaSJkiaI2mepNNDMz4HPizgXN4P5YwN7Z0v\n6S1JE1KnE2tPfDSxUxiRWS7pxkxxyFdntljFSTpA0nRJL0qaLam7pJaSVsfyNJe0VlJZpvwZ6t9M\nFJ+szOzT2Ns9gQ8KOK8NwLZs114wVNLfJb0qqU9of7URQUlTJfWTdAPQF5gg6ZfAk0DH8Bn1TYtT\nuaTKcN7TJbUt9FyBM4AHw3n/Hdg7djzAeqJr3pXYgAED6rsJuyyPbTI8rsnwuCbHY5sMj2vxSj2C\ndRBwoZnNCTf9lwF3pBIltQduBXoR3UzOCJ2Ut4COZtYj5GsZDnkYGGdmUyQ1pXYdxjKgKr7DzI4J\n9RwCXAsca2abJMVvTNuZWR9JBwNTgMnAZ8BgM9siaR9gTkgD6Ab8wMyWSnoJGBaOHxTqOBO4Dnja\nzEZK2huYK2mmmb0AvBDadATwIzO7JP1EUu02szHAmFDGs0Dqhj8+2hR/fRRwaGj/i5Kmmdn8VHm5\nFBirlHtD21dIOhq4x8wGhg5ZfzObDZwGPGFm2yTVyA8MTKv/ytRrSWOBF81sWnrFkgYDtwDtgO8W\ncF5DwnHlZL72AMrM7BhFo0YVwEmpwzOUd5OiqYlXmdkCSXcBU82sPJQ7MvxtTPR5DTKzDZKGAuOA\nkQWea0fgzdj7dWHfu+F9FdE1n9us2OsuQG2+unDOOeec24VUVlYWNGWy1CNYa81sTnj9ENE3+nFH\nAbPMbKOZVRF1oPoBK4GuYdTou8DHkvYEOqSmfpnZ52b2WTGNkSSgJ1EHLpMTgMfMbFOoY3Ms7a9h\n3zLgG6kigVskLQJmAh0kpdJWmdnS8HpJSAd4megWFuBkYLSkBUAl0BSotg7IzOZl6lxl8RBwh5kt\nzJNvhpltDvGbTM3PpRC5YoWkFsBxwGPh/H4PpEZWJgFnh9fDgEfz5M/IzMZk6lyFtL+a2cHA6cAf\nizivGtdeLG1y+DsP6Fxgefmef34QcBjRlwsLiDrdHdIz5TrXPNYB3SV9LWeu42Obd67qhM9hT47H\nNhke12R4XJPjsU2Gx3WHAQMGUFFRsX3Lpr7XYGVav1PjBtTMNkvqSTTycClwFnBFprzVCpIuA34Y\n6vknM1sfS2tEdPO8FXi8iHNI2ZqhzSOAfYFeZlal6AEQzTLkr4q9r2LH5yCiUa7Xa9GeaiRVEHVo\na0zVy6CQz2VnNQI2pUZs0kwBbpbUGigHniGaypctf62Z2XOSGkvax8w2FJA/07V3cUhOfYbb2PEZ\nfkn1Ly5qTIPNQ8ArZtanyONS1gH7x97vF/YBYGYrJS0D1kgaaGZLalmPc84555zLoNQjWJ0VLbwH\nOAf4v7T0uUA/SW0klQHDgdlhul2Zmf0FuB4oN7MtwJuSzgCQ1FTSHvHCzOxuM+tlZuXxzlVIqzKz\nLsBL7Bg9SfcMcJakNqGO1lnypTpYewPvhc7V8VQf1Sjkl1ufBC7ffkDsCXDFULR260RgVHpSlkNO\nktQqxG8w8HyGMpflqTZnrMzsY2CVpCGxMnuEtE+IPoc7gWkWyZq/WJK6xV6Xhzo3hPczw9TUbMfW\nuPayZQ1/VwOHK7I/cHSupmXYtxz4uqTeof7GYfploaYA54djewObzSw1PTAVw65Eo7/euSohn8Oe\nHI9tMjyuyfC4JsdjmwyPa/FK3cF6FfixpKVEC+1/F/annm63HhhNND1uAdEak6lEa0gqw5SpP4Y8\nEN1IXh6m5D1PnilkWbwGtMmUEKb03UzUyVsAjI+3N541/H0YOCq051xgWYY8mY5PuYnoQSCLFT2E\n4ufpGSQdEdYm5XIl0bSyF8NDFCry1DuXaLrbQqJpfvPT6twnT325YhV3LjBS0UNJXgEGxdIeJRoB\n/HNs34gc+WtQ9HCP0zIk/UDSK5LmE3XihoX8IlobtzFHsdmuvYzXgJk9T9TJWgL8imj6ILmOSTv+\nC2AIcJukhUT/HRxb6Lma2f8SdUzfIJpWeVlaltbA6jAF1znnnHPO1TGZfRWesp0cST8F9kl7TLuL\nkfQ9oKuZ/ba+21KXJB1K9NCVa+q7LaUSHprxfTMbniOPUVG6NtWpCmio/6ZVVlb6t4AJ8dgmw+Oa\nDI9rcjy2yfC4ZicJM6sxI6nUa7AaosnA/ZKmm9mp9d2YhsjMarNGrcELU+R2p87VeOA7wL/lzVyR\ndGuS0bZjbQaxnXPOOefqzm4/guWcq06S+b8LzjnnnHO5ZRvBKvUaLOecc84555zbZXkHyznnSsB/\nRyQ5HttkeFyT4XFNjsc2GR7X4nkHyznnnHPOOefqiK/Bcs5V42uwnHPOOefy8zVYzjnnnHPOOZcw\n72A551wJ+Bz25Hhsk+FxTYbHNTke22R4XIvnv4PlnKtBqjHavUtq27Yz69evru9mOOecc24X4muw\nnHPVSDLYXf5dEP5voHPOOedqw9dgOeecc84551zCStbBktRZ0stZ0mZJKi9VW9Lq7iRpvqTpsX2r\n6qMt2UjqL2liAflytlvSx+Fve0mTwusLJP2mNuUVWOcYSVflK6cY8TIlTZTUL0/+gyT9TdJnhbYl\nXJOd8qQXdc1KGiJpqaSnw/tHJC2UNCqcx5l5ji/kXM+RtChsz0nqEUsbL2mJpP7FtNvVDZ/DnhyP\nbTI8rsnwuCbHY5sMj2vxSr0GqyHOxRkMPGVmo2P7GmI7C2lTvjwGYGbvAEMLOK4u6mwINgA/Ifqs\n69NI4GIz+5ukdsCRZnYgRJ2nOqpjJdDPzD6UdApwL9AbwMyuljQXuAiYXUf1Oeecc865mFJPEWwi\n6aHwLf4kSc3SM0gaLmlx2G4N+xqFb+8Xh2/mR4X93STNCKMAL0nqWos2tQLeS9v3fqw954c6F0h6\nIOybKOlOSc9LeiM18iCphaSZoS2LJA0K+ztLWhaOWy7pYUknheOXSzoy5GsuaYKkOZLmSTo9NONz\n4MMCzuX9UM7Y0N75kt6SNCF1OrH2xEcTO4URmeWSbswUh3x1ZotVnKQDJE2X9KKk2ZK6S2opaXUs\nT3NJayWVZcqfof7NRPHJysw+MLN5wJcFnE/KBmBbtmsvGCrp75JeldQntL/aiKCkqZL6SboB6AtM\nkPRL4EmgY/iM+qbFqVxSZTjv6ZLaFnGuc8wsda3MATqmZVlPdM27EhswYEB9N2GX5bFNhsc1GR7X\n5Hhsk+FxLV6pR7AOAi40sznhpv8y4I5UoqT2wK1AL6KbyRmhk/IW0NHMeoR8LcMhDwPjzGyKpKbU\nrsNYBlTFd5jZMaGeQ4BrgWPNbJOk+I1pOzPrI+lgYAowGfgMGGxmWyTtQ3SDOyXk7wb8wMyWSnoJ\nGBaOHxTqOBO4DnjazEZK2huYK2mmmb0AvBDadATwIzO7JP1EUu02szHAmFDGs0Dqhj8+2hR/fRRw\naGj/i5Kmmdn8VHm5FBirlHtD21dIOhq4x8wGhg5ZfzObDZwGPGFm2yTVyA8MTKv/ytRrSWOBF81s\nWr52F3BeQ0KZ5WS+9gDKzOwYSacCFcBJqcMzlHeTpBOAq8xsgaS7gKlmVh7KHRn+Nib6vAaZ2QZJ\nQ4FxwMhanOvFwPS0fVVE13weFbHXA8LmnHPOObf7qqysLGjKZKlHsNaa2Zzw+iGib/TjjgJmmdlG\nM6si6kD1I5r21DWMGn0X+FjSnkAHM5sCYGafm9lnxTRGkoCeRB24TE4AHjOzTaGOzbG0v4Z9y4Bv\npIoEbpG0CJgJdJCUSltlZkvD6yUhHeBloEt4fTIwWtICoBJoClRbB2Rm8zJ1rrJ4CLjDzBbmyTfD\nzDaH+E2m5udSiFyxQlIL4DjgsXB+vwdSIzOTgLPD62HAo3nyZ2RmY+qic5WmxrUXS5sc/s4DOhdY\nXr7nnx8EHEb05cICok53h/RM+c5V0vHAhcC/piWtA7pL+lruZlTEtgF5muwK4XPYk+OxTYbHNRke\n1+R4bJPhcd1hwIABVFRUbN+yqe81WJnW79S4ATWzzZJ6At8FLgXOAq7IlLdaQdJlwA9DPf9kZutj\naY2Ibp63Ao8XcQ4pWzO0eQSwL9DLzKoUPQCiWYb8VbH3Vez4HEQ0yvV6LdpTjaQKog5tjal6GRTy\nueysRsCm1IhNminAzZJaA+XAM8CeOfKXTJZr7+KQnPoMt7HjM/yS6l9c1JgGm4eAV8ysT+1aDIoe\nbHEvcEqqw5tiZislLQPWSBpoZktqW49zzjnnnKup1CNYnSWlpp2dA/xfWvpcoJ+kNpLKgOHA7DDd\nrszM/gJcD5Sb2RbgTUlnAEhqKmmPeGFmdreZ9TKz8njnKqRVmVkX4CV2jJ6kewY4S1KbUEfrLPlS\nHay9gfdC5+p4qo9qFPLLrU8Cl28/QDq8gGNqNiZau3UiMCo9KcshJ0lqFeI3GHg+Q5nL8lSbM1Zm\n9jGwStKQWJk9QtonRJ/DncA0i2TNv5OqxUDRmrn2WTNnuPbylLsaOFyR/YGjC21LsBz4uqTeof7G\nYfplQRQ9+fC/gfPMbEWG9B5AV6LRX+9clZDPYU+OxzYZHtdkeFyT47FNhse1eKXuYL0K/FjSUqKF\n9r8L+1NPt1sPjCaaHreAaI3JVKKF+pVhytQfQx6A84HLw5S858kzhSyL14A2mRLClL6biTp5C4Dx\n8fbGs4a/DwNHhfacCyzLkCfT8Sk3ET0IZLGih1D8PD2DpCPC2qRcriSaVvZieIhCRZ565xJNd1tI\nNM1vflqd++SpL1es4s4FRip6KMkrwKBY2qNEI4B/ju0bkSN/DYoe7nFahv1tJb1JFJfrFD1EY88w\nRbQbsDFHsdmuvYzXgJk9T9TJWgL8imj6ILmOSTv+C2AIcJukhUT/HRxb6LkCNxBdz3eHtW1z09Jb\nA6vDFFznnHPOOVfHZPZVeMp2ciT9FNgn7THtLkbS94CuZvbb+m5LXZJ0KNFDV66p77aUSnhoxvfN\nbHiOPPbVePp+XRCl+jewsrLSvwVMiMc2GR7XZHhck+OxTYbHNTtJmFmNGUmlXoPVEE0G7pc03cxO\nre/GNERmVps1ag1emCK3O3WuxgPfAf6tgNxJN6dBaNu20GeTOOecc84VZrcfwXLOVSfJ/N8F55xz\nzrncso1glXoNlnPOOeecc87tsryD5ZxzJeC/I5Icj20yPK7J8Lgmx2ObDI9r8byD5ZxzzjnnnHN1\nxNdgOeeq8TVYzjnnnHP5+Ros55xzzjnnnEuYd7Ccc64EfA57cjy2yfC4JsPjmhyPbTI8rsXz38Fy\nztUg7R6/gxXXtmNb1r+1vr6b4ZxzzrmvOF+D5ZyrRpJRUd+tqAcV4P8eOuecc65QvgbLOeecc845\n5xJWsg6WpM6SXs6SNktSeanaklZ3J0nzJU2P7VtVH23JRlJ/SRMLyJez3ZI+Dn/bS5oUXl8g6Te1\nKa/AOsdIuipfOcWIlylpoqR+efIfJOlvkj4rtC3hmuyUJ72oa1bSEElLJT0d3j8iaaGkUeE8zsxz\nfN5zDfl+Len1UPbhsf3jJS2R1L+Ydru64XPYk+OxTYbHNRke1+R4bJPhcS1eqddgNcT5N4OBp8xs\ndGxfQ2xnIW3Kl8cAzOwdYGgBx9VFnQ3BBuAnRJ91fRoJXGxmf5PUDjjSzA6EqPNUFxVIOhXoZmYH\nSjoG+B3QG8DMrpY0F7gImF0X9TnnnHPOuepKPUWwiaSHwrf4kyQ1S88gabikxWG7NexrFL69Xyxp\nkaRRYX83STPCN/UvSepaiza1At5L2/d+rD3nhzoXSHog7Jso6U5Jz0t6IzXyIKmFpJmhLYskDQr7\nO0taFo5bLulhSSeF45dLOjLkay5pgqQ5kuZJOj0043PgwwLO5f1QztjQ3vmS3pI0IXU6sfbERxM7\nhRGZ5ZJuzBSHfHVmi1WcpAMkTZf0oqTZkrpLailpdSxPc0lrJZVlyp+h/s1E8cnKzD4ws3nAlwWc\nT8oGYFu2ay8YKunvkl6V1Ce0v9qIoKSpkvpJugHoC0yQ9EvgSaBj+Iz6psWpXFJlOO/pktoWeq7A\nGcCD4bz/DuwdOx5gPdE170pswIAB9d2EXZbHNhke12R4XJPjsU2Gx7V4pR7BOgi40MzmhJv+y4A7\nUomS2gO3Ar2IbiZnhE7KW0BHM+sR8rUMhzwMjDOzKZKaUrsOYxlQFd9hZseEeg4BrgWONbNNkuI3\npu3MrI+kg4EpwGTgM2CwmW2RtA8wJ6QBdAN+YGZLJb0EDAvHDwp1nAlcBzxtZiMl7Q3MlTTTzF4A\nXghtOgL4kZldkn4iqXab2RhgTCjjWSB1wx8fbYq/Pgo4NLT/RUnTzGx+qrxcCoxVyr2h7SskHQ3c\nY2YDQ4esv5nNBk4DnjCzbZJq5AcGptV/Zeq1pLHAi2Y2LV+7CzivIaHMcjJfewBlZnaMolGjCuCk\n1OEZyrtJ0gnAVWa2QNJdwFQzKw/ljgx/GxN9XoPMbIOkocA4YGSB59oReDP2fl3Y9254X0V0zec2\nK/a6C1Cbry6cc84553YhlZWVBU2ZLPUI1tr/z97dB1lV3fn+f39AGUVRwQcezIiG0hhujIpRY3S0\n9RcxMdF4URIfMliJo0lhqZiYKn6Z3HSjRIzGRJ0px2gcEgVrkHtJBjWKQLodRRGE5kFB1AkasYJ6\no6hMER3le/9Y3wO7T5/d54He3QjfV1VX77P32nt993fvA2edtdZuM1voy9NI3+hnHQu0mtlbZraZ\n1IA6GfgjcIj3Gp0BvCdpT2CYmc0GMLMPzOyv9QQjScCRpAZcJacBM83sba9jQ2bb73zdauCA0iGB\nKZKWA/OAYZJK29aa2Spffs63A6wkfYQFGA1MlNQOtAH9gA7zgMxsSaXGVY5pwM/NbFmVcnPNbIPn\nbxUygQ8AACAASURBVBadr0stusoVkvYAvgDM9PP7JVDqWbkf+IYvnw/MqFK+IjNr7o7GVZlO915m\n2yz/vQQYXuPxqj3//FPAZ0hfLrSTGt3Dygttw7m+Bhwm6W+6LHVq5icaV90ixrAXJ3JbjMhrMSKv\nxYncFiPyulVTUxMtLS1bfvL09hysSvN3On0ANbMNko4EzgC+C4wFJlQq2+FA0njgUq/nTDNbn9nW\nh/Th+X3goTrOoeT9CjFfBOwHHG1mm5UeALFbhfKbM683s/U6iNTL9WID8XQgqYXUoO00VK+CWq7L\ntuoDvF3qsSkzG/iJpIHAKOAPwJ5dlO8xOffeP/jm0jX8iK3X8EM6fnHRaRhsFQKeNbMTG4uY14C/\nzbz+hK8DwMz+KGk18Iqk/8/MnmuwnhBCCCGEUEFP92ANV5p4D3Ah8HjZ9kXAyZIGSeoLXAA85sPt\n+prZb4EfAaPMbCPwqqSvAUjqJ2n37MHM7HYzO9rMRmUbV75ts5kdDDzD1t6Tcn8Axkoa5HUMzClX\namDtDbzhjatT6dirUctfbp0DXLllh8wT4OqhNHfri8BV5Ztydjld0j6ev3OABRWOubpKtV3mysze\nA9ZKOi9zzM/6tv8iXYdbgQctyS2/jTrkQGnO3NDcwhXuvSrHfRk4SsnfAsfVGotbA+wv6fNe/y4+\n/LJWs4Fxvu/ngQ1mVhoeWMrhIaTe32hc9aAYw16cyG0xIq/FiLwWJ3JbjMhr/Xq6gfU8cLmkVaSJ\n9nf4+tLT7dYDE0nD49pJc0weIM0hafMhU/d6GUgfJK/0IXkLqDKELMcLwKBKG3xI309Ijbx24OZs\nvNmi/ns6cKzH801gdYUylfYvuY70IJAVSg+huLa8gKRjfG5SV64mDStb7A9RaKlS7yLScLdlpGF+\nS8vq3LdKfV3lKuubwCVKDyV5Fjg7s20GqQfw3zLrLuqifCdKD/f4aoX1gyW9SsrLPyo9RGNPHyI6\nAniri8Pm3XsV7wEzW0BqZD0H3EIaPkhX+5Tt/9/AecBPJS0jvQ9OqPVczez3pIbpS6RhlePLigwE\nXvYhuCGEEEIIoZvJ7OPwlO3iSPoBsG/ZY9pDhqSvAIeY2T/3dizdSdL/ID105ZrejqWn+EMz/qeZ\nXdBFGaOl52LabrRAkf8etrW1xbeABYncFiPyWozIa3Eit8WIvOaThJl1GpHU03OwtkezgF9LetjM\nvtzbwWyPzKyROWrbPR8itzM1rm4G/g74/6sWbik6mu3P4AMb6QAPIYQQQuhop+/BCiF0JMni34UQ\nQgghhK7l9WD19BysEEIIIYQQQthhRQMrhBB6QPwdkeJEbosReS1G5LU4kdtiRF7rFw2sEEIIIYQQ\nQugmMQcrhNBBzMEKIYQQQqgu5mCFEEIIIYQQQsGigRVCCD0gxrAXJ3JbjMhrMSKvxYncFiPyWr9o\nYIUQOpFU2M+QIQf39umFEEIIIRQm5mCFEDqQZFDkvwsi/t0JIYQQwsddzMEKIYQQQgghhIIV2sCS\nNFzSypxtrZJGFVl/HkkHSVoq6eHMurW9EUseSadImlpDubrjlnSVpN1ytl0s6TZfbpY0rsqxLpbU\nXKXMe/XGWE3pmH6PtdZQvlXS85La/drvV6V8l/n37Q/UGXM/SXO9/rGSTpL0rL8+PO+9ktm/6rlK\n2l3Sg5JWS1op6frMtsO8vhn1xB26R4xhL07kthiR12JEXosTuS1G5LV+PdGDtT2OBToHeNTMvpxZ\ntz3GWUtMjcQ9AejfwH6NxlBEbi1nuSsXmNnRZjbKzP5vnXU0sr3cKMC8/pnARcD1ZjYK2FTj8Wop\nc5OZfRo4GjhJ0hmkil8ws88AR0g6pM7YQwghhBBCDXqigbWrpGmSVkm6v1LPiaQLJK3wnxt8XR9J\nU33dcklX+foR3guwTNIzDX5Q3Ad4o2zdm5l4xnmd7ZJ+4+umSrpV0gJJL0ka4+v3kDTPY1ku6Wxf\nP9x7EaZKWiNpuqTTff81kj7n5fpLulvSQklLJJ3lYXwAvFPDubzpx5mU6Z1Z58fs770Z7Z7HsZKu\nAIYBrZLm+77f8pgWAidmjr2R9MG/K5u8HJIOkDTLr027pM+XUprJ7TWSFnmZZl83RdL4TJlmSd/L\nK1/mI+CtGvIE9d3vW/LvvVWl3C6RtIeXGSBppl/nezPxr5U0yJeP8d6z/YF7gWP9OJcBXweuy+7r\n+/SRdKOkp/28L631XM1sk5k95ssfAkuBT5QVe530Hgg9qKmpqbdD2GFFbosReS1G5LU4kdtiRF7r\nt0sP1PEp4FtmtlDS3cB44OeljZKGAjeQvm3fAMz1Rso64EAz+6yX28t3mU761n+2pH401kjsC2zO\nrjCz472ekcAPgRPM7G1J2Q+iQ8zsREmfBmYDs4C/AueY2UZJ+wILfRvACOBcM1sl6RngfN//bK9j\nDPCPwHwzu0TS3sAiSfPM7CngKY/pGOA7ZnZZ+YmU4jazZqDZj/EfwD8DXwJeM7Ov+nEGmNl7kq4G\nmvz8hgAtpPy/C7SRPpRjZjdXS6SZ3Z95eRvQZmZjJAnYs1TM6z8dONTMjvPtsyWdBMwAbgFu9/Jf\nB0bnlTezJ/BGm5mtA87z4w8F7iqdbwW/lvTfwCwzm1zlvLbkH/g+MN7MnpLUn3TNAY4CRgLrgQWS\nvmBmT9K5l8nM7E1J/wB838xKjfATgAfMbJak4ZnylwAbzOx4v8cXSHrUzF6p41zxe/csUm6zNpPe\nA11oySw3+U8IIYQQws6rra2tpiGTPdGD9SczW+jL04CTyrYfC7Sa2VtmtpnUgDoZ+CNwiPcanQG8\nJ2lPYJiZzQYwsw/M7K/UwT+oH0lqwFVyGjDTzN72OjZktv3O160GDigdEpgiaTkwDxgmqbRtrZmt\n8uXnfDvASuBgXx4NTJTUTmrc9AMOygZkZksqNa5yTANuNrN2r+d07yE6ycxKc6HE1l6l49ma/w9J\njZ1GnQb8i8dsmfpKRns8S0mNuE+RGlDLgP0lDZH0WeAtM3str3xe5Wb25y4aHBea2RHA3wF/J+mb\ndZzXAuAX3vs30O9TgEVepwHL2HpNOz1Npk6jgXF+TzwNDKLsvKucK5L6AvcBt5jZy2Wb15HeA11o\nyfw01R55yBVj2IsTuS1G5LUYkdfiRG6LEXndqqmpiZaWli0/eXqiB6vTt/kVynT6QGpmGyQdCZwB\nfBcYS5o71OWHVx9qdqnXc6aZrc9s60NquL0PPFTHOZS8XyHmi4D9gKPNbLPSQyd2q1B+c+b1Zrbm\nXqRerhcbiKcDSS2kBu09AGb2otKDRM4EJnvPWKWem21tEJRUmx8kYIqZ3VVh20zSNR7C1kZeV+Xr\nmv9kZn/23/8l6T7gOFJjtJZ9fyrpQeArpN6k0b4pe30/Yus1/ZCtX15UfJhIFQKuMLO5Dexbciew\nxsz+qcK2XwJzJB1nZt/ZhjpCCCGEEEKZnujBGi7peF++EHi8bPsi4GRJg/xb9wuAx3y4XV8z+y3w\nI2CUmW0EXpX0NdjyVLbdswczs9szDzJYX7Zts5kdDDwDfCMn3j8AYzNzaAbmlCs1SvYG3vDG1anA\n8AplujIHuHLLDtJRNezTOZg0d+uLwFWZdUOBTWZ2H3AT6SELkIYCloZcPk3K/0BJu5IaOZWOf3l2\nnlSO+aQhoKV5RANKu/vvOcC3S3OYJA3zuUkA9wPnA+eSGlt55fcrO2ZVkvr6/YSf41eBZ/31Oco8\naS9n/0+a2XNmdiOwGDi8SpVrgWN8+dxa48yYA4yXtIvXf2j5fV4l3snAXmZ2dU6Ra4BLonHVs2IM\ne3Eit8WIvBYj8lqcyG0xIq/164kG1vPA5ZJWkSbW3+HrDcAbQRNJw+PagcVm9gBwINDmw6Tu9TIA\n44ArfUjeAmBwAzG9QBp21YkP6fsJqZHXDpTmIeX1xE0nPbhgOfBNYHWFMpX2L7mO9CCQFUqP6b62\nvIA/KOHOLs4H4GrSwysW+0MUWoAjSHO62oEfA6Xeq7uARyTN9/xPIs0dexxY1enIyeHAX6rEMAE4\nVdIKUiN2pK8vXeu5pGFrT3mZmfg8Lc/7AGCdmb3eRfkB2WNmSRrqPU3l/obUY7OMNNRwnecA0jy5\nag8TmaD0yPPlpIdfPFyhTDaea4HbJC0i9WblybsnfkW6Dkv9nriDst7mvHOVdCBpft9IbX0wx7fL\nig0EXuoirhBCCCGE0CCl6SM7F0k/APY1s4lVCwcAJM0Gxvg8rR2GpHuAq82sWuNxh+BzEFcA55nZ\nmpwyVuxfLRA74787bW1t8S1gQSK3xYi8FiPyWpzIbTEir/kkYWadRlX1RA/W9mgWcKIyf2g4dM3M\nzt7RGlcAZjZuJ2pcHUbqJW4n9eKGEEIIIYRutlP2YIUQ8qUerOIMHjyc9etfLrKKEEIIIYTC5fVg\n9cRTBEMIHzPxxUsIIYQQQmN21iGCIYTQo+LviBQncluMyGsxIq/FidwWI/Jav2hghRBCCCGEEEI3\niTlYIYQOJFn8uxBCCCGE0LV4imAIIYQQQgghFCwaWCGE0ANiDHtxIrfFiLwWI/JanMhtMSKv9YsG\nVgghhBBCCCF0k5iDFULooLv+DtbgAwezft367jhUCCGEEMJ2J28OVjSwQggdSDJauuFALfH3tEII\nIYSw44qHXIQQQi+KMezFidwWI/JajMhrcSK3xYi81q/QBpak4ZJW5mxrlTSqyPrzSDpI0lJJD2fW\nre2NWPJIOkXS1BrK1R23pKsk7Zaz7WJJt/lys6RxVY51saTmKmXeqzfGakrH9HustYbyrZKel9Tu\n136/KuW7zL9vf6DOmPtJmuv1j5V0kqRn/fXhee+VzP61nusoSSskvSDplsz6w7y+GfXEHUIIIYQQ\natcTPVjb4xihc4BHzezLmXXbY5y1xNRI3BOA/g3s12gMReTWcpa7coGZHW1mo8zs/9ZZRyPby40C\nzOufCVwEXG9mo4BNNR6vljL/AlxiZocBh0k6g1TxC2b2GeAISYfUGXvYRk1NTb0dwg4rcluMyGsx\nIq/FidwWI/Jav55oYO0qaZqkVZLur9RzIukC/8Z9haQbfF0fSVN93XJJV/n6Ed4LsEzSMw1+UNwH\neKNs3ZuZeMZ5ne2SfuPrpkq6VdICSS9JGuPr95A0z2NZLulsXz9c0mrfb42k6ZJO9/3XSPqcl+sv\n6W5JCyUtkXSWh/EB8E4N5/KmH2dSpndmnR+zv6QHff0K7zW5AhgGtEqa7/t+y2NaCJyYOfZG0gf/\nrmzyckg6QNIsvzbtkj5fSmkmt9dIWuRlmn3dFEnjM2WaJX0vr3yZj4C3asgT1He/b8m/91aVcrtE\n0h5eZoCkmX6d783Ev1bSIF8+xnvP9gfuBY7141wGfB24Lruv79NH0o2SnvbzvrTWc5U0BBhgZot9\n1T2kLxSyXie9B0IIIYQQQjfbpQfq+BTwLTNbKOluYDzw89JGSUOBG4CjgQ3AXG+krAMONLPPerm9\nfJfppG/9Z0vqR2ONxL7A5uwKMzve6xkJ/BA4wczelpT9IDrEzE6U9GlgNjAL+CtwjpltlLQvsNC3\nAYwAzjWzVZKeAc73/c/2OsYA/wjMN7NLJO0NLJI0z8yeAp7ymI4BvmNml5WfSCluM2sGmv0Y/wH8\nM/Al4DUz+6ofZ4CZvSfpaqDJz28I0ELK/7tAG7DUj3lztUSa2f2Zl7cBbWY2RpKAPUvFvP7TgUPN\n7DjfPlvSScAM4Bbgdi//dWB0XnkzewJvtJnZOuA8P/5Q4K7S+Vbwa0n/Dcwys8lVzmtL/oHvA+PN\n7ClJ/UnXHOAoYCSwHlgg6Qtm9iSde5nMzN6U9A/A982s1Ag/AXjAzGZJGp4pfwmwwcyO93t8gaRH\nzeyVGs71QNJ7p2Sdr8vaTHoP5MsORDwYiP6ubdbW1hbfAhYkcluMyGsxIq/FidwWI/K6VVtbW01z\n0nqigfUnM1voy9OAK8g0sIBjgVYzewtA0nTgZGAycIikW4HfA49K2hMYZmazAczsg3qD8Q/qR3os\nlZwGzDSzt72ODZltv/N1qyUdUDokMEXSyaQPrsMy29aa2Spffg6Y58srSR9bAUYDZ0n6gb/uBxwE\nrClVamZLgE6NqxzTgJvNrF3SRuBnkqYAD3nDpBRzqVfpeDrmfwZwaI11lTsN+HuP2YDyuVejgdMl\nLfX69yA1oKZK2t8bewcAb5nZa5ImVCoPPEEFZvZnIK9xdaGZ/dl7n2ZJ+qaZ5d0D5RYAv/B7c5bH\nBrDI60TSMtI1fZJMj12DRpOG8Y3113uRzvuVUoEq51rNOtJ74JncEqc2eOQQQgghhB1UU1NTh8bm\npEmTKpbriQZWp2/zK5Tp9IHUzDZIOhI4A/guMJY0d6jLD68+1OxSr+dMM1uf2dYH+CPwPvBQHedQ\n8n6FmC8C9gOONrPNSg+d2K1C+c2Z15vZmnuRerlebCCeDiS1kBq09wCY2YtKDxI5E5jsPWOVem62\ntUFQUm1+kIApZnZXhW0zSdd4CKlHq1r5uuY/lRpCZvZfku4DjiO/kV2+708lPQh8hdSbNNo3Za/v\nR2y9ph+ytWe14sNEqhBwhZnNbWDf14C/zbz+hK/L+iUwR9JxZvadBuoIDYhv/4oTuS1G5LUYkdfi\nRG6LEXmtX0/MwRou6XhfvhB4vGz7IuBkSYMk9QUuAB7z4XZ9zey3wI+AUWa2EXhV0tdgy1PZds8e\nzMxuzzzIYH3Zts1mdjDpm/tv5MT7B2BsZg7NwJxypUbJ3sAb3rg6FRheoUxX5gBXbtlBOqqGfToH\nk+ZufRG4KrNuKLDJzO4DbiI9ZAHSUMDSkMunSfkfKGlXUiOn0vEvz86TyjGfNAS0NI9oQGl3/z0H\n+HZpDpOkYT43CeB+4HzgXFJjK6/8fmXHrEpSX7+f8HP8KvCsvz5H0vVV9v+kmT1nZjcCi4HDq1S5\nFjjGl8+tNc6MOcB4Sbt4/YeW3+d5/J5/R1JpWOU44N/Lil1DeghGNK5CCCGEELpZTzSwngcul7SK\nNLH+Dl9vsOUD4UTS3J92YLGZPUCaN9ImqZ30cICJvt844EpJy0lDtwY3ENMLwKBKG3xI309Ijbx2\noDQPKa8nbjrpwQXLgW8CqyuUqbR/yXWkB4GsUHpM97XlBZQelHBnF+cDcDXp4RWLlR6i0AIcQZrT\n1Q78mDTsEuAu4BFJ8z3/k0hzxx4HVnU6cnI48JcqMUwATpW0gtSIHenrS9d6LnAf8JSXmYnP0/K8\nDwDWmdnrXZQfkD1mlqSh3tNU7m9IPTbLSPPL1nkOIM2Tq/YwkQmSVvo1/gB4uEKZbDzXArdJWkTq\nzcqTd0/8inQdlvo9cQdlvc1dnCvA5cDdpPv8RTN7pGz7QOClLuIKBYi/I1KcyG0xIq/FiLwWJ3Jb\njMhr/ZSmyuxcfL7TvmY2sWrhAICk2cAYM+uqwfCxI+ke4Gozq9Z43CF4r9YK4DwzW5NTxmjphspa\nYGf89yVPTBIuTuS2GJHXYkReixO5LUbkNZ8kzKzTqKqdtYE1Avg1sLHsb2GFsMOSdBhpKOYK4GLL\nefNL6pZ/FAYfOJj169ZXLxhCCCGE8DEUDawQQk0k5bW9QgghhBCCy2tg9cQcrBBC2OnFGPbiRG6L\nEXktRuS1OJHbYkRe6xcNrBBCCCGEEELoJjFEMITQQQwRDCGEEEKoLoYIhhBCCCGEEELBooEVQgg9\nIMawFydyW4zIazEir8WJ3BYj8lq/aGCFEEIIIYQQQjeJOVghhA666+9g7QgGDx7O+vUv93YYIYQQ\nQtgOxd/BCiHUJDWw4t+FRMS/kSGEEEKoJB5yEUIIvSjGsBcncluMyGsxIq/FidwWI/Jav0IbWJKG\nS1qZs61V0qgi688j6SBJSyU9nFm3tjdiySPpFElTayhXd9ySrpK0W862iyXd5svNksZVOdbFkpqr\nlHmv3hirKR3T77HWGso/LKld0rOSfiVplyrlu8y/b3+gzpj7SZrr995YSSd5PEslHZ73XsnsX/Vc\nJe0u6UFJqyWtlHR9ZtthXt+MeuIOIYQQQgi164kerO1xfM05wKNm9uXMuu0xzlpiaiTuCUD/BvZr\nNIYicms5y3nGmtnRZvYZYB/gG3XW0cj2cqMAM7NRZjYTuAi43sxGAZtqPF4tZW4ys08DRwMnSTqD\nVPELfv5HSDqkztjDNmpqaurtEHZYkdtiRF6LEXktTuS2GJHX+vVEA2tXSdMkrZJ0f6WeE0kXSFrh\nPzf4uj6Spvq65ZKu8vUjvBdgmaRnGvyguA/wRtm6NzPxjPM62yX9xtdNlXSrpAWSXpI0xtfvIWme\nx7Jc0tm+frj3IkyVtEbSdEmn+/5rJH3Oy/WXdLekhZKWSDrLw/gAeKeGc3nTjzPJ410qaZ0fs7/3\nZrR7HsdKugIYBrRKmu/7fstjWgicmDn2RtIH/65s8nJIOkDSLL827ZI+X0ppJrfXSFrkZZp93RRJ\n4zNlmiV9L698mY+At6olycxKMe4K9AP+UmWXLfn33qpSbpdI2sPLDJA006/zvZn410oa5MvHKPXW\n7g/cCxzrx7kM+DpwXXZf36ePpBslPe3nfWmt52pmm8zsMV/+EFgKfKKs2Ouk90AIIYQQQuhuZlbY\nDzAc2Ax83l/fDXzPl1tJ3+gPBV4BBpEafPOBs33bo5lj7eW/FwJn+3I/YLcG4poETMjZNhJ4Hhjo\nr/fx31OBGb78aeBFX+4L7OnL+2bWDyd9SB/pr58B7vbls4FZvvwT4EJf3htYA+xeFtMxwJ01ntve\nwHJS78UY4JeZbQP89x8z5zckk/9dgCeA2xq83v8GXOnLytT3rv8+vRSPb38AOAk4CmjLHOc54MC8\n8v76vQr1DwUe7CK+R0gNqxl1ntds4ARf7u/36SnA216ngCeBL2TyOyhz7f7gy6cAszPHnQqMydwv\nK3z5UuCHmXt8MTC8nnMt3bvAfwIHl62fD3yui/0MmjM/rQa2k/5g3aW1tbXbjhU6itwWI/JajMhr\ncSK3xYi8btXa2mrNzc1bfvxzQqfPUl3OQ+kmfzKzhb48DbgC+Hlm+7FAq5m9BSBpOnAyMBk4RNKt\nwO+BRyXtCQwzs9mkM/qg3mAkCTjSY6nkNGCmmb3tdWzIbPudr1st6YDSIYEpkk4mNSaHZbatNbNV\nvvwcMM+XVwIH+/Jo4CxJP/DX/YCDSA0tvL4lwGU1nuI04GYza5e0EfiZpCnAQ2b2RCbmUq/S8XTM\n/wzg0BrrKnca8PceswHlc69GA6dLWur17wEcamZTJe0vaQhwAPCWmb0maUKl8qRGYCdm9mfgq3nB\nmdmXJPUD7pc0zszuqfG8FgC/8HtzlscGsMjrRNIy0jV9kkyPXYNGk4bxjfXXe5HO+5XMuXR5rpL6\nAvcBt5jZy2Wb15HeA8/kh9BSf9QhhBBCCDuwpqamDkMmJ02aVLFcTzSwyueMVJpD0ukDqZltkHQk\ncAbwXWAsae5Qlx9efajZpV7PmWa2PrOtD6l34X3goTrOoeT9CjFfBOwHHG1mm5UeOrFbhfKbM683\nszX3As41sxcbiKcDSS2kBu09AGb2otKDRM4EJkuaZ2aTK+26rXW7Ste2vJ4pZnZXhW0zSdd4CDCj\nhvLV6qocoNkHkv4PcBxQUwPLzH4q6UHgK8ACSaN9U/b6fsTWa/ohW4ffVnyYSBUCrjCzuQ3sW3In\nsMbM/qnCtl8CcyQdZ2bf2YY6Qh1iDHtxIrfFiLwWI/JanMhtMSKv9euJOVjDJR3vyxcCj5dtXwSc\nLGmQf+t+AfCYpH2Bvmb2W+BHwChL82helfQ12PJUtt2zBzOz2y09zGBUtnHl2zab2cGkb+7zHnLw\nB2BsZg7NwJxypUbJ3sAb3rg6lTTUq7xMV+YAV27ZQTqqhn06B5Pmbn0RuCqzbiiwyczuA24iDbsE\neJfUKwLwNCn/A31+0lgqkHR5dp5UjvnAeC/fR9KA0u7+ew7w7dIcJknDfG4SwP3A+cC5pMZWXvn9\nyo5ZldI8uSG+vAupobTMX5+jzJP2cvb/pJk9Z2Y3kobrHV6lyrWkoYH4+dRrDjDeY0XSoeX3eZV4\nJ5OG1F6dU+Qa4JJoXIUQQgghdL+eaGA9D1wuaRVpTsgdvj5N9kiNoIlAG9AOLDazB0hzcNoktZMe\nDjDR9xsHXClpOWno1uAGYnqBNOeoEx/S9xNSI68duDkbb7ao/55OenDBcuCbwOoKZSrtX3Id6UEg\nK5Qe031teQF/UMKdXZwPwNWkh1cs9ocotABHAIv8PH5MGnYJcBfwiKT5nv9JpLltjwOrOh05OZzq\nD4aYAJwqaQWpETvS15eu9VzSsLWnvMxMYE/ftgoYAKwzs9e7KD8ge8wsSUO9p6ncHsBsH8a3BHgV\n+FffNoLqDxOZoPTI8+WkeXUPVyiTjeda4DZJi0i9WXny7olfka7DUr8n7qCstznvXCUdCPwQGJl5\nMMe3y4oNBF7qIq5QgPg7IsWJ3BYj8lqMyGtxIrfFiLzWT2mqzM7F5zvta2YTqxYOAEiaTXogQ1cN\nho8dSfcAV5tZtcbjDsHnIK4AzjOzNTllbPv8qwW9QXTXv5FtbW0xzKIgkdtiRF6LEXktTuS2GJHX\nfJIws06jqnbWBtYI4NfARuv4t7BC2GFJOow0FHMFcLHlvPlTAysADB48nPXrX+7tMEIIIYSwHYoG\nVgihJpLy2l4hhBBCCMHlNbB6Yg5WCCHs9GIMe3Eit8WIvBYj8lqcyG0xIq/1iwZWCCGEEEIIIXST\nGCIYQugghgiGEEIIIVQXQwRDCCGEEEIIoWDRwAohhB4QY9iLE7ktRuS1GJHX4kRuixF5rV80sEII\nIYQQQgihm8QcrBBCBzvj38EafOBg1q9b39thhBBCCOFjJP4OVgihJpKMlt6Oooe1QPxbGEIIIYR6\nxEMuQgihF8UY9uJEbosReS1G5LU4kdtiRF7rV2gDS9JwSStztrVKGlVk/XkkHSRpqaSHM+vW0G7j\nMgAAIABJREFU9kYseSSdImlqDeXqjlvSVZJ2y9l2saTbfLlZ0rgqx7pYUnOVMu/VG2M1pWP6PdZa\nQ/mHJbVLelbSryTtUqV8l/n37Q/UGXM/SXP93hsr6SSPZ6mkw/PeK5n9az3XUZJWSHpB0i2Z9Yd5\nfTPqiTuEEEIIIdSuJ3qwtsdxN+cAj5rZlzPrtsc4a4mpkbgnAP0b2K/RGIrIreUs5xlrZkeb2WeA\nfYBv1FlHI9vLjQLMzEaZ2UzgIuB6MxsFbKrxeLWU+RfgEjM7DDhM0hmkil/w8z9C0iF1xh62UVNT\nU2+HsMOK3BYj8lqMyGtxIrfFiLzWrycaWLtKmiZplaT7K/WcSLrAv3FfIekGX9dH0lRft1zSVb5+\nhPcCLJP0TIMfFPcB3ihb92YmnnFeZ7uk3/i6qZJulbRA0kuSxvj6PSTN81iWSzrb1w+XtNr3WyNp\nuqTTff81kj7n5fpLulvSQklLJJ3lYXwAvFPDubzpx5nk8S6VtM6P2V/Sg75+hfeaXAEMA1olzfd9\nv+UxLQROzBx7I+mDf1c2eTkkHSBpll+bdkmfL6U0k9trJC3yMs2+boqk8ZkyzZK+l1e+zEfAW9WS\nZGalGHcF+gF/qbLLlvx7b1Upt0sk7eFlBkia6df53kz8ayUN8uVjlHpr9wfuBY7141wGfB24Lruv\n79NH0o2SnvbzvrTWc5U0BBhgZot91T2kLxSyXie9B0IIIYQQQjfrcphUN/kU8C0zWyjpbmA88PPS\nRklDgRuAo4ENwFxvpKwDDjSzz3q5vXyX6aRv/WdL6kdjjcS+wObsCjM73usZCfwQOMHM3paU/SA6\nxMxOlPRpYDYwC/grcI6ZbZS0L7DQtwGMAM41s1WSngHO9/3P9jrGAP8IzDezSyTtDSySNM/MngKe\n8piOAb5jZpeVn0gpbjNrBpr9GP8B/DPwJeA1M/uqH2eAmb0n6Wqgyc9vCNBCyv+7QBuw1I95c7VE\nmtn9mZe3AW1mNkaSgD1Lxbz+04FDzew43z5b0knADOAW4HYv/3VgdF55M3sCb7SZ2TrgPD/+UOCu\n0vmWk/QIcCwwz8weqXJeW/IPfB8Yb2ZPSepPuuYARwEjgfXAAklfMLMn6dzLZGb2pqR/AL5vZqVG\n+AnAA2Y2S9LwTPlLgA1mdrzf4wskPWpmr9RwrgeS3jsl63xd1mbSeyBfdiDiwUD0d22ztra2+Baw\nIJHbYkReixF5LU7kthiR163a2tpqmpPWEw2sP5nZQl+eBlxBpoFF+sDbamZvAUiaDpwMTAYOkXQr\n8HvgUUl7AsPMbDaAmX1QbzD+Qf1Ij6WS04CZZva217Ehs+13vm61pANKhwSmSDqZ9MF1WGbbWjNb\n5cvPAfN8eSXpYyvAaOAsST/w1/2Ag4A1pUrNbAnQqXGVYxpws5m1S9oI/EzSFOAhb5iUYi71Kh1P\nx/zPAA6tsa5ypwF/7zEbUD73ajRwuqSlXv8epAbUVEn7e2PvAOAtM3tN0oRK5YEnqMDM/gxUbFz5\n9i95g+V+SePM7J4az2sB8Au/N2d5bACLvE4kLSNd0yfJ9Ng1aDRpGN9Yf70X6bxfyZxLl+daxTrS\ne+CZ3BKnNnjkEEIIIYQdVFNTU4fG5qRJkyqW64kGVqdv8yuU6fSB1Mw2SDoSOAP4LjCWNHeoyw+v\nPtTsUq/nTDNbn9nWB/gj8D7wUB3nUPJ+hZgvAvYDjjazzUoPnditQvnNmdeb2Zp7kXq5Xmwgng4k\ntZAatPcAmNmLSg8SOROY7D1jkyvtuq11u2rzgwRMMbO7KmybSbrGQ0g9WtXKNzSvy8w+kPR/gONI\nw+dq2eenkh4EvkLqTRrtm7LX9yO2XtMP2dqzWvFhIlUIuMLM5jaw72vA32Zef8LXZf0SmCPpODP7\nTgN1hAbEt3/FidwWI/JajMhrcSK3xYi81q8n5mANl3S8L18IPF62fRFwsqRBkvoCFwCP+XC7vmb2\nW+BHwCifR/OqpK/Blqey7Z49mJnd7g8zGJVtXPm2zWZ2MOmb+7yHHPwBGJuZQzMwp1ypUbI38IY3\nrk4Fhlco05U5wJVbdpCOqmGfzsGkuVtfBK7KrBsKbDKz+4CbSA9ZgDQUsDTk8mlS/gf6/KSxVCDp\n8uw8qRzzSUNAS/OIBpR2999zgG+X5jBJGuZzkwDuB84HziU1tvLK71d2zKqU5skN8eVdSA2lZf76\nHEnXV9n/k2b2nJndCCwGDq9S5VrgGF8+t9Y4M+YA4z1WJB1afp/n8Xv+HUmlYZXjgH8vK3YN6SEY\n0bgKIYQQQuhmPdHAeh64XNIq0sT6O3y9wZYPhBNJc3/agcVm9gBp3kibpHbSwwEm+n7jgCslLScN\n3RrcQEwvAIMqbfAhfT8hNfLagdI8pLyeuOmkBxcsB74JrK5QptL+JdeRHgSyQukx3deWF1B6UMKd\nXZwPwNWkh1csVnqIQgtwBGlOVzvwY9KwS4C7gEckzff8TyLNHXscWNXpyMnhVH8wxATgVEkrSI3Y\nkb6+dK3nAvcBT3mZmfg8Lc/7AGCdmb3eRfkB2WNmSRrqPU3l9iDN31oGLAFeBf7Vt42g+sNEJkha\n6df4A+DhCmWy8VwL3CZpEak3K0/ePfEr0nVY6vfEHZT1NndxrgCXA3eT7vMXK8w3Gwi81EVcoQDx\nd0SKE7ktRuS1GJHX4kRuixF5rZ/SVJmdi8932tfMJlYtHACQNBsYY2ZdNRg+diTdA1xtZtUajzsE\n79VaAZxnZmtyyhgtPRpW72uBov8tjEnCxYncFiPyWozIa3Eit8WIvOaThJl1GlW1szawRgC/BjaW\n/S2sEHZYkg4jDcVcAVxsOW9+STvdPwqDDxzM+nXrqxcMIYQQQnDRwAoh1ERSXtsrhBBCCCG4vAZW\nT8zBCiGEnV6MYS9O5LYYkddiRF6LE7ktRuS1ftHACiGEEEIIIYRuEkMEQwgdxBDBEEIIIYTqYohg\nCCGEEEIIIRQsGlghhNADYgx7cSK3xYi8FiPyWpzIbTEir/WLBlYIIYQQQgghdJOYgxVC6GBn/DtY\nYccxePBw1q9/ubfDCCGEsBOIv4MVQqhJamDFvwvh40rE/2shhBB6QjzkIoQQelVbbwewA2vr7QB2\nSDHvohiR1+JEbosRea1foQ0sScMlrczZ1ippVJH155F0kKSlkh7OrFvbG7HkkXSKpKk1lKs7bklX\nSdotZ9vFkm7z5WZJ46oc62JJzVXKvFdvjNWUjun3WGsN5SdL+pOkd2s8fpf59+0P1B4xSOonaa7f\ne2MlnSTpWX99eN57JbN/1XOVtLukByWtlrRS0vWZbYd5fTPqiTuEEEIIIdSuJ3qwtsexGucAj5rZ\nlzPrtsc4a4mpkbgnAP0b2K/RGIrIreUs55kNHLsNdTSyvdwowMxslJnNBC4CrjezUcCmGo9XS5mb\nzOzTwNHASZLOIFX8gpl9BjhC0iF1xh62WVNvB7ADa+rtAHZITU1NvR3CDinyWpzIbTEir/XriQbW\nrpKmSVol6f5KPSeSLpC0wn9u8HV9JE31dcslXeXrR3gvwDJJzzT4QXEf4I2ydW9m4hnndbZL+o2v\nmyrpVkkLJL0kaYyv30PSPI9luaSzff1w70WYKmmNpOmSTvf910j6nJfrL+luSQslLZF0lofxAfBO\nDefyph9nkse7VNI6P2Z/781o9zyOlXQFMAxolTTf9/2Wx7QQODFz7I2kD/5d2eTlkHSApFl+bdol\nfb6U0kxur5G0yMs0+7opksZnyjRL+l5e+TIfAW9VS5KZLTKz16uVy9iSf++tKuV2iaQ9vMwASTP9\nOt+biX+tpEG+fIxSb+3+wL3AsX6cy4CvA9dl9/V9+ki6UdLTft6X1nquZrbJzB7z5Q+BpcAnyoq9\nTnoPhBBCCCGE7mZmhf0Aw4HNwOf99d3A93y5lfSN/lDgFWAQqcE3Hzjbtz2aOdZe/nshcLYv9wN2\nayCuScCEnG0jgeeBgf56H/89FZjhy58GXvTlvsCevrxvZv1w0of0kf76GeBuXz4bmOXLPwEu9OW9\ngTXA7mUxHQPcWeO57Q0sJ/VejAF+mdk2wH//MXN+QzL53wV4Aritwev9b8CVvqxMfe/679NL8fj2\nB4CTgKOAtsxxngMOzCvvr9+rUP9Q4MEqMb7bwHnNBk7w5f5+n54CvO11CngS+EImv4My1+4PvnwK\nMDtz3KnAmMz9ssKXLwV+mLnHFwPDGzjXfYD/BA4uWz8f+FwX+xk0Z35aDSx+tvkn8tgzucVC92ht\nbe3tEHZIkdfiRG6LEXndqrW11Zqbm7f8+P85lP/sQvH+ZGYLfXkacAXw88z2Y4FWM3sLQNJ04GRg\nMnCIpFuB3wOPStoTGGZms0ln9EG9wUgScKTHUslpwEwze9vr2JDZ9jtft1rSAaVDAlMknUxqTA7L\nbFtrZqt8+Tlgni+vBA725dHAWZJ+4K/7AQeRGlp4fUuAy2o8xWnAzWbWLmkj8DNJU4CHzOyJTMyl\nXqXj6Zj/GcChNdZV7jTg7z1mA8rnXo0GTpe01OvfAzjUzKZK2l/SEOAA4C0ze03ShErlSY3ATszs\nz8BXG4y9KwuAX/i9OctjA1jkdSJpGemaPkmmx65Bo0nD+Mb6671I5/1KqUC1c5XUF7gPuMXMXi7b\nvI70HngmP4SW+qMOIYQQQtiBNTU1dRgyOWnSpIrleqKBZVVeQ4UPpGa2QdKRwBnAd4GxpLlDXX54\n9aFml3o9Z5rZ+sy2PqTehfeBh+o4h5L3K8R8EbAfcLSZbVZ66MRuFcpvzrzezNbcCzjXzF5sIJ4O\nJLWQGrT3AJjZi0oPEjkTmCxpnplNrrTrttbtKl3b8nqmmNldFbbNJF3jIcCMGspXq6vbmNlPJT0I\nfAVYIGm0b8pe34/Yek0/ZOvw24oPE6lCwBVmNreReN2dwBoz+6cK234JzJF0nJl9ZxvqCHVp6u0A\ndmBNvR3ADinmXRQj8lqcyG0xIq/164k5WMMlHe/LFwKPl21fBJwsaZB/634B8JikfYG+ZvZb4EfA\nKDPbCLwq6Wuw5alsu2cPZma3m9nRlh4ksL5s22YzO5j0zf03cuL9AzA2M4dmYE65UqNkb+ANb1yd\nShrqVV6mK3OAK7fsIB1Vwz6dg0lzt74IXJVZNxTYZGb3ATeRhl0CvEvqFQF4mpT/gZJ2JTVyKh3/\n8uw8qRzzgfFevo+kAaXd/fcc4NulOUyShvncJID7gfOBc0mNrbzy+5Uds14d9pN0jjJP2qu4g/RJ\nM3vOzG4kDdc7vEoda0lDAyGdT73mAOMl7eL1H1p+n1eJdzJpSO3VOUWuAS6JxlUIIYQQQvfriQbW\n88DlklaR5oTc4esNwBtBE0l/yKQdWGxmD5Dm4LRJaic9HGCi7zcOuFLSctLQrcENxPQCac5RJz6k\n7yekRl47cHM23mxR/z2d9OCC5cA3gdUVylTav+Q60oNAVig9pvva8gL+oIQ7uzgfgKtJD69Y7A9R\naAGOABb5efyYNOwS4C7gEUnzPf+TSHPbHgdWdTpycjjwlyoxTABOlbSC1Igd6etL13ouadjaU15m\nJrCnb1sFDADWmT+MIqf8gOwxsyQN9Z6mTiT9VNKrwO5Kj2v/sW8aQfWHiUxQeuT5ctK8uocrlMnG\ncy1wm6RFpN6sPHn3xK9I12Gp3xN3UNbbnHeukg4EfgiMzDyY49tlxQYCL3URVyhEW28HsANr6+0A\ndkjxt2+KEXktTuS2GJHX+ilNldm5+Hynfc1sYtXCAQBJs0kPZOiqwfCxI+ke4Gozq9Z43CH4HMQV\nwHlmtianjPXgCMydSBsxlK0obWzNrdgZ/18rQltbWwwNKkDktTiR22JEXvNJwsw6jaraWRtYI4Bf\nAxut49/CCmGHJekw0lDMFcDFlvPmjwZW+HiLBlYIIYSeEQ2sEEJNUgMrhI+nwYOHs379y70dRggh\nhJ1AXgOrJ+ZghRA+Zir9TYf42baf1tbWXo9hR/3J5jYaV90n5l0UI/JanMhtMSKv9YsGVgghhBBC\nCCF0kxgiGELoQJLFvwshhBBCCF2LIYIhhBBCCCGEULBoYIUQQg+IMezFidwWI/JajMhrcSK3xYi8\n1i8aWCGEEEIIIYTQTWIOVgihg5iDFUIIIYRQXd4crF16I5gQwvZN6vRvRQg7ncEHDmb9uvW9HUYI\nIYSPmejBCiF0IMlo6e0odkBrgUN6O4gdVFG5bUl/E25n1dbWRlNTU2+HscOJvBYncluMyGu+eIpg\nCCGEEEIIIRSs0AaWpOGSVuZsa5U0qsj680g6SNJSSQ9n1q3tjVjySDpF0tQaytUdt6SrJO2Ws+1i\nSbf5crOkcVWOdbGk5ipl3qs3xmpKx/R7rLWG8pMl/UnSuzUev8v8+/YHao8YJPWTNNfvvbGSTpL0\nrL8+PO+9ktm/1nMdJWmFpBck3ZJZf5jXN6OeuEM3id6r4kRuCxHfWBcj8lqcyG0xIq/164kerO1x\nfMU5wKNm9uXMuu0xzlpiaiTuCUD/BvZrNIYicms5y3lmA8duQx2NbC83CjAzG2VmM4GLgOvNbBSw\nqcbj1VLmX4BLzOww4DBJZ5AqfsHMPgMcISk+koYQQgghFKAnGli7SpomaZWk+yv1nEi6wL9xXyHp\nBl/XR9JUX7dc0lW+foT3AiyT9EyDHxT3Ad4oW/dmJp5xXme7pN/4uqmSbpW0QNJLksb4+j0kzfNY\nlks629cPl7Ta91sjabqk033/NZI+5+X6S7pb0kJJSySd5WF8ALxTw7m86ceZ5PEulbTOj9lf0oO+\nfoX3mlwBDANaJc33fb/lMS0ETswceyPpg39XNnk5JB0gaZZfm3ZJny+lNJPbayQt8jLNvm6KpPGZ\nMs2SvpdXvsxHwFvVkmRmi8zs9WrlMrbk33urSrldImkPLzNA0ky/zvdm4l8raZAvH6PUW7s/cC9w\nrB/nMuDrwHXZfX2fPpJulPS0n/eltZ6rpCHAADNb7KvuIX2hkPU66T0QetJ21Ue+g4ncFiL+9k0x\nIq/FidwWI/Jav554iuCngG+Z2UJJdwPjgZ+XNkoaCtwAHA1sAOZ6I2UdcKCZfdbL7eW7TCd96z9b\nUj8aayT2BTZnV5jZ8V7PSOCHwAlm9rak7AfRIWZ2oqRPk3pEZgF/Bc4xs42S9gUW+jaAEcC5ZrZK\n0jPA+b7/2V7HGOAfgflmdomkvYFFkuaZ2VPAUx7TMcB3zOyy8hMpxW1mzUCzH+M/gH8GvgS8ZmZf\n9eMMMLP3JF0NNPn5DQFaSPl/F2gDlvoxb66WSDO7P/PyNqDNzMZIErBnqZjXfzpwqJkd59tnSzoJ\nmAHcAtzu5b8OjM4rb2ZP4I02M1sHnOfHHwrcVTrfbZHNP/B9YLyZPSWpP+maAxwFjATWAwskfcHM\nnqRzL5OZ2ZuS/gH4vpmVGuEnAA+Y2SxJwzPlLwE2mNnxfo8vkPSomb1Sw7keSHrvlKzzdVmbSe+B\nfNmBiAcTQ7BCCCGEsNNra2urqcHZEw2sP5nZQl+eBlxBpoFFGrbVamZvAUiaDpwMTAYOkXQr8Hvg\nUUl7AsPMbDaAmX1QbzD+Qf1Ij6WS04CZZva217Ehs+13vm61pANKhwSmSDqZ9MF1WGbbWjNb5cvP\nAfN8eSXpYyvAaOAsST/w1/2Ag4A1pUrNbAnQqXGVYxpws5m1S9oI/EzSFOAhb5iUYi71Kh1Px/zP\nAA6tsa5ypwF/7zEbUD73ajRwuqSlXv8epAbUVEn7e2PvAOAtM3tN0oRK5YEnqMDM/gxsc+OqggXA\nL/zenOWxASzyOpG0jHRNnyTTY9eg0aRhfGP99V6k836lVGAbz3Ud6T3wTG6JUxs8csgXjdTiRG4L\nEfMuihF5LU7kthiR162ampo65GPSpEkVy/VEA6vTt/kVynT6QGpmGyQdCZwBfBcYS5o71OWHVx9q\ndqnXc6aZrc9s6wP8EXgfeKiOcyh5v0LMFwH7AUeb2Walh07sVqH85szrzWzNvUi9XC82EE8HklpI\nDdp7AMzsRaUHiZwJTPaescmVdt3Wul21+UECppjZXRW2zSRd4yGkHq1q5XtszpyZ/VTSg8BXSL1J\no31T9vp+xNZr+iFbe1YrPkykCgFXmNncBvZ9DfjbzOtP+LqsXwJzJB1nZt9poI4QQgghhJCjJ+Zg\nDZd0vC9fCDxetn0RcLKkQZL6AhcAj/lwu75m9lvgR8AoM9sIvCrpa7DlqWy7Zw9mZreb2dH+IIH1\nZds2m9nBpG/uv5ET7x+AsZk5NANzypUaJXsDb3jj6lRgeIUyXZkDXLllB+moGvbpHEyau/VF4KrM\nuqHAJjO7D7iJ9JAFSEMBS0Munyblf6CkXUmNnErHvzw7TyrHfNIQ0NI8ogGl3f33HODbpTlMkob5\n3CSA+4HzgXNJja288vuVHbNeHfaTdI6k67vcQfqkmT1nZjcCi4HDq9SxFjjGl89tIMY5wHhJu3j9\nh5bf53n8nn9HUmlY5Tjg38uKXUN6CEY0rnpSzBMqTuS2EDHvohiR1+JEbosRea1fTzSwngcul7SK\nNLH+Dl9vsOUD4UTS3J92YLGZPUCaN9ImqZ30cICJvt844EpJy0lDtwY3ENMLwKBKG3xI309Ijbx2\noDQPKa8nbjrpwQXLgW8CqyuUqbR/yXWkB4GsUHpM97XlBfxBCXd2cT4AV5MeXrHYH6LQAhxBmtPV\nDvyYNOwS4C7gEUnzPf+TSHPHHgdWdTpycjjwlyoxTABOlbSC1Igd6etL13oucB/wlJeZic/T8rwP\nANaVHkaRU35A9phZkoZ6T1Mnkn4q6VVgd6XHtf/YN42g+sNEJkha6df4A+DhCmWy8VwL3CZpEak3\nK0/ePfEr0nVY6vfEHZT1Nnd1rsDlwN2k+/xFM3ukbPtA4KUu4gohhBBCCA3SzvhX6n2+075mNrFq\n4QCApNnAGDPrqsHwsSPpHuBqM6vWeNwheK/WCuA8M1uTU8Zo6dGwQtg+tcDO+H9kCCGE2kjCzDqN\nqtpZG1gjgF8DG8v+FlYIOyxJh5GGYq4ALracN7+kne8fhRAqGHzgYNavW1+9YAghhJ1SXgOrJ4YI\nbnfM7D/N7O+icRV2Jv6Hho8ys3F5jatM2fjp5p/W1tZej2FH/Skqtzt74yrmXRQj8lqcyG0xIq/1\n2ykbWCGEEEIIIYRQhJ1yiGAIIZ8ki38XQgghhBC6FkMEQwghhBBCCKFg0cAKIYQeEGPYixO5LUbk\ntRiR1+JEbosRea1fNLBCCCGEEEIIoZvEHKwQQgcxByuEEEIIobq8OVi79EYwIYTtW/p7xD1v8ODh\nrF//cq/UHUIIIYTQHWKIYAihAuuVn9dff6VHzq43xBj24kRuixF5LUbktTiR22JEXusXDawQQggh\nhBBC6Cbd0sCSNFzSypxtrZJGdUc99ZJ0kKSlkh7OrFvbG7HkkXSKpKk1lFvrv3NzXVZ+gKRXJd2W\nPYakQXXEVjVXfn0P6mL7xZL+qdY6a4zr4tJ5SWqWNK5K+WMltfvPcknfqKGOqZJOrrJ9TJ1xnyTp\nWb8n/0bSTZJWSvqpn8f3quxfy7l+UdIzfp6LJZ2a2fZ9Sc/Xcv6h+zU1NfV2CDusyG0xIq/FiLwW\nJ3JbjMhr/bpzDtb2OCv+HOBRM5uYWbc9xllLTJaznOc64LEG6tmW8kUfp1ErgWPMbLOkIcCzkv63\nmX3Uw3FcBFxvZvcBSLoUGGhmJqm5m+p4E/iqma2X9D+AOcAnAMzsZklPADcBM7qpvhBCCCGEkNGd\nQwR3lTRN0ipJ90varbyApAskrfCfG3xdH+8NWOHful/l60dImitpmX8jf0gDMe0DvFG27s1MPOO8\nznZJv/F1UyXdKmmBpJdKvRSS9pA0L9M7cLavHy5pte+3RtJ0Saf7/mskfc7L9Zd0t6SFkpZIOsvD\n+AB4p4ZzebN8haS7Mj0zb0j6X77+GOAA4NHyXYArvf7lkg7LnNu/+jVYJul/5tVZwV+Aj/w4X/Jj\nL5M0t0K8+0n635Ke9p8TlKyVtFem3AuS9q9UvkL9G4FNXQVoZn81s83+cnfgnRoaVxtI1wZJN3jP\n0zJJN2bKnFLhPjlF0gOZc/knv88u+X/s3X28XeOd///XWyZpSkhECNLmRvodlY6kSaiaKkfRjpmh\nqqXUXfvwQItSN50avRGlqlW+37T90dGaNCWtikGjmhE0J+0DKXIrEtFqOqhJUKIyo5Tz/v2xrs06\n+6x9d5yVHSef5+NxHvZe67PW+lyfvU/s61zXtTZwJHCRpGsl/QwYAiySdERVnXaRNDeNQC2ovE7A\nC020dZnttenxQ8BgSQNzIWuBoQ3aHkoQc9jLE7UtR9S1HFHX8kRtyxF1bV1fjmDtCnzK9kJJ1wCn\nAldUdkraCbgUmEz24fWO1El5Ahhle2KKq3zQnkX21/45kgbRu87gAKArv8H2Xuk6E4Dzgb1tPydp\nWC5sR9vvk7QbMAe4CfgLcJjtDZK2AxamfQDjgY/aXinpAeCodPyh6RqHA18E7rJ9oqShwH2S7rR9\nL3BvymkqcIrtk6sbUsm7attJ6bjRwFxghiQB3yIbLTmooCZP2Z4q6TPAucDJwJeB9bnXYGitaxbk\n8LF0zAjgamAf249V1bNiOnCF7XskvR243fYESbcAHwFmSnoP8AfbT0uaVR0PTKi6/uWVx5JOyTb5\n6uoLp/P+OzAO+EQT7TorHTec7HV/Z3q+TS6s6H0CBSN2tq+RtA9wq+2b0rn+bHtKepwfwbqa7H3w\naMr7KuAA2/nfp5ptzcV8DFhs+6+5zV009Xs/Lfe4I/2EEEIIIWy+Ojs7m+pw9mUH6zHbC9Pj64DP\nkutgAXsC820/C5A+PO8LXAyMkzQd+AUwT9IQYGfbcwBsv9xqMqmjMSnlUuQDwGzbz6VrrM/tuyVt\nWyVph8opga8rW5fTBeyc27fG9sr0+CHgzvT4QWBsevxB4BBJn0/PBwGjgdWVi9peRNYDx74QAAAg\nAElEQVThaaWdg4HZwOm2n5B0GnCb7SezElB9v+2b038XkXVqAA4EXluXY7uZEbVq7wUW2H4snWN9\nQcyBwG7ptQEYImlL4AbgK8BM4Chen75WK76Q7X+rs+8+4O8k7QrcLmm+7T830a7ngRcl/QC4Dfh5\nbl/R++QNkbQV8PfA7Fy7B1bH1WtrOs+7gK/Ts5P9DLC9pGE1XqNkWvNJh6bEHPbyRG3LEXUtR9S1\nPFHbckRdX9fR0dGtHhdeeGFhXJlrsIrW3fT4ch3b6yVNAj4EfBo4AvhcUWy3E0mnAiel6/xjZVpU\n2rcF8HvgJbIPxa16qSDnY4ARwOS0lmcNMLggviv3PD9aILJRrt/2Ip96rgJutD0/Pd8b2CfVZ2uy\nqZsv2D6/KtdX6fvvQWv05UkC9qoaUQG4V9mU0BFk6+a+Wi9eb+A7mmyvlvQo8H/IOpmN4l9No0gH\nkL03T0+Pofh98grdR1t7TJVtYAvgucrIVm9IehvZaNpxtv+Q32f7RUnXA7+X9HHbPaZyhhBCCCGE\n3uvLNVhjJFWmlH0C+HXV/vuAfSUNlzQAOBpYkKbbDbB9M/AlYIrtDcDjkj4MIGmQpLfmT2b7StuT\nbU/Jd67Svi7bY4EHyI3MVPklcESaAoakbWvEVT44DyWbXtel7M5sYwpi6rkdOOO1A6R3N3FMXWm0\naojtyyrbbB9re6ztXcimAP4o17mq5Q7gtNx5e0zvU7b+bKc651gIvF/SmBRfVM95wJm5c07K7buZ\nbMRzZW5kpV580ySNTe85Un7vAH6bns9UWidX49itgGG2/xM4G5hYKzT997+ACZIGpjoeUCM+f8xr\nbL8ArEnT+yo51LpmUb5DyUbZvpAbUc7vH0b2OzEqOlcbV8xhL0/UthxR13JEXcsTtS1H1LV1fdnB\nehg4TdJKsptLfC9tN0DqBJ0HdAJLgPtt3wqMAjolLQGuTTEAx5PdkGEZcDcwshc5PQIU3pY8Ten7\nGlknbwlQWctTayRuFrBnyudYYFVBTNHxFReRjSYtV3ab9a9WB0iaKqnmmpoC5wC7K7vJxWJJjaYX\n1srtYmC4sluGL6FqwU2aqjYeeLbmie1nyKY33pzOcX1B2JnAHspusLECOCW37wayUcLrm4zvQdIp\nNWqwD7BM0uJ0nZNz0wMnAk/WOe3WwM/T6/4r4Ky0vfB9YvuJdI0VqS2Lq2PqPK84FjhR2U01VgCH\nVgfUaevpZK/VV3LvixG5/UOBdbbr3iwjhBBCCCH0jux230G7PGm903ZVt2kPLUrreT5l+9x259KX\nJG0N/MD2ZvO9UGm643TbRXdkrMS4fXfWF/3536QQQggh9B+SsN1jRlJ/72CNB34IbLB9cJvTCaGt\nJJ1DNkp4me2f1Ilr2z8KI0eOYe3aP7Tr8iGEEEIITavVwerLKYKbHNuP2n5/dK5CyG5pn9Ys1uxc\n5WLb8tOfO1cxh708UdtyRF3LEXUtT9S2HFHX1vXrDlYIIYQQQgghbEz9eopgCKF1khz/LoQQQggh\n1LdZThEMIYQQQgghhI0pOlghhLARxBz28kRtyxF1LUfUtTxR23JEXVsXHawQQgghhBBC6COxBiuE\n0E2swQohhBBCaCzWYIUQmiapz392fNuO7W5WCCGEEELpYgQrhNCNJDOthBNPy75fa3PV2dlJR0dH\nu9Pol6K25Yi6liPqWp6obTmirrXFCFYIIYQQQgghlKxPOliSxkh6sMa++ZKm9MV1WiVptKTFkubm\ntq1pRy61SNpP0owm4tak/9asdVX81pIel/Tt/DkkDW8ht4a1Sq/v6Dr7T5D0nWav2WReJ1TaJekC\nScc3iN9T0pL0s0zSx5u4xgxJ+zbYf3iLee8jaUV6T75F0mWSHpT0jdSOsxsc37CtKe5fJf1W0ipJ\nH8xtP0fSw820P/S9+OtfeaK25Yi6liPqWp6obTmirq3ryxGsTXHuz2HAPNsH57Ztink2k5NrPK7l\nImBBL67zRuLLPk9vPQhMtT0Z+BDw/0ka0IY8jgEusT3F9kvAScBE21/oqwtI2g04EtgNOBi4UpIA\nbF8OnACc1lfXCyGEEEII3fVlB2ugpOskrZR0g6TB1QGSjpa0PP1cmrZtkUYDlqfRhTPT9vGS7pC0\nVNIDksb1IqdhwFNV257O5XN8uuYSSTPTthmSpku6W9LvKqMUkraSdGfKZZmkQ9P2MWmkYIak1ZJm\nSTooHb9a0h4pbktJ10haKGmRpENSGi8DzzfRlqerN0j6fm5k5ilJX07bpwI7APOqDwHOSNdfJulv\nc2379/QaLJX0kVrXLPAn4NV0nn9I514q6Y6CfEdIulHSb9LP3sqskbRNLu4RSdsXxRdcfwPwYr0E\nbf/Fdld6+lbgeduvNmjXerLXBkmXppGnpZK+mYvZr+B9sp+kW3Nt+U56n51I1vG5SNK1kn4GDAEW\nSTqiqk67SJor6X5JCyqvE/BCo7YCHwaut/2K7T8AvwXek9u/Fhja4ByhBPE9IuWJ2pYj6lqOqGt5\norbliLq27m/68Fy7Ap+yvVDSNcCpwBWVnZJ2Ai4FJpN9eL0jdVKeAEbZnpjiKh+0Z5H9tX+OpEH0\nrjM4AOjKb7C9V7rOBOB8YG/bz0kalgvb0fb70mjAHOAm4C/AYbY3SNoOWJj2AYwHPmp7paQHgKPS\n8YemaxwOfBG4y/aJkoYC90m60/a9wL0pp6nAKbZPrm5IJe+qbSel40YDc4EZabTiW2SjJQcV1OQp\n21MlfQY4FzgZ+DKwPvcaDK11zYIcPpaOGQFcDexj+7GqelZMB66wfY+ktwO3254g6RbgI8BMSe8B\n/mD7aUmzquOBCVXXv7zyWNIp2SZfXX3hdN5/B8YBn2iiXWel44aTve7vTM+3yYUVvU+gYMTO9jWS\n9gFutX1TOtefbU9Jjy/IhV9N9j54NOV9FXCA7fzvU622jiK9n5I/pm0VXTTzez8/93gsWdVCCCGE\nEDZjnZ2dTXU4+7KD9ZjthenxdcBnyXWwgD2B+bafBUgfnvcFLgbGSZoO/AKYJ2kIsLPtOQC2X241\nmdTRmJRyKfIBYLbt59I11uf23ZK2rZK0Q+WUwNeVrcvpAnbO7Vtje2V6/BBwZ3r8INnHU4APAodI\n+nx6PggYDayuXNT2IrIOTyvtHAzMBk63/YSk04DbbD+ZlYDqO5vcnP67iKxTA3Ag8Nq6HNvNjKhV\ney+wwPZj6RzrC2IOBHZLrw3AEElbAjcAXwFmAkcBP20QX8j2v9XZdx/wd5J2BW6XNN/2n5to1/PA\ni5J+ANwG/Dy3r+h98oZI2gr4e2B2rt0Dq+PqtbWBZ4DtJQ2r8Rpl9u/l2UNNMYe9PFHbckRdyxF1\nLU/UthxR19d1dHR0q8eFF15YGNeXHazqv9oXrbvpcRtD2+slTSJbG/Np4Ajgc0Wx3U4knUq2hsXA\nP9pem9u3BfB74CWyD8Wteqkg52OAEcBk213KbgAxuCC+K/c8P1ogslGu3/Yin3quAm60XRlz2BvY\nJ9Vna7Kpmy/YPr8q11fp29cfGrxmaf9etv9atf1eZVNCR5Ctm/tqvfjX+x2ts71a0qPA/yHrZDaK\nfzWNIh1A9t48PT2G4vfJK3Qfbe0xVbaBLYDnKiNbvfBH4O25529L2wCw/aKk64HfS/q47R5TOUMI\nIYQQQu/15RqsMZIqU8o+Afy6av99wL6Shiu7wcDRwII03W6A7ZuBLwFTbG8AHpf0YQBJgyS9NX8y\n21fanpxuGLC2al+X7bHAA+RGZqr8EjgiTQFD0rY14iofnIeSTa/rkrQ/MKYgpp7bgTNeO0B6dxPH\n1JVGq4bYvqyyzfaxtsfa3oVsCuCPcp2rWu4gd+ODoul9ytaf7VTnHAuB90sak+KL6jkPODN3zkm5\nfTeTjXiuzI2s1ItvmqSx6T1Hyu8dZGuTkDRTaZ1cjWO3AobZ/k/gbGBirdD03/8CJkgamOp4QI34\n/DGvsf0CsEbSx3I51LpmkTnAUel3ZhxZW+/LnWsY2e/EqOhcbVwxh708UdtyRF3LEXUtT9S2HFHX\n1vVlB+th4DRJK8luLvG9tN0AqRN0HtAJLAHut30r2fqQTklLgGtTDMDxZDdkWAbcDYzsRU6PAIW3\nJU9T+r5G1slbAlTW8tQaiZsF7JnyORZYVRBTdHzFRWSjScuV3Wb9q9UBkqZK6rF+qI5zgN2V3eRi\nsaRG0wtr5XYxMFzZLcOXAB1VeYlsndmzNU9sP0M2vfHmdI7rC8LOBPZQdoONFcApuX03kI0SXt9k\nfA+STqlRg32AZZIWp+ucnJseOBF4ss5ptwZ+nl73XwFnpe2F7xPbT6RrrEhtWVwdU+d5xbHAicpu\nqrECOLQ6oFZb0/v6BmAl2ZTbU939232HAutsN7pZRgghhBBC6AV1/+zVv6T1TtvZPq9hcKhJ0rvI\nbmBybrtz6UuStgZ+YHuz+V6oNN1xuu2iOzJWYsy0Ei4+DfrzvzchhBBC2LxIwnaPGUn9vYM1Hvgh\nsKHqu7BC2OxIOodslPAy2z+pE1fKPwojR41k7RNrGweGEEIIIbwJ1Opg9eUUwU2O7Udtvz86VyFk\nt7RPaxZrdq5ysX3+s7l3rmIOe3mituWIupYj6lqeqG05oq6t69cdrBBCCCGEEELYmPr1FMEQQusk\nOf5dCCGEEEKob7OcIhhCCCGEEEIIG1N0sEIIYSOIOezlidqWI+pajqhreaK25Yi6ti46WCGEEEII\nIYTQR2INVgihm1iDFUIIIYTQWK01WH/TjmRCCJs2qce/FWEjGTlyDGvX/qHdaYQQQgihl2KKYAih\ngOOnz3/mNxW3bt1/NfMChZxYH1COqGs5oq7lidqWI+rauuhghRBCCCGEEEIf6ZMOlqQxkh6ssW++\npCl9cZ1WSRotabGkublta9qRSy2S9pM0o4m4Nem/NWtdFb+1pMclfTt/DknDW8itYa3S6zu6zv4T\nJH2n2Ws2mdcJlXZJukDS8Q3ih0v6paQX8vVocMwMSfs22H94i3nvI2lFek++RdJlkh6U9I3UjrMb\nHN9MWw+U9ICkZZLul7R/bt85kh6W9PFW8g59paPdCfRbHR0d7U6hX4q6liPqWp6obTmirq3ryxGs\nTXFV/GHAPNsH57Ztink2k5NrPK7lImBBL67zRuLLPk9v/QX4EnBOm/M4BrjE9hTbLwEnARNtf6EP\nr/E08M+2JwGfBK6t7LB9OXACcFofXi+EEEIIIeT0ZQdroKTrJK2UdIOkwdUBko6WtDz9XJq2bZFG\nA5anv7qfmbaPl3SHpKXpL/LjepHTMOCpqm1P5/I5Pl1ziaSZadsMSdMl3S3pd5VRCklbSbozNzpw\naNo+RtKqdNxqSbMkHZSOXy1pjxS3paRrJC2UtEjSISmNl4Hnm2jL09UbJH0/5b5E0lOSvpy2TwV2\nAOZVHwKcka6/TNLf5tr27+k1WCrpI7WuWeBPwKvpPP+Qzr1U0h0F+Y6QdKOk36SfvZVZI2mbXNwj\nkrYvii+4/gbgxXoJ2v5f2/cALzXRnor1ZK8Nki5NI09LJX0zF7NfwftkP0m35trynfQ+OxE4ErhI\n0rWSfgYMARZJOqKqTrtImptGoBZUXifghSbausz22vT4IWCwpIG5kLXA0BbqEPpMZ7sT6LdifUA5\noq7liLqWJ2pbjqhr6/ryLoK7Ap+yvVDSNcCpwBWVnZJ2Ai4FJpN9eL0jdVKeAEbZnpjiKh+0Z5H9\ntX+OpEH0rjM4AOjKb7C9V7rOBOB8YG/bz0kalgvb0fb7JO0GzAFuIhsFOcz2BknbAQvTPoDxwEdt\nr5T0AHBUOv7QdI3DgS8Cd9k+UdJQ4D5Jd9q+F7g35TQVOMX2ydUNqeRdte2kdNxoYC4wQ5KAb5GN\nlhxUUJOnbE+V9BngXOBk4MvA+txrMLTWNQty+Fg6ZgRwNbCP7ceq6lkxHbjC9j2S3g7cbnuCpFuA\njwAzJb0H+IPtpyXNqo4HJlRd//LKY0mnZJt8daO8m2jXWemcw8le93em59vkworeJ1AwYmf7Gkn7\nALfavimd68+2p6THF+TCryZ7Hzya6nEVcIDt/O9Tw7ZK+hiw2PZfc5u7aOr3flrucQcxvS2EEEII\nm7vOzs6mOpx92cF6zPbC9Pg64LPkOljAnsB8288CpA/P+wIXA+MkTQd+AcyTNATY2fYcANsvt5pM\n6mhMSrkU+QAw2/Zz6Rrrc/tuSdtWSdqhckrg68rW5XQBO+f2rbG9Mj1+CLgzPX4QGJsefxA4RNLn\n0/NBwGhgdeWitheRdXhaaedgYDZwuu0nJJ0G3Gb7yawEVN9v++b030VknRqAA4HX1uXYbmZErdp7\ngQW2H0vnWF8QcyCwW3ptAIZI2hK4AfgKMBM4Cvhpg/hCtv+tF3k38jzwoqQfALcBP8/tK3qfvCGS\ntgL+Hpida/fA6rhGbZX0LuDr9OxkPwNsL2lYjdcomdZ80qFJHe1OoN+K9QHliLqWI+panqhtOaKu\nr+vo6OhWjwsvvLAwri87WNV/tS9ad9Pjy3Vsr5c0CfgQ8GngCOBzRbHdTiSdSraGxcA/VqZFpX1b\nAL8nmxJ2WwttqMhPJavkcQwwAphsu0vZDSAGF8R35Z7nRwtENsr1217kU89VwI2256fnewP7pPps\nTTZ18wXb51fl+ip9/z1ojb48ScBeVSMqAPcqmxI6gmzd3FfrxWsjfkeT7VfTKNIBZO/N09NjKH6f\nvEL30dYeU2Ub2AJ4rjKy1RuS3kY2mnac7T/k99l+UdL1wO8lfdx2j6mcIYQQQgih9/pyDdYYSZUp\nZZ8Afl21/z5gX2V3dBsAHA0sSNPtBti+mexGBFNsbwAel/RhAEmDJL01fzLbV9qenG4YsLZqX5ft\nscAD5EZmqvwSOCJNAUPStjXiKh+ch5JNr+tSdme2MQUx9dwOnPHaAdK7mzimrjRaNcT2ZZVtto+1\nPdb2LmRTAH+U61zVcge5Gx8UTe9Ttv5spzrnWAi8X9KYFF9Uz3nAmblzTsrtu5lsxHNlbmSlXnxv\ndXutJM1UWidXGJyNKA2z/Z/A2cDEBuf9L2CCpIGpjgfUiO+RC4DtF4A1aXpfJYda1yzKdyjZKNsX\nciPK+f3DyH4nRkXnamPrbHcC/VasDyhH1LUcUdfyRG3LEXVtXV92sB4GTpO0kuzmEt9L2w2QOkHn\nkX3KWALcb/tWYBTQKWkJ2R3PzkvHHU92Q4ZlwN3AyF7k9AhQeFvyNKXva2SdvCVAZS1PrZG4WcCe\nKZ9jgVUFMUXHV1xENpq0XNlt1r9aHSBpqqRW1g+dA+yu7CYXiyU1ml5YK7eLgeHKbhm+hKq5TGmq\n2njg2Zontp8hm954czrH9QVhZwJ7KLvBxgrglNy+G8hGCa9vMr4HSafUqkEacbwcOEHSY5LemXZN\nBJ6sc9qtgZ+n1/1XwFlpe+H7xPYTqS0rUlsWV8fUeV5xLHCisptqrAAOLWhPrbaeTvZafSX3vhiR\n2z8UWGe77s0yQgghhBBC78hu9x20y5PWO21n+7yGwaGmtJ7nU7bPbXcufUnS1sAPbG823wuVpjtO\nt110R8ZKjNt/Z/3NmejP/y6HEEII/YUkbPeYkdTfO1jjgR8CG6q+CyuEzY6kc8hGCS+z/ZM6cdHB\naqvoYIUQQghvBrU6WH05RXCTY/tR2++PzlUI2S3t05rFmp2r1yl+2vQzcmR+eWdoRqwPKEfUtRxR\n1/JEbcsRdW1dX99FLoTQD8QISt/r7OyMW92GEEIIm4F+PUUwhNA6SY5/F0IIIYQQ6tsspwiGEEII\nIYQQwsYUHawQQtgIYg57eaK25Yi6liPqWp6obTmirq2LDlYIIYQQQggh9JFYgxVC6CbWYIUQQggh\nNBZrsEIIIYQQQgihZNHBCiH0ICl+evGz49t2rFnTmMNenqhtOaKu5Yi6lidqW46oa+vie7BCCD1N\na3cCb07rpq1rdwohhBBCaLNYgxVC6EaSo4PVS9PiS5pDCCGEzUWpa7AkjZH0YI198yVN6YvrtErS\naEmLJc3NbVvTjlxqkbSfpBlNxK1J/61Z66r4rSU9Lunb+XNIGt5Cbg1rlV7f0XX2nyDpO81es8m8\nTqi0S9IFko5vED9c0i8lvZCvR4NjZkjat8H+w1vMex9JK9J78i2SLpP0oKRvpHac3eD4hm1Ncf8q\n6beSVkn6YG77OZIelvTxVvIOIYQQQgjN68s1WJvin20PA+bZPji3bVPMs5mcXONxLRcBC3pxnTcS\nX/Z5eusvwJeAc9qcxzHAJban2H4JOAmYaPsLfXUBSbsBRwK7AQcDV0oSgO3LgROA0/rqeqF5MYe9\nPFHbckRdyxF1LU/UthxR19b1ZQdroKTrJK2UdIOkwdUBko6WtDz9XJq2bZFGA5ZLWibpzLR9vKQ7\nJC2V9ICkcb3IaRjwVNW2p3P5HJ+uuUTSzLRthqTpku6W9LvKKIWkrSTdmXJZJunQtH1MGimYIWm1\npFmSDkrHr5a0R4rbUtI1khZKWiTpkJTGy8DzTbTl6eoNkr6fcl8i6SlJX07bpwI7APOqDwHOSNdf\nJulvc2379/QaLJX0kVrXLPAn4NV0nn9I514q6Y6CfEdIulHSb9LP3sqskbRNLu4RSdsXxRdcfwPw\nYr0Ebf+v7XuAl5poT8V6stcGSZemkaelkr6Zi9mv4H2yn6Rbc235TnqfnUjW8blI0rWSfgYMARZJ\nOqKqTrtImivpfkkLKq8T8EKjtgIfBq63/YrtPwC/Bd6T278WGNpCHUIIIYQQQgv68iYXuwKfsr1Q\n0jXAqcAVlZ2SdgIuBSaTfXi9I3VSngBG2Z6Y4ioftGeR/bV/jqRB9K4zOADoym+wvVe6zgTgfGBv\n289JGpYL29H2+9JowBzgJrJRkMNsb5C0HbAw7QMYD3zU9kpJDwBHpeMPTdc4HPgicJftEyUNBe6T\ndKfte4F7U05TgVNsn1zdkEreVdtOSseNBuYCM9JoxbfIRksOKqjJU7anSvoMcC5wMvBlYH3uNRha\n65oFOXwsHTMCuBrYx/ZjVfWsmA5cYfseSW8Hbrc9QdItwEeAmZLeA/zB9tOSZlXHAxOqrn955bGk\nU7JNvrpR3k2066x0zuFkr/s70/NtcmFF7xMoGLGzfY2kfYBbbd+UzvVn21PS4wty4VeTvQ8eTfW4\nCjjAdv73qVZbR5HeT8kf07aKLpr5vZ+fezwW6M2fN0I3HR0d7U6h34raliPqWo6oa3mituWIur6u\ns7OzqRG9vuxgPWZ7YXp8HfBZch0sYE9gvu1nAdKH532Bi4FxkqYDvwDmSRoC7Gx7DoDtl1tNJnU0\nJqVcinwAmG37uXSN9bl9t6RtqyTtUDkl8HVl63K6gJ1z+9bYXpkePwTcmR4/SPbxFOCDwCGSPp+e\nDwJGA6srF7W9iKzD00o7BwOzgdNtPyHpNOA2209mJaB64d3N6b+LyDo1AAcCr63Lsd3MiFq19wIL\nbD+WzrG+IOZAYLf02gAMkbQlcAPwFWAmcBTw0wbxhWz/Wy/ybuR54EVJPwBuA36e21f0PnlDJG0F\n/D0wO9fugdVxb6CtzwDbSxpW4zXK7N/Ls4cQQggh9FMdHR3dOpwXXnhhYVyZa7CK1t30uMtG+pA3\nCegEPg18v1ZstxNJp6apcYsl7Vi1bwtgDdk6lNuayr67/FSySh7HACOAybYnk009HFwQ35V7nh8t\nENko1+T0M872at64q4AbbVfGHPYGTpf0e7KRrOMkXVLQtlfp+9v0133N0v69cjUYnabv3QuMT6Ng\nhwH/US++j3Ouy/arZFPsbgT+GfjP3O6i98krdP+96jFVtoEtgOfSOq1Ku/+uheP/CLw99/xtaRsA\ntl8Ergd+L6lohDOUJOawlydqW46oazmiruWJ2pYj6tq6vuxgjZFUmVL2CeDXVfvvA/ZVdke3AcDR\nwII03W6A7ZvJbkQwxfYG4HFJHwaQNEjSW/Mns31l+vA5xfbaqn1dtscCD5AbmanyS+CINAUMSdvW\niKt8cB5KNr2uS9L+wJiCmHpuB8547QDp3U0cU1carRpi+7LKNtvH2h5rexeyKYA/sn1+g1PdQe7G\nB0XT+5StP9upzjkWAu+XNCbFF9VzHnBm7pyTcvtuJhvxXJkbWakX31vdXitJM5XWyRUGZyNKw2z/\nJ3A2MLHBef8LmCBpYKrjAc3mAmD7BWCNpI/lcqh1zSJzgKPS78w44B1kv3uVcw0j+50YZbvHOrkQ\nQgghhPDG9GUH62HgNEkryW4u8b203QCpE3Qe2UjVEuB+27eSrQ/plLQEuDbFABxPdkOGZcDdwMhe\n5PQIUHhb8jSl72tknbwlQGUtT62RuFnAnimfY4FVBTFFx1dcRHYjkOXKbrP+1eoASVMltbJ+6Bxg\n99xIXqPphbVyuxgYruyW4UuAjqq8RLbO7NmaJ7afIZveeHM6x/UFYWcCeyi7wcYK4JTcvhvIRgmv\nbzK+B0mn1KqBslvOXw6cIOkxSe9MuyYCT9Y57dbAz9Pr/ivgrLS98H1i+4nUlhWpLYurY+o8rzgW\nOFHZTTVWAIcWtKewrel9fQOwkmzK7anu/sVMQ4F1aSQrbEQxh708UdtyRF3LEXUtT9S2HFHX1vXr\nLxpO6522s31ew+BQk6R3kd3A5Nx259KXJG0N/MD2ZvO9UOmmGdNtF92RsRITXzTcW9Pii4ZDCCGE\nzYVqfNFwf+9gjQd+CGyo+i6sEDY7ks4hGyW8zPZP6sT1338USjZy1EjWPrG2cF9nZ2f8FbAkUdty\nRF3LEXUtT9S2HFHX2mp1sPr6JgebFNuPAu9vdx4hbArSLe0vbxhIjMKEEEIIIfRWvx7BCiG0TpLj\n34UQQgghhPpqjWD15U0uQgghhBBCCGGzFh2sEELYCOJ7RMoTtS1H1LUcUdfyRG3LEXVtXXSwQggh\nhBBCCKGPxBqsEEI3sQYrhBBCCKGxWIMVQgghhBBCCCWLDlYIoQdJ8dPPfnbcccPzrOIAACAASURB\nVGy731alifUB5Yi6liPqWp6obTmirq3r19+DFULorZgi2Pc6gY62XX3duh4zGEIIIYRQgliDFULo\nRpKjg9UfKb5AOoQQQuhDm9QaLEljJD1YY998SVM2dk7p2qMlLZY0N7dtTTtyqUXSfpJmNBG3Jv23\nZq2r4reW9Likb+e2zZc0usFxMyTt2yDfWxtdvxX5c0o6QdIFTRzzDUkPSlou6cgm4i+QdHyD/We3\nmPeukpZIWiRpnKQzJK2UdG1qx3caHN+wrZImSbontXVpvq2Sjpb0sKSzWsk7hBBCCCE0r51rsDbF\nP6UeBsyzfXBu26aYZzM5ucbjWi4CFvQunZZyKeOcdc8v6R+BdwMTgfcC50oaUkJOjRwGzLY91fYa\n4DPAgbaPS/tbfV2L/A9wnO3dgYOB/ydpGwDbPwH2A6KD1Rad7U6g34r1AeWIupYj6lqeqG05oq6t\na2cHa6Ck69Jf8G+QNLg6IP3FfXn6uTRt2yKNmiyXtEzSmWn7eEl3pL/aPyBpXC9yGgY8VbXt6Vw+\nx6drLpE0M22bIWm6pLsl/U7S4Wn7VpLuTLksk3Ro2j5G0qp03GpJsyQdlI5fLWmPFLelpGskLUwj\nHoekNF4Gnm+iLU9Xb5D0/ZT7EklPSfpy2j4V2AGYV3XIn4BXG1xnfcoJSXumdixNeW9Vdf3CNkm6\nV9Juubj5kqbUqUHei8CGBjlOAH7lzP8Cy4F/aHDMC+ncpJGmh1K7fpyLeVfK9XeSPptiu40YSjon\njXYdDHwO+IykuyRdBewCzK28h3PHjJB0o6TfpJ+9m22r7d/ZfjQ9/m+y9/P2uf3rgKEN2h5CCCGE\nEHqpnTe52BX4lO2Fkq4BTgWuqOyUtBNwKTCZ7EP8HamT8gQwyvbEFLdNOmQWcIntOZIG0bvO4wCg\nK7/B9l7pOhOA84G9bT8naVgubEfb70udhDnATcBfgMNsb5C0HbAw7QMYD3zU9kpJDwBHpeMPTdc4\nHPgicJftEyUNBe6TdKfte4F7U05TgVNsn1zdkEreVdtOSseNBuYCMyQJ+BZwDHBQVfzHGhXM9lnp\nnAOB64EjbC9OI0QvVoUXtikd93FgmqQdUz0XS/pajfj89W+oPE4dsKm2p1VddxnwFUlXAFsB+wMP\nNWjXFbmnXwDG2v5r7v0G2Xu4g6zDslrSlZXDe57OcyV9D3ihcm5JHwI60vvphFz8dOAK2/dIejtw\nOzChybaSi3kPMLDS4cpp4ncjf9oO2nlzhv6jo90J9FsdHR3tTqFfirqWI+panqhtOaKur+vs7Gxq\nRK+dHazHbC9Mj68DPkuugwXsCcy3/SyApFnAvsDFwDhJ04FfAPPSh/mdbc8BsP1yq8mkjsaklEuR\nD5BN73ouXWN9bt8tadsqSTtUTgl8Xdn6pC5g59y+NbZXpscPAZVOw4PA2PT4g8Ahkj6fng8CRgOr\nKxe1vQjo0blq0M7BwGzgdNtPSDoNuM32k1kJ6O2txnYFnrS9OOW2IV0vH1OrTbPJRs+mAUcCNzaI\nL2T7VqDHei/bd0jaE7iHbETnHhqPzOUtA34s6RbSa53cZvsV4E+S1gEjWzgnZLUuqveBwG56vXhD\nJG2ZRt+A2m197cTZHyh+BBxXsPtZSeMLOl450xomH0IIIYSwOeno6OjW4bzwwgsL4zalNVhFa0t6\nfPhMHZtJZAsaPg18v1ZstxNJp6apcYvTKEl+3xbAGmA34Lamsu/upYKcjwFGAJNtTyb7YD+4IL4r\n97yL1zu9Ihvlmpx+xtlezRt3FXCj7fnp+d7A6ZJ+TzaSdZykS3p57kads8I22X4SeEbS7mQjWT/N\nHdMnNbB9STrHh8je94+0cPg/Ad8FpgD3p/cL9Hwd/wZ4hWwktKLH1NcmCNgr1+7R+c5Vw4OlrYGf\nA/9q+/6CkOnAUkmf7EVuodc6251AvxXrA8oRdS1H1LU8UdtyRF1b184O1hhJlWlsnwB+XbX/PmBf\nScMlDQCOBhak6XYDbN8MfAmYkkZLHpf0YQBJgyS9NX8y21emD6tTbK+t2tdleyzwANkH/CK/BI6Q\nNDxdY9sacZVOxlDgKdtdkvYHxhTE1HM7cMZrB0jvbuKYutJo1RDbl1W22T7W9ljbuwDnAj+yfX7B\nsTOV1ofVsBrYMU1bRNKQ9Lrl1WvTT4F/AbaxvaKJ+KYpW7dXed0mAruT1ptJuqTyvqlxrIDRthcA\n5wHbAPVukLEO2F7StpLeAvxzL1KeB7y2LkvSpGYPTFM1bwFmpt+RIucD77D9w17kFkIIIYQQ6mhn\nB+th4DRJK8luLvG9tN0AqRN0HtmffZcA96dpUaOATklLgGtTDMDxwBmSlgF30/p0LchGNYYX7UhT\n+r5G1slbAlyezzcfmv47C9gz5XMssKogpuj4iovIbgSyPN004avVAZKmSrq6TnuqnQPsnhvJa2V6\n4UTgyVo7bf+VrHP6XUlLyToJb6kKq9em/6Dn6NXFdeJ7kHSIpGkFuwYCv5a0gux9dqztylq73YG1\nBcdUDACuS6/jImC67T8XxFXet6+kPO8n6yCuKojtdkyBM4E9lN0cZQVwSnVAnbYeCewDfDL3Ok+s\nihmUbnYRNqqOdifQb8X6gHJEXcsRdS1P1LYcUdfWxRcN56S1PtvZPq9h8GYkTTn7ge1ao3tvWpLm\nVt2Wv19L6wCX2d6pTkx80XC/FF80HEIIIfQlbUpfNLwJuwl4n3JfNBzA9gv9sXMFsJl1ro4mG1n8\nZhPR8dPPfkaOzM9S7l9ifUA5oq7liLqWJ2pbjqhr69p5F8FNTrqr2vvbnUcIZUhfNPyTJmNLzmbz\n09nZGdMsQgghhM1ATBEMIXQjyfHvQgghhBBCfTFFMIQQQgghhBBKFh2sEELYCGIOe3mituWIupYj\n6lqeqG05oq6tiw5WCCGEEEIIIfSRWIMVQugm1mCFEEIIITQWa7BCCCGEEEIIoWTRwQoh9CApfuKn\n9J8d37Zjn7xfY31AOaKu5Yi6lidqW46oa+vie7BCCD1Na3cC/dAaYFy7k9i0rJu2rt0phBBCCH0u\n1mCFELqR5OhghY1iWnypdQghhDcvaRNagyVpjKQHa+ybL2nKxs4pXXu0pMWS5ua2rWlHLrVI2k/S\njCbi1qT/1qx1VfzWkh6X9O3ctvmSRjc4boakfRvke2uj67cif05JJ0i6oIljviHpQUnLJR3ZRPwF\nko5vsP/sFvPeVdISSYskjZN0hqSVkq5N7fhOg+ObbesJkh6RtDrfBklHS3pY0lmt5B1CCCGEEJrX\nzjVYm+KfLQ8D5tk+OLdtU8yzmZxc43EtFwELepdOS7mUcc6655f0j8C7gYnAe4FzJQ0pIadGDgNm\n255qew3wGeBA28el/a2+rj1I2hb4CrAnsBdwgaShALZ/AuwHRAerHTapP9X0L7E+oBxR13JEXcsT\ntS1H1LV17exgDZR0XfoL/g2SBlcHpL+4L08/l6ZtW6RRk+WSlkk6M20fL+kOSUslPSCpN6sdhgFP\nVW17OpfP8emaSyTNTNtmSJou6W5Jv5N0eNq+laQ7Uy7LJB2ato+RtCodt1rSLEkHpeNXS9ojxW0p\n6RpJC9OIxyEpjZeB55toy9PVGyR9P+W+RNJTkr6ctk8FdgDmVR3yJ+DVBtdZn3JC0p6pHUtT3ltV\nXb+wTZLulbRbLm6+pCl1apD3IrChQY4TgF8587/AcuAfGhzzQjo3aaTpodSuH+di3pVy/Z2kz6bY\nbiOGks5Jo10HA58DPiPpLklXAbsAcyvv4dwxIyTdKOk36WfvFtr6IbI/Ejxvez3Za/paW22vA4Y2\nOEcIIYQQQuildt7kYlfgU7YXSroGOBW4orJT0k7ApcBksg/xd6ROyhPAKNsTU9w26ZBZwCW250ga\nRO86jwOArvwG23ul60wAzgf2tv2cpGG5sB1tvy91EuYANwF/AQ6zvUHSdsDCtA9gPPBR2yslPQAc\nlY4/NF3jcOCLwF22T0wjEPdJutP2vcC9KaepwCm2T65uSCXvqm0npeNGA3OBGZIEfAs4BjioKv5j\njQpm+6x0zoHA9cARthenEaIXq8IL25SO+zgwTdKOqZ6LJX2tRnz++jdUHqcO2FTb06quuwz4iqQr\ngK2A/YGHGrTritzTLwBjbf81936D7D3cQdZhWS3pysrhPU/nuZK+B7xQObekDwEd6f10Qi5+OnCF\n7XskvR24HZjQZFtHAY/nnv8xbctr/LsxP/d4LHFzhr4QNSxNR0dHu1Pol6Ku5Yi6lidqW46o6+s6\nOzubGtFrZwfrMdsL0+PrgM+S62CRTXGab/tZAEmzgH2Bi4FxkqYDvwDmpQ/zO9ueA2D75VaTSR2N\nSSmXIh8gm971XLrG+ty+W9K2VZJ2qJwS+Lqy9UldwM65fWtsr0yPHwIqnYYHyT7OAnwQOETS59Pz\nQcBoYHXlorYXAT06Vw3aORiYDZxu+wlJpwG32X4yKwE9Fuo1aVfgSduLU24b0vXyMbXaNJtspGUa\ncCRwY4P4QrZvBXqs97J9h6Q9gXvIRijvofHIXN4y4MeSbiG91slttl8B/iRpHTCyhXNCVuuieh8I\n7KbXizdE0pZp9A2o3dYmPStpvO1Ha0bs38szhxBCCCH0Ux0dHd06nBdeeGFh3Ka0BqtobUmPD5+p\nYzMJ6AQ+DXy/Vmy3E0mnpqlxi9MoSX7fFmQrJHYDbmsq++5eKsj5GGAEMNn2ZLIP9oML4rtyz7t4\nvdMrslGuyelnnO3VvHFXATfaroxR7A2cLun3ZCNZx0m6pJfnbtQ5K2yT7SeBZyTtTjaS9dPcMX1S\nA9uXpHN8iOx9/0gLh/8T8F1gCnB/er9Az9fxb4BXyEZCK3pMfW2CgL1y7R6d71w18Ee6d0Lflrbl\nTQeWSvpkL3ILvRVrsEoT6wPKEXUtR9S1PFHbckRdW9fODtYYSZVpbJ8Afl21/z5gX0nDJQ0AjgYW\npOl2A2zfDHwJmJJGSx6X9GEASYMkvTV/MttXpg+rU2yvrdrXZXss8ADZB/wivwSOkDQ8XWPbGnGV\nTsZQ4CnbXZL2B8YUxNRzO3DGawdI727imLrSaNUQ25dVttk+1vZY27sA5wI/sn1+wbEzldaH1bAa\n2DFNW0TSkPS65dVr00+BfwG2sb2iifimKVu3V3ndJgK7k9abSbqk8r6pcayA0bYXAOcB2wD1bpCx\nDthe0raS3gL8cy9Snge8ti5L0qQWjr0dOEjS0PQePShtyzsfeIftH/YitxBCCCGEUEc7O1gPA6dJ\nWkl2c4nvpe0GSJ2g88hGqpYA96dpUaOATklLgGtTDMDxwBmSlgF30/p0LchGNYYX7UhT+r5G1slb\nAlyezzcfmv47C9gz5XMssKogpuj4iovIbgSyPN004avVAZKmSrq6TnuqnQPsnhvJa2V64UTgyVo7\nbf+VrHP6XUlLyToJb6kKq9em/6Dn6NXFdeJ7kHSIpGkFuwYCv5a0gux9dqztylq73YG1BcdUDACu\nS6/jImC67T8XxFXet6+kPO8n69isKojtdkyBM4E9lN0cZQVwSnVArbamKawXkf2x4DfAhVXTWQEG\npZtdhI0p1mCVJtYHlCPqWo6oa3mituWIurYuvmg4J6312c72eQ2DNyOStgZ+YLvW6N6blqS5Vbfl\n79fSOsBltneqExNfNBw2jmnxRcMhhBDevFTji4ajg5UjaTzwQ2DD5vShO2weJB1NdkfEmbb/b524\n+EchbBQjR41k7RP1BpCb09nZGX9hLUHUtRxR1/JEbcsRda2tVgernXcR3OSku6q9v915hFCG9EXD\nP2kytuRsNj/xP6gQQghh8xAjWCGEbiQ5/l0IIYQQQqiv1ghWO29yEUIIIYQQQgj9SnSwQghhI4jv\nESlP1LYcUddyRF3LE7UtR9S1ddHBCiGEEEIIIYQ+EmuwQgjdxBqsEEIIIYTGYg1WCCGEEEIIIZQs\nbtMeQuhB6vHHmBBCCG8CI0eOYe3aP7Q7jbaIr8MoR9S1ddHBCiEUiCmCfa8T6GhzDv1VJ1HbMnQS\ndS1DJ2XWdd26+ANZCO0Wa7BCCN1IcnSwQgjhzUrxZfEhbCSb1BosSWMkPVhj33xJUzZ2TunaoyUt\nljQ3t21NO3KpRdJ+kmY0Ebcm/bdmravit5b0uKRv57bNlzS6wXEzJO3bIN9bG12/FflzSjpB0gVN\nHPNqem2XSLqlifgLJB3fYP/ZLea9a7r+IknjJJ0haaWka1M7vtPg+IZtlTRJ0j2SHpS0VNKRuX1H\nS3pY0lmt5B1CCCGEEJrXzptcbIp/XjkMmGf74Ny2TTHPZnJyjce1XAQs6F06LeVSxjmbOf//2J5i\ne7Ltw0rIpxmHAbNtT7W9BvgMcKDt49L+Vl/XIv8DHGd7d+Bg4P9J2gbA9k+A/YDoYLVFZ7sT6Mc6\n251AP9XZ7gT6qc52J9Bvxfc1lSPq2rp2drAGSrou/QX/BkmDqwPSX9yXp59L07Yt0qjJcknLJJ2Z\nto+XdEf6q/0Dksb1IqdhwFNV257O5XN8uuYSSTPTthmSpku6W9LvJB2etm8l6c6UyzJJh6btYySt\nSsetljRL0kHp+NWS9khxW0q6RtLCNOJxSErjZeD5JtrydPUGSd9PuS+R9JSkL6ftU4EdgHlVh/wJ\neLXBddannJC0Z2rH0pT3VlXXL2yTpHsl7ZaLmy9pSp0a5L0IbGiQI0Crk9JfSOcmjTQ9lNr141zM\nu1Kuv5P02RTbbcRQ0jlptOtg4HPAZyTdJekqYBdgbuU9nDtmhKQbJf0m/ezdbFtt/872o+nxf5O9\nn7fP7V8HDG2xFiGEEEIIoUntvMnFrsCnbC+UdA1wKnBFZaeknYBLgclkH+LvSJ2UJ4BRtiemuG3S\nIbOAS2zPkTSI3nUeBwBd+Q2290rXmQCcD+xt+zlJw3JhO9p+X+okzAFuAv4CHGZ7g6TtgIVpH8B4\n4KO2V0p6ADgqHX9ousbhwBeBu2yfKGkocJ+kO23fC9ybcpoKnGL75OqGVPKu2nZSOm40MBeYIUnA\nt4BjgIOq4j/WqGC2z0rnHAhcDxxhe7GkIaQOSk5hm9JxHwemSdox1XOxpK/ViM9f/4bK49QBm2p7\nWkGqb0m1fhn4hu2fNWjXFbmnXwDG2v5r7v0G2Xu4g6zDslrSlZXDe57OcyV9D3ihcm5JHwI60vvp\nhFz8dOAK2/dIejtwOzChhbZWYt4DDKx0uHKa+N3In7aDWOjeFzranUA/1tHuBPqpjnYn0E91tDuB\nfivudFeOqOvrOjs7mxrRa2cH6zHbC9Pj64DPkutgAXsC820/CyBpFrAvcDEwTtJ04BfAvPRhfmfb\ncwBsv9xqMqmjMSnlUuQDZNO7nkvXWJ/bd0vatkrSDpVTAl9Xtj6pC9g5t2+N7ZXp8UNApdPwIDA2\nPf4gcIikz6fng4DRwOrKRW0vAnp0rhq0czAwGzjd9hOSTgNus/1kVoKWR3oqdgWetL045bYhXS8f\nU6tNs8lGz6YBRwI3NogvZPtWoNZ6rzG2/zuNbP5S0vI0Ta8Zy4AfK1u7lV+/dZvtV4A/SVoHjGzy\nfBWiuN4HArvp9eINkbSl7f+tBDRoa+UPFD8CjivY/ayk8QUdr5xpDZMPIYQQQticdHR0dOtwXnjh\nhYVxm9IarKK1JT0+fKaOzSSyScyfBr5fK7bbiaRT09S4xWmUJL9vC2ANsBtwW1PZd/dSQc7HACOA\nybYnk03VGlwQ35V73sXrnV6RjXJNTj/jbK/mjbsKuNH2/PR8b+B0Sb8nG8k6TtIlvTx3o85ZYZts\nPwk8I2l3spGsn+aO6ZMapOlypE5VJ9nIaLP+CfguMAW4P71foOfr+DfAK2QjoRU9pr42QcBeuXaP\nzneuGh4sbQ38HPhX2/cXhEwHlkr6ZC9yC73W2e4E+rHOdifQT3W2O4F+qrPdCfRbsVaoHFHX1rWz\ngzVGUmUa2yeAX1ftvw/YV9JwSQOAo4EFabrdANs3A18CpqTRksclfRhA0iBJb82fzPaV6cPqFNtr\nq/Z12R4LPED2Ab/IL4EjJA1P19i2RlylkzEUeMp2l6T9gTEFMfXcDpzx2gHSu5s4pq40WjXE9mWV\nbbaPtT3W9i7AucCPbJ9fcOxMpfVhNawGdkzTFpE0JL1uefXa9FPgX4BtbK9oIr5pkoalaaNIGgG8\nD1iZnl9Sed/UOFbAaNsLgPOAbYAhdS63Dthe0raS3gL8cy9Snge8ti5L0qRmD0xTNW8BZqbfkSLn\nA++w/cNe5BZCCCGEEOpoZwfrYeA0SSvJbi7xvbTdAKkTdB7Zn3qWAPenaVGjgE5JS4BrUwzA8cAZ\nkpYBd9P6dC2AR4DhRTvSlL6vkXXylgCX5/PNh6b/zgL2TPkcC6wqiCk6vuIishuBLE83TfhqdYCk\nqZKurtOeaucAu+dG8lqZXjgReLLWTtt/JeucflfSUrJOwluqwuq16T/oOXp1cZ34HiQdImlawa7d\ngAfS63YX2Vq9h9O+3YG1BcdUDACuS6/jImC67T8XxFXet6+kPO8n6yCuKojtdkyBM4E9lN0cZQVw\nSnVAnbYeCewDfDL3Ok+sihmUbnYRNqqOdifQj3W0O4F+qqPdCfRTHe1OoN+KtULliLq2Lr5oOCet\n9dnO9nkNgzcjacrZD2zXGt1705I0t+q2/P1aWge4zPZOdWLii4ZDCOFNK75oOISNRZvSFw1vwm4C\n3qfcFw0HsP1Cf+xcAWxmnaujyUYWv9nuXDZPne1OoB/rbHcC/VRnuxPopzrbnUC/FWuFyhF1bV07\n7yK4yUl3VXt/u/MIoQzpi4Z/0lx0b28mGUIIoZ1GjhzTOCiEUKqYIhhC6EaS49+FEEIIIYT6Yopg\nCCGEEEIIIZQsOlghhLARxBz28kRtyxF1LUfUtTxR23JEXVsXHawQQgghhBBC6COxBiuE0E2swQoh\nhBBCaCzWYIUQQgghhBBCyaKDFUIIG0HMYS9P1LYcUddyRF3LE7UtR9S1dfE9WCGEHqT4HqwQQghh\nYxo5aiRrn1jb7jRCH4g1WCGEbiSZae3OIoQQQtjMTIP4XP7mEmuwQgghhBBCCKFkbelgSRoj6cEa\n++ZLmrKxc0rXHi1psaS5uW1r2pFLLZL2kzSjibg16b81a10Vv7WkxyV9O7dtvqTRDY6bIWnfBvne\n2uj6rcifU9IJki5o4phX02u7RNItTcRfIOn4BvvPbjHvXdP1F0kaJ+kMSSslXZva8Z0Gxzfb1hMk\nPSJpdb4Nko6W9LCks1rJO/SRTepfkn4maluOqGs5oq7lidqWItZgta6da7A2xTHQw4B5ts/LbdsU\n82wmJ9d4XMtFwILepdNSLmWcs5nz/4/ttnTccw4DZtu+BEDSZ4ADbD8p6QRaf117kLQt8BVgCiBg\nkaSf2f8/e2cfblVV7f/PFxJRUURNULqgcXu8Wr6B/MyrF4/mS2q+XJOMMK3bY5amll7NtKuH1NRK\nf3kzNZMfcgVNLVSUSEQ5ZCKCcDioIGlhvl3UUgrN9zN+f8yxYZ199uvpLDYcxud59nPWmnPMOccc\na7FZY40x57a/mtmtkh4E5gP/9x+ZSBAEQRAEQVCaRqYIbiRpkr/Bv11S32IBf+O+2D+Xe1kvj5os\nltQm6UwvHybpfkmLJD0maccu6LQl8EpR2asZfU70MVslTfSyCZKulvSwpGckHevlm0ma6bq0STrK\ny4dKWurtlkmaLOlgb79M0l4ut6mk8ZLmesTjSFfjXeCvNczl1eICST933VslvSLpv7x8BLAtMKOo\nyV+AD6qMs9J1QtJIn8ci13uzovFLzknSI5J2zsjNkjS8gg2yvAW8UUVHSM5GPazyvvFI05M+r1sy\nMh93XZ+RdLrLdogYSjrbo12HAd8Evi7pAUnXAR8Fphfu4UybbST9UtKj/tmnjrkeSnpJ8FczW0m6\npp8uVJrZy0D/Om0RdAdd+UYKaiNsmw9h13wIu+ZH2DYXmpqaGq3CekcjI1g7AV82s7mSxgOnAlcV\nKiVtB1wO7El6iL/fnZQXgMFmtpvLbeFNJgPfN7OpkvrQNeexN9CeLTCzvX2cXYDzgX3M7HVJW2bE\nBpnZvu4kTAWmAG8Dx5jZG5K2BuZ6HcAw4LNmtkTSY8Dnvf1RPsaxwAXAA2b2FUn9gXmSZprZI8Aj\nrtMI4BQz+2rxRAp6F5Wd7O2GANOBCZIE/AgYCxxcJH9cNYOZ2be8z42AXwCjzWyhpH64g5Kh5Jy8\n3fFAs6RBbs+Fki4tI58d//bCsTtgI8ysuYSqG7ut3wWuMLO7q8zrqszpt4EdzOy9zP0G6R5uIjks\nyyRdW2jeuTubLul6YFWhb0mHAk1+P52Ukb8auMrM5kj6J+A+YJca5zoYeD5z/qKXZan+b2NW5ngH\n4j+tIAiCIAg2eFpaWmpKmWykg/Wcmc3140nA6WQcLGAkMMvMXgOQNBkYBVwC7CjpauDXwAx/mN/e\nzKYCmNm79SrjjsburkspDiSld73uY6zM1N3lZUslbVvoErhMaX1SO7B9pm65mS3x4yeBgtPwOOlx\nFuAQ4EhJ5/h5H2AIsKwwqJktADo5V1Xm2Re4A/iGmb0g6TRgmqepFfTuCjsBL5nZQtftDR8vK1Nu\nTneQIi3NwOeAX1aRL4mZ3QOUW+811Mz+1yObD0pabGa1Zmu3Abcord3Krt+aZmbvA3+R9DIwsMb+\nCojS9j4I2FlrjNdP0qZm9veCQJW5VuM1ScPM7A9lJQ7oYs9BeZYTjmpehG3zIeyaD2HX/Ajb5kJL\nS0tEsZympqYOthg3blxJuXVpDVaptSWdHj7NbKWk3UmpUF8DRpNSryo6BpJOBU72cQ43sxWZul7A\nH4F3gGl1zKHAOyV0HgtsA+xpZu1Km070LSHfnjlvZ801ESnK9XQX9KnEdcAvzawQo9gH2M/tszkp\ndXOVmZ3fhb6rOWdl5yTpz5J2JUWyTslUdZL3KFddmNn/+t/lklpIkdFaif3A3AAAIABJREFUHawj\nSM79UcAFkj7h5cXX8UPA+6RIaIFOqa81IGBvM3uvC21fJEXVCnyEjvEoSBGyRZJON7ObujBGEARB\nEARBUIZGrsEaKqmQxvYF4KGi+nnAKElbSeoNjAFme7pdbzO7E/guMNyjJc9LOhpAUh9Jm2Q7M7Nr\nzWxPMxueda68rt3MdgAeIz3gl+JBYLSkrXyMAWXkCk5Gf+AVd64OAIaWkKnEfcAZqxtIe9TQpiIe\nrepnZj8slJnZCWa2g5l9FPhP4H9KOVeSJsrXh5VhGTDI0xaR1M+vW5ZKc7oNOBfYwsyeqEG+ZiRt\n6WmjSNoG2BdY4uffL9w3ZdoKGGJms4HzgC2AfhWGexn4sKQBkjYGPtMFlWcAq9dl+QuFWrkPOFhS\nf79HD/ayLOcD/xzO1Vom3qrmR9g2H8Ku+RB2zY+wbS5E9Kp+GulgPQWcJmkJaXOJ673cANwJOg9o\nAVqB+Z4WNRhokdQK3OwyACcCZ0hqAx6m/nQtgN8DW5Wq8JS+S0lOXitwZVbfrKj/nQyMdH1OAJaW\nkCnVvsDFpGjSYt804XvFApJGSLqhwnyKORvYVWmTi4WS6kkv3A14qVylR1uOB66RtIjkJGxcJFZp\nTr/y9rdlyi6pIN8JSUdKai5RtTPwmF+3B0hr9Z7yul2BSj+b3huY5NdxAXC1mf2thFzhvn3f9ZxP\ncmyWlpDt0KYEZwJ7KW2O8gQdI3pA+bl6CuvFpJcFjwLjitJZAfr4ZhdBEARBEARBN6P4xeg1+Fqf\nrYu2ad/gkbQ5cKOZlYvurbdImm5mhzVaj7WFrwNsM7PtKsgYzWtPpw2GWBuQH2HbfAi75kPYNT/W\nd9s2w7r4XB5rsMojCTPrlJnWyDVY6yJTgJs2tIfuapjZKsqnTq7XbEjXWdIY0o6IP6gq3Jy3NkEQ\nBEEQZBk4uCvJV8G6SESwgiDogCSL74UgCIIgCILKlItgNXINVhAEQRAEQRAEQY8iHKwgCIK1QC0/\nTBh0jbBtPoRd8yHsmh9h23wIu9ZPOFhBEARBEARBEATdRKzBCoKgA7EGKwiCIAiCoDqxBisIgiAI\ngiAIgiBnwsEKgiBYC0QOe36EbfMh7JoPYdf8CNvmQ9i1fuJ3sIIg6ITUKdodBEEQ9FAGDhzKihXP\nNlqNIOgxxBqsIAg6IMkgvheCIAg2HEQ8DwZB/cQarCAIgiAIgiAIgpxpiIMlaaikx8vUzZI0fG3r\n5GMPkbRQ0vRM2fJG6FIOSftLmlCD3HL/W9bWRfKbS3pe0n9nymZJGlKl3QRJo6roe0+18esh26ek\nkyRdVEOb6ZJelzS1xjEuknRilfqzatcaJO0kqVXSAkk7SjpD0hJJN/s8flKlfdW5Stpd0hxJj0ta\nJOlzmboxkp6S9K169A66i5ZGK9CDaWm0Aj2UlkYr0ENpabQCPZZYK5QPYdf6aWQEa12MRR8DzDCz\nwzJl66KetehkZY7LcTEwu2vq1KVLHn3W0v8PgBNy0KMejgHuMLMRZrYc+DpwkJl90evrva6leBP4\nopntChwG/FjSFgBmdiuwPxAOVhAEQRAEQU400sHaSNIkf4N/u6S+xQL+xn2xfy73sl4eNVksqU3S\nmV4+TNL9/tb+MUk7dkGnLYFXispezehzoo/ZKmmil02QdLWkhyU9I+lYL99M0kzXpU3SUV4+VNJS\nb7dM0mRJB3v7ZZL2crlNJY2XNNcjHke6Gu8Cf61hLq8WF0j6ueveKukVSf/l5SOAbYEZRU3+AnxQ\nZZyVrhOSRvo8FrnemxWNX3JOkh6RtHNGbpak4RVskOUt4I0qOmJms2qRy7DK+8YjTU/6vG7JyHzc\ndX1G0uku2yFiKOlsj3YdBnwT+LqkByRdB3wUmF64hzNttpH0S0mP+mefWudqZs+Y2R/8+H9J9/OH\nM/UvA/3rsEPQbTQ1WoEeTFOjFeihNDVagR5KU6MV6LE0NTU1WoUeSdi1fhq5i+BOwJfNbK6k8cCp\nwFWFSknbAZcDe5Ie4u93J+UFYLCZ7eZyW3iTycD3zWyqpD50zXnsDbRnC8xsbx9nF+B8YB8ze13S\nlhmxQWa2rzsJU4EpwNvAMWb2hqStgbleBzAM+KyZLZH0GPB5b3+Uj3EscAHwgJl9RVJ/YJ6kmWb2\nCPCI6zQCOMXMvlo8kYLeRWUne7shwHRggiQBPwLGAgcXyR9XzWBm9i3vcyPgF8BoM1soqR/uoGQo\nOSdvdzzQLGmQ23OhpEvLyGfHv71w7A7YCDNrrqZ3DfO6KnP6bWAHM3svc79BuoebSA7LMknXFpp3\n7s6mS7oeWFXoW9KhQJPfTydl5K8GrjKzOZL+CbgP2KXeuUr6P8BGBYcrQw3/NrLdNhEPBEEQBEEQ\nbOi0tLTUlDLZSAfrOTOb68eTgNPJOFjASGCWmb0GIGkyMAq4BNhR0tXAr4EZ/jC/vZlNBTCzd+tV\nxh2N3V2XUhxISu963cdYmam7y8uWStq20CVwmdL6pHZg+0zdcjNb4sdPAgWn4XFgBz8+BDhS0jl+\n3gcYAiwrDGpmC4BOzlWVefYF7gC+YWYvSDoNmGZmLyUT0NX9uXcCXjKzha7bGz5eVqbcnO4gRc+a\ngc8Bv6wiXxIzuwfo1vVeThtwi6S78GvtTDOz94G/SHoZGFhnv6K0vQ8CdtYa4/WTtKmZ/b0gUG2u\n/oLif4Avlqh+TdKwEo5Xhuaqygf10kI4qnnRQtg2D1oIu+ZBC2HXfGhpaYloSw6EXdfQ1NTUwRbj\nxo0rKddIB6vTW/4SMp0ePs1spaTdgUOBrwGjSalXFR0DSacCJ/s4h5vZikxdL+CPwDvAtDrmUOCd\nEjqPBbYB9jSzdqVNJ/qWkG/PnLez5pqIFOV6ugv6VOI64JeeMgewD7Cf22dzUurmKjM7vwt9V3PO\nys5J0p8l7UqKZJ2Sqeok71GutckRJOf+KOACSZ/w8uLr+CHgfVIktECn1NcaELC3mb3XhbZI2hy4\nF/iOmc0vIXI1sEjS6WZ2U1fGCIIgCIIgCErTyDVYQyUV0ti+ADxUVD8PGCVpK0m9gTHAbE+3621m\ndwLfBYZ7tOR5SUcDSOojaZNsZ2Z2rZntaWbDs86V17Wb2Q7AY6QH/FI8CIyWtJWPMaCMXMHJ6A+8\n4s7VAcDQEjKVuA84Y3UDaY8a2lTEo1X9zOyHhTIzO8HMdjCzjwL/CfxPKedK0kT5+rAyLAMGedoi\nkvr5dctSaU63AecCW5jZEzXId4VOESNJ3y/cNyUbpCjSEDObDZwHbAH0qzDGy8CHJQ2QtDHwmS7o\nOQNYvS7LXyjUhKdq3gVM9H8jpTgf+OdwrtY2TY1WoAfT1GgFeihNjVagh9LUaAV6LBFlyYewa/00\n0sF6CjhN0hLS5hLXe7kBuBN0HimW3grM97SowUCLpFbgZpcBOBE4Q1Ib8DD1p2sB/B7YqlSFp/Rd\nSnLyWoErs/pmRf3vZGCk63MCsLSETKn2BS4mRZMW+6YJ3ysWkDRC0g0V5lPM2cCuSptcLJRUT3rh\nbsBL5So92nI8cI2kRSQnYeMisUpz+pW3vy1TdkkF+U5IOlJSc5m633rfB0p6TlJhvdmuwIpSbZze\nwCS/jguAq83sbyXkCvft+67nfJKDuLSEbIc2JTgT2Etpc5Qn6BjRK8yn3Fw/B+wHfClznXcrkunj\nm10EQRAEQRAE3Yzil7vX4Gt9tjaz86oKb0B4ytmNZlYuurfeIml60bb8PRpfB9hmZttVkLF189cJ\n1ndaiDfXedFC2DYPWgi75kEL655dRU94Hoy1QvkQdi2PJMysU2ZaI9dgrYtMAW7a0B66q2Fmqyif\nOrlesyFdZ0ljSDsi/qAG6bzVCYIgCNYRBg4cWl0oCIKaiQhWEAQdkGTxvRAEQRAEQVCZchGsRq7B\nCoIgCIIgCIIg6FGEgxUEQbAWqOWHCYOuEbbNh7BrPoRd8yNsmw9h1/oJBysIgiAIgiAIgqCbiDVY\nQRB0INZgBUEQBEEQVCfWYAVBEARBEARBEORMOFhBEARrgchhz4+wbT6EXfMh7JofYdt8CLvWT/wO\nVhAEnZDid7CCIAiCoLsZOHggK15Y0Wg1gpyJNVhBEHRAktHcaC2CIAiCoAfSDPHs3XOINVhBEARB\nEARBEAQ50xAHS9JQSY+XqZslafja1snHHiJpoaTpmbLljdClHJL2lzShBrnl/resrYvkN5f0vKT/\nzpTNkjSkSrsJkkZV0feeauPXQ7ZPSSdJuqiGNtMlvS5pao1jXCTpxCr1Z9WuNUjaSVKrpAWSdpR0\nhqQlkm72efykSvta53qSpN9LWpadg6Qxkp6S9K169A66iXXqm6SHEbbNh7BrPoRd8yNsmwuxBqt+\nGhnBWhfjo8cAM8zssEzZuqhnLTpZmeNyXAzM7po6demSR5+19P8D4IQc9KiHY4A7zGyEmS0Hvg4c\nZGZf9Pp6r2snJA0ALgRGAnsDF0nqD2BmtwL7A+FgBUEQBEEQ5EQjHayNJE3yN/i3S+pbLOBv3Bf7\n53Iv6+VRk8WS2iSd6eXDJN0vaZGkxyTt2AWdtgReKSp7NaPPiT5mq6SJXjZB0tWSHpb0jKRjvXwz\nSTNdlzZJR3n5UElLvd0ySZMlHeztl0nay+U2lTRe0lyPeBzparwL/LWGubxaXCDp5657q6RXJP2X\nl48AtgVmFDX5C/BBlXFWuk5IGunzWOR6b1Y0fsk5SXpE0s4ZuVmShlewQZa3gDeq6IiZzapFLsMq\n7xuPND3p87olI/Nx1/UZSae7bIeIoaSzPdp1GPBN4OuSHpB0HfBRYHrhHs602UbSLyU96p996pjr\noaSXBH81s5Wka/rpjB1eBvrXYYegu+jKN1JQG2HbfAi75kPYNT/CtrnQ1NTUaBXWOxq5i+BOwJfN\nbK6k8cCpwFWFSknbAZcDe5Ie4u93J+UFYLCZ7eZyW3iTycD3zWyqpD50zXnsDbRnC8xsbx9nF+B8\nYB8ze13SlhmxQWa2rzsJU4EpwNvAMWb2hqStgbleBzAM+KyZLZH0GPB5b3+Uj3EscAHwgJl9xSMQ\n8yTNNLNHgEdcpxHAKWb21eKJFPQuKjvZ2w0BpgMTJAn4ETAWOLhI/rhqBjOzb3mfGwG/AEab2UJJ\n/XAHJUPJOXm744FmSYPcngslXVpGPjv+7YVjd8BGmFlzNb1rmNdVmdNvAzuY2XuZ+w3SPdxEcliW\nSbq20LxzdzZd0vXAqkLfkg4Fmvx+OikjfzVwlZnNkfRPwH3ALjXOdTDwfOb8RS/LUv3fxqzM8Q7E\nf1pBEARBEGzwtLS01JQy2UgH6zkzm+vHk4DTyThYpBSnWWb2GoCkycAo4BJgR0lXA78GZvjD/PZm\nNhXAzN6tVxl3NHZ3XUpxICm963UfY2Wm7i4vWypp20KXwGVK65Page0zdcvNbIkfPwkUnIbHSY+z\nAIcAR0o6x8/7AEOAZYVBzWwB0Mm5qjLPvsAdwDfM7AVJpwHTzOylZAK6uj/3TsBLZrbQdXvDx8vK\nlJvTHaRISzPwOeCXVeRLYmb3AN263stpA26RdBd+rZ1pZvY+8BdJLwMD6+xXlLb3QcDOWmO8fpI2\nNbO/FwT+wbm+JmmYmf2hrMQBXew5KM9ywlHNi7BtPoRd8yHsmh9h21xoaWmJKJbT1NTUwRbjxo0r\nKddIB6vTW/4SMp0ePs1spaTdSalQXwNGk1KvKjoGkk4FTvZxDjezFZm6XsAfgXeAaXXMocA7JXQe\nC2wD7Glm7UqbTvQtId+eOW9nzTURKcr1dBf0qcR1wC89ZQ5gH2A/t8/mpNTNVWZ2fhf6ruaclZ2T\npD9L2pUUyTolU9VJ3qNca5MjSM79UcAFkj7h5cXX8UPA+6RIaIFOqa81IGBvM3uvC21fJEXVCnyE\njvEoSBGyRZJON7ObujBGEARBEARBUIZGrsEaKqmQxvYF4KGi+nnAKElbSeoNjAFme7pdbzO7E/gu\nMNyjJc9LOhpAUh9Jm2Q7M7NrzWxPMxueda68rt3MdgAeIz3gl+JBYLSkrXyMAWXkCk5Gf+AVd64O\nAIaWkKnEfcAZqxtIe9TQpiIerepnZj8slJnZCWa2g5l9FPhP4H9KOVeSJsrXh5VhGTDI0xaR1M+v\nW5ZKc7oNOBfYwsyeqEG+K3SKGEn6fuG+KdkgRZGGmNls4DxgC6BfhTFeBj4saYCkjYHPdEHPGcDq\ndVn+QqFW7gMOltTf79GDvSzL+cA/h3O1lom3qvkRts2HsGs+hF3zI2ybCxG9qp9GOlhPAadJWkLa\nXOJ6LzcAd4LOA1qAVmC+p0UNBloktQI3uwzAicAZktqAh6k/XQvg98BWpSo8pe9SkpPXClyZ1Tcr\n6n8nAyNdnxOApSVkSrUvcDEpmrTYN034XrGApBGSbqgwn2LOBnZV2uRioaR60gt3A14qV+nRluOB\nayQtIjkJGxeJVZrTr7z9bZmySyrId0LSkZKay9T91vs+UNJzkgrrzXYFKv2kem9gkl/HBcDVZva3\nEnKF+/Z913M+ybFZWkK2Q5sSnAnspbQ5yhN0jOgV5lNyrp7CejHpZcGjwLiidFaAPr7ZRRAEQRAE\nQdDNKH5Neg2+1mdrMzuvqvAGhKTNgRvNrFx0b71F0vSibfl7NL4OsM3MtqsgYzSvPZ02GGJtQH6E\nbfMh7JoPYdf8WB9s2wzr27N3rMEqjyTMrFNmWiPXYK2LTAFu2tAeuqthZqsonzq5XrMhXWdJY0g7\nIv6gqnBz3toEQRAEwYbHwMFdSbAK1jcighUEQQckWXwvBEEQBEEQVKZcBKuRa7CCIAiCIAiCIAh6\nFOFgBUEQrAVq+WHCoGuEbfMh7JoPYdf8CNvmQ9i1fsLBCoIgCIIgCIIg6CZiDVYQBB2INVhBEARB\nEATViTVYQRAEQRAEQRAEORMOVhAEwVogctjzI2ybD2HXfAi75kfYNh/CrvUTv4MVBEEnpE7R7qCb\nGDhwKCtWPNtoNYIgCIIgyIlYgxUEQQckGcT3Qn6I+N4NgiAIgvWfWIMVBEEQBEEQBEGQMzU5WJKG\nSnq8TN0sScO7V63akDRE0kJJ0zNlyxuhSzkk7S9pQg1yyzPy95STkbRVN+pVcpxMfUW9/b6YVUWm\n2++PbJ+1XG9Ju0maI6lN0t2S+tXQpmK/klbVrvHqNj+U9LikKyRtI2mupAWS9qvl2tY41x9IWipp\nkaRfSdoiU/dbSfMkbVuv7kF30NJoBXossT4gH8Ku+RB2zY+wbT6EXeunngjWupjTcgwww8wOy5St\ni3rWopOVOa63n3qo1l+9ejeCWsa/ETjXzHYH7gTO7YZ+uzLvk4HdzOzbwEHAYjMbYWa/q7G/WmRm\nAB83sz2Ap4HvrG5sNgpYABxRt+ZBEARBEARBTdTjYG0kaZKkJZJul9S3WEDSGEmL/XO5l/WSNMHL\n2iSd6eXDJN3vb9ofk7RjF/TfEnilqOzVjD4n+pitkiZ62QRJV0t6WNIzko718s0kzXRd2iQd5eVD\nPSIwQdIySZMlHeztl0nay+U2lTQ+E5U40tV4F/hrDXN5NXPcX9K9kp6SdG2mfHWOp6SzPBqyOGPT\nTb1dq5eP9vKRru8i12+z7MCSpnkksFXSSklfrFHvD4DXvI9emQjNIkmnFQu73ea4jW9zfQ+VdHtG\nZnVkTdIhxfJV7FaOj7kTAzAT+GwNbV51HQZJmu32WSxp3zWq6hKf6xxJH/bCCYV7ys9X+d+7gX7A\nAknnAlcAx3i/fel4bcdKetTrrpNW7zhRda5mNtPM2v10LvCRIpEVpH83wVqnqdEK9FiampoarUKP\nJOyaD2HX/Ajb5kPYtX7q2UVwJ+DLZjZX0njgVOCqQqWk7YDLgT2BlcD97qS8AAw2s91crpCyNBn4\nvplNldSHrq0H6w20ZwvMbG8fZxfgfGAfM3tdUvahcpCZ7StpZ2AqMAV4GzjGzN6QtDXp4XSqyw8D\nPmtmSyQ9Bnze2x/lYxwLXAA8YGZfkdQfmCdpppk9AjziOo0ATjGzrxZPpKC3MxLYGXgOuE/SsWY2\npVCplB53ksv1Bh6V1OJ6vmhmn3G5zSVtBPwCGG1mC5XS494qGvuITL//D7jLzFYV9C6Hmb0AHOen\nXwWGkiI0VmRv3KbfBT5lZm+5k3EWcBnwM0mbmNlbwPHALS5/QQn5S8rZTdI04CtmtqJI1SclHWVm\nU4HP0dnpKDW3Qr9fAH5jZpe5o1Nw8jYD5pjZdyVdQYpOfb9UV97f0ZL+ZmaF1MaXgRFmdoafF+bw\nL26DfzWzDyT9FBgLTKpxrln+g3Tts7ST7pkqNGeOmwjnIAiCIAiCDZ2WlpaaUibrcWqeM7O5fjwJ\n2K+ofiQwy8xe8zfok4FRwB+BHT1qdCiwyh/yt/cHXszsXTN7uw5d8Ifd3UkOXCkOBO4ws9d9jJWZ\nuru8bClQWI8i4DJJbaQox/Zas1ZluZkt8eMnvR7gcWAHPz4EOE9SK2mxRR9gSFYhM1tQyrkqwTwz\n+5OlrcZupbOt9wPuNLO3zexNkoP4b67PwZIuk7SfO0k7AS+Z2ULX4Y1MhGM1krYBbgbGeLt6OQj4\nmetcbG+ATwK7AA+7jU4EhpjZB8BvgCMl9Salr00tJ19JATM7oozD8R/AaZLmkxyjd+uY13zgy5Iu\nJDmPb3r5O2b2az9ewJr7oJha9zsvpP99ChgOzPd5Hwh8tJNw+bmmQaULgPfM7JaiqheB3aqr05z5\nNFUXD2qgpdEK9FhifUA+hF3zIeyaH2HbfAi7rqGpqYnm5ubVn3LUE8EqXv9Raj1Ip4dJM1spaXfg\nUOBrwGjgm6VkO3QknUqKChhwePZhUlIvkuP2DjCtjjkUeKeEzmOBbYA9zaxdaUOBviXk2zPn7ayx\noUhRrqe7oE8xtdi6cyOzpz0KdThwsaQHSM5kNVv3Ijlyze505oFI6+XGlqi7DfgG8Dow38zedAe6\nnHxdmNnvSfcfkj5GHWuQzOwhSaO8zU2SrjSzScB7GbEPWHMfvI+/uPA5bFSnugImmtkFdbZb04H0\nJdI9cGCJ6inAhZKWmNkuXR0jCIIgCIIgKE09EayhkrJpUw8V1c8DRknayiMRY4DZnurV28zuJKWI\nDTezN4DnJR0NIKmPpE2ynZnZtWa2p5kNL35Tb2btZrYD8BgpnaoUDwKj5TuzSRpQRq7gfPQHXnHn\n6gBSuluxTCXuA85Y3UDao4Y25dhbae1XL9L8im39EGn9Tl+l9VT/DjzkaZpvedTiR6RIyDJgkKcn\nIqmfX58sVwBtZnZHKWWU1nBNrKLz/cAphb5L2HsusK+kYV6/qTs7ALNd15NZk9JWSb4uMuujepHu\nwev9fHtJM6u0HUK6L8aTNsso7IhY7p54FtjLj4+mo4NV6T4q1D0AHJfReYDrUBOSPg2cAxxlZu+U\nEDkRmB7OVSNoarQCPZZYH5APYdd8CLvmR9g2H8Ku9VOPg/UUKc1qCWmR/PVeXkgJWwGcR8qDaSVF\nIu4BBgMtnu50s8tAetA7w1PyHgYGdkH/3wMlt7b2lL5LSU5eK3BlVt+sqP+dDIx0fU4AlpaQKdW+\nwMWkjUAWK21p/71iAUkjJN1QYT4F5gHXkNIR/2Bmd2XHNrNW4CZS+tojwA1m1gbsSlr71QpcCFxi\nZu+RnLRrJC0i7TK3cdF4ZwOHKG1ysVDSZ4rqhwB/r6LzjcDzwGIff0yRzn8GvgTc6jaeQ0pfxFMW\n7wU+7X8rylPmGiht1jGoRNUYScuAJaQ1ajd5+XZ0jESVoglok7SQtH7rx5V0AH4O7O82+CTwZqau\nUiSyYKelJCdwhs97BtBpThXm+hPSZhr3+7W8tqh+AGl3wSAIgiAIgiAH5Etm1ksknQNsbWbnVRUO\nuoxv4nCzmT3RaF26E6WdDv9kZvc2Wpe1hW+asdjMflZBxhq/+35PpIXkr4v1+Xt3XaSlpSXesOZA\n2DUfwq75EbbNh7BreSRhZp0ylOpZg7UuMoW0LmZ60W9hBd2I/25Tj8PMftpoHdYmkmaT1g2W2u0w\nCIIgCIIg6AbW6whWEATdT4pgBXkxcOBQVqx4ttFqBEEQBEHwD9JTI1hBEORAvHgJgiAIgiDoGl35\ncd8gCIKgTuJ3RPIjbJsPYdd8CLvmR9g2H8Ku9RMOVhAEQRAEQRAEQTcRa7CCIOiAJIvvhSAIgiAI\ngsqUW4MVEawgCIIgCIIgCIJuIhysIAiCtUDksOdH2DYfwq75EHbNj7BtPoRd6yccrCAIgiAIgiAI\ngm4i1mAFQdCB+B2s9ZOBgwey4oUVjVYjCIIgCDYYyq3BCgcrCIIOSDKaG61FUDfN8ftlQRAEQbA2\niU0ugiAIGsnyRivQc4n1AfkQds2HsGt+hG3zIexaPzU5WJKGSnq8TN0sScO7V63akDRE0kJJ0zNl\n69RjjKT9JU2oQW55Rv6ecjKStupGvUqOk6mvqLffF7OqyHT7/ZHts5brLWk3SXMktUm6W1K/GtpU\n7FfSqto1Xt3mh5Iel3SFpG0kzZW0QNJ+tVzbGuc6QNIMScsk3Sepf6but5LmSdq2Xt2DIAiCIAiC\n2qgngrUu5p4cA8wws8MyZeuinrXoZGWO6+2nHqr1V6/ejaCW8W8EzjWz3YE7gXO7od+uzPtkYDcz\n+zZwELDYzEaY2e9q7K8WmfOAmWa2E/Ag8J3Vjc1GAQuAI+rWPPjH2bHRCvRcmpqaGq1CjyTsmg9h\n1/wI2+ZD2LV+6nGwNpI0SdISSbdL6lssIGmMpMX+udzLekma4GVtks708mGS7pe0SNJjkrry+LEl\n8EpR2asZfU70MVslTfSyCZKulvSwpGckHevlm0ma6bq0STrKy4dKWurtlkmaLOlgb79M0l4ut6mk\n8ZmoxJGuxrvAX2uYy6uZ4/6S7pX0lKRrM+WrczwlneXRkMUZm27q7Vq9fLSXj3R9F7l+m2UHljTN\nI4GtklZK+mKNen8AvOZ99MpEaBZJOq1Y2O02x218m+t7qKTbMzIqbMp2AAAgAElEQVSrI2uSDimW\nr2K3cnzMnRiAmcBna2jzquswSNJst89iSfuuUVWX+FznSPqwF04o3FN+vsr/3g30AxZIOhe4AjjG\n++1Lx2s7VtKjXnedpEJdLXM9GpjoxxNJLyGyrCD9uwmCIAiCIAhyoB4HayfgGjPbBVgFnJqtlLQd\ncDnQBOwBjHQnZQ9gsJnt5hGEQtrZZOAnZrYH8K/A/3ZB/95Ae7bAzPZ2fXYBzgeazGxP4MyM2CAz\n2xc4kvSgC/A2cIyZ7QUcCFyZkR8G/NCjAjsBn/f25/gYABcAD5jZJ739jyRtYmaPmNm3XKcRkm4o\nNZGC3s5I4DRgZ+Cfsw/s3s9w4CSX2wc4WdLuwKeBF81sTzPbDfiNpI2AXwCnu60PAt4qGvsIMxsO\nfAV4Frgrq3c5zOwFMzvOT78KDCVFaPYgXd+szlsD3wU+5TZeAJxFcnj+j6RNXPR44BaXv6CEfFm7\nuaM4qISqTxYcZuBzwEcqzauo3y8Av3H77A4s8vLNgDk+14dI0amSXXl/RwN/N7PhZvYD4ELgF37+\ndmYO/+I2+Fcfsx0YW8dctzWzl11+BVCcDthO+ndTmVmZzzqVdLseE3bMjVgfkA9h13wIu+ZH2DYf\nwq5raGlpobm5efWnHB+qo8/nzGyuH08CTgeuytSPBGaZWSGiMRkYBVwC7CjpauDXwAylNTDbm9lU\nADN7tw498P5FeuCdVEbkQOAOM3vdx1iZqbvLy5ZqzXoUAZdJGkV6CN0+U7fczJb48ZMkpwDgcWAH\nPz4EOFLSOX7eBxgCLCsMamYLSI5INeaZ2Z98nrcC+wFTMvX7AXcWHswlTQH+DbiP5NhdBkwzs99J\n+gTwkpktdB3e8DYdBpS0DXAzcJyZ1b2+iOS4XWe+jVmRvQE+CewCPOzXbiOSg/KBpN+QbPcrUvra\nOSRHvZN8JQXMrFzq238AP5H0X8BUUnSuVuYD491RvdvM2rz8HTP7tR8vIM2/FJ12lilDIf3vU8Bw\nYL7Puy/wcifh8nMt12+BF0m2rcwBNfYeBEEQBEGwgdDU1NQhZXLcuHEl5epxsIof1EqtB+n0MGlm\nKz26cijwNWA08M1Ssh06kk4lRQUMONzfxhfqegF/BN4BptUxhwLvlNB5LLANsKeZtSttKNC3hHx7\n5rydNTYU8Fkze7oL+hRTi607NzJ72qNbhwMXS3qA5ExWs3Uv4Fag2cyWdkHfWhBpvdzYEnW3Ad8A\nXgfmm9mb7lyUk68LM/s96f5D0seoYw2SmT3kTvcRwE2SrjSzScB7GbEPWHMfvI9HhjOOYT0ImGhm\nF9TZrsDLkgaa2cse4SpOoZ0CXChpiUejg7VFrMHKjVgfkA9h13wIu+ZH2DYfwq71U0+K4FBJ2bSp\nh4rq5wGjJG0lqTcwBpjtqV69zexOUorYcI+iPC/paABJfTIpYgCY2bWe6jY861x5XbuZ7QA8Rkqn\nKsWDwGj5zmySBpSRKzgf/YFX3Lk6gJTuVixTifuAM1Y3kPaooU059lZa+9WLNL9iWz9EWr/T19dT\n/TvwkKdpvmVmtwA/IkVClgGDJI1wvfr59clyBdBmZneUUkZpDdfEUnUZ7gdOKfRdwt5zgX0lDfP6\nTd3ZAZjtup5MSmesJl8XmfVRvUj34PV+vr2kmVXaDiHdF+NJm2UUdkQsd088C+zlx0fT0cGqdB8V\n6h4AjsvoPMB1qJWpwJf8+CTg7qL6E4Hp4VwFQRAEQRDkQz0O1lPAaZKWkBbJX+/lhZSwFaQdzFqA\nVlIk4h5gMNAiqZWUgnaetzsROENSG/AwMLAL+v8eKLm1taf0XUpy8lpZs6aqXHRoMmndWBtwArC0\nhEyp9gUuJm0EslhpS/vvFQtUWoNVxDzgGlI64h/M7K7s2GbWCtxESl97BLjBU9d2Beb5fC8ELjGz\n90hO2jWSFgEzgI2LxjsbOERpk4uFkj5TVD8E+HsVnW8EngcW+/hjinT+M+nB/1a38RzSejbMrB24\nl7SG7N5q8pS5BhXWJY2RtAxYQlqjdpOXb0fHSFQpmoA2SQtJ67d+XEkH4OfA/m6DTwJvZuoqRSIL\ndlpKcgJn+LxnAJ3mVGGuVwAH+3w/RVoXmWUA0B1R1qBeYg1WbsT6gHwIu+ZD2DU/wrb5EHatH/mS\nmfUSX++0tZmdV1U46DKSrgBuNrMnGq1Ld6K00+GfzOzeRuuytpD0U9L28D+rIGM0rz2dNhiWk2+a\nYDOsz9/n/wgtLS2RwpIDYdd8CLvmR9g2H8Ku5ZGEmXXKUFrfHaxhpEjOG0W/hRUEQRGSZpPWDZ5g\nZi9WkFt/vxQ2YAYOHsiKF1ZUFwyCIAiCoFvokQ5WEATdjySL74UgCIIgCILKlHOw6lmDFQRBEHSR\nyGHPj7BtPoRd8yHsmh9h23wIu9ZPOFhBEARBEARBEATdRKQIBkHQgUgRDIIgCIIgqE6kCAZBEARB\nEARBEORMOFhBEARrgchhz4+wbT6EXfMh7JofYdt8CLvWTzhYQRAEQRAEQRAE3USswQqCoAPxO1hB\nEAQwcOBQVqx4ttFqBEGwDhO/gxUEQU0kByu+F4Ig2NAR8YwUBEElYpOLIAiChtLSaAV6MC2NVqCH\n0tJoBXoksZ4lP8K2+RB2rZ+aHCxJQyU9XqZulqTh3atWbUgaImmhpOmZsuWN0KUckvaXNKEGueUZ\n+XvKyUjaqhv1KjlOpr6i3n5fzKoi0+33R7bPWq63pIskveD3ykJJn66hTcV+Ja2qXePVbX4o6XFJ\nV0jaRtJcSQsk7VfLta1xrj+QtFTSIkm/krRFpu63kuZJ2rZe3YMgCIIgCILaqCeCtS7GyY8BZpjZ\nYZmydVHPWnSyMsf19lMP1fqrV+9GUOv4V5nZcP/8phv67cq8TwZ2M7NvAwcBi81shJn9rsb+apGZ\nAXzczPYAnga+s7qx2ShgAXBE3ZoH3UBToxXowTQ1WoEeSlOjFeiRNDU1NVqFHkvYNh/CrvVTj4O1\nkaRJkpZIul1S32IBSWMkLfbP5V7WS9IEL2uTdKaXD5N0v79pf0zSjl3Qf0vglaKyVzP6nOhjtkqa\n6GUTJF0t6WFJz0g61ss3kzTTdWmTdJSXD/WIwARJyyRNlnSwt18maS+X21TS+ExU4khX413grzXM\n5dXMcX9J90p6StK1mfLVOZ6SzvJoyOKMTTf1dq1ePtrLR7q+i1y/zbIDS5rmkZ1WSSslfbFGvT8A\nXvM+emUiNIsknVYs7Hab4za+zfU9VNLtGZnVkTVJhxTLV7FbJTrlx1bhVddhkKTZbp/FkvZdo6ou\n8bnOkfRhL5xQuKf8fJX/vRvoByyQdC5wBXCM99uXjtd2rKRHve46SYW6qnM1s5lm1u6nc4GPFIms\nIP27CYIgCIIgCPLAzKp+gKFAO/BJPx8PnOXHs4DhwHbAn4CtSI7bA8BRXjcj09cW/ncucJQf9wH6\n1qJLkV7jgG+WqdsFeAoY4Odb+t8JwG1+vDPwtB/3Bvr58daZ8qEkZ2MXP38MGO/HRwFT/PhS4At+\n3B9YBmxSpNMI4IYqc9of+LuPK1JE4livW+72HQ60AX2BzYAngN2BY4GfZfraHNgI+AMw3Mv6+fXZ\nH5haNPZwYBGweReuxdeA21mzcUrB3oX7Y2tgdsEmwLnAd93uz2bKrwXGlJPP9llCh2nAoBLlF7nt\nFgE3Av3rmNdZwHf8WMBmftwOHO7HVwDnZ+6vYzPt/1bm+CTgvzPnhWv7L8BUoLeX/xQ4oda5FslM\nLdyTmbL/Av6zSjuDizKfWQYWn3/4E3YM265vnw3drlgezJo1K5d+g7BtXoRd1zBr1iy76KKLVn/8\ne4Liz4eonefMbK4fTwJOB67K1I8EZplZIaIxGRgFXALsKOlq4NfADEn9gO3NbCpJs3fr0APvXySn\nYlIZkQOBO8zsdR9jZabuLi9bmlmPIuAySaNID8/bZ+qWm9kSP34SmOnHjwM7+PEhwJGSzvHzPsAQ\nkqOFj7cA+GoN05tnZn/yed4K7AdMydTvB9xpZm+7zBTg34D7gB9JugyYZma/k/QJ4CUzW+g6vOFt\nOgwoaRvgZuA4M6t7fREp5e06MzMfZ2VR/SdJTu/Dfu02AuaY2QeSfkOy3a9I6WvnkHJTOslXUsDM\nyqW+XQt8z8xM0iWk+/YrNc5rPjBe0kbA3WbW5uXvmNmv/XgBaf6lqDVyZv73UySHdL7Puy/wcifh\n8nNNg0oXAO+Z2S1FVS9SU95Pc3WRIAiCIAiCDYimpqYOKZPjxo0rKVePg2VVzqHEw6SZrZS0O3Ao\nKcoxGvhmKdkOHUmnktasGClSsCJT1wv4I/AO6U1+vbxTQuexwDbAnmbWrrShQN8S8u2Z83bW2FDA\nZ83s6S7oU0wttu7cyOxppc0fDgculvQAyZmsZutewK1As5kt7YK+tSBSJHNsibrbgG8ArwPzzexN\ndy7KydeFmWVT634OlN3co0Tbh9zpPgK4SdKVZjYJeC8j9gFr7oP38dTbjGNYDwImmtkFdbZb04H0\nJdI9cGCJ6inAhZKWmNkuXR0j6ApNjVagB9PUaAV6KE2NVqBHEutZ8iNsmw9h1/qpZw3WUEl7+/EX\ngIeK6ucBoyRtJak3Kc1rtqStSelOd5JSwoZ7FOV5SUcDSOojaZNsZ2Z2rZntaWlTghVFde1mtgMp\nXe/4Mvo+CIyW78wmaUAZuYLz0R94xZ2rA0gpesUylbgPOGN1A2mPGtqUY2+ltV+9SPMrtvVDpPU7\nfX091b8DD0naDnjLoxY/IkVClgGDJI1wvfr59clyBdBmZneUUsbXcE2sovP9wCmFvkvYey6wr6Rh\nXr+ppI953WzX9WTgFzXI14WkQZnTY0kplUjaXtLM0q1Wtx1Cui/Gk9ILCzsilrsnngX28uOj6ehg\nVbqPCnUPAMdl1nQNcB1qQmmHxHNI6bfvlBA5EZgezlUQBEEQBEE+1ONgPQWcJmkJaZH89V5eSAlb\nAZxH+uGMVlIk4h5gMNAiqZWUgnaetzsROENSG/AwMLAL+v+etG6lE57SdynJyWsFrszqmxX1v5OB\nka7PCcDSEjKl2he4mLQRyGKlLe2/VywgaYSkGyrMp8A84BpSOuIfzOyu7Nhm1grcREpfe4S0rqsN\n2BWY5/O9ELjEzN4jOWnXSFpEWtO1cdF4ZwOHKG1ysVDSZ4rqh5DWhVXiRuB5YLGPP6ZI5z8DXwJu\ndRvPAXbyunbgXuDT/reiPGWugdJmHYNKVP3Ar8si0tqzb3n5dnSMRJWiCWiTtBD4HPDjSjqQImT7\nuw0+CbyZqasUiSzYaSnpRcQMn/cMoNOcKsz1J6R1dvf7tby2qH4AaXfBYK3T0mgFejAtjVagh9LS\naAV6JPGbQvkRts2HsGv9FDYkWC/x9U5bm9l5VYWDLiPpCuBmM3ui0bp0J0o7Hf7JzO5ttC5rC0k/\nJW0P/7MKMlZjVmpQFy1EylVetBC2zYMWNmy7ijyekVpaWiLlKifCtvkQdi2PJMysU4bS+u5gDSNF\nct6wjr+FFQRBEZJmk9YNnmBmL1aQW3+/FIIgCLqJgQOHsmLFs41WIwiCdZge6WAFQdD9SLL4XgiC\nIAiCIKhMOQernjVYQRAEQReJHPb8CNvmQ9g1H8Ku+RG2zYewa/2EgxUEQRAEQRAEQdBNRIpgEAQd\niBTBIAiCIAiC6kSKYBAEQRAEQRAEQc6EgxUEQbAWiBz2/Ajb5kPYNR/CrvkRts2HsGv9hIMVBEEQ\nBEEQBEHQTcQarCAIOhC/gxUEQbDuMXDwQFa8sKLRagRBkCF+BysIgpqQZDQ3WosgCIKgA80Qz2xB\nsG4Rm1wEQRA0kuWNVqAHE7bNh7BrPoRdcyPWCuVD2LV+anKwJA2V9HiZulmShnevWrUhaYikhZKm\nZ8rWqa8uSftLmlCD3PKM/D3lZCRt1Y16lRwnU19Rb78vZlWR6fb7I9tnLddb0kWSXvB7ZaGkT9fQ\npmK/klbVrvHqNj+U9LikKyRtI2mupAWS9qvl2tY41wGSZkhaJuk+Sf0zdb+VNE/StvXqHgRBEARB\nENRGPRGsdTEufQwww8wOy5Sti3rWopOVOa63n3qo1l+9ejeCWse/ysyG++c33dBvV+Z9MrCbmX0b\nOAhYbGYjzOx3NfZXi8x5wEwz2wl4EPjO6sZmo4AFwBF1ax784+zYaAV6MGHbfAi75kPYNTeampoa\nrUKPJOxaP/U4WBtJmiRpiaTbJfUtFpA0RtJi/1zuZb0kTfCyNklnevkwSfdLWiTpMUld+crZEnil\nqOzVjD4n+pitkiZ62QRJV0t6WNIzko718s0kzXRd2iQd5eVDJS31dsskTZZ0sLdfJmkvl9tU0vhM\nVOJIV+Nd4K81zOXVzHF/SfdKekrStZny1Tmeks7yaMjijE039XatXj7ay0e6votcv82yA0ua5pGd\nVkkrJX2xRr0/AF7zPnplIjSLJJ1WLOx2m+M2vs31PVTS7RmZ1ZE1SYcUy1exWyU65cdW4VXXYZCk\n2W6fxZL2XaOqLvG5zpH0YS+cULin/HyV/70b6AcskHQucAVwjPfbl47XdqykR73uOkmFulrmejQw\n0Y8nkl5CZFlB+ncTBEEQBEEQ5MCH6pDdCfiymc2VNB44FbiqUClpO+ByYE9gJXC/OykvAIPNbDeX\n28KbTAa+b2ZTJfWha+vBegPt2QIz29vH2QU4H9jHzF6XlH2oHGRm+0raGZgKTAHeBo4xszckbQ3M\n9TqAYcBnzWyJpMeAz3v7o3yMY4ELgAfM7CueljVP0kwzewR4xHUaAZxiZl8tnkhBb2cksDPwHHCf\npGPNbEqhUik97iSX6w08KqnF9XzRzD7jcptL2gj4BTDazBZK6ge8VTT2EZl+/x9wl5mtKuhdDjN7\nATjOT78KDCVFaKzI3rhNvwt8yszecifjLOAy4GeSNjGzt4DjgVtc/oIS8peUs5ukacBXzKzUNkvf\ncMfxMeBsM6voPGb6/QLwGzO7zB2dgpO3GTDHzL4r6QpSdOr7pbry/o6W9DczK6Q2vgyMMLMz/Lww\nh39xG/yrmX0g6afAWGBSjXPd1sxe9jFXqHM6YDvpnqlMNvFzB+KNa3ewnLBjXoRt8yHsmg9h19xo\naWmJaEsOhF3X0NLSUtOatHocrOfMbK4fTwJOJ+NgkR72Z5lZIaIxGRhFeiDeUdLVwK+BGf6Qv72Z\nTQUws3fr0APvX8DurkspDgTuMLPXfYyVmbq7vGxp5gFUwGWSRpEeQrfP1C03syV+/CQw048fJz1+\nAhwCHCnpHD/vAwwBlhUGNbMFJEekGvPM7E8+z1uB/UhOYIH9gDvN7G2XmQL8G3Af8CNJlwHTzOx3\nkj4BvGRmC12HN7xNhwElbQPcDBznzlW9HARcZ77FUZG9AT4J7AI87NduI5KD8oGk35Bs9ytS+to5\nQFMp+UoKFBzFElwLfM8dv0tI9+1XapzXfGC8O6p3m1mbl79jZr/24wWk+Zei1shZIf3vU8BwYL7P\nuy/wcifh8nMt12+BF0m2rcwBNfYeBEEQBEGwgdDU1NTB2Rw3blxJuXocrOIHtVLrQTo9TJrZSkm7\nA4cCXwNGA98sJduhI+lUUlTAgMOzb+ol9QL+CLwDTKtjDgXeKaHzWGAbYE8za1faUKBvCfn2zHk7\na2woUpTr6S7oU0wttu7cyOxpj0IdDlws6QGSM1nN1r2AW4FmM1vaBX1rQaT1cmNL1N0GfAN4HZhv\nZm+6c1FOvi7MLJta93Og7OYeJdo+5E73EcBNkq40s0nAexmxD1hzH7yPR2MzjmE9CJhoZhfU2a7A\ny5IGmtnLkgbROYV2CnChpCVmtksXxwi6Qryxzo+wbT6EXfMh7JobEWXJh7Br/dSTljdUUjZt6qGi\n+nnAKElbSeoNjAFme6pXbzO7k5QiNtyjKM9LOhpAUh9Jm2Q7M7NrzWxP35RgRVFdu5ntQEr3Or6M\nvg8Co+U7s0kaUEau4Hz0B15x5+oAUrpbsUwl7gPOWN1A2qOGNuXYW2ntVy/S/Ipt/RBp/U5fpfVU\n/w485Gmab5nZLcCPSJGQZcAgT09EUj+/PlmuANrM7I5Syiit4ZpYqi7D/cAphb5L2HsusK+kYV6/\nqaSPed1s1/VkUjpjNfm6cEejwLHAE16+vaSZpVutbjuEdF+MB250PaH8PfEssJcfH01HB6vSfVSo\newA4TmvWdA1wHWplKvAlPz4JuLuo/kRgejhXQRAEQRAE+VCPg/UUcJqkJaRF8td7eSElbAVpB7MW\noJUUibgHGAy0SGolpaCd5+1OBM6Q1AY8DAzsgv6/B0pube0pfZeSnLxW4MqsvllR/zsZGOn6nAAs\nLSFTqn2Bi0kbgSxW2tL+e8UCkkZIuqHCfArMA64hpSP+wczuyo5tZq3ATaT0tUeAGzx1bVfS2q9W\n4ELgEjN7j+SkXSNpETAD2LhovLOBQ5Q2uVgo6TNF9UOAv1fR+UbgeWCxjz+mSOc/kx78b3UbzyGt\n68PM2oF7gU/734rylLkGSpt1DCpR9QO/LouA/YFvefl2dIxElaIJaJO0EPgc8ONKOpAiZPu7DT4J\nvJmpqxSJLNhpKelFxAyf9wyg05wqzPUK4GBJy0jphpcX1Q8AuiPKGtTLOvUDEj2MsG0+hF3zIeya\nG/F7TfkQdq0frc+/Cu7rnbY2s/OqCgddxjdxuNnMnmi0Lt2J0k6HfzKzexuty9rCN81YbGY/qyBj\nNK89nTYYYmF7foRt8yHsmg9dtWszrM/PbGuD2IwhH8Ku5ZGEmXXKUFrfHaxhpEjOG0W/hRUEQRGS\nZpPWDZ5gZi9WkFt/vxSCIAh6KAMHD2TFC6U2yQ2CoFH0SAcrCILuR5LF90IQBEEQBEFlyjlYXfnt\nqSAIgqBOIoc9P8K2+RB2zYewa36EbfMh7Fo/4WAFQRAEQRAEQRB0E5EiGARBByJFMAiCIAiCoDqR\nIhgE/5+9O4+zo6rzPv75JoBsQ4CoCSAJTEZ5RIUQQFEwacSdVQw7A8OAy0tHVFyGEZUEQUAFjSAq\nYyagiQjMRHYkLOkMBENCCAmQEEHC6hPgUeIERxHTv+ePc25SffuuTVe603zfr9d9dd1Tp6p+9atK\n554+59Q1MzMzMyuZG1hmZuuBx7CXx7kth/NaDue1PM5tOZzX9rmBZWZmZmZm1kc8B8vMuvH3YJmZ\nlW/EiNGsXPl4f4dhZq+AvwfLzFqSGlj+vWBmVi7hz2BmGzY/5MLMrF919ncAg1hnfwcwSHX2dwCD\nVGd/BzBoea5QOZzX9rXUwJI0WtIDddbNljSub8NqjaRRku6TdHOhbEV/xFKPpAmSprVQb0Wh/vX1\n6kjatg/jqnmcwvqGcef7YnaTOn1+fxT32cr1ljRR0oOS1rQaS7P9SlrdWrTdtvm2pAcknS/ptZLm\nSVooab9Wrm2L5/otScsk3S/pvyRtVVj335LmS3p9u7GbmZmZWWva6cEaiP3YhwGzIuJDhbKBGGcr\nMUWd5Xb3045m+2s37v7QyvEfAD4CzOnD/fbmvD8G7BYR/wq8F1gSEXtGxF0t7q+VOrOAt0TEWOAR\n4N/WbhwxHlgIHNh25NYHOvo7gEGso78DGKQ6+juAQaqjvwMYtDo6Ovo7hEHJeW1fOw2sjSVNl7RU\n0lWSNq2uIOkYSUvy67xcNkTStFy2WNJnc/kYSbfmv7TfK2nnXsS/NfBcVdnzhXhOyMdcJOnyXDZN\n0hRJcyU9KunwXL6FpNtyLIslHZLLR+cegWmSlkuaIel9efvlkvbK9TaXNLXQK3FwDuOvwB9bOJfn\nC8vDJN0g6WFJlxTK147xlHRa7g1ZUsjp5nm7Rbn8iFy+d473/hzfFsUDS7ox9wQukrRK0j+2GPca\n4A95H0MKPTT3S/p0deWct7tzjq/M8X5A0lWFOmt71iS9v7p+k7zVFBHLI+KRYv5a8HyOYaSkOTk/\nSyTtuy5UnZ3P9W5Jr8uF0yr3VH6/Ov+8FtgSWCjpy8D5wGF5v5vS/doeJ+mevO6HkirrWjnX2yKi\nK7+dB7yhqspK0r8bMzMzMyvBRm3U3QU4KSLmSZoKfAq4sLJS0nbAecAewCrg1txIeRrYISJ2y/Uq\nQ5ZmAN+MiOskbULv5oMNBbqKBRHxjnycXYGvAO+MiBckFT9UjoyIfSW9GbgOmAn8BTgsIl6UNJz0\n4fS6XH8M8NGIWCrpXuDovP0h+RiHA2cAt0fEyZKGAfMl3RYRvwZ+nWPaE/hERHy8+kQqcWd7A28G\nngRukXR4RMysrFQa5nZirjcUuEdSZ47zmYg4KNf7O0kbA78AjoiI+yRtCfy56tgHFvb7H8A1EbG6\nEnc9EfE0MDG//TgwmtRDE1X5Juf0q8ABEfHn3Mg4DTgX+LGkzSLiz8BRwM9z/TNq1D+7Xt4k3Qic\nHBErG8XdisJ+jwV+FRHn5oZOpZG3BXB3RHxV0vmk3qlv1tpV3t+hkv4nIipDG58F9oyIU/P7yjn8\nn5yDd0XEGkk/AI4DpvfiXP+ZdO2Lukj3TBOTCssd+C+ufaET57EsnTi3ZejEeS1DJ85rOTo7O93b\nUgLndZ3Ozs6W5qS108B6MiLm5eXpwGcoNLBIH/ZnR0SlR2MGMJ70gXhnSVOAm4BZ+UP+9hFxHUBE\n/LWNOMj7F7B7jqWW9wBXR8QL+RirCuuuyWXLtG4+ioBzJY0nfQjdvrBuRUQszcsPAbfl5QeAnfLy\n+4GDJX0pv98EGAUsrxw0IhaSGiLNzI+IJ/J5XgHsR2oEVuwH/DIi/pLrzATeDdwCfEfSucCNEXGX\npLcCv4uI+3IML+Ztuh1Q0muBnwETc+OqXe8Ffhj5kUhV+QbYB9gVmJuv3cakBsoaSb8i5e6/SMPX\nvkT636dH/UYBVBqKfWwBMDU3VK+NiMW5/KWIuCkvLySdfy2t9ppVhv8dAIwDFuTz3hR4tkflJucq\n6Qzg5Yj4edWqZ2jpf/ZJzauYmZmZvYp0dHR0a2xOnjy5ZuJYhU4AACAASURBVL12GljV8z9qzQfp\n8WEyIlZJ2h34APBJ4Ajgc7XqdtuR9ClSr0AAHy7+pV7SEOAx4CXgxjbOoeKlGjEfB7wW2CMiupQe\nKLBpjfpdhfddrMuhSL1cj/Qinmqt5LrnRhGP5F6oDwPfkHQ7qTHZLNdDgCuASRGxrBfxtkKk+XLH\n1Vh3JfAvwAvAgoj4U25c1Ku/3kTEnbnRfSBwmaQLImI68HKh2hrW3Qd/I/fGFhqG7RBweUSc0duY\nJf0T6R54T43VM4GvS1oaEbv29hjWGx39HcAg1tHfAQxSHf0dwCDV0d8BDFruZSmH89q+dobljZZU\nHDZ1Z9X6+cB4SdtKGgocA8zJQ72GRsQvSUPExuVelKckHQogaRNJmxV3FhGXRMQeETGuehhURHRF\nxE7AvaThVLXcARyh/GQ2SdvUqVdpfAwDnsuNq/1Jw92q6zRyC3Dq2g2ksS1sU887lOZ+DSGdX3Wu\n7yTN39lUaT7VR4A78zDNP+dei++QekKWAyPz8EQkbZmvT9H5wOKIuLpWMEpzuC5vEvOtwCcq+66R\n73nAvpLG5PWbS3pjXjcnx/ox1g1pa1T/lSjOddpe0m0NK0ujSPfFVOAnOc5u+6nyOLBXXj6U7g2s\nRvdRZd3twMTCnK5tcgwtkfRBUg/gIRHxUo0qJwA3u3FlZmZmVo52GlgPA5+WtJQ0Sf5HubwyJGwl\ncDppcPEiUk/E9cAOQKekRaQhaKfn7U4ATpW0GJgLjOhF/L8Baj7aOg/pO4fUyFsEXFCMt1g1/5wB\n7J3jOR5YVqNOre0rvkF6EMgSpUfan1VdQdKeki5tcD4V84GLScMRfxsR1xSPHRGLgMtIw9d+DVya\nh669jTT3axHwdeDsiHiZ1Ei7WNL9pKfMvabqeF8A3q/0kIv7JB1UtX4U8L9NYv4J8BSwJB//mKqY\n/x/wT8AVOcd3k+b1kR/KcAPwwfyzYX3qXAOlh3WMrFF+mKSnSMMUb9C6x/pvR/eeqFo6gMWS7gOO\nBL7XKAbg34EJOQf7AH8qrGvUE1nJ0zLSHyJm5fOeBdQ6p5rnClxEepjGrflaXlK1fhvS0wVtvevs\n7wAGsc7+DmCQ6uzvAAapzv4OYNDy9zWVw3ltnzbkbxHP852GR8TpTStbr+WHOPwsIh7s71j6ktKT\nDp+IiBv6O5b1JT80Y0lE/LhBnej/p+8PRp14aFBZOnFuy9CJ81qGTlJexYb8GWwg8sMYyuG81ieJ\niOgxQmlDb2CNIfXkvFj1XVhmVkXSHNK8weMj4pkG9dzAMjMrnRtYZhu6QdnAMrO+lxpYZmZWphEj\nRrNy5eP9HYaZvQL1Gli9+e4pMxvkIsKvPn7Nnj2732MYrC/n1nndkF6VvLpx1fc8V6gczmv73MAy\nMzMzMzPrIx4iaGbdSAr/XjAzMzNrzEMEzczMzMzMSuYGlpnZeuAx7OVxbsvhvJbDeS2Pc1sO57V9\nbmCZmZmZmZn1Ec/BMrNuPAfLzMzMrLl6c7A26o9gzGxgk3r8rjAzs0FixA4jWPn0yv4Ow2zQcg+W\nmXUjKZjU31EMQiuAnfs7iEHKuS2H81qOgZDXSen7Dgebzs5OOjo6+juMQcd5rc9PETQzMzMzMytZ\nSw0sSaMlPVBn3WxJ4/o2rNZIGiXpPkk3F8pW9Ecs9UiaIGlaC/VWFOpfX6+OpG37MK6axymsbxh3\nvi9mN6nT5/dHcZ+tXG9JEyU9KGlNq7E026+k1a1F222bb0t6QNL5kl4raZ6khZL2a+Xatniu20ia\nJWm5pFskDSus+29J8yW9vt3YrQ/091+sBzPnthzOazmc19K4l6Uczmv72unBGoh9yYcBsyLiQ4Wy\ngRhnKzFFneV299OOZvtrN+7+0MrxHwA+Aszpw/325rw/BuwWEf8KvBdYEhF7RsRdLe6vlTqnA7dF\nxC7AHcC/rd04YjywEDiw7cjNzMzMrCXtNLA2ljRd0lJJV0natLqCpGMkLcmv83LZEEnTctliSZ/N\n5WMk3Srpfkn3SurN33S2Bp6rKnu+EM8J+ZiLJF2ey6ZJmiJprqRHJR2ey7eQdFuOZbGkQ3L5aEnL\n8nbLJc2Q9L68/XJJe+V6m0uaWuiVODiH8Vfgjy2cy/OF5WGSbpD0sKRLCuVrx3hKOi33hiwp5HTz\nvN2iXH5ELt87x3t/jm+L4oEl3Zh7AhdJWiXpH1uMew3wh7yPIYUemvslfbq6cs7b3TnHV+Z4PyDp\nqkKdtT1rkt5fXb9J3mqKiOUR8Ugxfy14PscwUtKcnJ8lkvZdF6rOzud6t6TX5cJplXsqv1+df14L\nbAkslPRl4HzgsLzfTel+bY+TdE9e90Np7RMnmp4rcChweV6+nPRHiKKVpH83tr4NqL71Qca5LYfz\nWg7ntTT+vqZyOK/ta+cpgrsAJ0XEPElTgU8BF1ZWStoOOA/YA1gF3JobKU8DO0TEbrneVnmTGcA3\nI+I6SZvQu/lgQ4GuYkFEvCMfZ1fgK8A7I+IFScUPlSMjYl9JbwauA2YCfwEOi4gXJQ0H5uV1AGOA\nj0bEUkn3Akfn7Q/JxzgcOAO4PSJOzsOy5ku6LSJ+Dfw6x7Qn8ImI+Hj1iVTizvYG3gw8Cdwi6fCI\nmFlZqTTM7cRcbyhwj6TOHOczEXFQrvd3kjYGfgEcERH3SdoS+HPVsQ8s7Pc/gGsiYnUl7noi4mlg\nYn77cWA0qYcmqvJNzulXgQMi4s+5kXEacC7wY0mbRcSfgaOAn+f6Z9Sof3a9vEm6ETg5Il7xo5EK\n+z0W+FVEnJsbOpVG3hbA3RHxVUnnk3qnvllrV3l/h0r6n4ioDG18FtgzIk7N7yvn8H9yDt4VEWsk\n/QA4Dpje4rm+PiKezcdcqZ7DAbtI90xjxYGfO+EhLWZmZvaq19nZ2VKDs50G1pMRMS8vTwc+Q6GB\nRfqwPzsiKj0aM4DxpA/EO0uaAtwEzMof8rePiOsAIuKvbcRB3r+A3XMstbwHuDoiXsjHWFVYd00u\nW1b4ACrgXEnjSR9Cty+sWxERS/PyQ8BtefkB0sdPgPcDB0v6Un6/CTAKWF45aEQsJDVEmpkfEU/k\n87wC2I/UCKzYD/hlRPwl15kJvBu4BfiOpHOBGyPiLklvBX4XEfflGF7M23Q7oKTXAj8DJubGVbve\nC/yw8gVKVfkG2AfYFZibr93GpAbKGkm/IuXuv0jD174EdNSq3yiASkOxjy0ApuaG6rURsTiXvxQR\nN+XlhaTzr6XVXrPK8L8DgHHAgnzemwLP9qjc+rlWDyt8hpTbxvZvce/WOjdSy+PclsN5LYfzWhrP\nFSqH87pOR0dHt3xMnjy5Zr12GljVH9RqzQfp8WEyIlZJ2h34APBJ4Ajgc7XqdtuR9ClSr0AAHy7+\npV7SEOAx4CXgxjbOoeKlGjEfB7wW2CMiupQeKLBpjfpdhfddrMuhSL1cj/Qinmqt5LrnRhGP5F6o\nDwPfkHQ7qTHZLNdDgCuASRGxrBfxtkKk+XLH1Vh3JfAvwAvAgoj4U25c1Ku/3kTEnbnRfSBwmaQL\nImI68HKh2hrW3Qd/I/fGFhqG7RBweUSc0cuQn5U0IiKelTSSnkNoZwJfl7Q0Inbt5THMzMzMrI52\nhuWNllQcNnVn1fr5wHhJ20oaChwDzMlDvYZGxC9JQ8TG5V6UpyQdCiBpE0mbFXcWEZdExB4RMa56\nGFREdEXETsC9pOFUtdwBHKH8ZDZJ29SpV2l8DAOey42r/UnD3arrNHILcOraDaSxLWxTzzuU5n4N\nIZ1fda7vJM3f2VRpPtVHgDvzMM0/R8TPge+QekKWAyPz8EQkbZmvT9H5wOKIuLpWMEpzuC6vta7g\nVuATlX3XyPc8YF9JY/L6zSW9Ma+bk2P9GGk4Y7P6r0RxrtP2km5rWFkaRbovpgI/yXF220+Vx4G9\n8vKhdG9gNbqPKutuByYW5nRtk2No1XXAP+XlE4Frq9afANzsxlU/8LyL8ji35XBey+G8lsZzhcrh\nvLavnQbWw8CnJS0lTZL/US6vDAlbSXqCWSewiNQTcT2wA9ApaRFpCNrpebsTgFMlLQbmAiN6Ef9v\ngJqPts5D+s4hNfIWARcU4y1WzT9nAHvneI4HltWoU2v7im+QHgSyROmR9mdVV5C0p6RLG5xPxXzg\nYtJwxN9GxDXFY0fEIuAy0vC1XwOX5qFrbyPN/VoEfB04OyJeJjXSLpZ0PzALeE3V8b4AvF/pIRf3\nSTqoav0o4H+bxPwT4ClgST7+MVUx/z/SB/8rco7vJs3rIyK6gBuAD+afDetT5xooPaxjZI3ywyQ9\nRRqmeIPWPdZ/O7r3RNXSASyWdB9wJPC9RjEA/w5MyDnYB/hTYV2jnshKnpaR/hAxK5/3LKDWOdU8\nV1Jj+X2SlpOGG55XtX4boC96Wc3MzMysBm3I3+Sd5zsNj4jTm1a2XssPcfhZRDzY37H0JaUnHT4R\nETf0dyzrS35oxpKI+HGDOsGk9ReTmZmtZ5NgQ/78ZzZQSCIieoxQ2tAbWGNIPTkvVn0XlplVkTSH\nNG/w+Ih4pkG9DfeXgpmZNTVihxGsfPoVP3DX7FVvUDawzKzvSQr/Xuh7nZ2dfhJTSZzbcjiv5XBe\ny+PclsN5ra9eA6s33z1lZmZmZmZmNbgHy8y6cQ+WmZmZWXPuwTIzMzMzMyuZG1hmZuuBv0ekPM5t\nOZzXcjiv5XFuy+G8ts8NLDMzMzMzsz7iOVhm1o3nYJmZmZk1V28O1kb9EYyZDWxSj98V9gqNGDGa\nlSsf7+8wzMzMrGQeImhmNYRfffx69tkn2rsE1jLPDyiH81oO57U8zm05nNf2NWxgSRot6YE662ZL\nGldOWI1JGiXpPkk3F8pW9Ecs9UiaIGlaC/UGVNxFrcTWrI6kMyWd1ndRdd+npGmSxjepv7WkmZIW\nS5onadcWjjFb0qgm69u6/yVNlLRU0u35/RWS7pf02XwehzfZvpVzPTaf52JJd0narbDuAkkPSZrQ\nTtxmZmZm1rpWerAG4mSMw4BZEfGhQtlAjLOVmAZi3BUbevwVXwEWRcTuwInA9/spjpOBUyLiAEkj\ngb0iYmxETOnDYzwGjM/nejZwaWVFRHwBOAv45z48nlm/6+jo6O8QBiXntRzOa3mc23I4r+1rpYG1\nsaTp+S/vV0natLqCpGMkLcmv83LZkPwX9yX5r+mfzeVjJN2a/3J/r6SdexH31sBzVWXPF+I5IR9z\nkaTLc9k0SVMkzZX0aKW3QNIWkm7LsSyWdEguHy1pWd5uuaQZkt6Xt18uaa9cb3NJU3PPyEJJB+cw\n/gr8sYVzeT7vZ6SkOblnbomkfXP5akln53zdLel1ufygwjFnFcrPlPTTXHe5pFNy+YS8/xskPSzp\nEiUnSfpuIXenSLqgOqfN4q+X9yJJfy/pZkkLcixvkrSVpMcLdTaX9KSkobXq1zj+KlKuG9kVuAMg\nIpYDO1Xy1cDvgTX17uPsSEn35HxWrteJki4qnM/1ksZL+hqwHzBV0reAW4Ad8vXerypP4yR15vO+\nWdKIVs81IuZFROW+mwfsUFVlJenfj5mZmZmVISLqvoDRQBewT34/FTgtL88GxgHbAU8A25IabLcD\nh+R1swr72ir/nAcckpc3ATZtFEOduCYDn6uzblfgYWCb/H7r/HMacGVefjPwSF4eCmyZl4cXykeT\nPszumt/fC0zNy4cAM/PyOcCxeXkYsBzYrCqmPYFLm5zTacC/5WUBW+TlLuDDefl84CuVYxW2PRn4\ndl4+E1iUczsceBIYCUwA/jefl4BZwOHAFsCjwNC8/VzgLb24JvXyfmbhnrkNGJOX3w7cnpd/CUzI\ny0dWctWg/tp91rgvDqpRfg5wQWE/fwX2aPG86t3Hsws5/xBwa14+Efh+of71pB6lyjZ7FO6vJYV6\n0/L12Chfg+GFfExt9Vyr6nyx+r4D3g3c0GS7gPCrz1+ElWP27Nn9HcKg5LyWw3ktj3NbDue1vvx/\ne4/PUq08RfDJiJiXl6cDnwEuLKzfG5gdEX8AkDQDGE8anrSzpCnATcAsSVsC20fEdaSImvU89CBJ\nwO45llreA1wdES/kY6wqrLsmly2T9PrKLoFzlea2dAHbF9atiIilefkh0gd+gAeAnfLy+4GDJX0p\nv98EGEVqaJGPtxD4eJNTW0Dq3dgYuDYiFufylyLipry8EHhvXt5R0lWkBu7GwIrCvq7Nuf29pDtI\njYo/AvMj4glI83+A/SJiptKcoIMkPQxsFBEPNYm1lkZ5R9IWwLuAq/M1JMcNcBVwFDAHOBr4QZP6\nNUXEmXVWnQdMkXQf6dotAta0eF6PUXUfF9bNzD8XkhpMrWj2eL5dgLcCt+bzHgL8rrpSg3NNB5H2\nB04i9ZoVPQO8SdJrIuKl+nuYVFjuyC8zMzOzV6/Ozs6WHvrRSgMrmryHGh8aI2KVpN2BDwCfBI4A\nPlerbrcdSZ8CPpaP8+GIWFlYN4T0gfcl4MYWYq9W/EBZieM44LWknoUupYc2bFqjflfhfRfrcifg\noxHxSC/iWSsi7syNvAOByyRdEBHTgZcL1dYUjnsR8J2IuFHpoQXFD9zFayRqX7NivamkeUoPk3pS\nyjAEeCEiaj0Y4jrgHEnbkHqM7gC2bFC/LRGxmsK8o3yNH2tx21r38Sl5deV+KF6Xv9F96G2PIbVN\nCHgwIvZtc7t1O0gPtrgU+GClwVsREY9JWgY8IemA+o3pSb09vNl65/kB5XBey+G8lse5LYfzuk5H\nR0e3fEyePLlmvVbmYI2W9I68fCxwZ9X6+cB4SdtKGgocA8yRNJw07OyXwFeBcRHxIvCUpEMBJG0i\nabPiziLikojYIyLGFRtXeV1XROxEGq53VJ147wCOkLRtPsY2depVGljDgOdy42p/uvdEtPJlQLcA\np67dQBrbwjY9g0lPrHsuIqYCPyE1NBrFsBXrejZOrFp3aM7tcNLQwAW5fG+luWVDSPm7CyAi5gM7\nkq7dFXXiW9bkFBrmPTdyVkiaWNjnbnndn0jXdApp+Fo0qt8uScNyzyCSPgbMyfciSvPvtmuwbY/7\nuF7V/PNxYGye37Yjqfew7u5rlC0HXidpn3z8jdTCUw8L8Y4C/gv4x4j4bY31uwE7k3qSe9NTaWZm\nZmYNtNLAehj4tKSlpMnxP8rlabJGagSdDnSShl4tiIjrSZPrOyUtAn6W6wCcAJwqaTFprkllAn87\nfkOa89VDHtJ3DqmRtwioPLChXk/cDFLDYzFwPLCsRp1a21d8g/QgkCVKj7Q/q7qCpD0lXdpz0246\ngMV5GNuRwPeaHHcy8J+SFtDzYRRLSNfjbuCsQkP1XuBi0nDH3+ZGQ8VVwNxY94CEYvzDm8TeKO9F\nxwMnKz2w40HSXLaKK0m9ib8olB3XoH4PkiZLOqjGqjcDD+ZG4geAygNXBIwB/tBgt/Xu45r3U0TM\nJTWyHiJdw4XVdeq8r2z/MjAROF/S/aR/U+9s41y/Rvq3cYnSw0bmV63fBng8IrpqbGu2QfJ3tJTD\neS2H81oe57Yczmv7lOZnbVjyfKfhEXF608qvMpLOBFZHxIVV5ROAL0REzUaKpOuBCyNido11BwI7\nR8TFZcTcXyS9BTgpIr7Y37GsL5KOBD4SEcc0qBP12/XWe2JD/H27Iejs7PQQlhI4r+VwXsvj3JbD\nea1PEhHRY0TShtrAGgNcBrwY3b8L61Wv3QaWpGGkYZ6LIuLo9ReprW9Kj99/N+lplbc3qOcGVinc\nwDIzMxtMBlUDy8zKkxpY1tdGjBjNypWP93cYZmZm1kfqNbBamYNlZq8ytb7Twa9X9vrFLy7r78s6\naHl+QDmc13I4r+VxbsvhvLbPDSwzMzMzM7M+4iGCZtaNpPDvBTMzM7PGPETQzMzMzMysZG5gmZmt\nBx7DXh7nthzOazmc1/I4t+VwXtvnBpaZmZmZmVkf8RwsM+vGc7DMzMzMmqs3B2uj/gjGzAY2qcfv\nChugRuwwgpVPr+zvMMzMzCxzD5aZdSMpmNTfUQxCK4CdS9jvpPS9Za9mnZ2ddHR09HcYg47zWg7n\ntTzObTmc1/p69RRBSaMlPVBn3WxJ4/oqwHZIGiXpPkk3F8pW9Ecs9UiaIGlaC/UGVNxFrcTWrI6k\nMyWd1ndRdd+npGmSxjepv7WkmZIWS5onadcWjjFb0qgm69u6/yVNlLRU0u35/RWS7pf02XwehzfZ\nvum55nrfl/RI3vfYQvkFkh6SNKGduM3MzMysda085GIg/mn0MGBWRHyoUDYQ42wlpoEYd8WGHn/F\nV4BFEbE7cCLw/X6K42TglIg4QNJIYK+IGBsRU/rqAJI+BIyJiDcCnwB+VFkXEV8AzgL+ua+OZ20o\no/fKAPyX1ZI4r+VwXsvj3JbDeW1fKw2sjSVNz395v0rSptUVJB0jaUl+nZfLhuS/uC/JPQefzeVj\nJN2a/7p+r6TefOzYGniuquz5Qjwn5GMuknR5LpsmaYqkuZIerfQWSNpC0m05lsWSDsnloyUty9st\nlzRD0vvy9ssl7ZXrbS5pau4ZWSjp4BzGX4E/tnAuz+f9jJQ0J/fMLZG0by5fLensnK+7Jb0ulx9U\nOOasQvmZkn6a6y6XdEoun5D3f4OkhyVdouQkSd8t5O4USRdU57RZ/PXyXiTp7yXdLGlBjuVNkraS\n9HihzuaSnpQ0tFb9GsdfRcp1I7sCdwBExHJgp0q+Gvg9sKbefZwdKemenM/K9TpR0kWF87le0nhJ\nXwP2A6ZK+hZwC7BDvt77VeVpnKTOfN43SxrRxrkeCvw0n+s9wLDC9gArSf9+zMzMzKwErTSwdgEu\njohdgdXAp4orJW0HnAd0AGOBvXMjZSywQ0TslnsOKsPlZgAXRcRY4F3A/+1F3EOBrmJBRLwjx7Mr\nqceiIyL2AIofiEdGxL7AwcD5uewvwGERsRfwHuCCQv0xwLcjYpech6Pz9l/KxwA4A7g9IvbJ239H\n0mYR8euI+HyOaU9Jl9Y6kUrcwLHAryJiHLA7cH8u3wK4O+frTuBjufzOiNgnIvYErgS+XNjt20jX\n413A15V6SwD2Bj4NvBn4B+AjwFXAwZKG5jonAf9RFVtdLea94lLgXyJib1IOfxgR/wMs0rphawfl\nPKypVb/G8T8fEfNyDJMlHVTjuIuBSoP67cAo4A1NzmtiRDxD/fsYYGg+/89Dt1lLPXr1IuIbwL3A\nsRHxZeAQ4NGIGBcRd1XqSdoIuAj4aD7vacA32zjXHYCnCu+fyWUVXaR/P7a+DdjBwBs+f0dLOZzX\ncjiv5XFuy+G8tq+Vpwg+WflQB0wHPgNcWFi/NzA7Iv4AIGkGMB44G9hZ0hTgJmCWpC2B7SPiOoCI\naPbX+B4kidQAmV6nynuAqyPihXyMVYV11+SyZZJeX9klcK7S3JYuYPvCuhURsTQvPwTclpcfAHbK\ny+8nNVC+lN9vQvoAv7xy0IhYCHy8yaktIPVubAxcGxGLc/lLEXFTXl4IvDcv7yjpKmA7YGO6f3y7\nNuf295LuAN5O6k2bHxFPQJr/A+wXETOV5gQdJOlhYKOIeKhJrLU0yjuStiA1+K7O15AcN6RG3lHA\nHOBo4AdN6tcUEWfWWXUeMEXSfaRrtwhY0+J5PUbVfVxYNzP/XAiMbnF/zR7PtwvwVuDWfN5DgN9V\nV2pwrs08A7xJ0msi4qW6tWYXlnfCw9vMzMzsVa+zs7OlBmcrDazqv8bXmnPT40NjRKyStDvwAeCT\nwBHA52rV7bYj6VOkXpoAPhwRKwvrhpA+8L4E3NhC7NWKHygrcRwHvBbYIyK6lB7asGmN+l2F912s\ny51IvQ2P9CKetSLiztzIOxC4TNIFETEdeLlQbU3huBcB34mIG3PvT/EDd/EaifrzpCrlU0m9Tw/T\nvYemLw0BXsg9dNWuA86RtA0wjjScb8sG9dsSEaspzDvK1/ixFretdR+fkldX7ofidfkb3XuGewyp\nbULAg7mntDeeAXYsvH9DLgMgIh6TtAx4QtIBdRvT+/fy6FafG6ml8fyAcjiv5XBey+PclsN5Xaej\no6NbPiZPnlyzXitDBEdLKg5ju7Nq/XxgvKRt8zCzY4A5koaThlD9EvgqMC4iXgSeknQogKRNJG1W\n3FlEXBIRe+ShUyur1nVFxE6koVZH1Yn3DuAISdvmY2xTp16lgTUMeC43rvane09EK18GdAtw6toN\nCk9ta4fSE+uei4ipwE9IDY1GMWzFup6NE6vWHZpzOxyYQOodgzR8c3RuqB4F3AUQEfNJH8qPAa6o\nE9+yJqfQMO+5kbNC0sTCPnfL6/5EuqZTgBsiqVu/XZKG5Z5BJH0MmJPvRZTm323XYNse93G9qvnn\n48BYJTuSeg/r7r5G2XLgdZL2ycffSC089bDgOuCEvO0+wKqIeLZwPruRPupv38ueSjMzMzNroJUG\n1sPApyUtJU2OrzyVLAByI+h0oJM09GpBRFxPmvfRKWkR8LNcB9KHv1MlLQbmAsUJ+K36DbBtrRV5\nSN85pEbeItbNqarXEzeD1PBYDBwPLKtRp9b2Fd8gPQhkidIj7c+qrtBoDlZBB7A4D2M7Evhek+NO\nBv5T0gJ6PoxiCel63A2cVWio3gtcTBru+NvcaKi4CpgbET0ezJEbGQ01yHvR8cDJSg/seJA0D6ni\nSlJv4i8KZcc1qN9Dg3lJbwYezI3ED5Dnh+UheGOAPzTYbb37uOb9FBFzSY2sh0jXcGF1nTrvK9u/\nDEwEzpd0P+nf1DtbPdc8nHSFpEeBH1M1ZxLYBng8Irqqt7WSeQ5WaTw/oBzOazmc1/I4t+VwXtu3\nQX7RcJ7vNDwiTm9a+VVG0pnA6oi4sKp8AvCFiKjZSJF0PXBhRMyuse5AYOeIuLiMmPuLpLcAJ0XE\nF/s7lvVF0pHARyLimAZ1/EXDZfAXDZfGX4JZDue1HM5reZzbcjiv9anOFw1vqA2sMcBlwItV34X1\nqtduA0vSMNIwz0URcfT6i9TWN6XH778b+LeIuL1BvGGqlgAAIABJREFUPTewNiST3MAyMzPrD4Oq\ngWVm5ZHkXwobkBE7jGDl0yubVzQzM7M+Va+B1cocLDN7lYkIv/r4NXv27FL268aV5weUxXkth/Na\nHue2HM5r+9zAMjMzMzMz6yMeImhm3UgK/14wMzMza8xDBM3MzMzMzErmBpaZ2XrgMezlcW7L4byW\nw3ktj3NbDue1fW5gmZmZmZmZ9RHPwTKzbjwHy8zMzKy5enOwNuqPYMxsYJN6/K4wszpGjBjNypWP\n93cYZmY2QHiIoJnVEH71+Wv2AIhhsL76N7fPPvsEg5HnXZTDeS2Pc1sO57V9DRtYkkZLeqDOutmS\nxpUTVmOSRkm6T9LNhbIV/RFLPZImSJrWQr0BFXdRK7E1qyPpTEmn9V1U3fcpaZqk8S1s831Jj0i6\nX9LYFurPljSqyfq27n9JEyUtlXR7fn9Fjuez+TwOb7J903OVdKykxfl1l6TdCusukPSQpAntxG1m\nZmZmrWtliGCUHkX7DgNmRcTphbKBGGcrMQ3EuCs29PgBkPQhYExEvFHSO4AfAfv0QygnA6dExN2S\nRgJ7RcQbc4xNG+MtegwYHxF/lPRB4FLyuUbEFyTNB/4ZmNNHx7OWdfR3AINYR38HMCh1dHT0dwiD\nkvNaHue2HM5r+1oZIrixpOn5L+9XSdq0uoKkYyQtya/zctmQ/Bf3Jfmv6Z/N5WMk3Zr/cn+vpJ17\nEffWwHNVZc8X4jkhH3ORpMtz2TRJUyTNlfRopbdA0haSbsuxLJZ0SC4fLWlZ3m65pBmS3pe3Xy5p\nr1xvc0lTJc2TtFDSwTmMvwJ/bOFcns/7GSlpTu6ZWyJp31y+WtLZOV93S3pdLj+ocMxZhfIzJf00\n110u6ZRcPiHv/wZJD0u6RMlJkr5byN0pki6ozmmz+OvlvUjS30u6WdKCHMubJG0l6fFCnc0lPSlp\naK36NY6/ipTrRg4FfgoQEfcAwySNaLLN74E19e7j7EhJ9+R8Vq7XiZIuKpzP9ZLGS/oasB8wVdK3\ngFuAHfL13q8qT+MkdebzvrkQa9NzjYh5EVG57+YBO1RVWUn692NmZmZmZYiIui9gNNAF7JPfTwVO\ny8uzgXHAdsATwLakBtvtwCF53azCvrbKP+cBh+TlTYBNG8VQJ67JwOfqrNsVeBjYJr/fOv+cBlyZ\nl98MPJKXhwJb5uXhhfLRpA+zu+b39wJT8/IhwMy8fA5wbF4eBiwHNquKaU/g0ibndBrwb3lZwBZ5\nuQv4cF4+H/hK5ViFbU8Gvp2XzwQW5dwOB54ERgITgP/N5yVgFnA4sAXwKDA0bz8XeEsvrkm9vJ9Z\nuGduI/UkAbwduD0v/xKYkJePrOSqQf21+6xxXxxUo/x64F2F97cB41o8r3r38exCzj8E3JqXTwS+\nX3Xs8YVt9ijcX0sK9abl67FRvgbDC/mY2uq5VtX5YvV9B7wbuKHJdgHhV5+/Zg+AGAbrq79zSwxG\ns2fP7u8QBiXntTzObTmc1/ry73+qX60MEXwyIubl5enAZ4ALC+v3BmZHxB8AJM0AxgNnAztLmgLc\nBMyStCWwfURcR4qoWc9DD5IE7J5jqeU9wNUR8UI+xqrCumty2TJJr6/sEjhXaW5LF7B9Yd2KiFia\nlx8ifTAHeADYKS+/HzhY0pfy+02AUaSGFvl4C4GPNzm1BaTejY2BayNicS5/KSJuyssLgffm5R0l\nXUVq4G4MrCjs69qc299LuoPUOPkjMD8inoA0/wfYLyJmKs0JOkjSw8BGEfFQk1hraZR3JG0BvAu4\nOl9DctwAVwFHkYatHQ38oEn9miLizF7E3cxjVN3HhXUz88+FpAZTK5o9nm8X4K3Arfm8hwC/q67U\n7Fwl7Q+cROo1K3oGeJOk10TES/X3MKmw3IGHYJmZmdmrXWdnZ0sP/ejNHKzq91DjQ2NErJK0O/AB\n4JPAEcDnatXttiPpU8DH8nE+HBErC+uGkD7wvgTc2ELs1YofKCtxHAe8ltSz0KX00IZNa9TvKrzv\nYl3uBHw0Ih7pRTxrRcSduZF3IHCZpAsiYjrwcqHamsJxLwK+ExE3Kj20oPiBu3iNRO1rVqw3FfgK\nqQdq2is5jwaGAC9ERK0HQ1wHnCNpG1KP0R3Alg3qt+sZYMfC+zfksqbq3Men5NWV+6F4Xf5G96G3\nPYbUNiHgwYjYt83t1u0gPdjiUuCDlQZvRUQ8JmkZ8ISkA+o3pif19vBWV0d/BzCIdfR3AIOS512U\nw3ktj3NbDud1nY6Ojm75mDx5cs16rczBGq30YACAY4E7q9bPB8ZL2lbSUOAYYI6k4aRhZ78Evkoa\nkvUi8JSkQwEkbSJps+LOIuKSiNgjIsYVG1d5XVdE7EQarndUnXjvAI6QtG0+xjZ16lUaWMOA53Lj\nan+690S08mVAtwCnrt2ghSfU1QwmPbHuuYiYCvyE1NBoFMNWrOvZOLFq3aE5t8NJQwMX5PK9leaW\nDSHl7y6AiJhPaoAcA1xRJ75lTU6hYd4jYjWwQtLEwj53y+v+RLqmU0jD16JR/V64Djgh72MfYFVE\nPJvf3yZpu3ob1rqP61XNPx8Hxub5bTuSeg/r7r5G2XLgdTlOJG0kadcG+6iOdxTwX8A/RsRva6zf\nDdiZ1JPcm55KMzMzM2uglQbWw8CnJS0lTY7/US4PgNwIOh3oJM39WRAR15Mm13dKWgT8LNeB9EH3\nVEmLSXNNmj1soJbfkOZ89ZCH9J1DauQtAioPbKjXEzeD1PBYDBwPLKtRp9b2Fd8gPQhkidIj7c+q\nriBpT0mXNjgfSH+CXSzpPtK8m+81Oe5k4D8lLaDnwyiWkK7H3cBZhYbqvcDFpOGOv82NhoqrgLmx\n7gEJxfiHN4m9Ud6LjgdOVnpgx4OkuWwVV5J6E39RKDuuQf0eJE2WdFCN2G4iNdYeBX4MfCrXFzAG\n+EOD3da7j2veTxExl9TIeoh0DRdW16nzvrL9y8BE4HxJ95P+Tb2z1XMFvkb6t3GJ0sNG5let3wZ4\nPCK6amxrpers7wAGsc7+DmBQ8nfflMN5LY9zWw7ntX1K87M2LHm+0/Do/ph2Iz1FEFgdERdWlU8A\nvhARNRspkq4HLoyI2TXWHQjsHBEXlxFzf5H0FuCkiPhif8eyvkg6EvhIRBzToE7Ub9db73XioWxl\n6aR/cys2xP9Lm+ns7PTQoBI4r+VxbsvhvNYniYjoMSJpQ21gjQEuA16MiA/1czgDSrsNLEnDSMM8\nF0XE0esvUlvflB6//27S0ypvb1DPDSyztgzOBpaZmTU2qBpYZlYeN7DM2uUGlpnZq1G9BlYrc7DM\n7FVHfvnlV4uvESNa/ZaGDYvnXZTDeS2Pc1sO57V9rTym3cxeZfzX+L7nMezlcW7NzGwg8RBBM+tG\nUvj3gpmZmVljHiJoZmZmZmZWMjewzMzWA49hL49zWw7ntRzOa3mc23I4r+1zA8vMzMzMzKyPeA6W\nmXXjOVhmZmZmzXkOlpmZmZmZWcn8mHYz60Hq8ccYMzMzsw3SiB1GsPLplevteB4iaGbdSAom9XcU\ng9AKYOf+DmKQcm7L4byWw3ktj3NbjsGQ10nlfMdnr4YIShot6YE662ZLGtdXAbZD0ihJ90m6uVC2\noj9iqUfSBEnTWqg3oOIuaiW2ZnUknSnptL6Lqvs+JU2TNL6Fbb4v6RFJ90sa20L92ZJGNVnf1v0v\naaKkpZJuz++vyPF8Np/H4U22f0XnKukCSQ9JmtBO3NZHNvT/nAYy57Yczms5nNfyOLflcF7b1soc\nrIHYxXUYMCsiPlQoG4hxthLTQIy7YkOPHwBJHwLGRMQbgU8AP+qnUE4GTomIAySNBPaKiLERMaWv\nDtDoXCPiC8BZwD/31fHMzMzMrLtWGlgbS5qe//J+laRNqytIOkbSkvw6L5cNyX9xXyJpsaTP5vIx\nkm7Nf12/V1Jv2sVbA89VlT1fiOeEfMxFki7PZdMkTZE0V9Kjld4CSVtIui3HsljSIbl8tKRlebvl\nkmZIel/efrmkvXK9zSVNlTRP0kJJB+cw/gr8sYVzeT7vZ6SkOblnbomkfXP5akln53zdLel1ufyg\nwjFnFcrPlPTTXHe5pFNy+YS8/xskPSzpEiUnSfpuIXenSLqgOqfN4q+X9yJJfy/pZkkLcixvkrSV\npMcLdTaX9KSkobXq1zj+KlKuGzkU+ClARNwDDJM0osk2vwfW1LuPsyMl3ZPzWbleJ0q6qHA+10sa\nL+lrwH7AVEnfAm4BdsjXe7+qPI2T1JnP++ZCrH1xritJ/35sfRuwfdWDgHNbDue1HM5reZzbcjiv\nbWulgbULcHFE7AqsBj5VXClpO+A8oAMYC+ydGyljgR0iYreI2B2oDJebAVwUEWOBdwH/txdxDwW6\nigUR8Y4cz67AV4COiNgDKH4gHhkR+wIHA+fnsr8Ah0XEXsB7gAsK9ccA346IXXIejs7bfykfA+AM\n4PaI2Cdv/x1Jm0XEryPi8zmmPSVdWutEKnEDxwK/iohxwO7A/bl8C+DunK87gY/l8jsjYp+I2BO4\nEvhyYbdvI12PdwFfV+otAdgb+DTwZuAfgI8AVwEHSxqa65wE/EdVbHW1mPeKS4F/iYi9STn8YUT8\nD7BI64atHZTzsKZW/RrH/3xEzMsxTJZ0UI3j7gA8VXj/TC5rdF4TI+IZ6t/HAEPz+X8eus1a6tGr\nFxHfAO4Fjo2ILwOHAI9GxLiIuKtST9JGwEXAR/N5TwO+2Yfn2kX692NmZmZmJWjlKYJPVj7UAdOB\nzwAXFtbvDcyOiD8ASJoBjAfOBnaWNAW4CZglaUtg+4i4DiAimv01vgdJIjVAptep8h7g6oh4IR9j\nVWHdNblsmaTXV3YJnKs0t6UL2L6wbkVELM3LDwG35eUHgJ3y8vtJDZQv5febAKOA5ZWDRsRC4ONN\nTm0BqXdjY+DaiFicy1+KiJvy8kLgvXl5R0lXAdsBG9P97wvX5tz+XtIdwNtJvWnzI+IJSPN/gP0i\nYqbSnKCDJD0MbBQRDzWJtZZGeUfSFqQG39X5GpLjhtTIOwqYAxwN/KBJ/Zoi4sxexN3MY1Tdx4V1\nM/PPhcDoFvfX7PF8uwBvBW7N5z0E+F11pVdwrs8Ab5L0moh4qW6t2YXlnfD4677gHJbHuS2H81oO\n57U8zm05nNe1Ojs76ezsbFqvlQZW9V/ja8256fGhMSJWSdod+ADwSeAI4HO16nbbkfQpUi9NAB+O\niJWFdUNIH3hfAm5sIfZqxQ+UlTiOA14L7BERXUoPbdi0Rv2uwvsu1uVOpN6GR3oRz1oRcWdu5B0I\nXCbpgoiYDrxcqLamcNyLgO9ExI2596f4gbt4jUT9eVKV8qmk3qeH6d5D05eGAC/kHrpq1wHnSNoG\nGAfcAWzZoH67ngF2LLx/Qy5rqs59fEpeXbkfitflb3TvGe4xpLYJAQ/mntLeaHiuEfGYpGXAE5IO\nqNuY3r+XRzczMzMbpDo6Oujo6Fj7fvLkyTXrtTJEcLSk4jC2O6vWzwfGS9o2DzM7BpgjaThpCNUv\nga8C4yLiReApSYcCSNpE0mbFnUXEJRGxRx46tbJqXVdE7EQaanVUnXjvAI6QtG0+xjZ16lUaWMOA\n53Ljan+690S08mVAtwCnrt2ghSfU1QwmPbHuuYiYCvyE1NBoFMNWrOvZOLFq3aE5t8OBCaTeMUjD\nN0fnhupRwF0AETGf9KH8GOCKOvEta3IKDfMeEauBFZImFva5W173J9I1nQLcEEnd+r1wHXBC3sc+\nwKqIeDa/vy0Pc62p1n1cr2r++TgwVsmOpN7DuruvUbYceF2OE0kb5eGXrap7rrlsN9LforbvZU+l\n9ZbHsJfHuS2H81oO57U8zm05nNe2tdLAehj4tKSlpMnxlaeSBUBuBJ0OdAKLgAURcT1p3kenpEXA\nz3IdSB/+TpW0GJgLNHvYQC2/AbattSIP6TuH1MhbxLo5VfV64maQGh6LgeOBZTXq1Nq+4hukB4Es\nUXqk/VnVFRrNwSroABZLug84Evhek+NOBv5T0gJ6PoxiCel63A2cVWio3gtcTBru+NvcaKi4Cpgb\nET0ezJEbGQ01yHvR8cDJSg/seJA0D6niSlJv4i8KZcc1qN9DvXlJeYjlCkmPAj8mzyPMQ/DGAH9o\nsNt693HN+yki5pIaWQ+RruHC6jp13le2fxmYCJwv6X7Sv6l3vtJzLdgGeDwiuqq3NTMzM7NXboP8\nouE832l4RJzetPKrjKQzgdURcWFV+QTgCxFRs5Ei6XrgwoiYXWPdgcDOEXFxGTH3F0lvAU6KiC/2\ndyzri6QjgY9ExDEN6viLhs3MzGzwmDSAvmh4AJsJ7KvCFw1b70gaJmk58KdajSuAiLhxsDWuACLi\noVdZ4+oC4IukIahmZmZmVoINsgfLzMojyb8UzMzMbNAYscMIVj69snnFNtXrwWrlKYJm9irjP7z0\nvc7Ozm5PHrK+49yWw3kth/NaHue2HM5r+9yDZWbdSAr/XjAzMzNrbLDNwTIzMzMzMxtw3MAyM1sP\nWvnmd+sd57Yczms5nNfyOLflcF7b5waWmZmZmZlZH/EcLDPrxnOwzMzMzJrzHCwzMzMzM7OSuYFl\nZj1I6vYa+YaR/R3SBs9j2Mvj3JbDeS2H81oe57Yczmv7/D1YZtbTpO5vn530bL+EYWZmZrah8Rws\n63eSVkfE3/V3HACSxgGXA/Mj4uRctiIidu6HWHYHto+Im/P7E4GdImJyk+1uBvYB7oyIQwrlxwBn\nAj+OiO822D6qG1hM8pcPm5mZmRV5DpYNZAPpk/vxwA8qjausrfgk9dW/q7HAh6vKWonlW6Tz6L5h\nxBXABODzrzw0MzMzM6vFDSwbMCRNlrRI0n2SnpY0VdJoScskTZO0XNIMSe+TNDe/3ytvu7ekuyUt\nlHSXpDf2Moytgeeqyp7Px5ggaY6kGyQ9LOmSQuyrJX1H0iJgH0njJHVKWiDpZkkjcr1TJT0k6X5J\nP89lm+dznZfjP1jSxsBZwJE5H0cA/wu82OwEImJ2vXoR8SwwrO2s2CvmMezlcW7L4byWw3ktj3Nb\nDue1fZ6DZQNGRJwJnClpGPDfwEV51RjgoxGxVNK9wNERsa+kQ4AzgI8Ay4D9IqJL0gHAucDEXoQx\nFOiqiusdhbd7A28GngRukXR4RMwEtgB+HRFflLQRMAc4JCJ+L+lI4JvAycC/kob5vSxpq7zPM4Db\nI+LkfO7zgduArwN7RsSp1UFKOjivm9SLc/QfVszMzMxK4gaWDUTTgQsi4n5Jo4EVEbE0r3uI1PgA\neAAYnZe3Bn6ae66CXtzbuWH0FtY17GqZHxFP5PpXAPsBM4E1+SfALsBbgVslidSg+V1etxj4uaRr\ngGty2fuBgyV9Kb/fBBjVKNaIuB64vvWz6+YPksZExG/r1phdWN6pl0exbjo6Ovo7hEHLuS2H81oO\n57U8zm05nNd1Ojs7W+rRcwPLBhRJk4AnI+KnheKXCstdhfddrLuHvwHcERGH50ZZsYlQ2ffZwIFA\nRMS4qnVvIPUcPRoR9zYIsXoOVOX9nwvfzivgwYjYt8b2BwLjgUOAMyS9Ldf/aEQ8UhXTPg3ieCWm\nAPdL+kxEXFazxv4lHdnMzMxsA9XR0dGtwTl5cu3njnmokA0EgrXD3t4LfLbW+iaGAc/k5ZNqVYiI\nr0bEHtWNq7zuaWCHFIY6Ghzn7Xle2BDgKODOGjEuB15XaSBJ2kjSrnndqIiYA5wObEUaWngLsHYY\noKSxeXF1rtMbon7evgL8Q93GlZXCY9jL49yWw3kth/NaHue2HM5r+9zAsoGg0vPzeWB7YEF+sMOk\nqvXVy0XfAs6TtJBe3te5B+pRYNsG1e4FLiYNVfxtRFSG+a2NKyJeJs3/Ol/S/cAi4J15COJ0SYuB\nhcCUiPgfUu/bxpKWSHqA9HALSL1wuxYecrFWfhDGpFoBSvpv4ErgPZKelPS+qiqb5IddmJmZmVkf\n8/dgmRVI+gHwQET8qMa6CcAXit8ttaGR9HpgcURs16COvwfLzMzMrAl/D5ZZa34KnCRpan8H0tfy\nFw3PIvX2mZmZmVkJ3MAyK4iIeyLiHVVfNFxZN2dD7r2KiCsiYmxEfLdp5UndXyN2GFFmaK8KHsNe\nHue2HM5rOZzX8ji35XBe2+enCJpZDx4OaGZmZtY7noNlZt1ICv9eMDMzM2vMc7DMzMzMzMxK5gaW\nmdl64DHs5XFuy+G8lsN5LY9zWw7ntX1uYJmZmZmZmfURz8Eys248B8vMzMysOc/BMjMzMzMzK5kb\nWGbWg6Rur5FvGNnfIW3wPIa9PM5tOZzXcjiv5XFuy+G8ts/fg2VmPU3q/vbZSc/2SxhmZmZmGxrP\nwTLrJ5JGAzdExNtarP8t4GDgJeD/t3fnQZaV9RnHvw8gIgjExDiYGVmMIooiM+CgAWJjAi4ERIyC\nQRHUbEyEimK5JTBdRaKE0kgBEhcyIehIobggZRREhgQVWYZNFiWyCMYZF1DAKBH45Y97Gk/3dPfM\n7XvP9CzfT1VXn/Oec+5971M9XfPr933P+R5wdFXd38f7PQtYAiwA3lNVH5zivJpYYLHYhw9LkiS1\nuQZLWjf1U7VcBOxaVbsDtwHv7vO9fgq8FTilz+skSZK0hiywpNn1uCSfSHJzkvOSbJFkjyTXJlme\n5IYkjwBU1Ver6tHmuiuAef28UVX9pKquAR4e8mfQGnAOe3fMthvm2g1z7Y7ZdsNc+2eBJc2uZwGn\nV9VzgAeAY6rqmqqaX1ULgC8z+YjTm4D/WIv9lCRJ0hpwDZY0S5o1WJdV1Y7N/n7AW6vq0Gb/MOAt\nwAHtB1MleS+woKpePcP3PRF4YNo1WC9uNewInO0aLEmStHFbtmzZuBG90dHRSddgeRdBaXZNrFoK\nIMlzgROAfScUV0cBrwBeMtmLJTkJOBCoZgRsZvab8ZWSJEkbpJGREUZGRh7bHx0dnfQ8pwhKs2uH\nJHs1238GXJ5kW2ApcGRV3Tt2YpKXAe8ADq6qhyZ7sar6u9b0wums8tcWdcs57N0x226YazfMtTtm\n2w1z7Z8jWNLsuhVYlGQJ8G3gTOC1wPbAx5KE34xGnQZsDlzca+aKqjpmTd8oyRzgamBr4NEkxwHP\nqaoHh/mBJEmSNmauwZI0js/BkiRJWj2fgyVJkiRJHbPAkrSqxeO/5sydM4ud2TA4h707ZtsNc+2G\nuXbHbLthrv1zDZakVTgdUJIkaWZcgyVpnCTl7wVJkqTpuQZLkiRJkjpmgSVJa4Fz2Ltjtt0w126Y\na3fMthvm2j8LLEmSJEkaEtdgSRrHNViSJEmr5xosSZIkSeqYBZakVSQZ6Gu7edvN9kdY5ziHvTtm\n2w1z7Ya5dsdsu2Gu/fM5WJJWtXiwy1cuXjmUbkiSJK1vXIMlrWOS3AHsUVX3Jrm8qvZJ8mLg+Ko6\naIDXPQv4E2BlVe02zXk1aIHFYh9WLEmSNmyuwZLWH49VJlW1z2TtM7QEeOmAryFJkqRpWGBJsyTJ\nXya5NsnyJLcnuWTsUOucB1qXbJvkwiS3Jvlwv+9XVZcD9w3Ybc2Qc9i7Y7bdMNdumGt3zLYb5to/\nCyxpllTVR6pqPrAQuBv4wGSntbZfACwCng08I8mh3fdSkiRJ/XANljTLmtGolVU12uy312DdX1Xb\nNGuwRqtqpDnnaOB5VfW2Pt9rB+CLq12D9eJWw47ATn19JNdgSZKkDc6yZcvGjeiNjo5OugbLuwhK\nsyjJUcDTquqYNTh9YsUybj/JQuAjTfsJVXXhjDu234yvlCRJ2iCNjIwwMjLy2P7o6Oik5zlFUJol\nSfYA3g68frrTWtt7JdkhySbAYcDl7ROr6sqqml9VC6YprjLhNbWWOIe9O2bbDXPthrl2x2y7Ya79\ns8CSZs8i4EnApc2NLj7atLdHptrbVwKnAzcB36uqz/XzZkmWAt8Adk7y/WaaoSRJkobINViSxvE5\nWJIkSavnc7AkSZIkqWMWWJK0FjiHvTtm2w1z7Ya5dsdsu2Gu/fMugpJWtXiwy+fMnTOUbkiSJK1v\nXIMlaZwk5e8FSZKk6bkGS5IkSZI6ZoElSWuBc9i7Y7bdMNdumGt3zLYb5to/CyxJkiRJGhLXYEka\nxzVYkiRJq+caLEmSJEnqmAWWJK0FzmHvjtl2w1y7Ya7dMdtumGv/fA6WpFUkq4x2S5IkbTTmzJ3D\nintWzOha12BJGidJDfqgYUmSpPXaYlhdneQaLGkNJHk0ySmt/bcnOWGW+rJD059FrbbTkhw5G/2R\nJEnS6llgSeM9BBya5LdnuyONHwHHJXE67/rujtnuwAbMbLthrt0w1+6YbTfMtW8WWNJ4DwMfBd42\n8UAzonRJkuuSXJxkXtO+JMmpSb6e5L+THNq65vgkVzbXnDiD/vwYuAQ4apL+7J7km81rn59k26b9\n0iTvT/KtJLcm2btp3yTJPzXt1yX58xn0R5IkSdOwwJLGK+AM4IgkW084dhqwpKp2B5Y2+2O2q6q9\ngYOAkwGS7A88s6oWAvOBPZPsM4P+nAwcn1XvPHE28I6mP98G2gXcplW1F/C38NiKqjcDP2vaFwJ/\nkWSHPvujmdpptjuwATPbbphrN8y1O2bbDXPtm9OOpAmq6sEkZwPHAb9sHXoR8Kpm+xyaQqrx+eba\nW5I8pWk7ANg/yXIgwFbAM4HL++zPnUmuAI4Ya0uyDbBtVY291tnAea3LPtt8vwYYK6IOAJ6X5DXN\n/jZNf+5a5U0vbW3viL9cJUnSRm/ZsmVrdNt6CyxpcqcCy4ElrbbpbiXzUGs7re/vq6qPTXVRkkPo\njTwV8JaqWj7Fqe8DPgMsm+R9puvPI/zm33mAt1bVxdNc17Pfas9Qv+7AQrUrZtsNc+2GuXbHbLth\nro8ZGRlhZGTksf3R0dFJz3OKoDReAKrqPnojQm9uHfsG8Lpm+/XAf033GsBXgDcl2Qogye8l+d32\niVX1+aqaX1ULpiiuxvrzHeBm4OBm/37g3rF0qtlUAAAIw0lEQVT1VcAbgMvWoD/HjN0wI8kzkzxh\nimskSZI0A45gSeO1R6k+ACxqtR0LLElyPL2bTxw9yTWP7VfVxUl2Ab7ZLJ96gF5h9uMZ9ucf6I2q\njTkK+JemSLp9df0BPk5vwt/yZj3Xj4BD+uiLBuFf/7pjtt0w126Ya3fMthvm2jcfNCxpHB80LEmS\nNnqLfdCwJK3bfI5Id8y2G+baDXPtjtl2w1z75giWpHGS+EtBkiRt1ObMncOKe1ZMe44jWJLWWFX5\nNeSvE088cdb7sKF+ma25rk9f5mq269vXxprr6oqr6VhgSZIkSdKQWGBJ0lpw5513znYXNlhm2w1z\n7Ya5dsdsu2Gu/XMNlqRxXIMlSZK0ZmqSNVgWWJIkSZI0JE4RlCRJkqQhscCSJEmSpCGxwJIkSZKk\nIbHAkgRAkpcluTXJd5O8c7b7s65LclaSlUluaLU9KclFSb6T5CtJtm0de3eS25LckuSAVvuCJDc0\nuX9obX+OdVGSeUm+luSmJDcmObZpN98BJHl8km8lubbJ9h+bdnMdgiSbJFme5IJm31yHIMmdSa5v\nfm6vbNrMdkBJtk3y6Sanm5LsZa7DY4EliSSbAKcDLwV2BV6XZJfZ7dU6bwm9vNreBXy1qp4FfA14\nN0CS5wCvBZ4NvBz4cJKxuw6dCby5qnYGdk4y8TU3Rg8Db6uqXYEXAYuan0fzHUBVPQTsV1Xzgd2A\nlyTZG3MdluOAm1v75jocjwIjVTW/qhY2bWY7uFOBL1XVs4HnA7dirkNjgSUJYCFwW1XdVVW/Bs4F\nXjnLfVqnVdXlwH0Tml8JnN1snw0c0mwfDJxbVQ9X1Z3AbcDCJNsBW1fVVc15/966ZqNVVSuq6rpm\n+0HgFmAe5juwqvrfZvPx9P4PcB/mOrAk84BXAB9vNZvrcIRV/79qtgNIsg2wb1UtAWjy+jnmOjQW\nWJIA5gJ3t/bvadrUn6dU1UroFQnAU5r2ifn+oGmbSy/rMeY+QZIdgd2BK4A55juYZhrbtcAKYFlV\n3Yy5DsM/A+8A2s++MdfhKODiJFcleUvTZraD2Qn4SZIlzbTWjybZEnMdGgssSeqODxocQJInAp8B\njmtGsibmab59qqpHmymC84B9k4xgrgNJciCwshl1XeWBoy3mOjN7V9UCeiOEi5Lsiz+zg9oMWACc\n0WT7C3rTA811SCywJEHvr1Hbt/bnNW3qz8okcwCaqRM/atp/ADytdd5YvlO1b/SSbEavuDqnqr7Q\nNJvvkFTV/cCXgD0x10HtDRyc5HbgU/TWtp0DrDDXwVXVD5vvPwY+T29Kuz+zg7kHuLuqrm72z6dX\ncJnrkFhgSQK4CnhGkh2SbA4cDlwwy31aH4Txf7G+ADiq2X4j8IVW++FJNk+yE/AM4MpmCsbPkyxs\nFgwf2bpmY/evwM1VdWqrzXwHkOTJY3cFS/IEYH/gWsx1IFX1nqravqqeTu9359eq6g3AFzHXgSTZ\nshnJJslWwAHAjfgzO5BmGuDdSXZumv4IuAlzHZrNZrsDkmZfVT2S5G+Ai+j94eWsqrpllru1Tkuy\nFBgBfifJ94ETgfcDn07yJuAuenddoqpuTnIevTuM/Ro4pqrGpl4sAv4N2ILeHZ2+vDY/x7qoubPd\nEcCNzXqhAt4DnAycZ74z9lTg7OY/QpvQGx28pMnYXIfv/ZjroOYAn0tS9P7P+smquijJ1ZjtoI4F\nPpnkccDtwNHAppjrUOQ3+UiSJEmSBuEUQUmSJEkaEgssSZIkSRoSCyxJkiRJGhILLEmSJEkaEgss\nSZIkSRoSCyxJkiRJGhILLEmStE5JcmmSBVMcOzfJ05vtO5NcNuH4dUluaLbfmOS0ad7nzCQvmuLY\nwUn+fuafQtLGygJLkiStF5L8PrBVVd3eNBWwdZK5zfFdmra26R74uRdwxRTHvgi8OslmA3RZ0kbI\nAkuSJE0ryZZJLkxybZIbkrymab8jyclN2xWtkaUnJ/lMkm81X3/Qep2zmnOvSXJw075Fkk8luSnJ\nZ4EtpujK4fQKn7bzmnaA1wFLJxzfvhkR+06SE1qfaRfgu1VVSY5t3vu6JEsBqqqAbwAHzDA2SRsp\nCyxJkrQ6LwN+UFXzq2o34MutY/c1bWcApzZtpwIfrKq9gD8FPt60vxe4pKpeCLwEOCXJE4C/Bn5R\nVbsCJwJ7TtGPfYCrW/sFnA+8qtk/iFULsBc0x58PvKY19fDlrc/xTmD3qtod+KvWtVcBfzhFXyRp\nUhZYkiRpdW4E9k/yviT7VNUDrWPnNt8/Bbyw2f5j4PQk1wIXAE9MsiW90aB3Ne3LgM2B7ekVMZ8A\nqKobgeun6McOwA8ntP0UuC/JYcDNwC8nHL+4qn5WVb8CPkuvSAN4Kb8psK4HliY5Anikde3/ADtO\n0RdJmpTziiVJ0rSq6rZm5OcVwElJvlpVJ40dbp/afN8E2Kuqft1+nSQAr66q2yZpH9c0VVemOHYe\nvRG0I6e4Ztx+M2q2bVWtaNoOpFfkHQy8N8lzq+rR5r2mW8MlSatwBEuSJE0ryVOBX1bVUuAUoH2H\nv8Oa74cD32y2vwIc17r++a32Y1vtuzeb/wkc0bQ9F9htiq7cBWzX7lrz/XPAycBFk1yzf5Lfaoqq\nQ4CvA/sBlzbvF2D7qroMeBewDfDE5tqnNu8pSWvMESxJkrQ6z6O3XupR4P8Yv07pSUmuB35F7yYT\n0CuuzmjaN6VXQB0DnAR8qLmNeoA76I0anQksSXITcAvj11m1XU5vfdbyZr8AqupBeoXfZKNhV9Kb\nGjgXOKeqlje3bv90c3xT4BNJtmn6dGpV3d8cWwhcuNp0JKklvZvkSJIk9SfJHcAeVXXvWnq/pwOn\nVdWBA77O1fSmMD4yzTmhV8i9oKoeHuT9JG1cnCIoSZJmaq3+lbZ5/tX9Y7eDH+B19pyuuGocBJxv\ncSWpX45gSZIkSdKQOIIlSZIkSUNigSVJkiRJQ2KBJUmSJElDYoElSZIkSUNigSVJkiRJQ/L/uDxL\nkuJrs2cAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "co_t, de_t = compression_decompression_times()\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression speed', fontsize=14, y=1.01)\n", - "\n", - "\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "xlim = (0, np.max((nbytes / 1000000) / np.array(co_t)) + 100)\n", - "ax.set_xlim(*xlim)\n", - "ax.set_ylim(0, len(co_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcVMW9///Xm00CiIAaVlkk6nUJyLgLARQ1eqOEGEQU\nlxjUeM034pqH1yWAC2oi/uJN1MQEUaNZ9F7iBQwqCIPRiKhsCkiUTUFxYXO7YmQ+vz+qGs709DbD\nzPQ0fJ6PRz+m+1Sdqs85dcRTXVWnZWY455xzzjnnnNtxjYodgHPOOeecc87tLLyD5ZxzzjnnnHO1\nxDtYzjnnnHPOOVdLvIPlnHPOOeecc7XEO1jOOeecc845V0u8g+Wcc84555xztcQ7WM4553Y6ks6X\n9HGx4ygWSXtKqpDUv9ixOOfcrsY7WM45VySSJsab4K2SvpT0vqSZki6V1KTY8ZW4PwP7FjuIIvMf\nunTOuSLwDpZzzhXXdKAD0A04EZgMjAX+LulrxQysrkhqWtd1mNkWM/uorutp4FTsAJxzblfkHSzn\nnCuuLWb2oZm9Z2aLzOyXwECgDPhpKpOkppLukPSOpM8kvSTppGRBkg6Q9L+SNkn6RNILkg6OaZJ0\no6S3JX0haZGkwYl9u8XRtDMllUv6XNI8Sd+Mr39I+lTSbEn7JPYbLek1SSMlrY77/VXSnok8EyVN\nkfRTSe8A7xRyTJKaSPovSWtjzKsljUukny5pYaxzvaRZkvaOaT+Q9Ena+fmRpDclbYl/L0xLr5B0\nkaTH4rEulzQiV+NJOkTSDEmb4zmfL2lATBsQy/xO3P5/kl6RVJZWxrHxnH8maY2keyXtnpbnp5Le\nise6MD0uSUfEsv9P0qvAUbnids45V3e8g+Wccw2MmS0GngK+n9j8IPAtYDhwMPAQMFnSNwEkdQSe\nB7YCg4DewH8BjeP+lwNXAdcAhwB/BSZJ6pVW/RjgNuBQYBPwR+BXwLXAEUCLWG5Sd2AEcFqsez9g\nQlqeAcA3gW/HPHmPCRgFfBcYBnwDOBNYFo+3PfAnYCLwb7GcPyTqMxJT5CR9Lx7HXbGuu4F7JX0n\nLc4b47npBfwFeEBSF7L7I/AucDjhnI8BvkjL8wvCeT8MWAFMkdQ8xvVN4GngiXh+vhfLeSAR+63A\nBcB/AAcS2uc3kk6J6S2BqcBbhI75tcCd+BRB55wrDjPzl7/85S9/FeFF6BxMzpJ2G/BpfN+T0HHq\nkpbnr8Cv4/tbgZVA4yzlrQGuT9s2C3g4vu8GVAAXJtK/E7d9N7HtfGBz4vNo4F9A58S2vnG/nonj\nfB9oksizbwHHdDcwPcvx9In775Ml/Xzg48Tn54HfZTj/zyU+VwC3JD43Bj4Dzs7RhpuBc7OkDYhl\nDk9sawlsBH4YPz+UIa5D4357ETq0nwN90/L8f8DU+P5iYAPwtUT6iHh++hf7OveXv/zlr13t5Yuo\nnXOuYRLbRyD6xM9LJCXX1TQDno3vDwWeN7OtVQoK0806Af9IS3oeOCVt22uJ9+/HGF5P29ZKUnMz\nS43UrDWztYk8LxE6CAcCy+O2183sq0SeshzHNDO+fxCYLumfwDPA34BpZmbAwnjsiyU9A8wA/tuy\nr7s6kKqjas8TRt0yHr+ZbZX0IfD1LGVCGBGbIOkHMZ7/MbNliXQD5iTK/EzSa8BBcdNhQE9JwxP7\npNo+1bFuDjxV+TTRhNChhjCCt8jM/i+R/iK+Bss554rCO1jOOdcwHUSYTgZhOncFYRraV2n5/o8d\nkz6N7F8Z0jJtq+4U88/SPuc9JjObL6kb26cVPgQsAE40swrgJElHAScBI4HbJPU3s9coXK7jT6Vn\nPVYzGyvpEUJH9WRgtKQfmdmDBdbfCPg9oaOW3iFaS5guCHAqce1ajlidc841AL4GyznnGhhJhxBu\n1h+Pm+YTbr47mtmKtNd7iTz9lOHx7mb2CWGdUN+0pH7AkloIubOkzonPR8V4c5VdyDFhZp+Z2SQz\n+zFhyuIgSd9IpL9kZjeb2RGEYzwzS31LqXr838oTY0HMbLmZ/drMTiWMkiUfniHg6G0fwnqpQxL1\nzgMONrOVGc7DlphvC9A9Q3qqw7UU+KYqP3XyGHwNlnPOFYWPYDnnXHHtFh/Y0AjYGzgB+E/gZWA8\ngJm9KemPwIOSribclLcjPG1wuZk9AdwL/Ah4PD4UYSPhoRRLzGwR4UELYyW9BbwKnEvoYPXJE18h\n08y+AB6SdBVhzdB9hPVBK7LtUMgxSboCeI8wavUVYV3RZmBNHLk6gfCAiPcJUw67AIuzVPkL4DFJ\n8wjTDU8BziI8VKJG4oMq7iR0hFcRHrffjzA9L+kGSR/FY/kZocP0p5h2B/CipPuA3wKfEKYznmpm\nl5jZp5LuBO6U1Ah4DmhF6LRtNbPfEx60cSswUdJNQGfgupoel3POuR3jHSznnCuuEwgjL1sJT+17\nnXAT/ru0NUs/AK4n3JB3ITzUYC5xvZKZvSupP6EjMZMwevEa4QEIEJ781yru357wNL7TzSy5virT\niEchoyArCT/sOwXYk9DpuaiA/XIeE6GzcQ3hCYJGGPU62cy+kLSZMCL1/4A2hOlzN5nZn8jAzP5X\n0k+AqwkPiFgN/IeZ/S3PseY6/q1AW8LDMjoC6wnn4Jq0/a8ldJb3J3QAv5NaL2Vmr8V2uwUoJzxY\nYwXhYR+p2G+UtI7wFMh7gY8Jnc6fx/TP4tMQ7yN0nt8gPOJ/co7YnXPO1RGFtcLOOedc9UkaDXzf\nzNIf977Li7+HNRPY28w2FDse55xz9cPXYDnnnHN1x5/k55xzuxjvYDnnnHN1x6eJOOfcLsanCDrn\nnHPOOedcLfERLOecc84555yrJd7Bcs4555xzzrla4h0s55xzzjnnnKsl/jtYzrlKJPnCTOecc865\nAphZlafFegfLOVeFP/ym9PzgBz/gwQcfLHYYrpq83UqTt1tp8nYrTQ253aTMv8ThUwSdc24n0L17\n92KH4GrA2600ebuVJm+30lSK7eYdLOecc84555yrJT5F0DlXRbYh711R+/bdWLduVbHDyKtNmzbF\nDsHVgLdbafJ2K03ebqWpFNvNO1jOuQx8DVbK+++XRmfz0EMPLXYIrga83UqTt1tp8nYrTaXYbvLF\n7M65pPAUQf93YTv5Qz+cc87Vqe7du7N69epih+Gy6NatG6tWraqyXVLGpwjW2xosSd0kvZYlbZak\nsvqKJa3urpLmSZqW2LayGLFkI2mApIkF5MsZt6RP4t+Okh6L78+X9KualFdgnaMlXZmvnOpIlilp\noqT+efIPkLQptvM8STcUUMcsSV3zpFfrmpU0VNISSc/Gz3+StEDSqHgcp+fZv5BjPVvSwvh6XlKv\nRNp4SYslDahO3M4555yrW6tXr8bM/NVAX9Xt/Nb3Qy4a4tfAQ4BnzOyUxLaGGGchMeXLYwBm9p6Z\nDStgv9qos6F4zszK4uuWIsUwErjQzAZJ6gAcbmaHmtndtVjHCqC/mfUGbgHuTyWY2VXATcAPa7E+\n10CUl5cXOwRXA95upcnbrTR5u7n6Ut8drKaSHonf4j8mqXl6BklnSVoUX7fHbY3it/eL4jfzo+L2\nnpKmx1GAVyT1qEFMbYAP0rZ9mIjnvFjnfEkPxW0TJd0t6QVJb6VGHiS1lDQjxrJQ0uC4vZukpXG/\nZZIelXRi3H+ZpMNjvhaSJkiaI+lVSafFML4ENhdwLB/GcsbGeOdJWiNpQupwEvEkRxO7xhGZZZJ+\nluk85Ksz27lKkrSvpGmSXpY0W9L+klpLWpXI00LS25IaZ8qfof5NhPOTT3UX0qwHtma79qJhkl6S\n9IakvjH+SiOCkqZI6i/pRqAfMEHSz4Gngc6xjfpVClQqk1Qej3uapPaFHquZzTGz1LUyB+iclmUd\n4Zp3zjnnnHN1oL4fcnEAcIGZzYk3/ZcCd6USJXUEbgf6EG4mp8dOyhqgs5n1ivlax10eBcaZ2WRJ\nzahZh7ExUJHcYGZHxXoOAq4DjjGzjZKSN6YdzKyvpAOBycAk4AtgiJl9KmlPwg3u5Ji/J/B9M1si\n6RVgeNx/cKzjdOB64FkzGylpD2CupBlm9iLwYozpMOBHZnZx+oGk4jaz0cDoWMZzQOqGPznalHx/\nBHBwjP9lSVPNbF6qvFwKPFcp98fYl0s6ErgvjubMlzTAzGYDpwJPmdlWSVXyA4PS6r8i9V7SWOBl\nM5uaoe5jJC0A1gLXmNmSPMc1NJZZRuZrD6CxmR0l6RRgDHBiavcM5d0s6XjgSjObL+keYIqZlcVy\nR8a/TQjtNdjM1ksaBowDRlbjWFMuBKalbasgXPN5jEm8HxhfriEbOHBgsUNwNeDtVpq83UqTt5vb\nUeXl5QWNhNZ3B+ttM5sT3z8C/IREB4twoz/LzDYASHoU6E+Y6tRD0t3A34BnJLUCOpnZZAAzK2QU\noxJJAnrHWDI5HnjczDbGOjYl0p6I25ZK+nqqSOA2hXUyFUCnRNrKxE39YmBGfP8a0D2+Pwk4TdI1\n8XMzoCuwLFWpmb0KVOlcZfEIcJeZLciTb3rq2CRNIoy0zCuwjpRc5wpJLYFjgcfjeQdoGv8+BpwJ\nzAaGA/fkyZ9R7Fhm8irQ1cw+j52hJ4BMo2GZrCDt2kukTUqU363A8vKNpB0AHEL4ckGELw3eTc+U\n41hDJdJxwAWEtkxaC+wvaTcz25K9hDF5wnTOOeec27UMHDiwUkd97NixGfMVew1WpvU7VW5A4816\nb6AcuAT4Xba8lQqSLk1MleuQltYIWAkcCDxZUPSVJW9OU3GMAPYC+phZH8LUw+YZ8lckPlewvaMr\nwihXn/jqYWbLqAFJYwgd2ipT9TIopF12VCNgY1wDlTq+Q2LaZOBkSW2BMmBmnvzVYmafmtnn8f00\nwlTVdgXum+3ag+1tuJXtbfgVlf+7qjINNg8BryeOu3fa+sD8BYQHW9xPGAXbmEwzsxXAUmC1pIOr\nGZtrwHxtQWnyditN3m6lqdTarUOH7kiqs1eHDt0LiqNHjx7MnDkzY9rzzz/PgQceWCvHm6uefL74\n4gtOO+002rRpw5lnngnADTfcwN57702nTp1YvXo1jRo1oqKiIk9JtaO+O1jdJKWmnZ0N/D0tfS7Q\nX1I7SY2Bs4DZcbpdYzP7K3ADUGZmnwLvSPougKRmkr6WLMzM7o03qWVmti4trcLMugOvEEZPMpkJ\nnJG6GY8dgExSHaw9gA/MrCKOIHTLkCeXp4HLtu0g1ejB/wprt04ARqUnZdnlRElt4vkbAryQocyl\nearNea7M7BNgpaShiTJ7xbTPCO1wNzDVgqz5qyuxhok41VCJUdIZcWpqtn2rXHvZssa/q4BDFewD\nHJkrtAzblgF7Szo61t8kTr8siMKTD/8HONfMlmdI7wX0IIz+Li60XOecc87Vr/ffX034zrtuXqH8\nHdOvXz+WLt1+i7gjnaQd8d///d98+OGHbNy4kb/85S+888473HXXXbzxxhu8+26YCLR9QlTdq+8O\n1hvAjyUtISy0/03cnnq63TrgWsJowXzCGpMphIX65ZLmA3+IeQDOAy6TtJDQKdh2I10N/wQyjmbE\nKX23Ejp584HxyXiTWePfR4EjYjznEEYK0vNk2j/lZsLoyiKFh1DclJ5B0mFxbVIuVwCdCOup5sXR\nrFz1ziVMd1tAmOZXaXpg7GTklONcJZ0DjFR4KMnrwOBE2l8II4B/TmwbkSN/FQoP9zg1Q9JQSa/H\nuH5JmIaYmiLaE9iQo9hs117Ga8DMXiB0shbHul5Nz5Plc2r/fwFDgTsU1ozNB46pxrHeSLie742j\nt3PT0tsCq8ysfr7CcfXG1xaUJm+30uTtVpq83XZeq1evZv/999/WiVq9ejV77bUXe+6Z9xa2buR6\n5vuu8AKuAW4vdhwN+QV8B/h/xY6jDo7rYODOYsdRz8c8DPhTnjwG5q9tL8w555yrS5n+X1P3/z8u\n7P9v3bt3t9tuu80OOugga9eunf3whz+0LVu2mJlZeXm5denSxczMzj33XGvUqJG1aNHCdt99d/vF\nL35RpayPPvrITj31VGvTpo21a9fO+vfvX6meO++803r16mVt2rSx4cOHb6vnwQcftH79+lUqS5It\nX77cRo8ebc2aNbOmTZva7rvvbr/97W/ta1/7mjVu3Nh23313u+CCC2zVqlXWqFEj27p1q5mZbd68\n2UaOHGkdO3a0Ll262A033GAVFRXVap/E9ir3UvU9gtUQTQL6KvFDw64yM3vSzH5d7Dhqm5ktNrOr\nix1HfZE0Hrga+H0Buf0VX+3bd8t/uhqAUltb4AJvt9Lk7VaavN1q7o9//CPTp09n+fLlLFu2jFtu\n2f6ToqlRo4cffpiuXbsydepUPv74Y66+uuot1vjx49lnn31Yv349H3zwAePGjauU/vjjj/PMM8+w\ncuVKFi5cyIMPPlilnvTPY8aM4brrrmP48OF8/PHHXHzxxUybNo1OnTrx8ccf88ADD1SJ4/zzz6dZ\ns2asWLGC+fPnM336dH7/+wJujwpU308RbHAsrFP5VrHjcK6uWfih4ULz1mUozjnnnCshP/nJT+jU\nqRMA119/PZdddhk33VRlJQuQ+x6iadOmvPfee6xcuZKePXvSt2/fSumjRo2iffuw4ue0005jwYLs\nD8Ku6b3K+++/z7Rp09i8eTO77bYbzZs35/LLL+f+++/noosuqlGZ6XwEyznndgK+tqA0ebuVJm+3\n0uTtVnNdunTZ9r5bt27bHhxRXT/96U/p2bMnJ510Et/4xje44447KqWnOlcALVq04NNPP61ZwDm8\n/fbb/Otf/6Jjx460a9eOtm3bcskll/DRRx/VWh27/AiWc84555xzLrt33nln2/vVq1dvG81Kl+9J\nfS1btuTOO+/kzjvvZMmSJRx33HEceeSRHHfccXn3+/zzz7d9XrduXY2fCrjPPvvQvHlz1q9fX2dP\nFvQRLOec2wn42oLS5O1WmrzdSpO3W83dc889rF27lg0bNjBu3DiGDx+eMV+HDh1YsWJF1nKefPJJ\nli8PvyKz++6706RJExo3bpy3/t69e7N48WIWLVrEli1bsv7Aby6pKYUdOnTgpJNO4oorruCTTz7B\nzFixYgXPPfdctcvMxjtYzjnnnHPONTDhIUvFf4iTJM4+++xt0/r2228/rr/++ox5r732Wm6++Wba\ntWvHXXfdVSX9zTff5IQTTmD33Xenb9++/PjHP6Z///7b6slmv/3242c/+xmDBg1i//3351vfqv7j\nE5LlP/zww3z55ZccdNBBtGvXjjPOOIN169bl2LuadflidudckiTzfxecc865+iPJHzDVgGVrn7i9\nSs/QR7Ccc84555xzrpZ4B8s5V4Ukf+0Crw5dOhT7Utvl+ZqQ0uTtVpq83Vx98acIOueqGlPsAFy1\nrQR6VG+X98e8XyehOOecc7syX4PlnKtEknkHaxcxxn9U2jnnGgJfg9WwNdg1WJK6SXotS9osSWX1\nFUta3V0lzZM0LbFtZTFiyUbSAEkTC8iXM25Jn8S/HSU9Ft+fL+lXNSmvwDpHS7oyXznVkSxT0kRJ\n/fPkHyBpU2zneZJuKKCOWZK65kmv1jUraaikJZKejZ//JGmBpFHxOE7Ps3/eY435/kvSm7HsQxPb\nx0taLGlAdeJ2zjnnnHOFq+81WA2xaz4EeMbMTklsa4hxFhJTvjwGYGbvmdmwAvarjTobiufMrCy+\nbilSDCOBC81skKQOwOFmdqiZ3V1bFUg6BehpZvsBPwJ+k0ozs6uAm4Af1lZ9rgFpUF8LuUL5mpDS\n5O1WmrzdXH2p7w5WU0mPxG/xH5PUPD2DpLMkLYqv2+O2RvHb+0WSFkoaFbf3lDQ9flP/iqRqrkAA\noA3wQdq2DxPxnBfrnC/pobhtoqS7Jb0g6a3UyIOklpJmxFgWShoct3eTtDTut0zSo5JOjPsvk3R4\nzNdC0gRJcyS9Kum0GMaXwOYCjuXDWM7YGO88SWskTUgdTiKe5Ghi1zgis0zSzzKdh3x1ZjtXSZL2\nlTRN0suSZkvaX1JrSasSeVpIeltS40z5M9S/iXB+8qnuT3WvB7Zmu/aiYZJekvSGpL4x/kojgpKm\nSOov6UagHzBB0s+Bp4HOsY36VQpUKpNUHo97mqT21TjW7wIPA5jZS8Aeif0B1hGueeecc845Vwfq\n+yEXBwAXmNmceNN/KbDtV8gkdQRuB/oQbianx07KGqCzmfWK+VrHXR4FxpnZZEnNqFmHsTFQkdxg\nZkfFeg4CrgOOMbONkpI3ph3MrK+kA4HJwCTgC2CImX0qaU9gTkwD6Al838yWSHoFGB73HxzrOB24\nHnjWzEZK2gOYK2mGmb0IvBhjOgz4kZldnH4gqbjNbDQwOpbxHJC64U+ONiXfHwEcHON/WdJUM5uX\nKi+XAs9Vyv0x9uWSjgTui6M58yUNMLPZwKnAU2a2VVKV/MCgtPqvSL2XNBZ42cymZqj7GEkLgLXA\nNWa2JM9xDY1llpH52gNobGZHxVGjMcCJqd0zlHezpOOBK81svqR7gClmVhbLHRn/NiG012AzWy9p\nGDAOGFngsXYG3kl8Xhu3pZ5mUEG45nOblXjfnWo/PMEVgbdRSRo4cGCxQ3A14O1Wmrzd3I4qLy8v\naCS0vkew3jazOfH9I4Rv9JOOAGaZ2QYzqyB0oPoDK4AecdTo28AnkloBncxsMoCZfWlmX1QnGEkC\nehM6cJkcDzxuZhtjHZsSaU/EbUuBr6eKBG6TtBCYAXSSlEpbmbipXxzTAV4j3MICnARcK2k+UA40\nAyqtAzKzVzN1rrJ4BLjLzBbkyTfdzDbF8zeJqu1SiFznCkktgWOBx+Px/RZIjaw8BpwZ3w8H/pIn\nf0ZmNjpL5+pVoKuZHQr8mth2Bapy7SXSJiXKL+zn0POPpB0AHEL4cmE+odPdKT1TjmPNZy2wv6Td\ncuY6LvHyG3fnnHOu3nXo0mGX+amO4447jgceeKDG+19wwQW0a9eOo48+GoD77ruPDh060Lp1azZs\n2ECjRo1YsWLFDsc5cOBAxowZs+2VTX2PYKV/s59p/U6VG1Az2ySpN/Bt4BLgDODyTHkrFSRdClwU\n6/l3M1uXSGtEuHneAjxZjWNI2ZIh5hHAXkAfM6tQeABE8wz5KxKfK9jeDiKMcr1Zg3gqkTSG0KGt\nMlUvg0LaZUc1AjamRmzSTAZuldQWKANmAq1y5K8WM/s08X6apHsltTOzDQXsm+nauzAmp9pwK9vb\n8Csqf3FRZRpsHgJeN7O+1dwvZS2wT+Jzl7gNADNbIWkpsFrSIDNbXMN6XENTg8e0u+IrLy/3b9VL\nkLdbaSq1dnt/7ft1+rMpO8tPdTz//PM8++yzvPvuuzRv3pyvvvqKq666irlz53LIIYcA4Wl/9am+\nR7C6SUpNOzsb+Hta+lygv6R2khoDZwGz43S7xmb2V+AGoCzeNL8j6bsAkppJ+lqyMDO718z6xAcb\nrEtLqzCz7sArbB89STcTOENSu1hH2yz5Uq22B/BB7FwdR+VRjUJa9mngsm07JJ4AVx0Ka7dOAEal\nJ2XZ5URJbeL5GwK8kKHMpXmqzXmuzOwTYKWkoYkye8W0zwjtcDcw1YKs+asruQYpTjVUqnOlsGau\nY459q1x72bLGv6uAQxXsAxyZK7QM25YBe0s6OtbfJE6/LNRk4Ly479HAJjPb9i9oPIc9CKO/3rly\nzjnnXElbtWoV3bt3p3nz8J32unXr2LJlCwceeOC2PPX9CPz67mC9AfxY0hLCQvvUE85ST7dbB1xL\nmB43n7DGZAphDUl5nDL1h5gHwo3kZXFK3gvkmUKWxT+BdpkS4pS+WwmdvPnA+GS8yazx76PAETGe\nc4ClGfJk2j/lZsKDQBYpPITipvQMkg6La5NyuYIwrexlhYcojMlT71zCdLcFhGl+89Lq3DNPfbnO\nVdI5wEiFh5K8DgxOpP2FMAL458S2ETnyV6HwcI9TMyQNlfR6jOuXhGmIqSmiPYFcI1nZrr2M14CZ\nvUDoZC2Odb2anifL59T+/wKGAncorBmbDxxT6LGa2d8IHdO3CNMqL03L0hZYFafgup2Jj16VpFL6\nNt1t5+1WmrzdaqZHjx6MHz+e3r1707ZtW8466yy+/HL7M7d+97vfsd9++7HXXnsxZMgQ3nvvvYzl\nbNmyhXPPPZe99tqLtm3bctRRR/Hhh9ufp7Zq1Sr69etH69atOfnkk9mwIdyezZ49m3322adSWT16\n9GDmzJk88MADXHTRRbz44ou0bt2aESNG8G//9m8AtG3blhNOOKFKHF9++SVXX3013bp1o2PHjlx6\n6aVs2bKlSr4dscv/0LCka4A9zezavJl3UZK+A/Qws18XO5baJOlgwkNXri52LPVF4aEZ3zOzs3Lk\n8R8a3lWM8R8ads65hkAZfshWUp1OESz0/wE9evSgffv2/O///i+77bYbxx57LJdffjkXX3wxM2fO\n5Mwzz2TGjBkcdNBBXHXVVSxcuJDZs2dXKef+++/nySef5LHHHqNZs2YsWLCA/fbbj1atWnHcccex\nZs0annrqKbp06cLJJ5/MMcccw7hx45g9ezbnnnsub7/9dqWYJkyYwPHHH89DDz3EhAkTeO655wBY\nvXo1++67L1999dW2qYGNGjXirbfeYt999+WKK65g5cqVPPTQQzRp0oSzzz6bQw45hFtvvTXrOcjU\nPontVWYk1fcarIZoEvCgpGlpv4XlIjOryRq1Bi9OkduVOlfjgW8B/5k385i6jsY1BO0712TQ39Wm\nUlsT4gJvt9Lk7VZzo0aNon378P+M0047jQULwvPT/vjHPzJy5Eh69+4NwG233Ubbtm15++236dq1\n0nPaaNq0KevXr+ef//wn3/zmN+nTp0+l9AsuuICePXsCMGzYMKZMmbJDMZtZxrVXv/vd73jttdfY\nY489ALj22msZMWJEzg5Wde3yHSwzW0646XRupxZ/aLjQvHUZiqsDfuPgnHOurqQ6VwAtWrTYNg3w\n3Xff5bDDDtuW1rJlS/bcc0/Wrl1bpYN13nnnsWbNGoYPH87mzZsZMWIE48aNo3Hj8OsxHTp0qFTH\np59+Sm1wALmwAAAgAElEQVT78MMP+fzzzyvFXFFRUev3PfW9Bss551wd8M5VafJ2K03ebqXJ2632\nderUidWrV2/7/Nlnn7F+/Xo6d+5cJW/jxo258cYbWbx4Mf/4xz+YOnUqDz/8cN46WrZsyeeff77t\n89atWyut3aqOvfbaixYtWrB48WI2bNjAhg0b2LRpE5s3b65Redl4B8s555xzzjlXbWeddRYTJ05k\n0aJFbNmyheuuu46jjz66yugVhJkWr7/+OhUVFbRq1YqmTZtuG73KZf/99+eLL75g2rRpfPXVV9xy\nyy2VHrKRSbYRKUlcdNFFXH755ds6aWvXruWZZ54p4GgLt8tPEXTOuZ2BTxEsTd5upcnbrTSVWru1\n79y+Tn+rqtB1uLl+Q2rQoEHcfPPNnH766WzatIljjz2WP//5zxnzrlu3jksuuYS1a9fSqlUrhg8f\nzjnnnJO3jtatW3PvvfcycuRIKioq+OlPf0qXLl2qFXPy8x133MHYsWM5+uijt422/cd//AcnnXRS\nzjKrY5d/iqBzrjJJ5v8ulJ5Su3FwgbdbafJ2K00Nud2yPaXONQzVfYqgd7Ccc5V4B8s555yrX97B\natiq28HyNVjOOeecc845V0u8g+Wcq0KSv3bBV4cO3Yt96e1yysvLix2CqwFvt9Lk7ebqiz/kwjmX\ngU9TKD3lwMAdKuH997MvMnbOOedcYXwNlnOuEknmHaxdla8BcM65YlCWNT6uYcjWPnF78dZgSeom\n6bUsabMkldVXLGl1d5U0T9K0xLaVxYglG0kDJE0sIF/OuCV9Ev92lPRYfH++pF/VpLwC6xwt6cp8\n5VRHskxJEyX1z5N/sKSFkuZLekXS8QXUMUtS1R9xqJxerWtW0lBJSyQ9Gz//SdICSaPicZyeZ/9C\njvXseKwLJT0vqVcibbykxZIGVCdu55xzztWtbt26FX2auL+yv7p161at9qzvNVgNsWs+BHjGzE5J\nbGuIcRYSU748BmBm75nZsAL2q406G4IZZtbbzPoAFwD3FymOkcCFZjZIUgfgcDM71MzursU6VgD9\nzaw3cAuJYzWzq4CbgB/WYn2uwSgvdgCuBnxNSGnyditNDbndVq1ahZn5K8Nr1qxZRY9h1apV1WrP\n+u5gNZX0SPwW/zFJzdMzSDpL0qL4uj1uaxS/vV8Uv5kfFbf3lDRdYRTgFUk9ahBTG+CDtG0fJuI5\nT9tHPx6K2yZKulvSC5LeUhx5kNRS0owYy0JJg+P2bpKWxv2WSXpU0olx/2WSDo/5WkiaIGmOpFcl\nnRbD+BLYXMCxfBjLGRvjnSdpjaQJqcNJxJMcTeyqMCKzTNLPMp2HfHVmO1dJkvaVNE3Sy5JmS9pf\nUmtJqxJ5Wkh6W1LjTPkz1L+JcH6yMrPPEx9bAR8VcFzrga3Zrr1omKSXJL0hqW+Mv9KIoKQpkvpL\nuhHoB0yQ9HPgaaBzbKN+aeepTFJ5PO5pklK/BFjIsc4xs9S1MgfonJZlHeGad84555xzdaC+H3Jx\nAHCBmc2JN/2XAnelEiV1BG4H+hBuJqfHTsoaoLOZ9Yr5WsddHgXGmdlkSc2oWYexMVCR3GBmR8V6\nDgKuA44xs42SkjemHcysr6QDgcnAJOALYIiZfSppT8IN7uSYvyfwfTNbIukVYHjcf3Cs43TgeuBZ\nMxspaQ9grqQZZvYi8GKM6TDgR2Z2cfqBpOI2s9HA6FjGc0Dqhj852pR8fwRwcIz/ZUlTzWxeqrxc\nCjxXKffH2JdLOhK4z8JoznxJA8xsNnAq8JSZbZVUJT8wKK3+K1LvJY0FXjazqekVSxoC3AZ0AL5d\nwHENjfuVkfnaA2hsZkdJOgUYA5yY2j1DeTcrTE280szmS7oHmGJmZbHckfFvE0J7DTaz9ZKGAeOA\nkYUea8KFwLS0bRWEaz6PMYn3A9nRhye4+jCw2AG4GmioP3rqcvN2K03ebqWpIbVbeXl5QSOh9d3B\netvM5sT3jwA/IdHBItzozzKzDQCSHgX6E6Y69ZB0N/A34BlJrYBOZjYZwMxyfrOfiSQBvWMsmRwP\nPG5mG2MdmxJpT8RtSyV9PVUkcJvCOpkKoFMibaWZLYnvFwMz4vvXgO7x/UnAaZKuiZ+bAV2BZalK\nzexVoErnKotHgLvMbEGefNNTxyZpEmGkZV6BdaTkOldIagkcCzwezztA0/j3MeBMYDYwHLgnT/6M\nYscyW9oTwBNxtOgPhM5+IVaQdu0l0ibFv68ChU7OzfeYtgOAQwhfLojwpcG76ZlyHSuApOMI0yH7\npSWtBfaXtJuZbclewpg8YTrnnHPO7VoGDhxYqcM3duzYjPmKvQYr0/qdKjeg8Wa9N2GRwSXA77Ll\nrVSQdGliqlyHtLRGwErgQODJgqKvLHlzmopjBLAX0MfCep8PgOYZ8lckPlewvaMrwihXn/jqYWbL\nqAFJYwgd2ipT9TIopF12VCNgo5mVJY7vkJg2GThZUlugDJiZJ3+NmdnzQJM4wlhI/mzXHmxvw61s\nb8OvqPzfVZVpsHkIeD1x3L2t8vrA/AWEB1vcTxgF25hMM7MVwFJgtaSDqxmba9DKix2Aq4GGvCbE\nZeftVpq83UpTKbZbfXewuklKTTs7G/h7WvpcoL+kdpIaA2cBs+PNcGMz+ytwA1BmZp8C70j6LoCk\nZpK+lizMzO6NN6llZrYuLa3CzLoDrxBGTzKZCZwhqV2so22WfKkO1h7AB2ZWEUcQumXIk8vTwGXb\ndpAOLWCfqsGEtVsnAKPSk7LscqKkNvH8DQFeyFDm0jzV5jxXZvYJsFLS0ESZvWLaZ4R2uBuYakHW\n/NUlqWfifVmsc338PCNOTc22b5VrL1vW+HcVcKiCfYAjc4WWYdsyYG9JR8f6m8TplwVRePLh/wDn\nmtnyDOm9gB6E0d/FhZbrnHPOOecKU98drDeAH0taQlho/5u4PfV0u3XAtYSvYucT1phMISzUL5c0\nnzC969q433nAZZIWEjoFqYcBVMc/gXaZEuKUvlsJnbz5wPhkvMms8e+jwBExnnMIIwXpeTLtn3Iz\n4UEgixQeQnFTegZJh8W1SblcAXQirKeaF0ezctU7lzDdbQFhml+l6YGFjPbkOFdJ5wAjFR5K8jow\nOJH2F8II4J8T20bkyF+FwsM9Ts2Q9H1Jr0uaR+jEDY/5RVgbtyFHsdmuvYzXgJm9QOhkLQZ+SZg+\nSK590vb/FzAUuEPSAsJ/B8dU41hvJFzP98bR27lp6W2BVWZWUXVXV9oGFjsAVwMNaW2BK5y3W2ny\nditNpdhuu/wPDcf1Tnua2bV5M++iJH0H6GFmvy52LLUpTpG7wMyuLnYs9SU+NON7ZnZWjjz+Q8O7\nLP+hS+ecc65QKvYPDTdgk4C+SvzQsKvMzJ7c2TpXAGa2eBfrXI0HrgZ+X+xYXF0oL3YArgZKcW2B\n83YrVd5upakU262+nyLY4MR1Kt8qdhzO1TULPzRcoEKWDLqdTfv21fuleuecc85VtctPEXTOVSbJ\n/N8F55xzzrncfIqgc84555xzztUx72A559xOoBTnqDtvt1Ll7VaavN1KUym2m3ewnHPOOeecc66W\n+Bos51wlvgbLOeeccy4/X4PlnHPOOeecc3XMO1jOObcTKMU56s7brVR5u5Umb7fSVIrttsv/DpZz\nrirJfwfLuZT2nduzbs26YofhnHOuRPgaLOdcJZKMMcWOwrkGZAz4/yudc86l8zVYzjnnnHPOOVfH\n6q2DJambpNeypM2SVFZfsaTV3VXSPEnTEttWFiOWbCQNkDSxgHw545b0SfzbUdJj8f35kn5Vk/IK\nrHO0pCvzlVMdyTIlTZTUP0/+wZIWSpov6RVJxxdQxyxJXfOkV+ualTRU0hJJz8bPf5K0QNKoeByn\n59k/77HGfP8l6c1Y9qGJ7eMlLZY0oDpxuxLRoP7VcoUqxbUFztutVHm7laZSbLf6XoPVEOdYDAGe\nMbNrE9saYpyFxJQvjwGY2XvAsAL2q406G4IZZjYZQNI3gb8C3yhCHCOBC83sH5I6AIeb2X4xrrwd\n6EJIOgXoaWb7SToK+A1wNICZXSVpLvBDYHZt1Oecc8455yqr7ymCTSU9Er/Ff0xS8/QMks6StCi+\nbo/bGsVv7xfFkYhRcXtPSdPjN/WvSOpRg5jaAB+kbfswEc95idGPh+K2iZLulvSCpLdSIw+SWkqa\nEWNZKGlw3N5N0tK43zJJj0o6Me6/TNLhMV8LSRMkzZH0qqTTYhhfApsLOJYPYzljY7zzJK2RNCF1\nOIl4kqOJXeOIzDJJP8t0HvLVme1cJUnaV9I0SS9Lmi1pf0mtJa1K5Gkh6W1JjTPlz1D/JsL5ycrM\nPk98bAV8VMBxrQe2Zrv2omGSXpL0hqS+Mf5KI4KSpkjqL+lGoB8wQdLPgaeBzrGN+qWdpzJJ5fG4\np0lqX+ixAt8FHo7H/RKwR2J/gHWEa97tbGryr58ruoEDBxY7BFcD3m6lydutNJViu9X3CNYBwAVm\nNife9F8K3JVKlNQRuB3oQ7iZnB47KWuAzmbWK+ZrHXd5FBhnZpMlNaNmHcbGQEVyg5kdFes5CLgO\nOMbMNkpK3ph2MLO+kg4EJgOTgC+AIWb2qaQ9gTkxDaAn8H0zWyLpFWB43H9wrON04HrgWTMbKWkP\nYK6kGWb2IvBijOkw4EdmdnH6gaTiNrPRwOhYxnNA6oY/OdqUfH8EcHCM/2VJU81sXqq8XAo8Vyn3\nx9iXSzoSuM/MBsUO2QAzmw2cCjxlZlslVckPDEqr/4rUe0ljgZfNbGp6xZKGALcBHYBvF3BcQ+N+\nZWS+9gAam9lRCqNGY4ATU7tnKO9mhamJV5rZfEn3AFPMrCyWOzL+bUJor8Fmtl7SMGAcMLLAY+0M\nvJP4vDZuez9+riBc87nNSrzvjt+8O+ecc26XV15eXtCUxfruYL1tZnPi+0eAn5DoYBFu9GeZ2QYA\nSY8C/YFbgB6S7gb+BjwjqRXQKTX1y8zyfbNfhSQBvWMsmRwPPG5mG2MdmxJpT8RtSyV9PVUkcJvC\nOpkKoFMibaWZLYnvFwMz4vvXCLewACcBp0m6Jn5uBnQFlqUqNbNXgSqdqyweAe4yswV58k1PHZuk\nSYSRlnkF1pGS61whqSVwLPB4PO8ATePfx4AzCdPWhgP35MmfUexYZkt7Angijhb9gdDZL8QK0q69\nRNqk+PdVoFuB5eV7/vkBwCGELxdE+NLg3fRMuY41j7XA/pJ2M7MtWXMdV8PSXfGsxDvCJai8vLwk\nv53d1Xm7lSZvt9LUkNpt4MCBlWIZO3ZsxnzFXoOVaf1OlRtQM9skqTdh5OES4Azg8kx5KxUkXQpc\nFOv5dzNbl0hrRLh53gI8WY1jSEnenKbiGAHsBfQxswqFB0A0z5C/IvG5gu3tIMIo15s1iKcSSWMI\nHdoqU/UyKKRddlQjYGNqxCbNZOBWSW2BMmAmYSpftvw1ZmbPS2oiaU8zW19A/kzX3oUxOdWGW9ne\nhl9ReSS1yjTYPAS8bmZ9q7lfylpgn8TnLnEbAGa2QtJSYLWkQWa2uIb1OOecc865DOp7DVY3hYX3\nAGcDf09Lnwv0l9ROUmPgLGB2nG7X2Mz+CtwAlJnZp8A7kr4LIKmZpK8lCzOze82sj5mVJTtXMa3C\nzLoDrxBGTzKZCZwhqV2so22WfKkO1h7AB7FzdRyVRzUK+eXWp4HLtu2QeAJcdSis3ToBGJWelGWX\nEyW1iedvCPBChjKX5qk257kys0+AlZKGJsrsFdM+I7TD3cBUC7Lmry5JPRPvy2Kd6+PnGXFqarZ9\nq1x72bLGv6uAQxXsAxyZK7QM25YBe0s6OtbfJE6/LNRk4Ly479HAJjNLTQ9MncMehNFf71ztTHz0\nqiQ1lG9lXfV4u5Umb7fSVIrtVt8drDeAH0taQlho/5u4PfV0u3XAtUA5MJ+wxmQKYQ1JuaT5hOld\nqSf+nQdcJmkhoVOQXMxfqH8C7TIlxCl9txI6efOB8cl4k1nj30eBI2I85wBLM+TJtH/KzYQHgSxS\neAjFTekZJB0W1yblcgXQibCeal4czcpV71zCdLcFhGl+laYHxk5GTjnOVdI5wEiFh5K8DgxOpP2F\nMAL458S2ETnyV6HwcI9TMyR9X9LrkuYROnHDY34R1sZtyFFstmsv4zVgZi8QOlmLgV8Spg+Sa5+0\n/f8FDAXukLSA8N/BMYUeq5n9jdAxfQv4LWGdY1JbYJWZVaTv65xzzjnndpx29V+nj+ud9kx7TLtL\nkPQdoIeZ/brYsdQmSQcTHrpydbFjqS/xoRnfM7OzcuQxxtRfTK6W+BqsujMG6ur/lQ1pbYErnLdb\nafJ2K00Nud0kYWZVZiTV9xqshmgS8KCkaWZ2SrGDaYjMrCZr1Bq8OEVuV+pcjQe+Bfxn3sxj6joa\n50pH+841mRzhnHNuV7XLj2A55yqTZP7vgnPOOedcbtlGsOp7DZZzzjnnnHPO7bS8g+WcczuBQn74\n0DU83m6lydutNHm7laZSbDfvYDnnnHPOOedcLfE1WM65SnwNlnPOOedcfr4GyznnnHPOOefqmHew\nnHNuJ1CKc9Sdt1up8nYrTd5upakU281/B8s5V4VUZbTb7eTat+/GunWrih2Gc845V/J8DZZzrhJJ\nBv7vwq5H+P8PnHPOucL5GiznnHPOOeecq2P11sGS1E3Sa1nSZkkqq69Y0uruKmmepGmJbSuLEUs2\nkgZImlhAvpxxS/ok/u0o6bH4/nxJv6pJeQXWOVrSlfnKqY5kmZImSuqfJ/8Bkv4h6YtCY4nXZNc8\n6dW6ZiUNlbRE0rPx858kLZA0Kh7H6Xn2L+RYz5a0ML6el9QrkTZe0mJJA6oTtysV5cUOwNVAKa4t\ncN5upcrbrTSVYrvV9xqshjj/ZAjwjJldm9jWEOMsJKZ8eQzAzN4DhhWwX23U2RCsB35CaOtiGglc\naGb/kNQBONzM9oPQeaqlOlYA/c1ss6STgfuBowHM7CpJc4EfArNrqT7nnHPOOZdQ31MEm0p6JH6L\n/5ik5ukZJJ0laVF83R63NYrf3i+K38yPitt7SpoeRwFekdSjBjG1AT5I2/ZhIp7zYp3zJT0Ut02U\ndLekFyS9lRp5kNRS0owYy0JJg+P2bpKWxv2WSXpU0olx/2WSDo/5WkiaIGmOpFclnRbD+BLYXMCx\nfBjLGRvjnSdpjaQJqcNJxJMcTewaR2SWSfpZpvOQr85s5ypJ0r6Spkl6WdJsSftLai1pVSJPC0lv\nS2qcKX+G+jcRzk9WZvaRmb0KfFXA8aSsB7Zmu/aiYZJekvSGpL4x/kojgpKmSOov6UagHzBB0s+B\np4HOsY36pZ2nMknl8binSWpfjWOdY2apa2UO0DktyzrCNe92OgOLHYCrgYEDBxY7BFcD3m6lydut\nNJViu9X3CNYBwAVmNife9F8K3JVKlNQRuB3oQ7iZnB47KWuAzmbWK+ZrHXd5FBhnZpMlNaNmHcbG\nQEVyg5kdFes5CLgOOMbMNkpK3ph2MLO+kg4EJgOTgC+AIWb2qaQ9CTe4k2P+nsD3zWyJpFeA4XH/\nwbGO04HrgWfNbKSkPYC5kmaY2YvAizGmw4AfmdnF6QeSitvMRgOjYxnPAakb/uRoU/L9EcDBMf6X\nJU01s3mp8nIp8Fyl3B9jXy7pSOA+MxsUO2QDzGw2cCrwlJltlVQlPzAorf4rUu8ljQVeNrOp+eIu\n4LiGxjLLyHztATQ2s6MknQKMAU5M7Z6hvJslHQ9caWbzJd0DTDGzsljuyPi3CaG9BpvZeknDgHHA\nyBoc64XAtLRtFYRrPo8xifcD8Zt355xzzu3qysvLC5qyWN8drLfNbE58/whh2tZdifQjgFlmtgFA\n0qNAf+AWoIeku4G/Ac9IagV0MrPJAGaW85v9TCQJ6B1jyeR44HEz2xjr2JRIeyJuWyrp66kigdsU\n1slUAJ0SaSvNbEl8vxiYEd+/BnSP708CTpN0TfzcDOgKLEtVGkdiqnSusngEuMvMFuTJNz11bJIm\nEUZa5hVYR0quc4WklsCxwOPxvAM0jX8fA84kTFsbDtyTJ39GsWNZ21aQdu0l0ibFv68C3QosL9/z\nzw8ADiF8uSDClwbvpmfKd6ySjgMuILRl0lpgf0m7mdmW7CWMyROma3jK8Y5w6SkvLy/Jb2d3dd5u\npcnbrTQ1pHYbOHBgpVjGjh2bMV+x12BlWr9T5QbUzDZJ6g18G7gEOAO4PFPeSgVJlwIXxXr+3czW\nJdIaEW6etwBPVuMYUpI3p6k4RgB7AX3MrELhARDNM+SvSHyuYHs7iDDK9WYN4qlE0hhCh7bKVL0M\nCmmXHdUI2JgasUkzGbhVUlugDJgJtMqRv95kufYujMmpNtzK9jb8isojqVWmweYh4HUz61uziEHh\nwRb3AyenOrwpZrZC0lJgtaRBZra4pvU455xzzrmq6nsNVjdJqWlnZwN/T0ufC/SX1E5SY+AsYHac\nbtfYzP4K3ACUmdmnwDuSvgsgqZmkryULM7N7zayPmZUlO1cxrcLMugOvEEZPMpkJnCGpXayjbZZ8\nqQ7WHsAHsXN1HJVHNQr55dangcu27SAdWsA+VYMJa7dOAEalJ2XZ5URJbeL5GwK8kKHMpXmqzXmu\nzOwTYKWkoYkye8W0zwjtcDcw1YKs+XdQpXOgsGauY9bMGa69POWuAg5VsA9wZKGxRMuAvSUdHetv\nEqdfFkThyYf/A5xrZsszpPcCehBGf71ztVMZWOwAXA00lG9lXfV4u5Umb7fSVIrtVt8drDeAH0ta\nQlho/5u4PfV0u3XAtYS5LvMJa0ymEBbql0uaD/wh5gE4D7hM0kJCpyD1MIDq+CfQLlNCnNJ3K6GT\nNx8Yn4w3mTX+fRQ4IsZzDrA0Q55M+6fcTHgQyCKFh1DclJ5B0mFxbVIuVwCdCOup5sXRrFz1ziVM\nd1tAmOZXaXpg7GTklONcJZ0DjFR4KMnrwOBE2l8II4B/TmwbkSN/FQoP9zg1w/b2kt4hnJfrFR6i\n0SpOwesJbMhRbLZrL+M1YGYvEDpZi4FfEqYPkmuftP3/BQwF7pC0gPDfwTGFHitwI+F6vjeubZub\nlt4WWGVmFVV3dc4555xzO0pmpfCU7boT1zvtmfaYdpcg6TtADzP7dbFjqU2SDiY8dOXqYsdSX+JD\nM75nZmflyGOl8fR9V1k5OzaKJXb1/x8UQ0NaW+AK5+1WmrzdSlNDbjdJmFmVGUn1vQarIZoEPChp\nmpmdUuxgGiIzq8katQYvTpHblTpX44FvAf9ZQO66Dsc1MO3bF/qcFuecc87lssuPYDnnKpNk/u+C\nc84551xu2Uaw6nsNlnPOOeecc87ttLyD5ZxzO4FCfvjQNTzebqXJ2600ebuVplJsN+9gOeecc845\n51wt8TVYzrlKfA2Wc84551x+vgbLOeecc8455+qYd7Ccc24nUIpz1J23W6nyditN3m6lqRTbzX8H\nyzlXheS/g+V2Xu07t2fdmnXFDsM559xOytdgOecqkWSMKXYUztWhMeD/73POObejfA2Wc84555xz\nztWxeutgSeom6bUsabMkldVXLGl1d5U0T9K0xLaVxYglG0kDJE0sIF/OuCV9Ev92lPRYfH++pF/V\npLwC6xwt6cp85VRHskxJEyX1z5P/AEn/kPRFobHEa7JrnvRqXbOShkpaIunZ+PlPkhZIGhWP4/Q8\n++c91pjvvyS9Gcs+NLF9vKTFkgZUJ25XIhrUv1quUKW4tsB5u5Uqb7fSVIrtVt9rsBrinIwhwDNm\ndm1iW0OMs5CY8uUxADN7DxhWwH61UWdDsB74CaGti2kkcKGZ/UNSB+BwM9sPQuepNiqQdArQ08z2\nk3QU8BvgaAAzu0rSXOCHwOzaqM8555xzzlVW31MEm0p6JH6L/5ik5ukZJJ0laVF83R63NYrf3i+S\ntFDSqLi9p6Tp8Zv6VyT1qEFMbYAP0rZ9mIjnvFjnfEkPxW0TJd0t6QVJb6VGHiS1lDQjxrJQ0uC4\nvZukpXG/ZZIelXRi3H+ZpMNjvhaSJkiaI+lVSafFML4ENhdwLB/GcsbGeOdJWiNpQupwEvEkRxO7\nxhGZZZJ+luk85Ksz27lKkrSvpGmSXpY0W9L+klpLWpXI00LS25IaZ8qfof5NhPOTlZl9ZGavAl8V\ncDwp64Gt2a69aJiklyS9IalvjL/SiKCkKZL6S7oR6AdMkPRz4Gmgc2yjfmnnqUxSeTzuaZLaF3qs\nwHeBh+NxvwTskdgfYB3hmnc7m5r86+eKbuDAgcUOwdWAt1tp8nYrTaXYbvU9gnUAcIGZzYk3/ZcC\nd6USJXUEbgf6EG4mp8dOyhqgs5n1ivlax10eBcaZ2WRJzahZh7ExUJHcYGZHxXoOAq4DjjGzjZKS\nN6YdzKyvpAOBycAk4AtgiJl9KmlPYE5MA+gJfN/Mlkh6BRge9x8c6zgduB541sxGStoDmCtphpm9\nCLwYYzoM+JGZXZx+IKm4zWw0MDqW8RyQuuFPjjYl3x8BHBzjf1nSVDOblyovlwLPVcr9Mfblko4E\n7jOzQbFDNsDMZgOnAk+Z2VZJVfIDg9LqvyL1XtJY4GUzm5ov7gKOa2gss4zM1x5AYzM7SmHUaAxw\nYmr3DOXdLOl44Eozmy/pHmCKmZXFckfGv00I7TXYzNZLGgaMA0YWeKydgXcSn9fGbe/HzxWEa/7/\nZ+/to6wsrnz/zxeUURQVfOHFjK1haQw3RsWoMTra+ouYmGi8KMaXDC7jaLJwqZiYtbyZ/AIoEaMx\nUWd+jtE4JAquQe4lGdQoAul2FEUQmhcFUW/QiDet3igqs4iO9v79UfvA06fPWx+b7n6692ets049\nVbuqdtV+TvezT9WuU5mmTPpA4uE9CIIgCIJ+T3Nzc01bFrvbwfqTmS319CzStq2fZ8qPBprM7G0A\nSbOBE4HpwEGSbgN+DzwmaXdglJnNBzCzat/sd0CSgMNdl1KcAsw1s3e8j82Zst953npJ+xWaBGYo\nxTnXmyQAACAASURBVMm0AaMyZRvNbJ2nnwcWeXot6REWYBxwhqQf+PUg4ABgQ6FTX4np4FyVYRbw\nczNbVUVuYWFskuaRVlpW1thHgUpzhaTdgC8Bc33eAXb29weAb5K2rZ0H/H9V5EvijmVX80eK7r1M\n2Tx/XwE01NhetfPPPwN8jvTlgkhfGvyfYqFPMNbXgUMk/Y2ZfVBW6uQ6Ww96jo2EI5xDmpubc/nt\nbH8n7JZPwm75pDfZrbGxsZ0u06ZNKynX0zFYpeJ3OjyAmtlmSYcDpwHfBSYAk0vJtmtImgRc6v2c\nbmatmbIBpIfnD4CHOzGGAtmH04IeFwL7AEeaWZvSARC7lJBvy1y3sd0OIq1yvVSHPu2QNJXk0HbY\nqleCWuzySRkAvFNYsSliPvATSUOBscAfgN0ryHcbZe69f/Digg0/ZrsNP6L9SmqHbbBVEPCcmR1f\nn8a8Dvxt5vpTngeAmf1R0nrgVUn/j5k9X2c/QRAEQRAEQQm6OwarQSnwHuAC4Imi8mXAiZKGSRoI\nnA887tvtBprZb4EfAWPNbAvwmqRvAEgaJGnXbGNmdoeZHWlmY7POlZe1mdmBwLOk1ZNS/AGYIGmY\n9zG0jFzBwdoTeNOdq5Npv6pRyy+3LgCu3FYhcwJcZ1CK3foycFVxUZkqp0ray+fvLGBJiTbXV+m2\n4lyZ2fvARknnZNr8vJf9J8kOtwEPWaKs/Cek3RwoxcyNLCtc4t6r0u4rwBFK/C1wTK26OBuAfSV9\n0fvfybdf1sp8YKLX/SKw2cwK2wMLc3gQafU3nKu+RKxe5ZLe8q1s0DnCbvkk7JZP8mi37nawXgAu\nl7SOFGh/p+cXTrdrBa4FmoEWUozJg6QYkmZJLcB9LgPpQfJKSatJTkE2mL9WXgSGlSrwLX0/ITl5\nLcAtWX2zov4+Gzja9fkWsL6ETKn6Ba4nHQSyRukQiuuKBSQd5bFJlbgaGEWKp1rpq1mV+l1G2u62\nirTNr932QHcyKlJhrrJ8C7hE6VCS54AzM2VzSCuA/5bJu7CCfAeUDvf4eon84ZJeI83LPyodorG7\nb8EbDbxdodly917Je8DMlpCcrOeBW0nbB6lUp6j+fwHnAD+VtIr0OTiu1rGa2e9JjunLwC9JcY5Z\nhgKvmFlbcd0gCIIgCILgk6P+/mv2Hu+0d9Ex7UEGSV8DDjKzf+5pXboSSf+NdOjKNT2tS3fhh2b8\ndzM7v4KMMbX7dAq6iIjBqp2p0Fv+9/Wm2IKgdsJu+STslk96s90kYWYddiR1dwxWb2Qe8GtJj5jZ\nV3tamd6ImdUTo9br8S1y/cm5ugX4O+B/VBWeuqO1CYKeY/j+9Wx2CIIgCILa6PcrWEEQtEeSxd+F\nIAiCIAiCypRbweruGKwgCIIgCIIgCII+SzhYQRAEfYBafvgw6H2E3fJJ2C2fhN3ySR7tFg5WEARB\nEARBEARBFxExWEEQtCNisIIgCIIgCKoTMVhBEARBEARBEAQ7mHCwgiAI+gB53KMehN3yStgtn4Td\n8kke7Ra/gxUEQQekDqvdQcDw4Q20tr7S02oEQRAEQa8mYrCCIGiHJIP4uxCUQsT/jCAIgiBIRAxW\nEARBEARBEATBDmaHOliSGiStLVPWJGnsjuy/HJIOkLRS0iOZvI09oUs5JJ0kaWYNcp3WW9JVknYp\nU3aRpNs9PUXSxCptXSRpShWZ9zurYzUKbfo91lSDfJOkFyS1uO33qSJfcf69/MFO6jxI0kLvf4Kk\nEyQ959eHlvusZOpXHaukXSU9JGm9pLWSbsiUHeL9zemM3kFeaO5pBYI6yGNsQRB2yytht3ySR7t1\nxwpWb9xPchbwmJl9NZPXG/WsRad69J4MDK6jXr067Ii5tTLpSpxvZkea2Vgz+7+d7KOe8mLGAub9\nzwUuBG4ws7HA1hrbq0XmZjP7LHAkcIKk00gdv2hmnwMOk3RQJ3UPgiAIgiAIaqA7HKydJc2StE7S\nA6VWTiSdL2mNv270vAGSZnreaklXef5oXwVYJenZOh8U9wLeLMp7K6PPRO+zRdJvPG+mpNskLZH0\nsqTxnr+bpEWuy2pJZ3p+g68izJS0QdJsSad6/Q2SvuBygyXdI2mppBWSznA1PgTerWEsb3k70zKr\nM5u8zcG+mtHi8zhB0hXAKKBJ0mKve7HrtBQ4PtP2FtKDfyW2uhyS9pM0z23TIumLhSnNzO01kpa5\nzBTPmyFpUkZmiqTvlZMv4mPg7RrmCTp3v2+bf1+tKsztCkm7ucwQSXPdzvdl9N8oaZinj/LVs32B\n+4CjvZ3LgHOB67N1vc4ASTdJesbHfWmtYzWzrWb2uKc/AlYCnyoSe4P0GQj6FI09rUBQB42NjT2t\nQlAHYbd8EnbLJ3m0W3ecIvgZ4GIzWyrpHmAS8PNCoaSRwI2kb9s3AwvdSdkE7G9mn3e5PbzKbNK3\n/vMlDaI+J3Eg0JbNMLNjvZ8xwA+B48zsHUnZB9ERZna8pM8C84F5wF+Bs8xsi6S9gaVeBjAaONvM\n1kl6FjjP65/pfYwH/hFYbGaXSNoTWCZpkZk9DTztOh0FfMfMLiseSEFvM5sCTPE2/gP4Z+ArwOtm\n9nVvZ4iZvS/paqDRxzcCmEqa//dI+4xWepu3VJtIM3sgc3k70Gxm4yUJ2L0g5v2fChxsZsd4+XxJ\nJwBzgFuBO1z+XGBcOXkzexJ32sxsE3COtz8SuLsw3hL8WtJ/AfPMbHqVcW2bf+D7wCQze1rSYJLN\nAY4AxgCtwBJJXzKzp+i4ymRm9pakfwC+b2YFJ/w44EEzmyepISN/CbDZzI71e3yJpMfM7NVOjBW/\nd88gzW2WNtJnoAJTM+lG4uE9CIIgCIL+TnNzc01bFrtjBetPZrbU07OAE4rKjwaazOxtM2sjOVAn\nAn8EDvJVo9OA9yXtDowys/kAZvahmf2VTuAP6oeTHLhSnALMNbN3vI/NmbLfed56YL9Ck8AMSauB\nRcAoSYWyjWa2ztPPeznAWuBAT48DrpXUQnJuBgEHZBUysxWlnKsyzAJuMbMW7+dUXyE6wcwKsVBi\n+6rSsWyf/49Izk69nAL8i+tsmf4KjHN9VpKcuM+QHKhVwL6SRkj6PPC2mb1eTr5c52b25woOxwVm\ndhjwd8DfSfpWJ8a1BPiFr/4N9fsUYJn3acAqttv0k55xPg6Y6PfEM8AwisZdZaxIGgjcD9xqZq8U\nFW8ifQYqMDXzaqxd86AHae5pBYI6yGNsQRB2yytht3zSm+zW2NjI1KlTt73K0R0rWB2+zS8h0+GB\n1Mw2SzocOA34LjCBFDtU8eHVt5pd6v2cbmatmbIBJMftA+DhToyhwAcldL4Q2Ac40szalA6d2KWE\nfFvmuo3tcy/SKtdLdejTDklTSQ7tvQBm9pLSQSKnA9N9ZazUyk1X/ehRtfggATPM7O4SZXNJNh7B\ndievknyn4p/M7M/+/p+S7geOITmjtdT9qaSHgK+RVpPGeVHWvh+z3aYfsf3Li5KHiVRBwBVmtrCO\nugXuAjaY2T+VKPslsEDSMWb2nU/QRxAEQRAEQVBEd6xgNUg61tMXAE8UlS8DTpQ0zL91Px943Lfb\nDTSz3wI/Asaa2RbgNUnfgG2nsu2abczM7sgcZNBaVNZmZgcCzwLfLKPvH4AJmRiaoWXkCk7JnsCb\n7lydDDSUkKnEAuDKbRWkI2qo01GZFLv1ZeCqTN5IYKuZ3Q/cTDpkAdJWwMKWy2dI8z9U0s4kJ6dU\n+5dn46TKsJi0BbQQRzSkUN3fFwDfLsQwSRrlsUkADwDnAWeTnK1y8vsUtVkVSQP9fsLH+HXgOb8+\nS5mT9srU/7SZPW9mNwHLgUOrdLkROMrTZ9eqZ4YFwCRJO3n/Bxff51X0nQ7sYWZXlxG5BrgknKu+\nRmNPKxDUQR5jC4KwW14Ju+WTPNqtOxysF4DLJa0jBdbf6fkG4E7QtaT9LS3AcjN7ENgfaPZtUve5\nDMBE4ErfkrcEGF6HTi+Stl11wLf0/YTk5LUAhTikcitxs0kHF6wGvgWsLyFTqn6B60kHgaxROqb7\numIBPyjhrgrjAbiadHjFcj9EYSpwGCmmqwX4MVBYvbobeFTSYp//aaTYsSeAdR1aThwK/KWKDpOB\nkyWtITmxYzy/YOuFpG1rT7vMXDxOy+d9CLDJzN6oID8k22YWSSN9pamYvyGt2KwibTXc5HMAKU6u\n2mEik5WOPF9NOvzikRIyWX2uA26XtIy0mlWOcvfEr0h2WOn3xJ0UrTaXG6uk/UnxfWO0/WCObxeJ\nDQVerqBXEARBEARBUCdK4SP9C0k/APY2s2urCgcASJoPjPc4rT6DpHuBq82smvPYJ/AYxDXAOWa2\noYyM9c5fLQgq08yOX8US/fF/xo6kubk5l9/O9nfCbvkk7JZPerPdJGFmHXZVdccKVm9kHnC8Mj80\nHFTGzM7sa84VgJlN7EfO1SGkVeIW0ipuEARBEARB0MX0yxWsIAjKk1awgqAjw4c30Nr6Sk+rEQRB\nEAS9gnIrWN1ximAQBDkjvngJgiAIgiCoj/66RTAIgqBP0Zt+JySonbBbPgm75ZOwWz7Jo93CwQqC\nIAiCIAiCIOgiIgYrCIJ2SLL4uxAEQRAEQVCZOEUwCIIgCIIgCIJgBxMOVhAEQR8gj3vUg7BbXgm7\n5ZOwWz7Jo93CwQqCIAiCIAiCIOgiIgYrCIJ2xO9gBUHQ1xi+/3BaN7X2tBpBEPQxysVghYMVBEE7\nJBlTe1qLIAiCLmRq/L5fEARdTxxyEQRB0JfZ2NMKBHURdssleYwJCcJueSWPdtuhDpakBklry5Q1\nSRq7I/svh6QDJK2U9Egmr1f9m5N0kqSZNch1Wm9JV0napUzZRZJu9/QUSROrtHWRpClVZN7vrI7V\nKLTp91hTDfJNkl6Q1OK236eKfMX59/IHO6nzIEkLvf8Jkk6Q9JxfH1rus5KpX+tYx0paI+lFSbdm\n8g/x/uZ0Ru8gCIIgCIKgdrpjBas3rsmfBTxmZl/N5PVGPWvRqR69JwOD66hXrw47Ym6tTLoS55vZ\nkWY21sz+byf7qKe8mLGAef9zgQuBG8xsLLC1xvZqkfkX4BIzOwQ4RNJppI5fNLPPAYdJOqiTuge9\nnbBoPgm75ZLGxsaeViGog7BbPsmj3brDwdpZ0ixJ6yQ9UGrlRNL5/o37Gkk3et4ASTM9b7Wkqzx/\ntK8CrJL0bJ0PinsBbxblvZXRZ6L32SLpN543U9JtkpZIelnSeM/fTdIi12W1pDM9v0HSeq+3QdJs\nSad6/Q2SvuBygyXdI2mppBWSznA1PgTerWEsb3k70zKrM5u8zcGSHvL8Nb5qcgUwCmiStNjrXuw6\nLQWOz7S9hfTgX4mtLoek/STNc9u0SPpiYUozc3uNpGUuM8XzZkialJGZIul75eSL+Bh4u4Z5gs7d\n79vm31erCnO7QtJuLjNE0ly3830Z/TdKGubpo3z1bF/gPuBob+cy4Fzg+mxdrzNA0k2SnvFxX1rr\nWCWNAIaY2XLPupf0hUKWN0ifgSAIgiAIgqCL2akb+vgMcLGZLZV0DzAJ+HmhUNJI4EbgSGAzsNCd\nlE3A/mb2eZfbw6vMJn3rP1/SIOpzEgcCbdkMMzvW+xkD/BA4zszekZR9EB1hZsdL+iwwH5gH/BU4\ny8y2SNobWOplAKOBs81snaRngfO8/pnex3jgH4HFZnaJpD2BZZIWmdnTwNOu01HAd8zssuKBFPQ2\nsynAFG/jP4B/Br4CvG5mX/d2hpjZ+5KuBhp9fCOAqaT5fw9oBlZ6m7dUm0gzeyBzeTvQbGbjJQnY\nvSDm/Z8KHGxmx3j5fEknAHOAW4E7XP5cYFw5eTN7EnfazGwTcI63PxK4uzDeEvxa0n8B88xsepVx\nbZt/4PvAJDN7WtJgks0BjgDGAK3AEklfMrOn6LjKZGb2lqR/AL5vZgUn/DjgQTObJ6khI38JsNnM\njvV7fImkx8zs1RrGuj/ps1Ngk+dlaSN9BsqT3Yh4IPEtex7YSNgpj4Tdcklzc3Muv1Xv74Td8klv\nsltzc3NNMWHd4WD9ycyWenoWcAUZBws4Gmgys7cBJM0GTgSmAwdJug34PfCYpN2BUWY2H8DMPuys\nMv6gfrjrUopTgLlm9o73sTlT9jvPWy9pv0KTwAxJJ5IeXEdlyjaa2TpPPw8s8vRa0mMrwDjgDEk/\n8OtBwAHAhkKnZrYC6OBclWEWcIuZtUjaAvxM0gzgYXdMCjoXVpWOpf38zwEOrrGvYk4B/t51NqA4\n9moccKqkld7/biQHaqakfd3Z2w9428xelzS5lDzwJCUwsz8D5ZyrC8zsz776NE/St8ys3D1QzBLg\nF35vznPdAJZ5n0haRbLpU2RW7OpkHGkb3wS/3oM07lcLAlXGWo1NpM/As2UlTq6z5SAIgiAIgj5K\nY2NjO2dv2rRpJeW6w8Hq8G1+CZkOD6RmtlnS4cBpwHeBCaTYoYoPr77V7FLv53Qza82UDQD+CHwA\nPNyJMRT4oITOFwL7AEeaWZvSoRO7lJBvy1y3sX3uRVrleqkOfdohaSrJob0XwMxeUjpI5HRguq+M\nlVq5+aQOQYFq8UECZpjZ3SXK5pJsPIK0olVNvlPxTwVHyMz+U9L9wDGUd7KL6/5U0kPA10irSeO8\nKGvfj9lu04/YvrJa8jCRKgi4wswW1lH3deBvM9ef8rwsvwQWSDrGzL5TRx9BbyRWQfJJ2C2X9JZv\n04POEXbLJ3m0W3fEYDVIOtbTFwBPFJUvA06UNEzSQOB84HHfbjfQzH4L/AgYa2ZbgNckfQO2ncq2\na7YxM7sjc5BBa1FZm5kdSPrm/ptl9P0DMCETQzO0jFzBKdkTeNOdq5OBhhIylVgAXLmtgnREDXU6\nKpNit74MXJXJGwlsNbP7gZtJhyxA2gpY2HL5DGn+h0rameTklGr/8mycVBkWk7aAFuKIhhSq+/sC\n4NuFGCZJozw2CeAB4DzgbJKzVU5+n6I2qyJpoN9P+Bi/Djzn12dJuqFK/U+b2fNmdhOwHDi0Spcb\ngaM8fXatemZYAEyStJP3f3DxfV4Ov+fflVTYVjkR+PcisWtIh2CEcxUEQRAEQdDFdIeD9QJwuaR1\npMD6Oz3fYNsD4bWk2J8WYLmZPUiKG2mW1EI6HOBarzcRuFLSatLWreF16PQiMKxUgW/p+wnJyWsB\nCnFI5VbiZpMOLlgNfAtYX0KmVP0C15MOAlmjdEz3dcUCSgcl3FVhPABXkw6vWK50iMJU4DBSTFcL\n8GPStkuAu4FHJS32+Z9Gih17AljXoeXEocBfqugwGThZ0hqSEzvG8wu2XgjcDzztMnPxOC2f9yHA\nJjN7o4L8kGybWSSN9JWmYv6GtGKzihRftsnnAFKcXLXDRCZLWus2/hB4pIRMVp/rgNslLSOtZpWj\n3D3xK5IdVvo9cSdFq80VxgpwOXAP6T5/ycweLSofCrxcQa8gj/SqH5oIaibslkvy+Ls8Qdgtr+TR\nbuqPv2zu8U57m9m1VYUDACTNB8abWSWHIXdIuhe42syqOY99Al/VWgOcY2YbysgYU7tVraAriMMS\n8knYrXuYCl35vNObgu6D2gm75ZPebDdJmFmHXVX91cEaDfwa2FL0W1hB0GeRdAhpK+Ya4CIr8+GX\n1P/+KARB0KcZvv9wWje1VhcMgiDoBOFgBUFQE5LK+V5BEARBEASBU87B6o4YrCAIgmAHk8c96kHY\nLa+E3fJJ2C2f5NFu4WAFQRAEQRAEQRB0EbFFMAiCdsQWwSAIgiAIgurEFsEgCIIgCIIgCIIdTDhY\nQRAEfYA87lEPwm55JeyWT8Ju+SSPdgsHKwiCIAiCIAiCoIuIGKwgCNoRv4MV5JHhwxtobX2lp9UI\ngiAI+hHxO1hBENREcrDi70KQN0T8PwuCIAi6kzjkIgiCoE/T3NMKBHWQx9iCIOyWV8Ju+SSPdtuh\nDpakBklry5Q1SRq7I/svh6QDJK2U9Egmb2NP6FIOSSdJmlmDXKf1lnSVpF3KlF0k6XZPT5E0sUpb\nF0maUkXm/c7qWI1Cm36PNdUg/4ikFknPSfqVpJ2qyFecfy9/sJM6D5K00O+9CZJOcH1WSjq03Gcl\nU7/qWCXtKukhSeslrZV0Q6bsEO9vTmf0DoIgCIIgCGqnO1aweuOejbOAx8zsq5m83qhnLTrVo/dk\nYHAd9erVYUfMrZVJl2OCmR1pZp8D9gK+2ck+6ikvZixgZjbWzOYCFwI3mNlYYGuN7dUic7OZfRY4\nEjhB0mmkjl/08R8m6aBO6h70ehp7WoGgDhobG3tahaAOwm75JOyWT/Jot+5wsHaWNEvSOkkPlFo5\nkXS+pDX+utHzBkia6XmrJV3l+aN9FWCVpGfrfFDcC3izKO+tjD4Tvc8WSb/xvJmSbpO0RNLLksZ7\n/m6SFrkuqyWd6fkNvoowU9IGSbMlner1N0j6gssNlnSPpKWSVkg6w9X4EHi3hrG85e1Mc31XStrk\nbQ721YwWn8cJkq4ARgFNkhZ73Ytdp6XA8Zm2t5Ae/Cux1eWQtJ+keW6bFklfLExpZm6vkbTMZaZ4\n3gxJkzIyUyR9r5x8ER8Db1ebJDMr6LgzMAj4S5Uq2+bfV6sKc7tC0m4uM0TSXLfzfRn9N0oa5umj\nlFZr9wXuA472di4DzgWuz9b1OgMk3STpGR/3pbWO1cy2mtnjnv4IWAl8qkjsDdJnIAiCIAiCIOhq\nzGyHvYAGoA34ol/fA3zP002kb/RHAq8Cw0gO32LgTC97LNPWHv6+FDjT04OAXerQaxowuUzZGOAF\nYKhf7+XvM4E5nv4s8JKnBwK7e3rvTH4D6SF9jF8/C9zj6TOBeZ7+CXCBp/cENgC7Ful0FHBXjWPb\nE1hNWr0YD/wyUzbE3/+YGd+IzPzvBDwJ3F6nvf8NuNLTyvT3nr+fWtDHyx8ETgCOAJoz7TwP7F9O\n3q/fL9H/SOChCvo9SnKs5nRyXPOB4zw92O/Tk4B3vE8BTwFfyszvsIzt/uDpk4D5mXZnAuMz98sa\nT18K/DBzjy8HGjoz1sK9C/xv4MCi/MXAFyrUM5iSeTUZWLx6/au/2wnLI01NTT2tQlAHYbd8EnbL\nJ73Jbk1NTTZlypRtL//fQ/GrYhxKF/EnM1vq6VnAFcDPM+VHA01m9jaApNnAicB04CBJtwG/Bx6T\ntDswyszmk0b0YWeVkSTgcNelFKcAc83sHe9jc6bsd563XtJ+hSaBGZJOJDmTozJlG81snaefBxZ5\nei1woKfHAWdI+oFfDwIOIDlaeH8rgMtqHOIs4BYza5G0BfiZpBnAw2b2ZEbnwqrSsbSf/znAwTX2\nVcwpwN+7zgYUx16NA06VtNL73w042MxmStpX0ghgP+BtM3td0uRS8iQnsANm9mfg6+WUM7OvSBoE\nPCBpopndW+O4lgC/8HtznusGsMz7RNIqkk2fIrNiVyfjSNv4Jvj1HqRxv5oZS8WxShoI3A/camav\nFBVvIn0Gni2vwtTOax0EQRAEQdCHaWxsbLdlcdq0aSXlusPBsirXUOKB1Mw2SzocOA34LjCBFDtU\n8eHVt5pd6v2cbmatmbIBpNWFD4CHOzGGAh+U0PlCYB/gSDNrUzp0YpcS8m2Z6za2z72As83spTr0\naYekqSSH9l4AM3tJ6SCR04HpkhaZ2fRSVT9p304p2xb3M8PM7i5RNpdk4xHAnBrkq/VVWkGzDyX9\nL+AYoCYHy8x+Kukh4GvAEknjvChr34/ZbtOP2L79tuRhIlUQcIWZLayjboG7gA1m9k8lyn4JLJB0\njJl95xP0EfQqGntagaAO8hhbEITd8krYLZ/k0W7dEYPVIOlYT18APFFUvgw4UdIw/9b9fOBxSXsD\nA83st8CPgLGW4mhek/QN2HYq267ZxszsDkuHGYzNOlde1mZmB5K+uS93yMEfgAmZGJqhZeQKTsme\nwJvuXJ1M2upVLFOJBcCV2ypIR9RQp6MyKXbry8BVmbyRwFYzux+4mbTtEuA90qoIwDOk+R/q8UkT\nKIGky7NxUmVYDExy+QGShhSq+/sC4NuFGCZJozw2CeAB4DzgbJKzVU5+n6I2q6IUJzfC0zuRHKVV\nfn2WMiftlan/aTN73sxuIm3XO7RKlxtJWwPx8XSWBcAk1xVJBxff51X0nU7aUnt1GZFrgEvCuQqC\nIAiCIOh6usPBegG4XNI6UkzInZ5vAO4EXUv6EZcWYLmZPUiKwWmW1EI6HOBarzcRuFLSatLWreF1\n6PQiKeaoA76l7yckJ68FuCWrb1bU32eTDi5YDXwLWF9CplT9AteTDgJZo3RM93XFAn5Qwl0VxgNw\nNenwiuV+iMJU4DBgmY/jx6RtlwB3A49KWuzzP40U2/YEsK5Dy4lDqX4wxGTgZElrSE7sGM8v2Hoh\nadva0y4zF9jdy9YBQ4BNZvZGBfkh2TazSBrpK03F7AbM9218K4DXgH/1stFUP0xkstKR56tJcXWP\nlJDJ6nMdcLukZaTVrHKUuyd+RbLDSr8n7qRotbncWCXtD/wQGJM5mOPbRWJDgZcr6BXkkuaeViCo\ngzz+vksQdssrYbd8kke7KYXK9C883mlvM7u2qnAAgKT5pAMZKjkMuUPSvcDVZlbNeewTeAziGuAc\nM9tQRsbq3IEZ9CjN9O9tgiKP/8+am5tzuf2lvxN2yydht3zSm+0mCTPrsKuqvzpYo4FfA1us/W9h\nBUGfRdIhpK2Ya4CLrMyHPzlYQZAvhg9voLX1lZ5WIwiCIOhHhIMVBEFNSCrnewVBEARBEAROOQer\nO2KwgiAIgh1MHveoB2G3vBJ2yydht3ySR7uFgxUEQRAEQRAEQdBFxBbBIAjaEVsEgyAIgiAIqhNb\nBIMgCIIgCIIgCHYw4WAFQRD0AfK4Rz0Iu+WVsFs+CbvlkzzaLRysIAiCIAiCIAiCLiJisIIgaEf8\nDlYQBL2B4fsPp3VTa0+rEQRBUJb4HawgCGpCkjG1p7UIgqDfMxXiGSUIgt5MHHIRBEHQl9nYsMjF\nvgAAIABJREFU0woEdRF2yyV5jAkJwm55JY9226EOlqQGSWvLlDVJGrsj+y+HpAMkrZT0SCavV/2b\nk3SSpJk1yHVab0lXSdqlTNlFkm739BRJE6u0dZGkKVVk3u+sjtUotOn3WFMN8o9IapH0nKRfSdqp\ninzF+ffyBzup8yBJC/3emyDpBNdnpaRDy31WMvVrHetYSWskvSjp1kz+Id7fnM7oHQRBEARBENRO\nd6xg9cb1/bOAx8zsq5m83qhnLTrVo/dkYHAd9erVYUfMrZVJl2OCmR1pZp8D9gK+2ck+6ikvZixg\nZjbWzOYCFwI3mNlYYGuN7dUi8y/AJWZ2CHCIpNNIHb/o4z9M0kGd1D3o7YRF80nYLZc0Njb2tApB\nHYTd8kke7dYdDtbOkmZJWifpgVIrJ5LO92/c10i60fMGSJrpeaslXeX5o30VYJWkZ+t8UNwLeLMo\n762MPhO9zxZJv/G8mZJuk7RE0suSxnv+bpIWuS6rJZ3p+Q2S1nu9DZJmSzrV62+Q9AWXGyzpHklL\nJa2QdIar8SHwbg1jecvbmeb6rpS0ydscLOkhz1/jqyZXAKOAJkmLve7FrtNS4PhM21tID/6V2Opy\nSNpP0jy3TYukLxamNDO310ha5jJTPG+GpEkZmSmSvldOvoiPgberTZKZFXTcGRgE/KVKlW3z76tV\nhbldIWk3lxkiaa7b+b6M/hslDfP0UUqrtfsC9wFHezuXAecC12frep0Bkm6S9IyP+9JaxyppBDDE\nzJZ71r2kLxSyvEH6DARBEARBEARdTMVtUl3EZ4CLzWyppHuAScDPC4WSRgI3AkcCm4GF7qRsAvY3\ns8+73B5eZTbpW//5kgZRn5M4EGjLZpjZsd7PGOCHwHFm9o6k7IPoCDM7XtJngfnAPOCvwFlmtkXS\n3sBSLwMYDZxtZuskPQuc5/XP9D7GA/8ILDazSyTtCSyTtMjMngaedp2OAr5jZpcVD6Sgt5lNAaZ4\nG/8B/DPwFeB1M/u6tzPEzN6XdDXQ6OMbAUwlzf97QDOw0tu8pdpEmtkDmcvbgWYzGy9JwO4FMe//\nVOBgMzvGy+dLOgGYA9wK3OHy5wLjysmb2ZO402Zmm4BzvP2RwN2F8RYj6VHgaGCRmT1aZVzb5h/4\nPjDJzJ6WNJhkc4AjgDFAK7BE0pfM7Ck6rjKZmb0l6R+A75tZwQk/DnjQzOZJasjIXwJsNrNj/R5f\nIukxM3u1hrHuT/rsFNjkeVnaSJ+B8mQ3Ih5IfMueBzYSdsojYbdc0tzcnMtv1fs7Ybd80pvs1tzc\nXFNMWHc4WH8ys6WengVcQcbBIj3wNpnZ2wCSZgMnAtOBgyTdBvweeEzS7sAoM5sPYGYfdlYZf1A/\n3HUpxSnAXDN7x/vYnCn7neetl7RfoUlghqQTSQ+uozJlG81snaefBxZ5ei3psRVgHHCGpB/49SDg\nAGBDoVMzWwF0cK7KMAu4xcxaJG0BfiZpBvCwOyYFnQurSsfSfv7nAAfX2FcxpwB/7zobUBx7NQ44\nVdJK7383kgM1U9K+7uztB7xtZq9LmlxKHniSEpjZn4GSzpWXf8UdlgckTTSze2sc1xLgF35vznPd\nAJZ5n0haRbLpU2RW7OpkHGkb3wS/3oM07lczY6k41ipsIn0Gni0rcXKdLQdBEARBEPRRGhsb2zl7\n06ZNKynXHQ5Wh2/zS8h0eCA1s82SDgdOA74LTCDFDlV8ePWtZpd6P6ebWWumbADwR+AD4OFOjKHA\nByV0vhDYBzjSzNqUDp3YpYR8W+a6je1zL9Iq10t16NMOSVNJDu29AGb2ktJBIqcD031lbHqpqp+0\nb6dafJCAGWZ2d4myuSQbjyCtaFWTryuuy8w+lPS/gGNI2+dqqfNTSQ8BXyOtJo3zoqx9P2a7TT9i\n+8pqycNEqiDgCjNbWEfd14G/zVx/yvOy/BJYIOkYM/tOHX0EvZFYBcknYbdc0lu+TQ86R9gtn+TR\nbt0Rg9Ug6VhPXwA8UVS+DDhR0jBJA4Hzgcd9u91AM/st8CNgrMfRvCbpG7DtVLZds42Z2R1+mMHY\nrHPlZW1mdiDpm/tyhxz8AZiQiaEZWkau4JTsCbzpztXJQEMJmUosAK7cVkE6ooY6HZVJsVtfBq7K\n5I0EtprZ/cDNpEMWIG0FLGy5fIY0/0M9PmkCJZB0eTZOqgyLSVtAC3FEQwrV/X0B8O1CDJOkUR6b\nBPAAcB5wNsnZKie/T1GbVVGKkxvh6Z1IjtIqvz5L0g1V6n/azJ43s5uA5cChVbrcCBzl6bNr1TPD\nAmCS64qkg4vv83L4Pf+upMK2yonAvxeJXUM6BCOcqyAIgiAIgi6mOxysF4DLJa0jBdbf6fkG2x4I\nryXF/rQAy83sQVLcSLOkFtLhANd6vYnAlZJWk7ZuDa9DpxeBYaUKfEvfT0hOXgtQiEMqtxI3m3Rw\nwWrgW8D6EjKl6he4nnQQyBqlY7qvKxZQOijhrgrjAbiadHjFcqVDFKYCh5FiulqAH5O2XQLcDTwq\nabHP/zRS7NgTwLoOLScOpfrBEJOBkyWtITmxYzy/YOuFwP3A0y4zF4/T8nkfAmwyszcqyA/JtplF\n0khfaSpmN1L81ipgBfAa8K9eNprqh4lMlrTWbfwh8EgJmaw+1wG3S1pGWs0qR7l74lckO6z0e+JO\nilabK4wV4HLgHtJ9/lKJeLOhwMsV9ArySK/6oYmgZsJuuSSPv8sThN3ySh7tpv74K+ke77S3mV1b\nVTgAQNJ8YLyZVXIYcoeke4Grzaya89gn8FWtNcA5ZrahjIwxtVvVCrqCOCwhn4TdyjMVeuszSm8K\nug9qJ+yWT3qz3SRhZh12VfVXB2s08GtgS9FvYQVBn0XSIaStmGuAi6zMh19S//ujEARBr2P4/sNp\n3dRaXTAIgqCHCAcrCIKakFTO9wqCIAiCIAiccg5Wd8RgBUEQBDuYPO5RD8JueSXslk/Cbvkkj3YL\nBysIgiAIgiAIgqCLiC2CQRC0I7YIBkEQBEEQVCe2CAZBEARBEARBEOxgwsEKgiDoA+Rxj3oQdssr\nYbd8EnbLJ3m0WzhYQRAEQRAEQRAEXUTEYAVB0I74HaygtzN8eAOtra/0tBpBEARBPyd+BysIgppI\nDlb8XQh6MyL+dwVBEAQ9TRxyEQRB0Kdp7mkFgjrIY2xBEHbLK2G3fJJHu+1QB0tSg6S1ZcqaJI3d\nkf2XQ9IBklZKeiSTt7EndCmHpJMkzaxBrtN6S7pK0i5lyi6SdLunp0iaWKWtiyRNqSLzfmd1rEah\nTb/HmmqQny7pT5Leq7H9ivPv5Q/WrjFIGiRpod97EySdIOk5vz603GclU7/qWCXtKukhSeslrZV0\nQ6bsEO9vTmf0DoIgCIIgCGqnO1aweuM+jrOAx8zsq5m83qhnLTrVo/dkYHAd9erVYUfMrZVJl2M+\ncPQn6KOe8mLGAmZmY81sLnAhcIOZjQW21theLTI3m9lngSOBEySdRur4RTP7HHCYpIM6qXvQ62ns\naQWCOmhsbOxpFYI6CLvlk7BbPsmj3brDwdpZ0ixJ6yQ9UGrlRNL5ktb460bPGyBppuetlnSV54/2\nVYBVkp6t80FxL+DNory3MvpM9D5bJP3G82ZKuk3SEkkvSxrv+btJWuS6rJZ0puc3+CrCTEkbJM2W\ndKrX3yDpCy43WNI9kpZKWiHpDFfjQ+DdGsbylrczzfVdKWmTtznYVzNafB4nSLoCGAU0SVrsdS92\nnZYCx2fa3kJ68K/EVpdD0n6S5rltWiR9sTClmbm9RtIyl5nieTMkTcrITJH0vXLyRXwMvF1tksxs\nmZm9UU0uw7b599WqwtyukLSbywyRNNftfF9G/42Shnn6KKXV2n2B+4CjvZ3LgHOB67N1vc4ASTdJ\nesbHfWmtYzWzrWb2uKc/AlYCnyoSe4P0GQiCIAiCIAi6GjPbYS+gAWgDvujX9wDf83QT6Rv9kcCr\nwDCSw7cYONPLHsu0tYe/LwXO9PQgYJc69JoGTC5TNgZ4ARjq13v5+0xgjqc/C7zk6YHA7p7eO5Pf\nQHpIH+PXzwL3ePpMYJ6nfwJc4Ok9gQ3ArkU6HQXcVePY9gRWk1YvxgO/zJQN8fc/ZsY3IjP/OwFP\nArfXae9/A670tDL9vefvpxb08fIHgROAI4DmTDvPA/uXk/fr90v0PxJ4qIqO79UxrvnAcZ4e7Pfp\nScA73qeAp4AvZeZ3WMZ2f/D0ScD8TLszgfGZ+2WNpy8Ffpi5x5cDDXWMdS/gfwMHFuUvBr5QoZ7B\nlMyrycDi1etf/clOWF+hqampp1UI6iDslk/CbvmkN9mtqanJpkyZsu3l/48ofu3EjudPZrbU07OA\nK4CfZ8qPBprM7G0ASbOBE4HpwEGSbgN+DzwmaXdglJnNJ43ow84qI0nA4a5LKU4B5prZO97H5kzZ\n7zxvvaT9Ck0CMySdSHImR2XKNprZOk8/Dyzy9FrgQE+PA86Q9AO/HgQcQHK08P5WAJfVOMRZwC1m\n1iJpC/AzSTOAh83syYzOhVWlY2k//3OAg2vsq5hTgL93nQ0ojr0aB5wqaaX3vxtwsJnNlLSvpBHA\nfsDbZva6pMml5ElOYAfM7M/A1+vUvRJLgF/4vTnPdQNY5n0iaRXJpk+RWbGrk3GkbXwT/HoP0rhf\nLQhUG6ukgcD9wK1m9kpR8SbSZ+DZ8ipM7bzWQRAEQRAEfZjGxsZ2WxanTZtWUq47HCyrcg0lHkjN\nbLOkw4HTgO8CE0ixQxUfXn2r2aXez+lm1popG0BaXfgAeLgTYyjwQQmdLwT2AY40szalQyd2KSHf\nlrluY/vcCzjbzF6qQ592SJpKcmjvBTCzl5QOEjkdmC5pkZlNL1X1k/btlLJtcT8zzOzuEmVzSTYe\nAcypQb5aX12Gmf1U0kPA14AlksZ5Uda+H7Pdph+xffttycNEqiDgCjNbWI++zl3ABjP7pxJlvwQW\nSDrGzL7zCfoIehWNPa1AUAd5jC0Iwm55JeyWT/Jot+6IwWqQdKynLwCeKCpfBpwoaZh/634+8Lik\nvYGBZvZb4EfAWDPbArwm6Ruw7VS2XbONmdkdZnakpYMEWovK2szsQNI3998so+8fgAmZGJqhZeQK\nTsmewJvuXJ1M2upVLFOJBcCV2ypIR9RQp6MyKXbry8BVmbyRwFYzux+4mbTtEuA90qoIwDOk+R8q\naWeSk1Oq/cuzcVJlWAxMcvkBkoYUqvv7AuDbhRgmSaM8NgngAeA84GySs1VOfp+iNjtLu3qSzlLm\npL2SFaRPm9nzZnYTabveoVX62EjaGghpPJ1lATBJ0k7e/8HF93kVfaeTttReXUbkGuCScK6CIAiC\nIAi6nu5wsF4ALpe0jhQTcqfnG4A7QdeSfsSlBVhuZg+SYnCaJbWQDge41utNBK6UtJq0dWt4HTq9\nSIo56oBv6fsJyclrAW7J6psV9ffZpIMLVgPfAtaXkClVv8D1pINA1igd031dsYAflHBXhfEAXE06\nvGK5H6IwFTgMWObj+DFp2yXA3cCjkhb7/E8jxbY9Aazr0HLiUOAvVXSYDJwsaQ3JiR3j+QVbLyRt\nW3vaZeYCu3vZOmAIsMn8MIoy8kOybWaRNNJXmjog6aeSXgN2VTqu/cdeNJrqh4lMVjryfDUpru6R\nEjJZfa4Dbpe0jLSaVY5y98SvSHZY6ffEnRStNpcbq6T9gR8CYzIHc3y7SGwo8HIFvYJc0tzTCgR1\nkMffdwnCbnkl7JZP8mg3pVCZ/oXHO+1tZtdWFQ4AkDSfdCBDJYchd0i6F7jazKo5j30Cj0FcA5xj\nZhvKyFg37sAMuoxm+s82QdFX/nc1NzfncvtLfyfslk/CbvmkN9tNEmbWYVdVf3WwRgO/BrZY+9/C\nCoI+i6RDSFsx1wAXWZkPfzhYQe+n7zhYQRAEQX4JBysIgppIDlYQ9F6GD2+gtfWVnlYjCIIg6OeU\nc7C6IwYrCIKcUeo3HeLVu19NTU09rkN3vfqSc5XH2IIg7JZXwm75JI92CwcrCIIgCIIgCIKgi4gt\ngkEQtEOSxd+FIAiCIAiCysQWwSAIgiAIgiAIgh1MOFhBEAR9gDzuUQ/Cbnkl7JZPwm75JI92Cwcr\nCIIgCIIgCIKgi4gYrCAI2hExWEEQBEEQBNUpF4O1U08oEwRB70bq8LciCIKgJMP3H07rptaeViMI\ngqDXECtYQRC0Q5Ixtae1CDrNRuCgnlYi6DR9wW5T02/n9Seam5tpbGzsaTWCThJ2yye92W5ximAQ\nBEEQBEEQBMEOZoc6WJIaJK0tU9YkaeyO7L8ckg6QtFLSI5m8jT2hSzkknSRpZg1yndZb0lWSdilT\ndpGk2z09RdLEKm1dJGlKFZn3O6tjNQpt+j3WVIP8dEl/kvReje1XnH8vf7B2jUHSIEkL/d6bIOkE\nSc/59aHlPiuZ+rWOdaykNZJelHRrJv8Q729OZ/QOckLeV0H6K2G3XNJbv00PKhN2yyd5tFt3rGD1\nxn0DZwGPmdlXM3m9Uc9adKpH78nA4Drq1avDjphbK5Mux3zg6E/QRz3lxYwFzMzGmtlc4ELgBjMb\nC2ytsb1aZP4FuMTMDgEOkXQaqeMXzexzwGGS4rEuCIIgCIJgB9AdDtbOkmZJWifpgVIrJ5LO92/c\n10i60fMGSJrpeaslXeX5o30VYJWkZ+t8UNwLeLMo762MPhO9zxZJv/G8mZJuk7RE0suSxnv+bpIW\nuS6rJZ3p+Q2S1nu9DZJmSzrV62+Q9AWXGyzpHklLJa2QdIar8SHwbg1jecvbmeb6rpS0ydscLOkh\nz1/jqyZXAKOAJkmLve7FrtNS4PhM21tID/6V2OpySNpP0jy3TYukLxamNDO310ha5jJTPG+GpEkZ\nmSmSvldOvoiPgberTZKZLTOzN6rJZdg2/75aVZjbFZJ2c5khkua6ne/L6L9R0jBPH6W0WrsvcB9w\ntLdzGXAucH22rtcZIOkmSc/4uC+tdaySRgBDzGy5Z91L+kIhyxukz0DQl+hVa/BBzYTdckkef5cn\nCLvllTzarTtOEfwMcLGZLZV0DzAJ+HmhUNJI4EbgSGAzsNCdlE3A/mb2eZfbw6vMJn3rP1/SIOpz\nEgcCbdkMMzvW+xkD/BA4zszekZR9EB1hZsdL+ixpRWQe8FfgLDPbImlvYKmXAYwGzjazdZKeBc7z\n+md6H+OBfwQWm9klkvYElklaZGZPA0+7TkcB3zGzy4oHUtDbzKYAU7yN/wD+GfgK8LqZfd3bGWJm\n70u6Gmj08Y0AppLm/z2gGVjpbd5SbSLN7IHM5e1As5mNlyRg94KY938qcLCZHePl8yWdAMwBbgXu\ncPlzgXHl5M3sSdxpM7NNwDne/kjg7sJ4PwnZ+Qe+D0wys6clDSbZHOAIYAzQCiyR9CUze4qOq0xm\nZm9J+gfg+2ZWcMKPAx40s3mSGjLylwCbzexYv8eXSHrMzF6tYaz7kz47BTZ5XpY20megPNmNiAcS\n25iCIAiCIOj3NDc31+TwdYeD9SczW+rpWcAVZBws0ratJjN7G0DSbOBEYDpwkKTbgN8Dj0naHRhl\nZvMBzOzDzirjD+qHuy6lOAWYa2bveB+bM2W/87z1kvYrNAnMkHQi6cF1VKZso5mt8/TzwCJPryU9\ntgKMA86Q9AO/HgQcAGwodGpmK4AOzlUZZgG3mFmLpC3AzyTNAB52x6Sgc2FV6Vjaz/8c4OAa+yrm\nFODvXWcDimOvxgGnSlrp/e9GcqBmStrXnb39gLfN7HVJk0vJA09SAjP7M/CJnasSLAF+4ffmPNcN\nYJn3iaRVJJs+RWbFrk7GkbbxTfDrPUjjfrUg8AnHuon0GXi2rMTJdbYc9BzhBOeTsFsuyWNMSBB2\nyyu9yW6NjY3t9Jk2bVpJue5wsDp8m19CpsMDqZltlnQ4cBrwXWACKXao4sOrbzW71Ps53cxaM2UD\ngD8CHwAPd2IMBT4oofOFwD7AkWbWpnToxC4l5Nsy121sn3uRVrleqkOfdkiaSnJo7wUws5eUDhI5\nHZjuK2PTS1X9pH071eKDBMwws7tLlM0l2XgEaUWrmny3xcyZ2U8lPQR8jbSaNM6Lsvb9mO02/Yjt\nK6slDxOpgoArzGxhHXVfB/42c/0pz8vyS2CBpGPM7Dt19BEEQRAEQRCUoTtisBokHevpC4AnisqX\nASdKGiZpIHA+8LhvtxtoZr8FfgSMNbMtwGuSvgHbTmXbNduYmd1hZkf6QQKtRWVtZnYg6Zv7b5bR\n9w/AhEwMzdAycgWnZE/gTXeuTgYaSshUYgFw5bYK0hE11OmoTIrd+jJwVSZvJLDVzO4HbiYdsgBp\nK2Bhy+UzpPkfKmlnkpNTqv3Ls3FSZVhM2gJaiCMaUqju7wuAbxdimCSN8tgkgAeA84CzSc5WOfl9\nitrsLO3qSTpL0g0VK0ifNrPnzewmYDlwaJU+NgJHefrsOnRcAEyStJP3f3DxfV4Ov+fflVTYVjkR\n+PcisWtIh2CEc9WXiFiefBJ2yyV5jAkJwm55JY926w4H6wXgcknrSIH1d3q+wbYHwmtJsT8twHIz\ne5AUN9IsqYV0OMC1Xm8icKWk1aStW8Pr0OlFYFipAt/S9xOSk9cCFOKQyq3EzSYdXLAa+BawvoRM\nqfoFricdBLJG6Zju64oF/KCEuyqMB+Bq0uEVy/0QhanAYaSYrhbgx6RtlwB3A49KWuzzP40UO/YE\nsK5Dy4lDgb9U0WEycLKkNSQndoznF2y9ELgfeNpl5uJxWj7vQ4BNhcMoysgPybaZRdJIX2nqgKSf\nSnoN2FXpuPYfe9Foqh8mMlnSWrfxh8AjJWSy+lwH3C5pGWk1qxzl7olfkeyw0u+JOylaba40VuBy\n4B7Sff6SmT1aVD4UeLmCXkEQBEEQBEGdqL/9+jqAxzvtbWbXVhUOAJA0HxhvZpUchtwh6V7gajOr\n5jz2CXxVaw1wjpltKCNjTO1WtYIgyDNToT8+SwRBEEjCzDrsquqvDtZo4NfAlqLfwgqCPoukQ0hb\nMdcAF1mZD7+k/vdHIQiCuhm+/3BaN7VWFwyCIOhjhIMVBEFNSCrnewW9mObm5l510lJQG2G3fBJ2\nyydht3zSm+1WzsHqjhisIAiCIAiCIAiCfkGsYAVB0I5YwQqCIAiCIKhOrGAFQRAEQRAEQRDsYMLB\nCoIg6APk8XdCgrBbXgm75ZOwWz7Jo93CwQqCIAiCIAiCIOgiIgYrCIJ2RAxWEARBEARBdcrFYO3U\nE8oEQdC7Sb9HHATBJ2X48AZaW1/paTWCIAiCbiS2CAZBUAKLV+5eTb1Ah3gVv95441UqkcfYgiDs\nllfCbvkkj3YLBysIgiAIgiAIgqCL6BIHS1KDpLVlypokje2KfjqLpAMkrZT0SCZvY0/oUg5JJ0ma\nWYPcRn8vO9dF8kMkvSbp9mwbkoZ1Qreqc+X2PaBC+UWS/qnWPmvU66LCuCRNkTSxivzRklr8tVrS\nN2voY6akE6uUj++k3idIes7vyb+RdLOktZJ+6uP4XpX6tYz1y5Ke9XEul3Rypuz7kl6oZfxBHmns\naQWCOmhsbOxpFYI6CLvlk7BbPsmj3boyBsu6sK2u4izgMTO7NpPXG/WsRScrky7H9cDjdfTzSeR3\ndDv1shY4yszaJI0AnpP0P83s427W40LgBjO7H0DSpcBQMzNJU7qoj7eAr5tZq6T/BiwAPgVgZrdI\nehK4GZjTRf0FQRAEQRAEGbpyi+DOkmZJWifpAUm7FAtIOl/SGn/d6HkDfDVgjX/rfpXnj5a0UNIq\n/0b+oDp02gt4syjvrYw+E73PFkm/8byZkm6TtETSy4VVCkm7SVqUWR040/MbJK33ehskzZZ0qtff\nIOkLLjdY0j2SlkpaIekMV+ND4N0axvJWcYakuzMrM29K+n89/yhgP+Cx4irAld7/akmHZMb2r26D\nVZL+e7k+S/AX4GNv5yve9ipJC0vou4+k/ynpGX8dp8RGSXtk5F6UtG8p+RL9bwG2VlLQzP5qZm1+\nuSvwbg3O1WaSbZB0o688rZJ0U0bmpBL3yUmSHsyM5Z/8PrsEOBe4XtJ9kv4d2B1YIWlC0Tx9WtIj\nvgL1eMFOwPs1jHW1mbV6+nlgF0k7Z0RagT2rjD3IJc09rUBQB3mMLQjCbnkl7JZP8mi3rlzB+gxw\nsZktlfT/s3fvUXZUZf7/3x8yiQiBhBAIkDEX4ncwcUhMAiIjQiOgw8yAiIAgN10sQC6CXBwZvBAE\nEUX4fqP+wOEyMUIECQNMEDMkQDq6IBFyJyQEgTgQmXARgmQGQejP74/aB6tP17l006G7Os9rrV59\nTtVTVc+uXZ2cffbeVTcApwNXVVZK2hm4HJhI9uF1bmqkrAOG2x6f4ioftGeQfds/S9IAutYY7Ae0\n5RfY3isdZxxwIbC37ZclDc6F7WT7o5LGArOA24E/AYfZ3ihpe2BhWgcwBviM7VWSFgFHp+0PTcc4\nHPgacJ/tkyQNAh6SdK/tBcCClNNk4FTbp1QXpJJ31bKT03YjgNnANEkCvk/WW3JQwTl53vZkSacB\n5wOnAN8ANuTqYFCtYxbkcETaZihwLbCP7aerzmfFVOAq2w9Keh9wj+1xku4EPg1Ml/Rh4He2X5A0\nozoeGFd1/CsrryWdmi3ytdUHTvv9N2A08LkmynVO2m4IWb1/IL3fNhdWdJ1AQY+d7Rsk7QPcZfv2\ntK8/2p6UXud7sK4luw6eTHlfAxxgO//3VLOsuZgjgCW2/5xb3EZTf/dTcq9biOFnIYQQQtjctba2\nNtXg684G1tO2F6bXNwFfItfAAvYE5tl+CSB9eN4XuBQYLWkq8EtgjqSBwC62ZwHYfqOzyaSGxoSU\nS5GPAzNtv5yOsSG37s60bLWkHSu7BL6jbF5OG7BLbt1a26vS60eBe9PrR4BR6fUngEMkfSW9HwCM\nANZUDmp7MVmDpzPl3BKYCZxpe52kM4C7bT+bnQKq77d9R/q9mKxRA3Ag8Pa8HNvN9KhEg1wQAAAg\nAElEQVRV+wgw3/bTaR8bCmIOBMamugEYKGkr4Fbgm8B04Gj+MnytVnwh2/9aZ91DwN9K2g24R9I8\n239solyvAK9Juh64G/hFbl3RdfKOSNoa+DtgZq7c/avj6pU17eeDwHfo2Mh+EdhB0uAadZRMaT7p\n0Eu09HQCoQvKOLcgRL2VVdRbOfWmemtpaWmXz8UXX1wYtynnYBXNu+nwcB3bGyRNAD4JfBE4Evhy\nUWy7HUmnAyen4/xDZVhUWrcF8BTwOtmH4s56vSDnY4GhwMQ0l2ctsGVBfFvufb63QGS9XL/tQj71\nXAPcZnteer83sE86P9uQDd181faFVbm+Rfc/B63Rw5ME7FXVowKwQNmQ0KFk8+a+VS9e7+AZTbbX\nSHoS+D9kjcxG8W+lXqQDyK7NM9NrKL5O3qR9b2uHobINbAG8XOnZ6gpJf03Wm3a87d/l19l+TdIt\nwFOSPmu7w1DOEEIIIYTQdd05B2ukpMqQss8Bv65a/xCwr6QhkvoBxwDz03C7frbvAL4OTLK9EXhG\n0qcAJA2Q9N78zmxfbXui7Un5xlVa12Z7FLCIXM9MlfuBI9MQMCRtVyOu8sF5ENnwujZld2YbWRBT\nzz3AWW9vIH2oiW3qSr1VA21fUVlm+zjbo2zvSjYE8Ke5xlUtc4EzcvvtMLxP2fyznevsYyHwMUkj\nU3zR+ZwDnJ3b54TcujvIejxX5XpW6sU3TdKodM2R8ns/8Nv0frrSPLka224NDLb9n8C5wPhaoen3\nfwHjJPVP5/GAGvH5bd5m+1VgbRreV8mh1jGL8h1E1sv21VyPcn79YLK/ieHRuOprWns6gdAFZZxb\nEKLeyirqrZzKWG/d2cB6DDhD0iqym0v8OC03QGoEXUD2KWAp8LDtu4DhQKukpcCNKQbgBLIbMiwH\nHgCGdSGnx4HC25KnIX3fJmvkLQUqc3lq9cTNAPZM+RwHrC6IKdq+4hKy3qQVym6z/q3qAEmTJdWc\nU1PgPGB3ZTe5WCKp0fDCWrldCgxRdsvwpVSNNUpD1cYAL9Xcsf0i2fDGO9I+bikIOxvYQ9kNNlYC\np+bW3UrWS3hLk/EdSDq1xjnYB1guaUk6zim54YHjgWfr7HYb4Bep3n8FnJOWF14nttelY6xMZVlS\nHVPnfcVxwEnKbqqxEji0OqBOWc8kq6tv5q6Lobn1g4DnbNe9WUYIIYQQQuga2T19B+1NJ8132r7q\nNu2hk9J8ni/YPr+nc+lOkrYBrre92TwXKg13nGq76I6MlRj3/J31Q+grRF/+fzaEEDZnkrDdYURS\nX29gjQF+Amy0fXAPpxNCj5J0Hlkv4RW2b64T13f/UQjhXTZs2EjWr/9dT6cRQghhE9gsG1ghhM6T\n5Ph3oXxaW1t71Z2WQnOi3sop6q2cot7KqTfXW60GVnfOwQohhBBCCCGEzVr0YIUQ2okerBBCCCGE\nxqIHK4QQQgghhBA2sWhghRBCH1DG54SEqLeyinorp6i3cipjvUUDK4QQQgghhBC6SczBCiG0E3Ow\nQgghhBAaqzUH6696IpkQQu8mdfi3IoQQQgibyLDhw1i/bn1PpxG6SfRghRDakWSm9HQWodPWAqN7\nOonQaVFv5RT1Vk69ud6mQHwmLxbPwQohhBBCCCGEzVi3NLAkjZT0SI118yRN6o7jdJakEZKWSJqd\nW7a2J3KpRdJ+kqY1Ebc2/a55rqvit5H0jKQf5PchaUgncmt4rlL9jqiz/kRJP2z2mE3mdWKlXJIu\nknRCg/g9JS1NP8slfbaJY0yTtG+D9Yd3Mu99JK1M1+R7JF0h6RFJ303lOLfB9g3LmuL+RdJvJa2W\n9Inc8vMkPdZM+UMJ9dZvZUN9UW/lFPVWTlFvpdRbe6/q6c4erN7Yr3kYMMf2wbllvTHPZnJyjde1\nXALM78Jx3kn8pt5PVz0CTLY9Efgk8P9J6tcDeRwLXGZ7ku3XgZOB8ba/2l0HkDQWOAoYCxwMXK00\nocr2lcCJwBnddbwQQgghhNBedzaw+ku6SdIqSbdK2rI6QNIxklakn8vTsi1Sb8CK1Ltwdlo+RtJc\nScskLZLUle8dBgPPVy17IZfPCemYSyVNT8umSZoq6QFJT1R6KSRtLenelMtySYem5SNTT8E0SWsk\nzZB0UNp+jaQ9UtxWkm6QtFDSYkmHpDTeAF5poiwvVC+QdF2uZ+Z5Sd9IyycDOwJzqjcBzkrHXy7p\nb3Jl+7dUB8skfbrWMQv8AXgr7efv076XSZpbkO9QSbdJ+k362VuZtZK2zcU9LmmHoviC428EXquX\noO0/2W5Lb98LvGL7rQbl2kBWN0i6PPU8LZP0vVzMfgXXyX6S7sqV5YfpOjuJrOFziaQbJf0HMBBY\nLOnIqvO0q6TZkh6WNL9ST8CrjcoKfAq4xfabtn8H/Bb4cG79emBQg32EMupVffOhaVFv5RT1Vk5R\nb6VUxudgdeddBHcDvmB7oaQbgNOBqyorJe0MXA5MJPvwOjc1UtYBw22PT3GVD9ozyL7tnyVpAF1r\nDPYD2vILbO+VjjMOuBDY2/bLkgbnwnay/dHUGzALuB34E3CY7Y2StgcWpnUAY4DP2F4laRFwdNr+\n0HSMw4GvAffZPknSIOAhSffaXgAsSDlNBk61fUp1QSp5Vy07OW03ApgNTEu9Fd8n6y05qOCcPG97\nsqTTgPOBU4BvABtydTCo1jELcjgibTMUuBbYx/bTVeezYipwle0HJb0PuMf2OEl3Ap8Gpkv6MPA7\n2y9ImlEdD4yrOv6VldeSTs0W+drqA6f9/hvZAIHPNVGuc9J2Q8jq/QPp/ba5sKLrBAp67GzfIGkf\n4C7bt6d9/dH2pPT6olz4tWTXwZMp72uAA2zn/55qlXU46XpKfp+WVbTRzN/9vNzrUcSwihBCCCFs\n9lpbW5tq8HVnA+tp2wvT65uAL5FrYAF7AvNsvwSQPjzvC1wKjJY0FfglMEfSQGAX27MAbL/R2WRS\nQ2NCyqXIx4GZtl9Ox9iQW3dnWrZa0o6VXQLfUTYvpw3YJbdure1V6fWjwL3p9SNkH08BPgEcIukr\n6f0AYASwpnJQ24vJGjydKeeWwEzgTNvrJJ0B3G372ewUUH1nkzvS78VkjRqAA4G35+XYbqZHrdpH\ngPm2n0772FAQcyAwNtUNwEBJWwG3At8EpgNHAz9vEF/I9r/WWfcQ8LeSdgPukTTP9h+bKNcrwGuS\nrgfuBn6RW1d0nbwjkrYG/g6YmSt3/+q4emVt4EVgB0mDa9RRZv8u7j30nGgEl1PUWzlFvZVT1Fsp\n9aY5WC0tLe3yufjiiwvjurOBVf2tfdG8mw63MbS9QdIEsrkxXwSOBL5cFNtuR9LpZHNYDPyD7fW5\ndVsATwGvk30o7qzXC3I+FhgKTLTdpuwGEFsWxLfl3ud7C0TWy/XbLuRTzzXAbbYrfQ57A/uk87MN\n2dDNV21fWJXrW3T/c9AaPTxJwF62/1y1fIGyIaFDyebNfatevN7BM5psr5H0JPB/yBqZjeLfSr1I\nB5Bdm2em11B8nbxJ+97WDkNlG9gCeLnSs9UFvwfel3v/12kZALZfk3QL8JSkz9ruMJQzhBBCCCF0\nXXfOwRopqTKk7HPAr6vWPwTsK2mIshsMHAPMT8Pt+tm+A/g6MMn2RuAZSZ8CkDRA0nvzO7N9te2J\n6YYB66vWtdkeBSwi1zNT5X7gyDQEDEnb1YirfHAeRDa8rk3S/sDIgph67gHOensD6UNNbFNX6q0a\naPuKyjLbx9keZXtXsiGAP801rmqZS+7GB0XD+5TNP9u5zj4WAh+TNDLFF53POcDZuX1OyK27g6zH\nc1WuZ6VefNMkjUrXHCm/95PNTULSdKV5cjW23RoYbPs/gXOB8bVC0+//AsZJ6p/O4wE14vPbvM32\nq8BaSUfkcqh1zCKzgKPT38xosrI+lNvXYLK/ieHRuOpjYm5BOUW9lVPUWzlFvZVSGedgdWcD6zHg\nDEmryG4u8eO03ACpEXQB0AosBR62fRfZ/JBWSUuBG1MMwAlkN2RYDjwADOtCTo8DhbclT0P6vk3W\nyFsKVOby1OqJmwHsmfI5DlhdEFO0fcUlZL1JK5TdZv1b1QGSJkvqMH+ojvOA3ZXd5GKJpEbDC2vl\ndikwRNktw5cCLVV5iWye2Us1d2y/SDa88Y60j1sKws4G9lB2g42VwKm5dbeS9RLe0mR8B5JOrXEO\n9gGWS1qSjnNKbnjgeODZOrvdBvhFqvdfAeek5YXXie116RgrU1mWVMfUeV9xHHCSsptqrAQOrQ6o\nVdZ0Xd8KrCIbcnu62z+5cBDwnO1GN8sIIYQQQghdoL781Og032l72xc0DA41Sfog2Q1Mzu/pXLqT\npG2A621vNs+FSsMdp9ouuiNjJcZMefdyCiGEEDZ7U6AvfybvqyRhu8OIpL7ewBoD/ATYWPUsrBA2\nO5LOI+slvML2zXXi+u4/CiGEEEIvNGz4MNavW984MPQqm2UDK4TQeZIc/y6UT2tra6+601JoTtRb\nOUW9lVPUWzn15nqr1cDqzjlYIYQQQgghhLBZix6sEEI70YMVQgghhNBY9GCFEEIIIYQQwiYWDawQ\nQugDyvickBD1VlZRb+UU9VZOZay3aGCFEEIIIYQQQjeJOVghhHZiDlYIIYQQQmO15mD9VU8kE0Lo\n3aQO/1aEsFkaNmwk69f/rqfTCCGEUCIxRDCEUMDxU7qfeb0gh77389xz/8WmVMa5BSHqrayi3sqp\njPUWDawQQgghhBBC6Cbd0sCSNFLSIzXWzZM0qTuO01mSRkhaIml2btnansilFkn7SZrWRNza9Lvm\nua6K30bSM5J+kN+HpCGdyK3huUr1O6LO+hMl/bDZYzaZ14mVckm6SNIJDeKHSLpf0qv589Fgm2mS\n9m2w/vBO5r2PpJXpmnyPpCskPSLpu6kc5zbYvpmyHihpkaTlkh6WtH9u3XmSHpP02c7kHcqipacT\nCF3Q0tLS0ymELoh6K6eot3IqY711Zw+Wu3Ff3eUwYI7tg3PLemOezeTkGq9ruQSY34XjvJP4Tb2f\nrvoT8HXgvB7O41jgMtuTbL8OnAyMt/3VbjzGC8A/2Z4AfB64sbLC9pXAicAZ3Xi8EEIIIYSQ050N\nrP6SbpK0StKtkrasDpB0jKQV6efytGyL1BuwIn3rfnZaPkbSXEnL0jfyo7uQ02Dg+aplL+TyOSEd\nc6mk6WnZNElTJT0g6YlKL4WkrSXdm+sdODQtHylpddpujaQZkg5K26+RtEeK20rSDZIWSlos6ZCU\nxhvAK02U5YXqBZKuS7kvlfS8pG+k5ZOBHYE51ZsAZ6XjL5f0N7my/Vuqg2WSPl3rmAX+ALyV9vP3\nad/LJM0tyHeopNsk/Sb97K3MWknb5uIel7RDUXzB8TcCr9VL0Pb/2n4QeL2J8lRsIKsbJF2eep6W\nSfpeLma/gutkP0l35cryw3SdnQQcBVwi6UZJ/wEMBBZLOrLqPO0qaXbqgZpfqSfg1SbKutz2+vT6\nUWBLSf1zIeuBQZ04D6E0Wns6gdAFZZxbEKLeyirqrZzKWG/deRfB3YAv2F4o6QbgdOCqykpJOwOX\nAxPJPrzOTY2UdcBw2+NTXOWD9gyyb/tnSRpA1xqD/YC2/ALbe6XjjAMuBPa2/bKkwbmwnWx/VNJY\nYBZwO1kvyGG2N0raHliY1gGMAT5je5WkRcDRaftD0zEOB74G3Gf7JEmDgIck3Wt7AbAg5TQZONX2\nKdUFqeRdtezktN0IYDYwTZKA75P1lhxUcE6etz1Z0mnA+cApwDeADbk6GFTrmAU5HJG2GQpcC+xj\n++mq81kxFbjK9oOS3gfcY3ucpDuBTwPTJX0Y+J3tFyTNqI4HxlUd/8rKa0mnZot8baO8myjXOWmf\nQ8jq/QPp/ba5sKLrBAp67GzfIGkf4C7bt6d9/dH2pPT6olz4tWTXwZPpfFwDHGA7//fUsKySjgCW\n2P5zbnEbTf3dT8m9biGGn4UQQghhc9fa2tpUg687G1hP216YXt8EfIlcAwvYE5hn+yWA9OF5X+BS\nYLSkqcAvgTmSBgK72J4FYPuNziaTGhoTUi5FPg7MtP1yOsaG3Lo707LVknas7BL4jrJ5OW3ALrl1\na22vSq8fBe5Nrx8BRqXXnwAOkfSV9H4AMAJYUzmo7cVkDZ7OlHNLYCZwpu11ks4A7rb9bHYKqL7f\n9h3p92KyRg3AgcDb83JsN9OjVu0jwHzbT6d9bCiIORAYm+oGYKCkrYBbgW8C04GjgZ83iC9k+1+7\nkHcjrwCvSboeuBv4RW5d0XXyjkjaGvg7YGau3P2r4xqVVdIHge/QsZH9IrCDpME16iiZ0nzSoZdo\n6ekEQheUcW5BiHorq6i3cupN9dbS0tIun4svvrgwrjsbWNXf2hfNu+nwcB3bGyRNAD4JfBE4Evhy\nUWy7HUmnk81hMfAPlWFRad0WwFNkQ8Lu7kQZKvJDySp5HAsMBSbablN2A4gtC+Lbcu/zvQUi6+X6\nbRfyqeca4Dbb89L7vYF90vnZhmzo5qu2L6zK9S26/zlojR6eJGCvqh4VgAXKhoQOJZs396168XoX\nn9Fk+63Ui3QA2bV5ZnoNxdfJm7Tvbe0wVLaBLYCXKz1bXSHpr8l60463/bv8OtuvSboFeErSZ213\nGMoZQgghhBC6rjvnYI2UVBlS9jng11XrHwL2VXZHt37AMcD8NNyun+07yG5EMMn2RuAZSZ8CkDRA\n0nvzO7N9te2J6YYB66vWtdkeBSwi1zNT5X7gyDQEDEnb1YirfHAeRDa8rk3ZndlGFsTUcw9w1tsb\nSB9qYpu6Um/VQNtXVJbZPs72KNu7kg0B/GmucVXLXHI3Piga3qds/tnOdfaxEPiYpJEpvuh8zgHO\nzu1zQm7dHWQ9nqtyPSv14ruqXV1Jmq40T64wOOtRGmz7P4FzgfEN9vtfwDhJ/dN5PKBGfIdcAGy/\nCqxNw/sqOdQ6ZlG+g8h62b6a61HOrx9M9jcxPBpXfU1rTycQuqCMcwtC1FtZRb2VUxnrrTsbWI8B\nZ0haRXZziR+n5QZIjaALyD4FLAUetn0XMBxolbSU7I5nF6TtTiC7IcNy4AFgWBdyehwovC15GtL3\nbbJG3lKgMpenVk/cDGDPlM9xwOqCmKLtKy4h601aoew269+qDpA0WVJn5g+dB+yu7CYXSyQ1Gl5Y\nK7dLgSHKbhm+lKqxRmmo2hjgpZo7tl8kG954R9rHLQVhZwN7KLvBxkrg1Ny6W8l6CW9pMr4DSafW\nOgepx/FK4ERJT0v6QFo1Hni2zm63AX6R6v1XwDlpeeF1YntdKsvKVJYl1TF13lccB5yk7KYaK4FD\nC8pTq6xnktXVN3PXxdDc+kHAc7br3iwjhBBCCCF0jeyevoP2ppPmO21v+4KGwaGmNJ/nC7bP7+lc\nupOkbYDrbW82z4VKwx2n2i66I2Mlxj1/Z/0QegvRl/+fDCGE0HWSsN1hRFJfb2CNAX4CbKx6FlYI\nmx1J55H1El5h++Y6cdHACuFt0cAKIYRQrFYDqzuHCPY6tp+0/bFoXIWQ3dI+zVms2bj6C8VP/MQP\nYtiw/HTb7lfGuQUh6q2sot7KqYz11t13kQsh9AHxjX35tLa29qpb2YYQQgibqz49RDCE0HmSHP8u\nhBBCCCHUt1kOEQwhhBBCCCGEd1M0sEIIoQ8o4xj1EPVWVlFv5RT1Vk5lrLdoYIUQQgghhBBCN4k5\nWCGEdmIOVgghhBBCYzEHK4QQQgghhBA2sbhNewihA6nDlzEhhBBCqQwbPoz169a//T4eZ1FOZay3\naGCFEDqa0tMJhE5bC4zu6SRCp0W9lVPUWyk8N+W5nk4hbKZiDlYIoR1JjgZWCCGE0psC8Tk3bEqb\ndA6WpJGSHqmxbp6kSd1xnM6SNELSEkmzc8vW9kQutUjaT9K0JuLWpt81z3VV/DaSnpH0g/w+JA3p\nRG4Nz1Wq3xF11p8o6YfNHrPJvE6slEvSRZJOaBA/RNL9kl7Nn48G20yTtG+D9Yd3Mu99JK1M1+R7\nJF0h6RFJ303lOLfB9g3LmuL+RdJvJa2W9Inc8vMkPSbps53JO4QQQgghNK87b3LRG78iOAyYY/vg\n3LLemGczObnG61ouAeZ34TjvJH5T76er/gR8HTivh/M4FrjM9iTbrwMnA+Ntf7W7DiBpLHAUMBY4\nGLhaaUKV7SuBE4Ezuut4oRfpVV8dhaZFvZVT1FsplfF5SqGc9dadDaz+km6StErSrZK2rA6QdIyk\nFenn8rRsi9QbsELScklnp+VjJM2VtEzSIkldGe08GHi+atkLuXxOSMdcKml6WjZN0lRJD0h6otJL\nIWlrSfemXJZLOjQtH5l6CqZJWiNphqSD0vZrJO2R4raSdIOkhZIWSzokpfEG8EoTZXmheoGk61Lu\nSyU9L+kbaflkYEdgTvUmwFnp+Msl/U2ubP+W6mCZpE/XOmaBPwBvpf38fdr3MklzC/IdKuk2Sb9J\nP3srs1bStrm4xyXtUBRfcPyNwGv1ErT9v7YfBF5vojwVG8jqBkmXp56nZZK+l4vZr+A62U/SXbmy\n/DBdZyeRNXwukXSjpP8ABgKLJR1ZdZ52lTRb0sOS5lfqCXi1UVmBTwG32H7T9u+A3wIfzq1fDwzq\nxHkIIYQQQgid0J03udgN+ILthZJuAE4HrqqslLQzcDkwkezD69zUSFkHDLc9PsVVPmjPIPu2f5ak\nAXStMdgPaMsvsL1XOs444EJgb9svSxqcC9vJ9kdTb8As4HayXpDDbG+UtD2wMK0DGAN8xvYqSYuA\no9P2h6ZjHA58DbjP9kmSBgEPSbrX9gJgQcppMnCq7VOqC1LJu2rZyWm7EcBsYFrqrfg+WW/JQQXn\n5HnbkyWdBpwPnAJ8A9iQq4NBtY5ZkMMRaZuhwLXAPrafrjqfFVOBq2w/KOl9wD22x0m6E/g0MF3S\nh4Hf2X5B0ozqeGBc1fGvrLyWdGq2yNc2yruJcp2T9jmErN4/kN5vmwsruk6goMfO9g2S9gHusn17\n2tcfbU9Kry/KhV9Ldh08mc7HNcABtvN/T7XKOpx0PSW/T8sq2mjm735e7vUoYjJ3GUQdlVPUWzlF\nvZVS2e5EFzK9qd5aW1ub6lHrzgbW07YXptc3AV8i18AC9gTm2X4JIH143he4FBgtaSrwS2COpIHA\nLrZnAdh+o7PJpIbGhJRLkY8DM22/nI6xIbfuzrRstaQdK7sEvqNsXk4bsEtu3Vrbq9LrR4F70+tH\nyD6eAnwCOETSV9L7AcAIYE3loLYXkzV4OlPOLYGZwJm210k6A7jb9rPZKaB64t0d6fdiskYNwIHA\n2/NybDfTo1btI8B820+nfWwoiDkQGJvqBmCgpK2AW4FvAtOBo4GfN4gvZPtfu5B3I68Ar0m6Hrgb\n+EVuXdF18o5I2hr4O2Bmrtz9q+PeQVlfBHaQNLhGHWX27+LeQwghhBD6qJaWlnYNvosvvrgwblPO\nwSqad9PhLhvpQ94EoBX4InBdrdh2O5JOT0PjlkjaqWrdFmQjpMeSfSjurPxQskoexwJDgYm2J5IN\nPdyyIL4t9z7fWyCyXq6J6We07TW8c9cAt9mu9DnsDZwp6SmynqzjJV1WULa36P7b9Dd6eJKAvXLn\nYEQavrcAGJN6wQ4D/r1efDfnXJftt8iG2N0G/BPwn7nVRdfJm7T/u+owVLaBLYCX0zytSrn/thPb\n/x54X+79X6dlANh+DbgFeEpSUQ9nKKuYE1JOUW/lFPVWSmWcyxPKWW/d2cAaKakypOxzwK+r1j8E\n7Kvsjm79gGOA+Wm4XT/bd5DdiGCS7Y3AM5I+BSBpgKT35ndm++r04XOS7fVV69psjwIWkeuZqXI/\ncGQaAoak7WrEVT44DyIbXtcmaX9gZEFMPfcAZ729gfShJrapK/VWDbR9RWWZ7eNsj7K9K9kQwJ/a\nvrDBruaSu/FB0fA+ZfPPdq6zj4XAxySNTPFF53MOcHZunxNy6+4g6/FcletZqRffVe3qStJ0pXly\nhcFZj9Jg2/8JnAuMb7Df/wLGSeqfzuMBzeYCYPtVYK2kI3I51DpmkVnA0elvZjTwfrK/vcq+BpP9\nTQy33WGeXAghhBBCeGe6s4H1GHCGpFVkN5f4cVpugNQIuoCsp2op8LDtu8jmh7RKWgrcmGIATiC7\nIcNy4AFgWBdyehwovC15GtL3bbJG3lKgMpenVk/cDGDPlM9xwOqCmKLtKy4huxHICmW3Wf9WdYCk\nyZI6M3/oPGD3XE9eo+GFtXK7FBii7JbhS4GWqrxENs/spZo7tl8kG954R9rHLQVhZwN7KLvBxkrg\n1Ny6W8l6CW9pMr4DSafWOgfKbjl/JXCipKclfSCtGg88W2e32wC/SPX+K+CctLzwOrG9LpVlZSrL\nkuqYOu8rjgNOUnZTjZXAoQXlKSxruq5vBVaRDbk93e0fAjIIeC71ZIW+JOaElFPUWzlFvZVSb5rL\nE5pXxnrr0w8aTvOdtrd9QcPgUJOkD5LdwOT8ns6lO0naBrje9mbzXKh004yptovuyFiJiQcNhxBC\nKL8p8aDhsGmpxoOG+3oDawzwE2Bj1bOwQtjsSDqPrJfwCts314nru/8ohBBC2GwMGz6M9ev+Mouk\ntbW1lL0hm7veXG+1GljdfZODXsX2k8DHejqPEHqDdEv7KxsGEt/4lVFv/g8o1Bb1Vk5RbyGEevp0\nD1YIofMkOf5dCCGEEEKor1YPVnfe5CKEEEIIIYQQNmvRwAohhD6gjM8JCVFvZRX1Vk5Rb+VUxnqL\nBlYIIYQQQgghdJOYgxVCaCfmYIUQQgghNBZzsEIIIYQQQghhE4sGVgihA0nxU9KfnXYa1dOXT+iE\nMs4tCFFvZRX1Vk5lrLc+/RysEEJXxRDB8mkFWnjuuQ4jFUIIIYTwLoo5WCGEdiQ5GlhlpnhQdAgh\nhPAu6FVzsCSNlPRIjXXzJE16t3NKxx4haYmk2blla3sil1ok7SdpWhNxa9Pvmh8kNx0AACAASURB\nVOe6Kn4bSc9I+kFu2TxJIxpsN03Svg3yvavR8Tsjv09JJ0q6qIltvivpEUkrJB3VRPxFkk5osP7c\nTua9m6SlkhZLGi3pLEmrJN2YyvHDBts3LKukCZIeTGVdli+rpGMkPSbpnM7kHUIIIYQQmteTc7B6\n41eshwFzbB+cW9Yb82wmJ9d4XcslwPyupdOpXDbFPuvuX9I/AB8CxgMfAc6XNHAT5NTIYcBM25Nt\nrwVOAw60fXxa39l6LfI/wPG2dwcOBv6fpG0BbN8M7AdEA6tPau3pBEIXlHFuQYh6K6uot3IqY731\nZAOrv6Sb0jf4t0rasjogfeO+Iv1cnpZtkXpNVkhaLunstHyMpLnpW/tFkkZ3IafBwPNVy17I5XNC\nOuZSSdPTsmmSpkp6QNITkg5Py7eWdG/KZbmkQ9PykZJWp+3WSJoh6aC0/RpJe6S4rSTdIGlh6vE4\nJKXxBvBKE2V5oXqBpOtS7kslPS/pG2n5ZGBHYE7VJn8A3mpwnA0pJyTtmcqxLOW9ddXxC8skaYGk\nsbm4eZIm1TkHea8BGxvkOA74lTP/C6wA/r7BNq+mfZN6mh5N5fpZLuaDKdcnJH0pxbbrMZR0Xurt\nOhj4MnCapPskXQPsCsyuXMO5bYZKuk3Sb9LP3s2W1fYTtp9Mr/+b7HreIbf+OWBQg7KHEEIIIYQu\n6smbXOwGfMH2Qkk3AKcDV1VWStoZuByYSPYhfm5qpKwDhtsen+K2TZvMAC6zPUvSALrWeOwHtOUX\n2N4rHWcccCGwt+2XJQ3Ohe1k+6OpkTALuB34E3CY7Y2StgcWpnUAY4DP2F4laRFwdNr+0HSMw4Gv\nAffZPknSIOAhSffaXgAsSDlNBk61fUp1QSp5Vy07OW03ApgNTJMk4PvAscBBVfFHNDphts9J++wP\n3AIcaXtJ6iF6rSq8sExpu88CUyTtlM7nEknfrhGfP/6tldepATbZ9pSq4y4HvinpKmBrYH/g0Qbl\nuir39qvAKNt/zl1vkF3DLWQNljWSrq5s3nF3ni3px8CrlX1L+iTQkq6nE3PxU4GrbD8o6X3APcC4\nJstKLubDQP9Kgyunib+N/G5b0k/o3Vp6OoHQBS0tLT2dQuiCqLdyinorp95Ub62trU31qPVkA+tp\n2wvT65uAL5FrYAF7AvNsvwQgaQawL3ApMFrSVOCXwJz0YX4X27MAbL/R2WRSQ2NCyqXIx8mGd72c\njrEht+7OtGy1pB0ruwS+o2x+UhuwS27dWtur0utHgUqj4RFgVHr9CeAQSV9J7wcAI4A1lYPaXgx0\naFw1KOeWwEzgTNvrJJ0B3G372ewU0NVbkO0GPGt7ScptYzpePqZWmWaS9Z5NAY4CbmsQX8j2XUCH\n+V6250raE3iQrEfnQRr3zOUtB34m6U5SXSd3234T+IOk54BhndgnZOe66HwfCIzVX07eQElbpd43\noHZZ395x9gXFT4HjC1a/JGlMQcMrZ0rD5EMIIYQQNictLS3tGnwXX3xxYVxvmoNVNLekw4fP1LCZ\nQDbh4IvAdbVi2+1IOj0NjVuSekny67YA1gJjgbubyr691wtyPhYYCky0PZHsg/2WBfFtufdt/KXR\nK7JeronpZ7TtNbxz1wC32Z6X3u8NnCnpKbKerOMlXdbFfTdqnBWWyfazwIuSdifryfp5bptuOQe2\nL0v7+CTZdf94Jzb/R+BHwCTg4XS9QMd6/CvgTbKe0IoOQ1+bIGCvXLlH5BtXDTeWtgF+AfyL7YcL\nQqYCyyR9vgu5hV6rtacTCF1QxrkFIeqtrKLeyqmM9daTDayRkirD2D4H/Lpq/UPAvpKGSOoHHAPM\nT8Pt+tm+A/g6MCn1ljwj6VMAkgZIem9+Z7avTh9WJ9leX7WuzfYoYBHZB/wi9wNHShqSjrFdjbhK\nI2MQ8LztNkn7AyMLYuq5Bzjr7Q2kDzWxTV2pt2qg7Ssqy2wfZ3uU7V2B84Gf2r6wYNvpSvPDalgD\n7JSGLSJpYKq3vHpl+jnwz8C2tlc2Ed80ZfP2KvU2HtidNN9M0mWV66bGtgJG2J4PXABsC9S7QcZz\nwA6StpP0HuCfupDyHODteVmSJjS7YRqqeScwPf2NFLkQeL/tn3QhtxBCCCGEUEdPNrAeA86QtIrs\n5hI/TssNkBpBF5B9LbsUeDgNixoOtEpaCtyYYgBOAM6StBx4gM4P14KsV2NI0Yo0pO/bZI28pcCV\n+Xzzoen3DGDPlM9xwOqCmKLtKy4huxHIinTThG9VB0iaLOnaOuWpdh6we64nrzPDC8cDz9ZaafvP\nZI3TH0laRtZIeE9VWL0y/Tsde68urRPfgaRDJE0pWNUf+LWklWTX2XG2K3PtdgfWF2xT0Q+4KdXj\nYmCq7T8WxFWu2zdTng+TNRBXF8S226bA2cAeym6OshI4tTqgTlmPAvYBPp+r5/FVMQPSzS5Cn9LS\n0wmELuhNcwtC86LeyinqrZzKWG/xoOGcNNdne9sXNAzejKQhZ9fbrtW7V1qSZlfdlr9PS/MAl9ve\nuU5MPGi41OJBwyGEEMK7Qb3pQcO92O3AR5V70HAA26/2xcYVwGbWuDqGrGfxe01Ex09Jf4YNy49G\nDr1dGecWhKi3sop6K6cy1ltP3kWw10l3VftYT+cRwqaQHjR8c5Oxmzib0N1aW1tLOYwihBBC6Gti\niGAIoR1Jjn8XQgghhBDqiyGCIYQQQgghhLCJRQMrhBD6gDKOUQ9Rb2UV9VZOUW/lVMZ6iwZWCCGE\nEEIIIXSTmIMVQmgn5mCFEEIIITQWc7BCCCGEEEIIYROL27SHEDqQOnwZE0IouWHDh7F+3fqeTqNP\niMcilFPUWzmVsd6igRVC6GhKTycQOm0tMLqnkwid9i7W23NTnnt3DhRCCJu5mIMVQmhHkqOBFUIf\nNCUeIh5CCN2pV83BkjRS0iM11s2TNOndzikde4SkJZJm55at7YlcapG0n6RpTcStTb9rnuuq+G0k\nPSPpB7ll8ySNaLDdNEn7Nsj3rkbH74z8PiWdKOmiJrb5rqRHJK2QdFQT8RdJOqHB+nM7mfdukpZK\nWixptKSzJK2SdGMqxw8bbN9sWU+U9LikNfkySDpG0mOSzulM3iGEEEIIoXk9eZOL3vg12mHAHNsH\n55b1xjybyck1XtdyCTC/a+l0KpdNsc+6+5f0D8CHgPHAR4DzJQ3cBDk1chgw0/Zk22uB04ADbR+f\n1ne2XjuQtB3wTWBPYC/gIkmDAGzfDOwHRAOrL+pVXwWFpkW9lVIZn8sTot7Kqoz11pMNrP6Sbkrf\n4N8qacvqgPSN+4r0c3latkXqNVkhabmks9PyMZLmSlomaZGkroxqHww8X7XshVw+J6RjLpU0PS2b\nJmmqpAckPSHp8LR8a0n3plyWSzo0LR8paXXabo2kGZIOStuvkbRHittK0g2SFqYej0NSGm8ArzRR\nlheqF0i6LuW+VNLzkr6Rlk8GdgTmVG3yB+CtBsfZkHJC0p6pHMtS3ltXHb+wTJIWSBqbi5snaVKd\nc5D3GrCxQY7jgF8587/ACuDvG2zzato3qafp0VSun+ViPphyfULSl1Jsux5DSeel3q6DgS8Dp0m6\nT9I1wK7A7Mo1nNtmqKTbJP0m/ezdibJ+kuxLgldsbyCr07fLavs5YFCDfYQQQgghhC7qyZtc7AZ8\nwfZCSTcApwNXVVZK2hm4HJhI9iF+bmqkrAOG2x6f4rZNm8wALrM9S9IAutZ47Ae05RfY3isdZxxw\nIbC37ZclDc6F7WT7o6mRMAu4HfgTcJjtjZK2BxamdQBjgM/YXiVpEXB02v7QdIzDga8B99k+KfVA\nPCTpXtsLgAUpp8nAqbZPqS5IJe+qZSen7UYAs4FpkgR8HzgWOKgq/ohGJ8z2OWmf/YFbgCNtL0k9\nRK9VhReWKW33WWCKpJ3S+Vwi6ds14vPHv7XyOjXAJtueUnXc5cA3JV0FbA3sDzzaoFxX5d5+FRhl\n+8+56w2ya7iFrMGyRtLVlc077s6zJf0YeLWyb0mfBFrS9XRiLn4qcJXtByW9D7gHGNdkWYcDz+Te\n/z4ty2v8tzEv93oUcfOEMog6Kqeot1Iq2x3NQibqrZx6U721trY21aPWkw2sp20vTK9vAr5EroFF\nNsRpnu2XACTNAPYFLgVGS5oK/BKYkz7M72J7FoDtNzqbTGpoTEi5FPk42fCul9MxNuTW3ZmWrZa0\nY2WXwHeUzU9qA3bJrVtre1V6/ShQaTQ8QvZxFuATwCGSvpLeDwBGAGsqB7W9GOjQuGpQzi2BmcCZ\nttdJOgO42/az2Smgq/fn3g141vaSlNvGdLx8TK0yzSTraZkCHAXc1iC+kO27gA7zvWzPlbQn8CBZ\nD+WDNO6Zy1sO/EzSnaS6Tu62/SbwB0nPAcM6sU/IznXR+T4QGKu/nLyBkrZKvW9A7bI26SVJY2w/\nWTNi/y7uOYQQQgihj2ppaWnX4Lv44osL43rTHKyiuSUdPnymhs0EoBX4InBdrdh2O5JOT0PjlqRe\nkvy6LchGwo8F7m4q+/ZeL8j5WGAoMNH2RLIP9lsWxLfl3rfxl0avyHq5Jqaf0bbX8M5dA9xmu9JH\nsTdwpqSnyHqyjpd0WRf33ahxVlgm288CL0ranawn6+e5bbrlHNi+LO3jk2TX/eOd2PwfgR8Bk4CH\n0/UCHevxr4A3yXpCKzoMfW2CgL1y5R6Rb1w18HvaN0L/Oi3Lmwosk/T5LuQWequYy1NOUW+lVMY5\nISHqrazKWG892cAaKakyjO1zwK+r1j8E7CtpiKR+wDHA/DTcrp/tO4CvA5NSb8kzkj4FIGmApPfm\nd2b76vRhdZLt9VXr2myPAhaRfcAvcj9wpKQh6Rjb1YirNDIGAc/bbpO0PzCyIKaee4Cz3t5A+lAT\n29SVeqsG2r6issz2cbZH2d4VOB/4qe0LC7adrjQ/rIY1wE5p2CKSBqZ6y6tXpp8D/wxsa3tlE/FN\nUzZvr1Jv44HdSfPNJF1WuW5qbCtghO35wAXAtkC9G2Q8B+wgaTtJ7wH+qQspzwHenpclaUIntr0H\nOEjSoHSNHpSW5V0IvN/2T7qQWwghhBBCqKMnG1iPAWdIWkV2c4kfp+UGSI2gC8h6qpYCD6dhUcOB\nVklLgRtTDMAJwFmSlgMP0PnhWpD1agwpWpGG9H2brJG3FLgyn28+NP2eAeyZ8jkOWF0QU7R9xSVk\nNwJZkW6a8K3qAEmTJV1bpzzVzgN2z/XkdWZ44Xjg2Vorbf+ZrHH6I0nLyBoJ76kKq1emf6dj79Wl\ndeI7kHSIpCkFq/oDv5a0kuw6O852Za7d7sD6gm0q+gE3pXpcDEy1/ceCuMp1+2bK82Gyhs3qgth2\n2xQ4G9hD2c1RVgKnVgfUKmsawnoJ2ZcFvwEurhrOCjAg3ewi9CUxl6ecot5KqTfNCQnNi3orpzLW\nWzxoOCfN9dne9gUNgzcjkrYBrrddq3evtCTNrrotf5+W5gEut71znZh40HAIfdGUeNBwCCF0J9V4\n0HA0sHIkjQF+AmzcnD50h82DpGPI7og43fb/rRMX/yiE0AcNGz6M9evqddiHZrW2tpbyW/XNXdRb\nOfXmeqvVwOrJuwj2Oumuah/r6TxC2BTSg4ZvbjJ2E2cTultv/g8o1Bb1FkIIfU/0YIUQ2pHk+Hch\nhBBCCKG+Wj1YPXmTixBCCCGEEELoU6KBFUIIfUAZnxMSot7KKuqtnKLeyqmM9RYNrBBCCCGEEELo\nJjEHK4TQTszBCiGEEEJoLOZghRBCCCGEEMImFrdpDyF0IHX4MiaEsAlst90wXnopnk1VNnF7/XKK\neiunMtZbNLBCCAViiGD5tAItPZxD6KyXX44vM0IIoa+JOVghhHYkORpYIbxbFA/2DiGEkupVc7Ak\njZT0SI118yRNerdzSsceIWmJpNm5ZWt7IpdaJO0naVoTcWvT75rnuip+G0nPSPpBbtk8SSMabDdN\n0r4N8r2r0fE7I79PSSdKuqiJbd5KdbtU0p1NxF8k6YQG68/tZN67peMvljRa0lmSVkm6MZXjhw22\nb1hWSRMkPSjpEUnLJB2VW3eMpMckndOZvEMIIYQQQvN68iYXvfEru8OAObYPzi3rjXk2k5NrvK7l\nEmB+19LpVC6bYp/N7P9/bE+yPdH2YZsgn2YcBsy0Pdn2WuA04EDbx6f1na3XIv8DHG97d+Bg4P9J\n2hbA9s3AfkA0sPqk1p5OIITNRhmfyxOi3sqqjPXWkw2s/pJuSt/g3yppy+qA9I37ivRzeVq2Reo1\nWSFpuaSz0/Ixkuamb+0XSRrdhZwGA89XLXshl88J6ZhLJU1Py6ZJmirpAUlPSDo8Ld9a0r0pl+WS\nDk3LR0panbZbI2mGpIPS9msk7ZHitpJ0g6SFqcfjkJTGG8ArTZTlheoFkq5LuS+V9Lykb6Tlk4Ed\ngTlVm/wBeKvBcTaknJC0ZyrHspT31lXHLyyTpAWSxubi5kmaVOcc5L0GbGyQI0BnJzq8mvZN6ml6\nNJXrZ7mYD6Zcn5D0pRTbrsdQ0nmpt+tg4MvAaZLuk3QNsCswu3IN57YZKuk2Sb9JP3s3W1bbT9h+\nMr3+b7LreYfc+ueAQZ08FyGEEEIIoUk9eZOL3YAv2F4o6QbgdOCqykpJOwOXAxPJPsTPTY2UdcBw\n2+NT3LZpkxnAZbZnSRpA1xqP/YC2/ALbe6XjjAMuBPa2/bKkwbmwnWx/NDUSZgG3A38CDrO9UdL2\nwMK0DmAM8BnbqyQtAo5O2x+ajnE48DXgPtsnSRoEPCTpXtsLgAUpp8nAqbZPqS5IJe+qZSen7UYA\ns4FpkgR8HzgWOKgq/ohGJ8z2OWmf/YFbgCNtL5E0kNRAySksU9rus8AUSTul87lE0rdrxOePf2vl\ndWqATbY9pSDV96Rz/QbwXdv/0aBcV+XefhUYZfvPuesNsmu4hazBskbS1ZXNO+7OsyX9GHi1sm9J\nnwRa0vV0Yi5+KnCV7QclvQ+4BxjXibJWYj4M9K80uHKa+NvI77aFuHlCGbT0dAIhbDbKdkezkIl6\nK6feVG+tra1N9aj1ZAPradsL0+ubgC+Ra2ABewLzbL8EIGkGsC9wKTBa0lTgl8Cc9GF+F9uzAGy/\n0dlkUkNjQsqlyMfJhne9nI6xIbfuzrRstaQdK7sEvqNsflIbsEtu3Vrbq9LrR4FKo+ERYFR6/Qng\nEElfSe8HACOANZWD2l4MdGhcNSjnlsBM4Ezb6ySdAdxt+9nsFHS6p6diN+BZ20tSbhvT8fIxtco0\nk6z3bApwFHBbg/hCtu8Cas33Gmn7v1PP5v2SVqRhes1YDvxM2dyt/Pytu22/CfxB0nPAsCb3VyGK\nz/eBwFj95eQNlLSV7f+tBDQoa+ULip8CxxesfknSmIKGV86UhsmHEEIIIWxOWlpa2jX4Lr744sK4\n3jQHq2huSYcPn6lhM4FswsEXgetqxbbbkXR6Ghq3JPWS5NdtAawFxgJ3N5V9e68X5HwsMBSYaHsi\n2VCtLQvi23Lv2/hLo1dkvVwT089o22t4564BbrM9L73fGzhT0lNkPVnHS7qsi/tu1DgrLJPtZ4EX\nJe1O1pP189w23XIO0nA5UqOqlaxntFn/CPwImAQ8nK4X6FiPfwW8SdYTWtFh6GsTBOyVK/eIfOOq\n4cbSNsAvgH+x/XBByFRgmaTPdyG30Gu19nQCIWw2yjgnJES9lVUZ660nG1gjJVWGsX0O+HXV+oeA\nfSUNkdQPOAaYn4bb9bN9B/B1YFLqLXlG0qcAJA2Q9N78zmxfnT6sTrK9vmpdm+1RwCKyD/hF7geO\nlDQkHWO7GnGVRsYg4HnbbZL2B0YWxNRzD3DW2xtIH2pim7pSb9VA21dUltk+zvYo27sC5wM/tX1h\nwbbTleaH1bAG2CkNW0TSwFRvefXK9HPgn4Ftba9sIr5pkganYaNIGgp8FFiV3l9WuW5qbCtghO35\nwAXAtsDAOod7DthB0naS3gP8UxdSngO8PS9L0oRmN0xDNe8Epqe/kSIXAu+3/ZMu5BZCCCGEEOro\nyQbWY8AZklaR3Vzix2m5AVIj6AKyr2WXAg+nYVHDgVZJS4EbUwzACcBZkpYDD9D54VoAjwNDilak\nIX3fJmvkLQWuzOebD02/ZwB7pnyOA1YXxBRtX3EJ2Y1AVqSbJnyrOkDSZEnX1ilPtfOA3XM9eZ0Z\nXjgeeLbWStt/Jmuc/kjSMrJGwnuqwuqV6d/p2Ht1aZ34DiQdImlKwaqxwKJUb/eRzdV7LK3bHVhf\nsE1FP+CmVI+Lgam2/1gQV7lu30x5PkzWQFxdENtumwJnA3souznKSuDU6oA6ZT0K2Af4fK6ex1fF\nDEg3uwh9SktPJxDCZqM3zQkJzYt6K6cy1ls8aDgnzfXZ3vYFDYM3I2nI2fW2a/XulZak2VW35e/T\n0jzA5bZ3rhMTDxoO4V0TDxoOIYSyUm960HAvdjvwUeUeNBzA9qt9sXEFsJk1ro4h61n8Xk/nEjaF\n1p5OIITNRhnnhISot7IqY7315F0Ee510V7WP9XQeIWwK6UHDNzcX3dWbSYYQOmO77boymj2EEEJv\nFkMEQwjtSHL8uxBCCCGEUF8MEQwhhBBCCCGETSwaWCGE0AeUcYx6iHorq6i3cop6K6cy1ls0sEII\nIYQQQgihm8QcrBBCOzEHK4QQQgihsZiDFUIIIYQQQgibWDSwQgihDyjjGPUQ9VZWUW/lFPVWTmWs\nt3gOVgihAymegxVCCGHzNWz4MNavW9/TaYSSijlYIYR2JJkpPZ1FCCGE0IOmQHxGDo3EHKwQQggh\nhBBC2MR6pIElaaSkR2qsmydp0rudUzr2CElLJM3OLVvbE7nUImk/SdOaiFubftc811Xx20h6RtIP\ncsvmSRrRYLtpkvZtkO9djY7fGfl9SjpR0kVNbPNWqtulku5sIv4iSSc0WH9uJ/PeLR1/saTRks6S\ntErSjakcP2ywfbNlPVHS45LW5Msg6RhJj0k6pzN5h5LoVf9ShaZFvZVT1Fs5Rb2VUszB6pze2O96\nGDDH9gW5Zb0xz2Zyco3XtVwCzO9aOp3KZVPss5n9/4/tHmm45xwGzLR9GYCk04ADbD8r6UQ6X68d\nSNoO+CYwCRCwWNJ/2H7F9s2S7gceBv7vOylICCGEEEIo1pNDBPtLuil9g3+rpC2rA9I37ivSz+Vp\n2Rap12SFpOWSzk7Lx0iaK2mZpEWSRnchp8HA81XLXsjlc0I65lJJ09OyaZKmSnpA0hOSDk/Lt5Z0\nb8pluaRD0/KRklan7dZImiHpoLT9Gkl7pLitJN0gaWHq8TgkpfEG8EoTZXmheoGk61LuSyU9L+kb\naflkYEdgTtUmfwDeanCcDSknJO2ZyrEs5b111fELyyRpgaSxubh5kibVOQd5rwEbG+QIWWOjM15N\n+yb1ND2ayvWzXMwHU65PSPr/2Tv3cC2rMv9/vpCIipKHBGUGNKbL0UlN0J85Org1D6l5GJOMNK3x\nMhtNOziao05hmodKr/E3jlrpoCNoanlAiUSMTaYiChvQQLKiPA1ooxSa5/39/fGsF5797vf82/ju\nB+7Pde1rP+9a91rrXut+2Dz3c99rvacn2R4RQ0lnpmjXIcBXgH+W9ICka4APAtNL93CuzVaSfizp\n0fSzVxNzPZjsJcGfbK8ks+nHS5W2VwBDm1yLoAi08hcvaD9ht2ISdismYbdC0tHR0W4VmqadEawd\ngM/bniPpeuBU4IpSpaRtgEuB3cge4u9PTspzwAjbuyS5zVKTKcDFtqdKGkRrzuNAoDtfYHvPNM5O\nwLnAXrZfkfT+nNhw23snJ2EqcAfwBnCU7VclbQnMSXUAo4FP2l4s6XHg06n9EWmMo4HzgAdsnyRp\nKDBX0kzbjwCPJJ3GAqfY/kL5REp6l5WdnNqNBKYDkyQJ+B5wHHBgmfwx9RbM9ldTnxsAPwLG254v\naQjJQclRcU6p3bHAREnD03rOl/TtKvL58W8rXScHbKztiRVU3TCt9VvAZbbvrjOvK3Ifvw5sZ/vt\n3P0G2T3cQeawLJV0dal57+48XdK1wKpS35IOBjrS/XRiTv5K4ArbD0v6a+A+YKcG5zoCeDb3+flU\nlqf+v41ZuevtiP+UgiAIgiBY7+ns7GwoZbGdDtYztuek68nA6eQcLGAPYJbtlwEkTQHGARcB20u6\nEvgpMCM9zG9reyqA7beaVSY5GrsmXSqxP1l61ytpjJW5urtS2RJJW5e6BC5Rtj+pG9g2V7fM9uJ0\n/Sug5DQ8QfY4C3AQcLiks9LnQcBIYGlpUNvzgF7OVZ15DgZuB75k+zlJpwHTUppaSe9W2AF4wfb8\npNuraby8TLU53U4WaZkIfAr4cR35iti+B6i232uU7f9Jkc2fS1pku9Fs7IXAzcr2buX3b02z/Q7w\nv5JWAMMa7K+EqLzeBwA7as3iDZG0se2/lATqzLUeL0sabfu3VSX2a7HnoH0sIxzhIhJ2KyZht2IS\ndisknZ2d/SaK1dHR0UOXCy64oKJcf9qDVWlvSa+HT9srJe1Klgr1RWA8WepVTcdA0qnAyWmcQ20v\nz9UNAH4HvAlMa2IOJd6soPNxwFbAbra7lR06MbiCfHfuczdrbCKyKNfTLehTi2uAH9suxSj2AvZJ\n67MpWermKtvnttB3Pees6pwk/VHSzmSRrFNyVb3kU5SrKWz/T/q9TFInWWS0UQfrMDLn/gjgPEkf\nTuXldnwf8A5ZJLREr9TXBhCwp+23W2j7PFlUrcRf0TMeBVmEbIGk023f0MIYQRAEQRAEQRXauQdr\nlKRSGttngAfL6ucC4yRtIWkgMAGYndLtBtq+EzgfGJOiJc9KOhJA0iBJG+U7s3217d1sj8k7V6mu\n2/Z2wONkD/iV+DkwXtIWaYzNq8iVnIyhwIvJudoPGFVBphb3AWesbiB9pIE2NUnRqiG2v1sqs328\n7e1sfxD4F+C/KzlXkm5U2h9WhaXA8JS2iKQhyW55as3pVuBsYDPbTzYg70Ac1gAAIABJREFU3zCS\n3p/SRpG0FbA3sDh9vrh031RpK2Ck7dnAOcBmwJAaw60APiBpc0kbAp9oQeUZwOp9WemFQqPcBxwo\naWi6Rw9MZXnOBf4mnKt1jHgrW0zCbsUk7FZMwm6FpL9Er5qhnQ7WU8BpkhaTHS5xbSo3QHKCzgE6\ngS7gsZQWNQLolNQF3JRkAE4AzpC0EHiI5tO1AH4NbFGpIqX0fZvMyesCLs/rmxdNv6cAeyR9jgeW\nVJCp1L7EhWTRpEXp0IRvlQtIGivpBzXmU86ZwM7KDrmYL6mZ9MJdgBeqVaZoy7HAVZIWkDkJG5aJ\n1ZrTT1L7W3NlF9WQ74WkwyVNrFC1I/B4stsDZHv1nkp1OwO1vqp9IDA52XEecKXtP1eQK9237yQ9\nHyNzbJZUkO3RpgJfBnZXdjjKk/SM6AHV55pSWC8ke1nwKHBBWTorwKB02EUQBEEQBEHQxyi+pXoN\naa/PlmXHtK/3SNoUuM52teheYZE03fYh7dbjvSLtA1xoe5saMmbie6dT0EfE3oJiEnYrJmG3YtKM\n3SZCPCP3D/rTHqxyJGG7V2ZaO/dg9UfuAG5Y3x6662F7FdVTJwvN+mRnSRPITkT8Tl3hiWtbmyAI\ngiDovwwb0UoiVBBkRAQrCIIeSHL8XQiCIAiCIKhNtQhWO/dgBUEQBEEQBEEQrFOEgxUEQbAO0MgX\nHwb9j7BbMQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9iD1YQRAEQRAE9Yk9WEEQBEEQBEEQ\nBGuZcLCCIAjWAYqYox6E3YpK2K2YhN2KSRHtFt+DFQRBL6Re0e4gCNrIsGGjWL789+1WIwiCIGiA\n2IMVBEEPJBni70IQ9C9E/H8dBEHQv4g9WEEQBEEQBEEQBGuZtjhYkkZJeqJK3SxJY95rndLYIyXN\nlzQ9V7asHbpUQ9K+kiY1ILcs/a661mXym0p6VtL/zZXNkjSyTrtJksbV0feeeuM3Q75PSSdK+mYD\nbaZLekXS1AbH+KakE+rUf61xrUHSDpK6JM2TtL2kMyQtlnRTmsd/1Glfd66SdpX0sKQnJC2Q9Klc\n3QRJT0n6ajN6B0Whs90KBC3R2W4FghYo4p6QIOxWVIpot3ZGsPpjrsNRwAzbh+TK+qOejejkKtfV\nuBCY3Zo6TemyNvpspP/vAMevBT2a4SjgdttjbS8D/hk4wPZnU32zdq3Ea8Bnbe8MHAL8u6TNAGzf\nAuwLhIMVBEEQBEGwlming7WBpMnpDf5tkgaXC6Q37ovSz6WpbECKmiyStFDSl1P5aEn3p7f2j0va\nvgWd3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0dCrfRNLMpMtCSUek8lGSlqR2SyVNkXRgar9U\n0u5JbmNJ10uakyIehyc13gL+1MBcXiovkPTDpHuXpBcl/VsqHwtsDcwoa/K/wLt1xlmZdELSHmke\nC5Lem5SNX3FOkh6RtGNObpakMTXWIM/rwKt1dMT2rEbkcqxKfZMiTb9K87o5J/N3SdffSDo9yfaI\nGEo6M0W7DgG+AvyzpAckXQN8EJheuodzbbaS9GNJj6afvRqdq+3f2P5tuv4fsvv5A7n6FcDQJtYh\nKAwd7VYgaImOdisQtEBHR0e7VQhaIOxWTIpot3aeIrgD8HnbcyRdD5wKXFGqlLQNcCmwG9lD/P3J\nSXkOGGF7lyS3WWoyBbjY9lRJg2jNeRwIdOcLbO+ZxtkJOBfYy/Yrkt6fExtue+/kJEwF7gDeAI6y\n/aqkLYE5qQ5gNPBJ24slPQ58OrU/Io1xNHAe8IDtkyQNBeZKmmn7EeCRpNNY4BTbXyifSEnvsrKT\nU7uRwHRgkiQB3wOOAw4skz+m3oLZ/mrqcwPgR8B42/MlDSE5KDkqzim1OxaYKGl4Ws/5kr5dRT4/\n/m2l6+SAjbU9sZ7eDczritzHrwPb2X47d79Bdg93kDksSyVdXWreuztPl3QtsKrUt6SDgY50P52Y\nk78SuML2w5L+GrgP2KnZuUr6P8AGJYcrRwP/NvLddhAPgUEQBEEQrO90dnY2lLLYTgfrGdtz0vVk\n4HRyDhawBzDL9ssAkqYA44CLgO0lXQn8FJiRHua3tT0VwPZbzSqTHI1dky6V2J8sveuVNMbKXN1d\nqWyJpK1LXQKXKNuf1A1sm6tbZntxuv4VUHIangC2S9cHAYdLOit9HgSMBJaWBrU9D+jlXNWZ52Dg\nduBLtp+TdBowzfYL2RLQ6vncOwAv2J6fdHs1jZeXqTan28miZxOBTwE/riNfEdv3AH263yuxELhZ\n0l0kWyem2X4H+F9JK4BhTfYrKq/3AcCOWrN4QyRtbPsvJYF6c00vKP4b+GyF6pclja7geOWYWFf5\noL/RSTjCRaSTsFvx6OzsLORb9fWdsFsx6U926+jo6KHLBRdcUFGunQ5Wr7f8FWR6PXzaXilpV+Bg\n4IvAeLLUq5qOgaRTgZPTOIfaXp6rGwD8DngTmNbEHEq8WUHn44CtgN1sdys7dGJwBfnu3Odu1thE\nZFGup1vQpxbXAD9OKXMAewH7pPXZlCx1c5Xtc1vou55zVnVOkv4oaWeySNYpuape8inK9V5yGJlz\nfwRwnqQPp/JyO74PeIcsElqiV+prAwjY0/bbLbRF0qbAvcC/2n6sgsiVwAJJp9u+oZUxgiAIgiAI\ngsq0cw/WKEmlNLbPAA+W1c8FxknaQtJAYAIwO6XbDbR9J3A+MCZFS56VdCSApEGSNsp3Zvtq27vZ\nHpN3rlJdt+3tgMfJHvAr8XNgvKQt0hibV5ErORlDgReTc7UfMKqCTC3uA85Y3UD6SANtapKiVUNs\nf7dUZvt429vZ/iDwL8B/V3KuJN2otD+sCkuB4SltEUlDkt3y1JrTrcDZwGa2n2xAvhV6RYwkXVy6\nbyo2yKJII23PBs4BNgOG1BhjBfABSZtL2hD4RAt6zgBW78tKLxQaIqVq3gXcmP6NVOJc4G/CuVrX\n6Gi3AkFLdLRbgaAF+svb9KA5wm7FpIh2a6eD9RRwmqTFZIdLXJvKDZCcoHPI8ie6gMdSWtQIoFNS\nF3BTkgE4AThD0kLgIZpP1wL4NbBFpYqU0vdtMievC7g8r29eNP2eAuyR9DkeWFJBplL7EheSRZMW\npUMTvlUuIGmspB/UmE85ZwI7KzvkYr6kZtILdwFeqFaZoi3HAldJWkDmJGxYJlZrTj9J7W/NlV1U\nQ74Xkg6XNLFK3S9S3/tLekZSab/ZzsDySm0SA4HJyY7zgCtt/7mCXOm+fSfp+RiZg7ikgmyPNhX4\nMrC7ssNRnqRnRK80n2pz/RSwD/C5nJ13KZMZlA67CIIgCIIgCPoYxTfDryHt9dnS9jl1hdcjUsrZ\ndbarRfcKi6TpZcfyr9OkfYALbW9TQ8b989sJgtp0EtGQItJJY3YT8f91/6E/7QkJGifsVkz6s90k\nYbtXZlo792D1R+4AbljfHrrrYXsV1VMnC836ZGdJE8hORPxOA9JrW50gCJpg2LBR9YWCIAiCfkFE\nsIIg6IEkx9+FIAiCIAiC2lSLYLVzD1YQBEEQBEEQBME6RThYQRAE6wCNfPFh0P8IuxWTsFsxCbsV\nkyLaLRysIAiCIAiCIAiCPiL2YAVB0IPYgxUEQRAEQVCf2IMVBEEQBEEQBEGwlgkHKwiCYB2giDnq\nQditqITdiknYrZgU0W7xPVhBEPRCiu/BCoIgCIK+ZNiIYSx/bnm71QjeA2IPVhAEPZBkJrZbiyAI\ngiBYx5gI8dy9bhF7sIIgCIIgCIIgCNYybXGwJI2S9ESVulmSxrzXOqWxR0qaL2l6rmxZO3SphqR9\nJU1qQG5Z+l11rcvkN5X0rKT/myubJWlknXaTJI2ro+899cZvhnyfkk6U9M0G2kyX9IqkqQ2O8U1J\nJ9Sp/1rjWoOkHSR1SZonaXtJZ0haLOmmNI//qNO+0bmeKOnXkpbm5yBpgqSnJH21Gb2DgtCv/lIF\nDRN2KyZht2ISdiskRdyD1c4IVn+MkR4FzLB9SK6sP+rZiE6ucl2NC4HZranTlC5ro89G+v8OcPxa\n0KMZjgJutz3W9jLgn4EDbH821Tdr115I2hz4BrAHsCfwTUlDAWzfAuwLhIMVBEEQBEGwlming7WB\npMnpDf5tkgaXC6Q37ovSz6WpbECKmiyStFDSl1P5aEn3S1og6XFJ27eg0/uBF8vKXsrpc0Ias0vS\njalskqQrJT0k6TeSjk7lm0iamXRZKOmIVD5K0pLUbqmkKZIOTO2XSto9yW0s6XpJc1LE4/CkxlvA\nnxqYy0vlBZJ+mHTvkvSipH9L5WOBrYEZZU3+F3i3zjgrk05I2iPNY0HSe5Oy8SvOSdIjknbMyc2S\nNKbGGuR5HXi1jo7YntWIXI5VqW9SpOlXaV4352T+Lun6G0mnJ9keEUNJZ6Zo1yHAV4B/lvSApGuA\nDwLTS/dwrs1Wkn4s6dH0s1cTcz2Y7CXBn2yvJLPpx3PrsAIY2sQ6BEWhlb94QfsJuxWTsFsxCbsV\nko6Ojnar0DTtPEVwB+DztudIuh44FbiiVClpG+BSYDeyh/j7k5PyHDDC9i5JbrPUZApwse2pkgbR\nmvM4EOjOF9jeM42zE3AusJftVyS9Pyc23PbeyUmYCtwBvAEcZftVSVsCc1IdwGjgk7YXS3oc+HRq\nf0Qa42jgPOAB2yelCMRcSTNtPwI8knQaC5xi+wvlEynpXVZ2cmo3EpgOTJIk4HvAccCBZfLH1Fsw\n219NfW4A/AgYb3u+pCEkByVHxTmldscCEyUNT+s5X9K3q8jnx7+tdJ0csLG2J9bTu4F5XZH7+HVg\nO9tv5+43yO7hDjKHZamkq0vNe3fn6ZKuBVaV+pZ0MNCR7qcTc/JXAlfYfljSXwP3ATs1ONcRwLO5\nz8+nsjz1/23Myl1vR/ynFARBEATBek9nZ2dDKYvtdLCesT0nXU8GTifnYJGlOM2y/TKApCnAOOAi\nYHtJVwI/BWakh/ltbU8FsP1Ws8okR2PXpEsl9idL73oljbEyV3dXKlsiaetSl8AlyvYndQPb5uqW\n2V6crn8FlJyGJ8geZwEOAg6XdFb6PAgYCSwtDWp7HtDLuaozz8HA7cCXbD8n6TRgmu0XsiWg1fO5\ndwBesD0/6fZqGi8vU21Ot5NFWiYCnwJ+XEe+IrbvAfp0v1diIXCzpLtItk5Ms/0O8L+SVgDDmuxX\nVF7vA4AdtWbxhkja2PZfSgL/n3N9WdJo27+tKrFfiz0H7WMZ4QgXkbBbMQm7FZOwWyHp7OzsN1Gs\njo6OHrpccMEFFeXa6WD1estfQabXw6ftlZJ2JUuF+iIwniz1qqZjIOlU4OQ0zqG2l+fqBgC/A94E\npjUxhxJvVtD5OGArYDfb3coOnRhcQb4797mbNTYRWZTr6Rb0qcU1wI9TyhzAXsA+aX02JUvdXGX7\n3Bb6ruecVZ2TpD9K2pksknVKrqqXfIpyvZccRubcHwGcJ+nDqbzcju8D3iGLhJbolfraAAL2tP12\nC22fJ4uqlfgresajIIuQLZB0uu0bWhgjCIIgCIIgqEI792CNklRKY/sM8GBZ/VxgnKQtJA0EJgCz\nU7rdQNt3AucDY1K05FlJRwJIGiRpo3xntq+2vZvtMXnnKtV1294OeJzsAb8SPwfGS9oijbF5FbmS\nkzEUeDE5V/sBoyrI1OI+4IzVDaSPNNCmJilaNcT2d0tlto+3vZ3tDwL/Avx3JedK0o1K+8OqsBQY\nntIWkTQk2S1PrTndCpwNbGb7yQbkW6FXxEjSxaX7pmKDLIo00vZs4BxgM2BIjTFWAB+QtLmkDYFP\ntKDnDGD1vqz0QqFR7gMOlDQ03aMHprI85wJ/E87VOka8lS0mYbdiEnYrJmG3QtJfolfN0E4H6yng\nNEmLyQ6XuDaVGyA5QecAnUAX8FhKixoBdErqAm5KMgAnAGdIWgg8RPPpWgC/BraoVJFS+r5N5uR1\nAZfn9c2Lpt9TgD2SPscDSyrIVGpf4kKyaNKidGjCt8oFJI2V9IMa8ynnTGBnZYdczJfUTHrhLsAL\n1SpTtOVY4CpJC8ichA3LxGrN6Sep/a25sotqyPdC0uGSJlap+0Xqe39Jz0gq7TfbGaj1teoDgcnJ\njvOAK23/uYJc6b59J+n5GJljs6SCbI82FfgysLuyw1GepGdErzSfinNNKawXkr0seBS4oCydFWBQ\nOuwiCIIgCIIg6GMU3yi9hrTXZ0vb59QVXo+QtClwne1q0b3CIml62bH86zRpH+BC29vUkDET3zud\ngj4i9hYUk7BbMQm7FZN2220ixHN38/SnPVjlSMJ2r8y0du7B6o/cAdywvj1018P2KqqnThaa9cnO\nkiaQnYj4nbrCE9e2NkEQBEGwfjFsRCvJVUERiQhWEAQ9kOT4uxAEQRAEQVCbahGsdu7BCoIgCIIg\nCIIgWKcIBysIgmAdoJEvPgz6H2G3YhJ2KyZht2JSRLuFgxUEQRAEQRAEQdBHxB6sIAh6EHuwgiAI\ngiAI6hN7sIIgCIIgCIIgCNYy4WAFQRCsAxQxRz0IuxWVsFsxCbsVkyLaLb4HKwiCXki9ot3BOsaw\nYaNYvvz37VYjCIIgCNY5Yg9WEAQ9kGSIvwvrPiL+/gdBEARB68QerCAIgiAIgiAIgrVMQw6WpFGS\nnqhSN0vSmL5VqzEkjZQ0X9L0XNmyduhSDUn7SprUgNyynPw91WQkbdGHelUcJ1dfU+90X8yqI9Pn\n90e+z0bsLWkXSQ9LWijpbklDGmhTs19JqxrXeHWb70p6QtJlkraSNEfSPEn7NGLbBuf6HUlLJC2Q\n9BNJm+XqfiFprqStm9U9KAKd7VYgaIEi7i0Iwm5FJexWTIpot2YiWP0xl+QoYIbtQ3Jl/VHPRnRy\nletm+2mGev01q3c7aGT864Czbe8K3Amc3Qf9tjLvk4FdbH8dOABYZHus7V822F8jMjOAv7P9EeBp\n4F9XN7bHAfOAw5rWPAiCIAiCIGiIZhysDSRNlrRY0m2SBpcLSJogaVH6uTSVDZA0KZUtlPTlVD5a\n0v3pTfvjkrZvQf/3Ay+Wlb2U0+eENGaXpBtT2SRJV0p6SNJvJB2dyjeRNDPpslDSEal8VIoITJK0\nVNIUSQem9ksl7Z7kNpZ0fS4qcXhS4y3gTw3M5aXc9VBJ90p6StLVufLVOZ6SvpaiIYtya7pxateV\nysen8j2SvguSfpvkB5Y0LUUCuyStlPTZBvV+F3g59TEgF6FZIOm0cuG0bg+nNb416XuwpNtyMqsj\na5IOKpevs27V+FByYgBmAp9soM1LSYfhkman9Vkkae81quqiNNeHJX0gFU4q3VPp86r0+25gCDBP\n0tnAZcBRqd/B9LTtcZIeTXXXSKtPnKg7V9szbXenj3OAvyoTWU727yZY5+hotwJBC3R0dLRbhaAF\nwm7FJOxWTIpot2ZOEdwB+LztOZKuB04FrihVStoGuBTYDVgJ3J+clOeAEbZ3SXKllKUpwMW2p0oa\nRGv7wQYC3fkC23umcXYCzgX2sv2KpPxD5XDbe0vaEZgK3AG8ARxl+1VJW5I9nE5N8qOBT9peLOlx\n4NOp/RFpjKOB84AHbJ8kaSgwV9JM248AjySdxgKn2P5C+URKeif2AHYEngHuk3S07TtKlcrS405M\ncgOBRyV1Jj2ft/2JJLeppA2AHwHjbc9Xlh73etnYh+X6/S/gLturSnpXw/ZzwDHp4xeAUWQRGpet\nN2lNzwc+Zvv15GR8DbgE+L6kjWy/DhwL3Jzkz6sgf1G1dZM0DTjJ9vIyVX8l6QjbU4FP0dvpqDS3\nUr+fAX5m+5Lk6JScvE2Ah22fL+kysujUxZW6Sv0dKenPtkupjSuAsbbPSJ9Lc/jbtAZ/b/tdSf8J\nHAdMbnCuef6JzPZ5usnumTpMzF13EA/vQRAEQRCs73R2djaUstiMU/OM7TnpejKwT1n9HsAs2y+n\nN+hTgHHA74DtU9ToYGBVesjfNj3wYvst2280oQvpYXdXMgeuEvsDt9t+JY2xMld3VypbApT2owi4\nRNJCsijHtlqzV2WZ7cXp+lepHuAJYLt0fRBwjqQuss0Qg4CReYVsz6vkXFVgru0/ODvi6xZ6r/U+\nwJ2237D9GpmD+A9JnwMlXSJpn+Qk7QC8YHt+0uHVXIRjNZK2Am4CJqR2zXIA8P2kc/l6A3wU2Al4\nKK3RCcBI2+8CPwMOlzSQLH1tajX5WgrYPqyKw/FPwGmSHiNzjN5qYl6PAZ+X9A0y5/G1VP6m7Z+m\n63msuQ/KafS881L638eAMcBjad77Ax/sJVx9rtmg0nnA27ZvLqt6HtilvjoTcz8d9cWDfkBnuxUI\nWqCIewuCsFtRCbsVk/5kt46ODiZOnLj6pxrNRLDK939U2g/S62HS9kpJuwIHA18ExgNfqSTboyPp\nVLKogIFD8w+TkgaQOW5vAtOamEOJNyvofBywFbCb7W5lBwoMriDfnfvczZo1FFmU6+kW9CmnkbXu\n3ch+OkWhDgUulPQAmTNZb60HkDlyE5PTuTYQ2X654yrU3Qp8CXgFeMz2a8mBribfFLZ/TXb/IelD\nNLEHyfaDksalNjdIutz2ZODtnNi7rLkP3iG9uEhz2KBJdQXcaPu8Jtut6UD6HNk9sH+F6juAb0ha\nbHunVscIgiAIgiAIKtNMBGuUpHza1INl9XOBcZK2SJGICcDslOo10PadZCliY2y/Cjwr6UgASYMk\nbZTvzPbVtnezPab8Tb3tbtvbAY+TpVNV4ufAeKWT2SRtXkWu5HwMBV5MztV+ZOlu5TK1uA84Y3UD\n6SMNtKnGnsr2fg0gm1/5Wj9Itn9nsLL9VP8IPJjSNF9PUYvvkUVClgLDU3oikoYk++S5DFho+/ZK\nyijbw3VjHZ3vB04p9V1hvecAe0saneo3Ts4OwOyk68msSWmrJd8Uuf1RA8juwWvT520lzazTdiTZ\nfXE92WEZpRMRq90Tvwd2T9dH0tPBqnUfleoeAI7J6bx50qEhJH0cOAs4wvabFUROAKaHc7Uu0tFu\nBYIWKOLegiDsVlTCbsWkiHZrxsF6iizNajHZJvlrU3kpJWw5cA5ZnkoXWSTiHmAE0JnSnW5KMpA9\n6J2RUvIeAoa1oP+vgYpHW6eUvm+TOXldwOV5ffOi6fcUYI+kz/HAkgoyldqXuJDsIJBFyo60/1a5\ngKSxkn5QYz4l5gJXkaUj/tb2XfmxbXcBN5Clrz0C/MD2QmBnsr1fXcA3gItsv03mpF0laQHZKXMb\nlo13JnCQskMu5kv6RFn9SOAvdXS+DngWWJTGn1Cm8x+BzwG3pDV+mCx9kZSyeC/w8fS7pjxVbKDs\nsI7hFaomSFoKLCbbo3ZDKt+GnpGoSnQACyXNJ9u/9e+1dAB+COyb1uCjwGu5ulqRyNI6LSFzAmek\nec8Aes2pxlz/g+wwjfuTLa8uq9+c7HTBIAiCIAiCYC2gtGWmkEg6C9jS9jl1hYOWSYc43GT7yXbr\n0pcoO+nwD7bvbbcu7xXp0IxFtr9fQ8btP30/aJ5OmotiiSL//V9X6OzsLOTb2fWdsFsxCbsVk/5s\nN0nY7pWh1MwerP7IHWT7YqaXfRdW0Iek721a57D9n+3W4b1E0myyfYOVTjsMgiAIgiAI+oBCR7CC\nIOh7sghWsK4zbNgoli//fbvVCIIgCILCsq5GsIIgWAvEi5cgCIIgCILWaOXLfYMgCIJ+Rn/6npCg\nccJuxSTsVkzCbsWkiHYLBysIgiAIgiAIgqCPiD1YQRD0QJLj70IQBEEQBEFtqu3BighWEARBEARB\nEARBHxEOVhAEwTpAEXPUg7BbUQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9iO/BCoKgGsNG\nDGP5c8vbrUYQBEG/oNoerHCwgiDogSQzsd1aBEHQL5kY35MXBEFQIg65CIIgWJdZ1m4FgpYIuxWS\nIu4JCcJuRaWIdmvIwZI0StITVepmSRrTt2o1hqSRkuZLmp4r61f/XUnaV9KkBuSW5eTvqSYjaYs+\n1KviOLn6mnqn+2JWHZk+vz/yfTZib0m7SHpY0kJJd0sa0kCbmv1KWtW4xqvbfFfSE5Iuk7SVpDmS\n5knapxHbNjjXzSXNkLRU0n2ShubqfiFprqStm9U9CIIgCIIgaIxmIlj9MSfgKGCG7UNyZf1Rz0Z0\ncpXrZvtphnr9Nat3O2hk/OuAs23vCtwJnN0H/bYy75OBXWx/HTgAWGR7rO1fNthfIzLnADNt7wD8\nHPjX1Y3tccA84LCmNQ/6P9u3W4GgJcJuhaSjo6PdKgQtEHYrJkW0WzMO1gaSJktaLOk2SYPLBSRN\nkLQo/VyaygZImpTKFkr6ciofLel+SQskPS6plf9m3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0\ndCrfRNLMpMtCSUek8lGSlqR2SyVNkXRgar9U0u5JbmNJ1+eiEocnNd4C/tTAXF7KXQ+VdK+kpyRd\nnStfneMp6WspGrIot6Ybp3ZdqXx8Kt8j6bsg6bdJfmBJ01IksEvSSkmfbVDvd4GXUx8DchGaBZJO\nKxdO6/ZwWuNbk74HS7otJ7M6sibpoHL5OutWjQ8lJwZgJvDJBtq8lHQYLml2Wp9FkvZeo6ouSnN9\nWNIHUuGk0j2VPq9Kv+8GhgDzJJ0NXAYclfodTE/bHifp0VR3jaRSXSNzPRK4MV3fSPYSIs9ysn83\nQRAEQRAEwVrgfU3I7gB83vYcSdcDpwJXlColbQNcCuwGrATuT07Kc8AI27skuc1SkynAxbanShpE\na/vBBgLd+QLbe6ZxdgLOBfay/Yqk/EPlcNt7S9oRmArcAbwBHGX7VUlbAnNSHcBo4JO2F0t6HPh0\nan9EGuNo4DzgAdsnpbSsuZJm2n4EeCTpNBY4xfYXyidS0juxB7Aj8Axwn6Sjbd9RqlSWHndikhsI\nPCqpM+n5vO1PJLlNJW0A/AgYb3u+svS418vGPizX738Bd9leVdKW2VSCAAAgAElEQVS7GrafA45J\nH78AjCKL0LhsvUlrej7wMduvJyfja8AlwPclbWT7deBY4OYkf14F+YuqrZukacBJtsuPuPqVpCNs\nTwU+BfxVrXmV9fsZ4Ge2L0mOTsnJ2wR42Pb5ki4ji05dXKmr1N+Rkv5su5TauAIYa/uM9Lk0h79N\na/D3tt+V9J/AccDkBue6te0Vaczl6p0O2E12z9Qmn/i5HfGWvQgsI+xURMJuhaSzs7OQb9XXd8Ju\nxaQ/2a2zs7OhPWHNOFjP2J6TricDp5NzsMge9mfZLkU0pgDjyB6It5d0JfBTYEZ6yN82PfBi+60m\n9CD1L2DXpEsl9gdut/1KGmNlru6uVLYk9wAq4BJJ48geQrfN1S2zvThd/4osCgLwBNnjJ8BBwOGS\nzkqfBwEjgaWlQW3PI3NE6jHX9h/SPG8B9iFzAkvsA9xp+40kcwfwD8B9wPckXQJMs/1LSR8GXrA9\nP+nwamrTY0BJWwE3Acck56pZDgCucTpeqmy9AT4K7AQ8lGy3AZmD8q6kn5Gt3U/I0tfOAjoqyddS\noOQoVuCfgP+Q9G9kTnMz99tjwPXJUb3b9sJU/qbtn6breWTzr0Svk2WqUEr/+xgwBngszXswsKKX\ncPW5Vuu3xPNka1ub/RrsPQiCIAiCYD2ho6Ojh7N3wQUXVJRrxsEqf1CrtB+k18Ok7ZWSdgUOBr4I\njAe+Ukm2R0fSqWRRAQOH5t/USxoA/A54E5jWxBxKvFlB5+OArYDdbHcrO1BgcAX57tznbtasocii\nXE+3oE85jax170b20ykKdShwoaQHyJzJems9ALgFmGh7SQv6NoLI9ssdV6HuVuBLwCvAY7ZfS85F\nNfmmsP1rsvsPSR+iiT1Ith9MTvdhwA2SLrc9GXg7J/Yua+6Dd0jR2Jxj2AwCbrR9XpPtSqyQNMz2\nCknD6Z1CewfwDUmLbe/U4hhBfySiIMUk7FZI+svb9KA5wm7FpIh2ayYtb5SkfNrUg2X1c4FxkraQ\nNBCYAMxOqV4Dbd9JliI2JkVRnpV0JICkQZI2yndm+2rbu9keU54GZbvb9nbA42TpVJX4OTBe6WQ2\nSZtXkSs5H0OBF5NztR9Zulu5TC3uA85Y3UD6SANtqrGnsr1fA8jmV77WD5Lt3xmsbD/VPwIPpjTN\n123fDHyPLBKyFBie0hORNCTZJ89lwELbt1dSRtkerhsr1eW4Hzil1HeF9Z4D7C1pdKrfODk7ALOT\nrieTpTPWk2+K3P6oAWT34LXp87aSZtZpO5Lsvrie7LCM0omI1e6J3wO7p+sj6elg1bqPSnUPAMfk\ndN486dAoU4HPpesTgbvL6k8ApodzFQRBEARBsHZoxsF6CjhN0mKyTfLXpvJSSthyshPMOoEuskjE\nPcAIoFNSF1kK2jmp3QnAGZIWAg8Bw1rQ/9dAxaOtU0rft8mcvC7g8ry+edH0ewqwR9LneGBJBZlK\n7UtcSHYQyCJlR9p/q1xA0lhJP6gxnxJzgavI0hF/a/uu/Ni2u4AbyNLXHgF+kFLXdibb+9UFfAO4\nyPbbZE7aVZIWADOADcvGOxM4SNkhF/MlfaKsfiTwlzo6Xwc8CyxK408o0/mPZA/+t6Q1fphsXx+2\nu4F7gY+n3zXlqWIDZYd1DK9QNUHSUmAx2R61G1L5NvSMRFWiA1goaT7Z/q1/r6UD8ENg37QGHwVe\ny9XVikSW1mkJmRM4I817BtBrTjXmehlwYJrvx8j2RebZHOiLKGvQ3+hXX1ARNEzYrZAU8Xt5grBb\nUSmi3VTkb2RP+522tH1OXeGgZdIhDjfZfrLduvQlyk46/IPte9uty3tFOjRjke3v15AxE987nYI+\nIg5LKCZFs9tEKPJzQ1/RnzbdB40Tdism/dlukrDdK0Op6A7WaLJIzqtl34UVBEEZkmaT7Rs83vbz\nNeSK+0chCIK1yrARw1j+XPnhpUEQBOsn66SDFQRB3yPJ8XchCIIgCIKgNtUcrFa+eyoIgiDoZxQx\nRz0IuxWVsFsxCbsVkyLaLRysIAiCIAiCIAiCPiJSBIMg6EGkCAZBEARBENQnUgSDIAiCIAiCIAjW\nMuFgBUEQrAMUMUc9CLsVlbBbMQm7FZMi2i0crCAIgiAIgiAIgj4i9mAFQdCD+B6sIOjfDBs2iuXL\nf99uNYIgCNZ74nuwgiBoiMzBir8LQdB/EfF/dxAEQfuJQy6CIAjWaTrbrUDQEp3tViBogSLuCQnC\nbkWliHZryMGSNErSE1XqZkka07dqNYakkZLmS5qeK1vWDl2qIWlfSZMakFuWk7+nmoykLfpQr4rj\n5Opr6p3ui1l1ZPr8/sj32Yi9JX1T0nPpXpkv6eMNtKnZr6RVjWu8us13JT0h6TJJW0maI2mepH0a\nsW2Dc/2OpCWSFkj6iaTNcnW/kDRX0tbN6h4EQRAEQRA0RjMRrP6Yj3AUMMP2Ibmy/qhnIzq5ynWz\n/TRDvf6a1bsdNDr+FbbHpJ+f9UG/rcz7ZGAX218HDgAW2R5r+5cN9teIzAzg72x/BHga+NfVje1x\nwDzgsKY1DwpAR7sVCFqio90KBC3Q0dHRbhWCFgi7FZMi2q0ZB2sDSZMlLZZ0m6TB5QKSJkhalH4u\nTWUDJE1KZQslfTmVj5Z0f3rT/rik7VvQ//3Ai2VlL+X0OSGN2SXpxlQ2SdKVkh6S9BtJR6fyTSTN\nTLoslHREKh+VIgKTJC2VNEXSgan9Ukm7J7mNJV2fi0ocntR4C/hTA3N5KXc9VNK9kp6SdHWufHWO\np6SvpWjIotyabpzadaXy8al8j6TvgqTfJvmBJU1LkZ0uSSslfbZBvd8FXk59DMhFaBZIOq1cOK3b\nw2mNb036HizptpzM6siapIPK5eusWy165cfW4aWkw3BJs9P6LJK09xpVdVGa68OSPpAKJ5XuqfR5\nVfp9NzAEmCfpbOAy4KjU72B62vY4SY+mumsklerqztX2TNvd6eMc4K/KRJaT/bsJgiAIgiAI1gLv\na0J2B+DztudIuh44FbiiVClpG+BSYDdgJXB/clKeA0bY3iXJlVKWpgAX254qaRCt7QcbCHTnC2zv\nmcbZCTgX2Mv2K5LyD5XDbe8taUdgKnAH8AZwlO1XJW1J9nA6NcmPBj5pe7Gkx4FPp/ZHpDGOBs4D\nHrB9kqShwFxJM20/AjySdBoLnGL7C+UTKemd2APYEXgGuE/S0bbvKFUqS487MckNBB6V1Jn0fN72\nJ5LcppI2AH4EjLc9X9IQ4PWysQ/L9ftfwF22V5X0robt54Bj0scvAKPIIjQuW2/Smp4PfMz268nJ\n+BpwCfB9SRvZfh04Frg5yZ9XQf6iausmaRpwku3lFdT9UnIcHwfOtF3Tecz1+xngZ7YvSY5Oycnb\nBHjY9vmSLiOLTl1cqavU35GS/my7lNq4Ahhr+4z0uTSHv01r8Pe235X0n8BxwOQm5lrin8hsn6eb\n7J6pw8TcdQfxlr0IdBJ2KiKdhN2KR2dnZyHfqq/vhN2KSX+yW2dnZ0N7wppxsJ6xPSddTwZOJ+dg\nkT3sz7JdimhMAcaRPRBvL+lK4KfAjPSQv63tqQC232pCD1L/AnZNulRif+B226+kMVbm6u5KZUu0\nZj+KgEskjSN7CN02V7fM9uJ0/StgZrp+AtguXR8EHC7prPR5EDASWFoa1PY8MkekHnNt/yHN8xZg\nHzInsMQ+wJ2230gydwD/ANwHfE/SJcA027+U9GHgBdvzkw6vpjY9BpS0FXATcExyrprlAOAap6Ot\nytYb4KPATsBDyXYbkDko70r6Gdna/YQsfe0ssieOXvK1FCg5ihW4GvhWcvwuIrtvT2pwXo8B1ydH\n9W7bC1P5m7Z/mq7nkc2/Eo1Gzkrpfx8DxgCPpXkPBlb0Eq4+12xQ6Tzgbds3l1U9T0NPcxPriwRB\nEARBEKxHdHR09HD2LrjggopyzThY5fs/Ku0H6fUwaXulpF2Bg4EvAuOBr1SS7dGRdCpZVMDAofk3\n9ZIGAL8D3gSmNTGHEm9W0Pk4YCtgN9vdyg4UGFxBvjv3uZs1ayiyKNfTLehTTiNr3buR/XSKQh0K\nXCjpATJnst5aDwBuASbaXtKCvo0gsv1yx1WouxX4EvAK8Jjt15JzUU2+KWznU+t+CFQ93KNC2weT\n030YcIOky21PBt7Oib3LmvvgHVI0NucYNoOAG22f12S7NR1InyO7B/avUH0H8A1Ji23v1OoYQX+k\no90KBC3R0W4FghboL2/Tg+YIuxWTItqtmbS8UZLyaVMPltXPBcZJ2kLSQGACMDuleg20fSdZitiY\nFEV5VtKRAJIGSdoo35ntq23vlg4lWF5W1217O7J0r2Or6PtzYLzSyWySNq8iV3I+hgIvJudqP7J0\nt3KZWtwHnLG6gfSRBtpUY09le78GkM2vfK0fJNu/M1jZfqp/BB5MaZqvp6jF98giIUuB4Sk9EUlD\nkn3yXAYstH17JWWU7eG6sY7O9wOnlPqusN5zgL0ljU71G0v6UKqbnXQ9mTUpbbXkm0LS8NzHo4En\nU/m2kmZWbrW67Uiy++J64LqkJ1S/J34P7J6uj6Sng1XrPirVPQAcozV7ujZPOjSEshMSzwKOsP1m\nBZETgOnhXAVBEARBEKwdmnGwngJOk7SYbJP8tam8lBK2HDiHLKG8iywScQ8wAuiU1EWWgnZOancC\ncIakhcBDwLAW9P81UPFo65TS920yJ68LuDyvb140/Z4C7JH0OR5YUkGmUvsSF5IdBLJI2ZH23yoX\nkDRW0g9qzKfEXOAqsnTE39q+Kz+27S7gBrL0tUeAH6TUtZ3J9n51Ad8ALrL9NpmTdpWkBWSnzG1Y\nNt6ZwEHKDrmYL+kTZfUjgb/U0fk64FlgURp/QpnOfwQ+B9yS1vhhsn19pEMZ7gU+nn7XlKeKDZQd\n1jG8QtV3kl0WAPsCX03l29AzElWJDmChpPnAp4B/r6UDWYRs37QGHwVey9XVikSW1mkJ2YuIGWne\nM4Bec6ox1/8gO0zj/mTLq8vqNyc7XTBY5+hstwJBS3S2W4GgBYr4vTxB2K2oFNFuKvK3waf9Tlva\nPqeucNAy6RCHm2w/2W5d+hJlJx3+wfa97dblvSIdmrHI9vdryLj9p+8HzdNJpJsVkU6at5so8v/d\n6wL9adN90Dhht2LSn+0mCdu9MpSK7mCNJovkvFr2XVhBEJQhaTbZvsHjbT9fQ664fxSCYD1g2LBR\nLF/++3arEQRBsN6zTjpYQRD0PZIcfxeCIAiCIAhqU83BauW7p4IgCIJ+RhFz1IOwW1EJuxWTsFsx\nKaLdwsEKgiAIgiAIgiDoIyJFMAiCHkSKYBAEQRAEQX0iRTAIgiAIgiAIgmAtEw5WEATBOkARc9SD\nsFtRCbsVk7BbMSmi3cLBCoIgCIIgCIIg6CNiD1YQBD2I78EKgiAIho0YxvLnlrdbjSDo18T3YAVB\n0BCSzMR2axEEQRC0lYkQz4hBUJs45CIIgmBdZlm7FQhaIuxWTMJuhaSIe3mCYtqtIQdL0ihJT1Sp\nmyVpTN+q1RiSRkqaL2l6rqxf/dmTtK+kSQ3ILcvJ31NNRtIWfahXxXFy9TX1TvfFrDoyfX5/5Pts\nxN6SvinpuXSvzJf08Qba1OxX0qrGNV7d5ruSnpB0maStJM2RNE/SPo3YtsG5bi5phqSlku6TNDRX\n9wtJcyVt3azuQRAEQRAEQWM0E8Hqj3Hio4AZtg/JlfVHPRvRyVWum+2nGer116ze7aDR8a+wPSb9\n/KwP+m1l3icDu9j+OnAAsMj2WNu/bLC/RmTOAWba3gH4OfCvqxvb44B5wGFNax70f7ZvtwJBS4Td\niknYrZB0dHS0W4WgBYpot2YcrA0kTZa0WNJtkgaXC0iaIGlR+rk0lQ2QNCmVLZT05VQ+WtL9khZI\nelxSK3+u3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0dCrfRNLMpMtCSUek8lGSlqR2SyVNkXRg\nar9U0u5JbmNJ1+eiEocnNd4C/tTAXF7KXQ+VdK+kpyRdnStfneMp6WspGrIot6Ybp3ZdqXx8Kt8j\n6bsg6bdJfmBJ01Jkp0vSSkmfbVDvd4GXUx8DchGaBZJOKxdO6/ZwWuNbk74HS7otJ7M6sibpoHL5\nOutWi175sXV4KekwXNLstD6LJO29RlVdlOb6sKQPpMJJpXsqfV6Vft8NDAHmSTobuAw4KvU7mJ62\nPU7So6nuGkmlukbmeiRwY7q+kewlRJ7lZP9ugiAIgiAIgrXA+5qQ3QH4vO05kq4HTgWuKFVK2ga4\nFNgNWAncn5yU54ARtndJcpulJlOAi21PlTSI1vaDDQS68wW290zj7AScC+xl+xVJ+YfK4bb3lrQj\nMBW4A3gDOMr2q5K2BOakOoDRwCdtL5b0OPDp1P6INMbRwHnAA7ZPSmlZcyXNtP0I8EjSaSxwiu0v\nlE+kpHdiD2BH4BngPklH276jVKksPe7EJDcQeFRSZ9LzedufSHKbStoA+BEw3vZ8SUOA18vGPizX\n738Bd9leVdK7GrafA45JH78AjCKL0LhsvUlrej7wMduvJyfja8AlwPclbWT7deBY4OYkf14F+Yuq\nrZukacBJtisde/Sl5Dg+Dpxpu6bzmOv3M8DPbF+SHJ2Sk7cJ8LDt8yVdRhadurhSV6m/IyX92XYp\ntXEFMNb2GelzaQ5/m9bg722/K+k/geOAyQ3OdWvbK9KYy9U7HbCb7J6pTT7xczvibW0RWEbYqYiE\n3YpJ2K2QdHZ2FjIasr7Tn+zW2dnZ0J6wZhysZ2zPSdeTgdPJOVhkD/uzbJciGlOAcWQPxNtLuhL4\nKTAjPeRva3sqgO23mtCD1L+AXZMuldgfuN32K2mMlbm6u1LZktwDqIBLJI0jewjdNle3zPbidP0r\nYGa6foLs8RPgIOBwSWelz4OAkcDS0qC255E5IvWYa/sPaZ63APuQOYEl9gHutP1GkrkD+AfgPuB7\nki4Bptn+paQPAy/Ynp90eDW16TGgpK2Am4BjknPVLAcA1zgdOVS23gAfBXYCHkq224DMQXlX0s/I\n1u4nZOlrZwEdleRrKVByFCtwNfCt5PhdRHbfntTgvB4Drk+O6t22F6byN23/NF3PI5t/JRqNnJXS\n/z4GjAEeS/MeDKzoJVx9rtX6LfE82drWZr8Gew+CIAiCIFhP6Ojo6OHsXXDBBRXlmnGwyh/UKu0H\n6fUwaXulpF2Bg4EvAuOBr1SS7dGRdCpZVMDAofk39ZIGAL8D3gSmNTGHEm9W0Pk4YCtgN9vdyg4U\nGFxBvjv3uZs1ayiyKNfTLehTTiNr3buR/XSKQh0KXCjpATJnst5aDwBuASbaXtKCvo0gsv1yx1Wo\nuxX4EvAK8Jjt15JzUU2+KWznU+t+CFQ93KNC2weT030YcIOky21PBt7Oib3LmvvgHVI0NucYNoOA\nG22f12S7EiskDbO9QtJweqfQ3gF8Q9Ji2zu1OEbQH4m36cUk7FZMwm6FpL9EQYLmKKLdmknLGyUp\nnzb1YFn9XGCcpC0kDQQmALNTqtdA23eSpYiNSVGUZyUdCSBpkKSN8p3Zvtr2bulQguVldd22tyNL\n9zq2ir4/B8YrncwmafMqciXnYyjwYnKu9iNLdyuXqcV9wBmrG0gfaaBNNfZUtvdrANn8ytf6QbL9\nO4OV7af6R+DBlKb5uu2bge+RRUKWAsNTeiKShiT75LkMWGj79krKKNvDdWOluhz3A6eU+q6w3nOA\nvSWNTvUbS/pQqpuddD2ZLJ2xnnxTJEejxNHAk6l8W0kzK7da3XYk2X1xPXBd0hOq3xO/B3ZP10fS\n08GqdR+V6h4AjtGaPV2bJx0aZSrwuXR9InB3Wf0JwPRwroIgCIIgCNYOzThYTwGnSVpMtkn+2lRe\nSglbTnaCWSfQRRaJuAcYAXRK6iJLQTsntTsBOEPSQuAhYFgL+v8aqHi0dUrp+zaZk9cFXJ7XNy+a\nfk8B9kj6HA8sqSBTqX2JC8kOAlmk7Ej7b5ULSBor6Qc15lNiLnAVWTrib23flR/bdhdwA1n62iPA\nD1Lq2s5ke7+6gG8AF9l+m8xJu0rSAmAGsGHZeGcCByk75GK+pE+U1Y8E/lJH5+uAZ4FFafwJZTr/\nkezB/5a0xg+T7evDdjdwL/Dx9LumPFVsoOywjuEVqr6T7LIA2Bf4airfhp6RqEp0AAslzQc+Bfx7\nLR3IImT7pjX4KPBarq5WJLK0TkvIXkTMSPOeAfSaU425XgYcKGkpWbrhpWX1mwN9EWUN+hv96gsq\ngoYJuxWTsFshKeL3KQXFtJuK/C3dab/TlrbPqSsctEw6xOEm20+2W5e+RNlJh3+wfW+7dXmvSIdm\nLLL9/RoyZuJ7p1PQR8Sm+2ISdism64PdJkKRnxEr0Z8OSwgapz/bTRK2e2UoFd3BGk0WyXm17Luw\ngiAoQ9Jssn2Dx9t+voZccf8oBEEQBH3CsBHDWP5cpUN5gyAosU46WEEQ9D2SHH8XgiAIgiAIalPN\nwWrlu6eCIAiCfkYRc9SDsFtRCbsVk7BbMSmi3cLBCoIgCIIgCIIg6CMiRTAIgh5EimAQBEEQBEF9\nIkUwCIIgCIIgCIJgLRMOVhAEwTpAEXPUg7BbUQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9\niO/BCoJ1g2HDRrF8+e/brUYQBME6S3wPVhAEDZE5WPF3IQiKj4j/44MgCNYecchFEATBOk1nuxUI\nWqKz3QoELVDEPSFB2K2oFNFuDTlYkkZJeqJK3SxJY/pWrcaQNFLSfEnTc2XL2qFLNSTtK2lSA3LL\ncvL3VJORtEUf6lVxnFx9Tb3TfTGrjkyf3x/5Phuxt6RjJD0p6d1GdanXr6RVjWnbo813JT0h6TJJ\nW0maI2mepH0asW2Dc/2OpCWSFkj6iaTNcnW/kDRX0tbN6v7/2Lv3eKuqeu/jny+oeTuiUoGaoIdT\nPlkpopalwTbLynvmXY8e0y4vO1nZ5VhWgJfUSk+kWfnEQQsy9RzybiLK4ihGICKoIGnitQf1lBR2\nyoz9e/4YY8Hca6/b3m7ce+L3/Xrt155rzDHn/M0xJps11rgsMzMzM2tPT3qwBuI4g8OAGRHx4ULa\nQIyznZiiwXZPz9MTrc7X07j7QzvXfwD4CDC7D8/bm/v+OLBLRPwb8H5gcUTsHhF3t3m+dvLMAN4W\nEaOBR4CvrDk4YiywADiwx5FbCXT0dwDWKx39HYD1QkdHR3+HYL3geiunMtZbTxpYG0qaKmmJpGsk\nbVybQdKxkhbnnwty2iBJU3LaIkmfzemjJN2eP2m/V9KOvYh/S+C5mrTnC/GcmK+5UNKVOW2KpEmS\n5kh6VNLhOX0zSTNzLIskHZLTR+YegSmSlkmaJukD+fhlkvbI+TaVNLnQK3FwDuNvwB/buJfnC9tD\nJN0k6WFJlxXS14zxlHRG7g1ZXCjTTfNxC3P6kTl9zxzv/Tm+zYoXlnRz7glcKGmlpH9uM+7VwB/y\nOQYVemjul/Tp2sy53O7JZXx1jveDkq4p5FnTsyZp/9r8LcqtrohYFhGPFMuvDc/nGIZLmp3LZ7Gk\nvdeGqnPzvd4j6Q05cUr1mcqvV+Xf1wObAwskfRm4EDgsn3djutbt8ZJ+nff9QFJ1Xzv3OjMiOvPL\nucCbarKsIP27MTMzM7N1YIMe5N0JODki5kqaDJwGXFzdKWkb4AJgN2AlcHtupDwNbBcRu+R81SFL\n04BvRsQNkjaid/PBBgOdxYSIeFe+zs7AV4F3R8QLkopvKodHxN6S3grcAEwH/gocFhEvShpKenN6\nQ84/CvhoRCyRdC9wTD7+kHyNw4GzgDsi4hRJQ4B5kmZGxK+AX+WYdgc+GRGfqL2RatzZnsBbgSeB\n2yQdHhHTqzuVhrmdlPMNBn4tqZLjfCYiDsr5/kHShsDPgSMj4j5JmwN/qbn2gYXz/gdwXUSsqsbd\nSEQ8DRyRX34CGEnqoYma8iaX6deA/SLiL7mRcQZwPvAjSZtExF+Ao4Gf5fxn1cl/bqNyk3QzcEpE\nrGgWdzsK5z0O+GVEnJ8bOtVG3mbAPRHxNUkXknqnvlnvVPl8h0r6U0RUhzY+C+weEafn19V7+D+5\nDN4TEaslfR84Hpjai3v9GKnuizpJz0wLEwrbHfhT9jKo4Hoqowqut/KpVCql/FT9tc71Vk4Dqd4q\nlUpbc8J60sB6MiLm5u2pwGcoNLBIb/ZnRUS1R2MaMJb0hnhHSZOAW4AZ+U3+thFxA0BE/K0HcZDP\nL2DXHEs97wOujYgX8jVWFvZdl9OWau18FAHnSxpLehO6bWHf8ohYkrcfAmbm7QeAHfL2/sDBkr6U\nX28EjACWVS8aEQtIDZFW5kXEE/k+rwL2ITUCq/YBfhERf815pgPvBW4DviPpfODmiLhb0tuB30XE\nfTmGF/MxXS4o6fXAT4EjcuOqp94P/CDyklU15Q2wF7AzMCfX3YakBspqSb8kld1/kYavfYn0jqNb\n/mYBVBuKfWw+MDk3VK+PiEU5/aWIuCVvLyDdfz3t9ppVh//tB4wB5uf73hh4tlvmFvcq6Szg5Yj4\nWc2uZ2jr3dyE1lnMzMzMXkM6Ojq6NPYmTpxYN19PGli18z/qzQfp9mYyIlZK2hX4IPAp4Ejgc/Xy\ndjmRdBqpVyCAA4qf1EsaBDwGvATc3IN7qHqpTszHA68HdouITqUFBTauk7+z8LqTtWUoUi/XI72I\np1Y7Zd39oIhHci/UAcA5ku4gNSZblfUg4CpgQkQs7UW87RBpvtzxdfZdDfwr8AIwPyL+nBsXjfK/\naiLirtzoPhC4QtJFETEVeLmQbTVrn4O/k3tjCw3DnhBwZUSc1duYJf0L6Rl4X53d04FvSFoSETv3\n9ho2EHX0dwDWKx39HYD1wkD5NN16xvVWTmWst54MyxspqThs6q6a/fOAsZK2ljQYOBaYnYd6DY6I\nX5CGiI3JvShPSToUQNJGkjYpniwiLouI3SJiTO0wqIjojDHZmKEAACAASURBVIgdgHtJw6nquRM4\nUnllNklbNchXbXwMAZ7Ljat9ScPdavM0cxtw+poDpNFtHNPIu5Tmfg0i3V9tWd9Fmr+zsdJ8qo8A\nd+Vhmn/JvRbfIfWELAOG5+GJSNo810/RhcCiiLi2XjBKc7iubBHz7cAnq+euU95zgb0ljcr7N5X0\n5rxvdo7146wd0tYs/ytRnOu0raSZTTNLI0jPxWTgxznOLuep8TiwR94+lK4NrGbPUXXfHcARhTld\nW+UY2iLpQ6QewEMi4qU6WU4EbnXjyszMzGzd6EkD62Hg05KWkCbJ/zCnV4eErQDOJA0oX0jqibgR\n2A6oSFpIGoJ2Zj7uROB0SYuAOcCwXsT/G6Du0tZ5SN95pEbeQuCiYrzFrPn3NGDPHM8JwNI6eeod\nX3UOaSGQxUpL2p9dm0HS7pIub3I/VfOAS0nDEX8bEdcVrx0RC4ErSMPXfgVcnoeuvYM092sh8A3g\n3Ih4mdRIu1TS/aRV5l5Xc70vAPsrLXJxn6SDavaPAP63Rcw/Bp4CFufrH1sT8/8A/wJclcv4HtK8\nPvKiDDcBH8q/m+anQR0oLdYxvE76YZKeIg1TvElrl/Xfhq49UfV0AIsk3QccBXy3WQzA/wXG5TLY\nC/hzYV+znshqOS0lfRAxI9/3DKDePdW9V+AS0mIat+e6vKxm/1ak1QVtvVPp7wCsVyr9HYD1Qhm/\nl8dcb2VVxnpTmb/lPc93GhoRZ7bMbL2WF3H4aUQ82N+x9CWllQ6fiIib+juWV0teNGNxRPyoSZ7o\n/9X3recqeLhZGVVYd/Umyvx//EA2kCbdW/tcb+U0kOtNEhHRbYRS2RtYo0g9OS/WfBeWmdWQNJs0\nb/CEiHimST43sMzWC25gmZmtS+tlA8vM+l5qYJlZ2Q0bNpIVKx7v7zDMzNZbjRpYvfnuKTNbz0WE\nf0r2M2vWrH6PwT8Dq97cuFp3yjgnxFxvZVXGenMDy8zMzMzMrI94iKCZdSEp/HfBzMzMrDkPETQz\nMzMzM1vH3MAyM1sPlHGMurneysr1Vk6ut3IqY725gWVmZmZmZtZHPAfLzLrwHCwzMzOz1hrNwdqg\nP4Ixs4FN6va3wszMrEeGbTeMFU+v6O8wzF517sEysy4kBRP6OwrrseXAjv0dhPWY662cXG/tmZC+\nV3GgqFQqdHR09HcY1kMDud68iqCZmZmZmdk61lYDS9JISQ802DdL0pi+Das9kkZIuk/SrYW05f0R\nSyOSxkma0ka+5YX8NzbKI2nrPoyr7nUK+5vGnZ+LWS3y9PnzUTxnO/Ut6QhJD0pa3W4src4raVV7\n0XY55tuSHpB0oaTXS5oraYGkfdqp2zbvdStJMyQtk3SbpCGFff8taZ6kN/Y0disBf5peTq63cnK9\nldJA7QWx5spYbz3pwRo4fbxrHQbMiIgPF9IGYpztxBQNtnt6np5odb6ext0f2rn+A8BHgNl9eN7e\n3PfHgV0i4t+A9wOLI2L3iLi7zfO1k+dMYGZE7ATcCXxlzcERY4EFwIE9jtzMzMzM2tKTBtaGkqZK\nWiLpGkkb12aQdKykxfnngpw2SNKUnLZI0mdz+ihJt0u6X9K9knrzedCWwHM1ac8X4jkxX3OhpCtz\n2hRJkyTNkfSopMNz+maSZuZYFkk6JKePlLQ0H7dM0jRJH8jHL5O0R863qaTJhV6Jg3MYfwP+2Ma9\nPF/YHiLpJkkPS7qskL5mjKekM3JvyOJCmW6aj1uY04/M6XvmeO/P8W1WvLCkm3NP4EJJKyX9c5tx\nrwb+kM8xqNBDc7+kT9dmzuV2Ty7jq3O8H5R0TSHPmp41SfvX5m9RbnVFxLKIeKRYfm14PscwXNLs\nXD6LJe29NlSdm+/1HklvyIlTqs9Ufr0q/74e2BxYIOnLwIXAYfm8G9O1bo+X9Ou87wfSmhUnWt4r\ncChwZd6+kvQhRNEK0r8bW98MqL57a5vrrZxcb6VUxu9TsnLWW09WEdwJODki5kqaDJwGXFzdKWkb\n4AJgN2AlcHtupDwNbBcRu+R8W+RDpgHfjIgbJG1E7+aDDQY6iwkR8a58nZ2BrwLvjogXJBXfVA6P\niL0lvRW4AZgO/BU4LCJelDQUmJv3AYwCPhoRSyTdCxyTjz8kX+Nw4Czgjog4JQ/LmidpZkT8CvhV\njml34JMR8YnaG6nGne0JvBV4ErhN0uERMb26U2mY20k532Dg15IqOc5nIuKgnO8fJG0I/Bw4MiLu\nk7Q58Jeaax9YOO9/ANdFxKpq3I1ExNPAEfnlJ4CRpB6aqClvcpl+DdgvIv6SGxlnAOcDP5K0SUT8\nBTga+FnOf1ad/Oc2KjdJNwOnRMQrXrKocN7jgF9GxPm5oVNt5G0G3BMRX5N0Ial36pv1TpXPd6ik\nP0VEdWjjs8DuEXF6fl29h/+Ty+A9EbFa0veB44Gpbd7rGyPi2XzNFeo+HLCT9Mw0Vxz4uQMeDmNm\nZmaveZVKpa0GX08aWE9GxNy8PRX4DIUGFunN/qyIqPZoTAPGkt4Q7yhpEnALMCO/yd82Im4AiIi/\n9SAO8vkF7Jpjqed9wLUR8UK+xsrCvuty2tLCG1AB50saS3oTum1h3/KIWJK3HwJm5u0HSG8/AfYH\nDpb0pfx6I2AEsKx60YhYQGqItDIvIp7I93kVsA+pEVi1D/CLiPhrzjMdeC9wG/AdSecDN0fE3ZLe\nDvwuIu7LMbyYj+lyQUmvB34KHJEbVz31fuAH1S9QqilvgL2AnYE5ue42JDVQVkv6Jans/os0fO1L\nQEe9/M0CqDYU+9h8YHJuqF4fEYty+ksRcUveXkC6/3ra7TWrDv/bDxgDzM/3vTHwbLfM7d9r7bDC\nZ0hl29y+bZ7dBg43gsvJ9VZOrrdSKuNcHhtY9dbR0dElnokTJ9bN15MGVu0btXrzQbq9mYyIlZJ2\nBT4IfAo4EvhcvbxdTiSdRuoVCOCA4if1kgYBjwEvATf34B6qXqoT8/HA64HdIqJTaUGBjevk7yy8\n7mRtGYrUy/VIL+Kp1U5Zdz8o4pHcC3UAcI6kO0iNyVZlPQi4CpgQEUt7EW87RJovd3ydfVcD/wq8\nAMyPiD/nxkWj/K+aiLgrN7oPBK6QdFFETAVeLmRbzdrn4O/k3thCw7AnBFwZEWf1MuRnJQ2LiGcl\nDaf7ENrpwDckLYmInXt5DTMzMzNroCfD8kZKKg6buqtm/zxgrKStJQ0GjgVm56FegyPiF6QhYmNy\nL8pTkg4FkLSRpE2KJ4uIyyJit4gYUzsMKiI6I2IH4F7ScKp67gSOVF6ZTdJWDfJVGx9DgOdy42pf\n0nC32jzN3AacvuYAaXQbxzTyLqW5X4NI91db1neR5u9srDSf6iPAXXmY5l8i4mfAd0g9IcuA4Xl4\nIpI2z/VTdCGwKCKurReM0hyuK+vtK7gd+GT13HXKey6wt6RRef+mkt6c983OsX6cNJyxVf5XojjX\naVtJM5tmlkaQnovJwI9znF3OU+NxYI+8fShdG1jNnqPqvjuAIwpzurbKMbTrBuBf8vZJwPU1+08E\nbnXjaj3kOSHl5HorJ9dbKZVxLo+Vs9560sB6GPi0pCWkSfI/zOnVIWErSCuYVYCFpJ6IG4HtgIqk\nhaQhaGfm404ETpe0CJgDDOtF/L8B6i5tnYf0nUdq5C0ELirGW8yaf08D9szxnAAsrZOn3vFV55AW\nAlmstKT92bUZJO0u6fIm91M1D7iUNBzxtxFxXfHaEbEQuII0fO1XwOV56No7SHO/FgLfAM6NiJdJ\njbRLJd0PzABeV3O9LwD7Ky1ycZ+kg2r2jwD+t0XMPwaeAhbn6x9bE/P/kN74X5XL+B7SvD4iohO4\nCfhQ/t00Pw3qQGmxjuF10g+T9BRpmOJNWrus/zZ07YmqpwNYJOk+4Cjgu81iAP4vMC6XwV7Anwv7\nmvVEVstpKemDiBn5vmcA9e6p7r2SGssfkLSMNNzwgpr9WwF90ctqZmZmZnVoIH3Ddk/l+U5DI+LM\nlpmt1/IiDj+NiAf7O5a+pLTS4RMRcVN/x/JqyYtmLI6IHzXJE0x49WIyM7P11AQo8/tMs1YkERHd\nRiiVvYE1itST82LNd2GZWQ1Js0nzBk+IiGea5CvvHwUzMxswhm03jBVPv+KFfc0GrPWygWVmfU9S\n+O9C+VQqlQG10pK1x/VWTq63cnK9ldNArrdGDazefPeUmZmZmZmZ1eEeLDPrwj1YZmZmZq25B8vM\nzMzMzGwdcwPLzGw9UMbvCTHXW1m53srJ9VZOZaw3N7DMzMzMzMz6iOdgmVkXnoNlZmZm1lqjOVgb\n9EcwZjawSd3+VlgfGDZsJCtWPN7fYZiZmdk65CGCZlZH+Gcd/Dz77BM9qoWeKOMYdXO9lZXrrZxc\nb+VUxnpr2sCSNFLSAw32zZI0Zt2E1ZykEZLuk3RrIW15f8TSiKRxkqa0kW9AxV3UTmyt8kgaL+mM\nvouq6zklTZE0tkX+LSVNl7RI0lxJO7dxjVmSRrTY36PnX9IRkpZIuiO/vkrS/ZI+m+/j8BbHt3Ov\nx+X7XCTpbkm7FPZdJOkhSeN6EreZmZmZta+dHqyBOBnjMGBGRHy4kDYQ42wnpoEYd1XZ46/6KrAw\nInYFTgK+109xnAKcGhH7SRoO7BERoyNiUh9e4zFgbL7Xc4HLqzsi4gvA2cDH+vB6NkAM1G+5t+Zc\nb+Xkeisn11s5lbHe2mlgbShpav7k/RpJG9dmkHSspMX554KcNih/4r44f5r+2Zw+StLt+ZP7eyXt\n2Iu4twSeq0l7vhDPifmaCyVdmdOmSJokaY6kR6u9BZI2kzQzx7JI0iE5faSkpfm4ZZKmSfpAPn6Z\npD1yvk0lTc49IwskHZzD+Bvwxzbu5fl8nuGSZueeucWS9s7pqySdm8vrHklvyOkHFa45o5A+XtJP\nct5lkk7N6ePy+W+S9LCky5ScLOnfC2V3qqSLasu0VfyNyr1I0j9KulXS/BzLWyRtIenxQp5NJT0p\naXC9/HWuv5JU1s3sDNwJEBHLgB2q5dXE74HVjZ7j7ChJv87lWa2vkyRdUrifGyWNlfR1YB9gsqRv\nAbcB2+X63qemnMZIquT7vlXSsHbvNSLmRkT1uZsLbFeTZQXp34+ZmZmZrQsR0fAHGAl0Anvl15OB\nM/L2LGAMsA3wBLA1qcF2B3BI3jejcK4t8u+5wCF5eyNg42YxNIhrIvC5Bvt2Bh4Gtsqvt8y/pwBX\n5+23Ao/k7cHA5nl7aCF9JOnN7M759b3A5Lx9CDA9b58HHJe3hwDLgE1qYtoduLzFPZ0BfCVvC9gs\nb3cCB+TtC4GvVq9VOPYU4Nt5ezywMJftUOBJYDgwDvjffF8CZgCHA5sBjwKD8/FzgLf1ok4alfv4\nwjMzExiVt98J3JG3fwGMy9tHVcuqSf4156zzXBxUJ/084KLCef4G7NbmfTV6jmcVyvzDwO15+yTg\ne4X8N5J6lKrH7FZ4vhYX8k3J9bFBroOhhfKY3O691uT5Yu1zB7wXuKnFcQHhn3XyQ6wrs2bNWmfn\ntnXH9VZOrrdycr2V00Cut/z/erf3Uu2sIvhkRMzN21OBzwAXF/bvCcyKiD8ASJoGjCUNT9pR0iTg\nFmCGpM2BbSPiBlJErXoeupEkYNccSz3vA66NiBfyNVYW9l2X05ZKemP1lMD5SnNbOoFtC/uWR8SS\nvP0Q6Q0/wAPADnl7f+BgSV/KrzcCRpAaWuTrLQA+0eLW5pN6NzYEro+IRTn9pYi4JW8vAN6ft7eX\ndA2pgbshsLxwrutz2f5e0p2kRsUfgXkR8QSk+T/APhExXWlO0EGSHgY2iIiHWsRaT7NyR9JmwHuA\na3MdkuMGuAY4GpgNHAN8v0X+uiJifINdFwCTJN1HqruFwOo27+sxap7jwr7p+fcCUoOpHa2W59sJ\neDtwe77vQcDvajM1udd0EWlf4GRSr1nRM8BbJL0uIl5qfIYJhe2O/GNmZmb22lWpVNpadKOdBla0\neA113jRGxEpJuwIfBD4FHAl8rl7eLieSTgM+nq9zQESsKOwbRHrD+xJwcxux1yq+oazGcTzwelLP\nQqfSog0b18nfWXjdydqyE/DRiHikF/GsERF35UbegcAVki6KiKnAy4VsqwvXvQT4TkTcrLRoQfEN\nd7GORP06K+abTJqn9DCpJ2VdGAS8EBH1Foa4AThP0lakHqM7gc2b5O+RiFhFYd5RruPH2jy23nN8\nat5dfR6K9fJ3ug697TaktgUBD0bE3j08bu0J0sIWlwMfqjZ4qyLiMUlLgSck7de4MT2ht5e3flLG\nMermeisr11s5ud7KaSDVW0dHR5d4Jk6cWDdfO3OwRkp6V94+DrirZv88YKykrSUNBo4FZksaShp2\n9gvga8CYiHgReErSoQCSNpK0SfFkEXFZROwWEWOKjau8rzMidiAN1zu6Qbx3AkdK2jpfY6sG+aoN\nrCHAc7lxtS9deyLa+TKg24DT1xwgjW7jmO7BpBXrnouIycCPSQ2NZjFswdqejZNq9h2ay3YoaWjg\n/Jy+p9LcskGk8rsbICLmAduT6u6qBvEtbXELTcs9N3KWSzqicM5d8r4/k+p0Emn4WjTL31OShuSe\nQSR9HJidn0WU5t9t0+TYbs9xo6z59+PA6Dy/bXtS72HD09dJWwa8QdJe+fobqI1VDwvxjgD+C/jn\niPhtnf27ADuSepJ701NpZmZmZk2008B6GPi0pCWkyfE/zOlpskZqBJ0JVEhDr+ZHxI2kyfUVSQuB\nn+Y8ACcCp0taRJprUp3A3xO/Ic356iYP6TuP1MhbCFQXbGjUEzeN1PBYBJwALK2Tp97xVeeQFgJZ\nrLSk/dm1GSTtLuny7od20QEsysPYjgK+2+K6E4H/lDSf7otRLCbVxz3A2YWG6r3ApaThjr/NjYaq\na4A5sXaBhGL8Q1vE3qzci04ATlFasONB0ly2qqtJvYk/L6Qd3yR/N5ImSjqozq63Ag/mRuIHgeqC\nKwJGAX9octpGz3Hd5yki5pAaWQ+R6nBBbZ4Gr6vHvwwcAVwo6X7Sv6l39+Bev076t3GZ0mIj82r2\nbwU8HhGddY61Eivj94SY662sXG/l5HorpzLWm9L8rHLJ852GRsSZLTO/xkgaD6yKiItr0scBX4iI\nuo0USTcCF0fErDr7DgR2jIhL10XM/UXS24CTI+KL/R3Lq0XSUcBHIuLYJnmicbveXhmxrv7mViqV\nATWMwtrjeisn11s5ud7KaSDXmyQiotuIpLI2sEYBVwAvRtfvwnrN62kDS9IQ0jDPhRFxzKsXqb3a\nlJbffy9ptco7muRzA2udWXcNLDMzM3t1rVcNLDNbd1IDy9aFYcNGsmLF4/0dhpmZmfWBRg2sduZg\nmdlrTL3vdPDPK/9Zl42rMo5RN9dbWbneysn1Vk5lrDc3sMzMzMzMzPqIhwiaWReSwn8XzMzMzJrz\nEEEzMzMzM7N1zA0sM7P1QBnHqJvrraxcb+XkeiunMtabG1hmZmZmZmZ9xHOwzKwLz8EyMzMza63R\nHKwN+iMYMxvYpG5/K8z61LDthrHi6RX9HYaZmVmfcw+WmXUhKZjQ31FYjy0HduzvIHpgQvq+tde6\nSqVCR0dHf4dhPeR6KyfXWzkN5Hrr1SqCkkZKeqDBvlmSxvRVgD0haYSk+yTdWkhb3h+xNCJpnKQp\nbeQbUHEXtRNbqzySxks6o++i6npOSVMkjW2Rf0tJ0yUtkjRX0s5tXGOWpBEt9vfo+Zd0hKQlku7I\nr6+SdL+kz+b7OLzF8S3vNef7nqRH8rlHF9IvkvSQpHE9idvMzMzM2tfOIhcD8SPGw4AZEfHhQtpA\njLOdmAZi3FVlj7/qq8DCiNgVOAn4Xj/FcQpwakTsJ2k4sEdEjI6ISX11AUkfBkZFxJuBTwI/rO6L\niC8AZwMf66vr2QBSpt4rW2Ogfiprzbneysn1Vk5lrLd2GlgbSpqaP3m/RtLGtRkkHStpcf65IKcN\nyp+4L849B5/N6aMk3Z4/Xb9XUm/eFmwJPFeT9nwhnhPzNRdKujKnTZE0SdIcSY9WewskbSZpZo5l\nkaRDcvpISUvzccskTZP0gXz8Mkl75HybSpqce0YWSDo4h/E34I9t3Mvz+TzDJc3OPXOLJe2d01dJ\nOjeX1z2S3pDTDypcc0Yhfbykn+S8yySdmtPH5fPfJOlhSZcpOVnSvxfK7lRJF9WWaav4G5V7kaR/\nlHSrpPk5lrdI2kLS44U8m0p6UtLgevnrXH8lqayb2Rm4EyAilgE7VMurid8Dqxs9x9lRkn6dy7Na\nXydJuqRwPzdKGivp68A+wGRJ3wJuA7bL9b1PTTmNkVTJ932rpGE9uNdDgZ/ke/01MKRwPMAK0r8f\nMzMzM1sH2mlg7QRcGhE7A6uA04o7JW0DXAB0AKOBPXMjZTSwXUTsknsOqsPlpgGXRMRo4D3A/+tF\n3IOBzmJCRLwrx7MzqceiIyJ2A4pviIdHxN7AwcCFOe2vwGERsQfwPuCiQv5RwLcjYqdcDsfk47+U\nrwFwFnBHROyVj/+OpE0i4lcR8fkc0+6SLq93I9W4geOAX0bEGGBX4P6cvhlwTy6vu4CP5/S7ImKv\niNgduBr4cuG07yDVx3uAbyj1lgDsCXwaeCvwT8BHgGuAgyUNznlOBv6jJraG2iz3qsuBf42IPUll\n+IOI+BOwUGuHrR2Uy2F1vfx1rv/5iJibY5go6aA6110EVBvU7wRGAG9qcV9HRMQzNH6OAQbn+/88\ndJm11K1XLyLOAe4FjouILwOHAI9GxJiIuLuaT9IGwCXAR/N9TwG+2YN73Q54qvD6mZxW1Un692Pr\nmwE72NiaKeP3u5jrraxcb+VUxnprZxXBJ6tv6oCpwGeAiwv79wRmRcQfACRNA8YC5wI7SpoE3ALM\nkLQ5sG1E3AAQEa0+je9GkkgNkKkNsrwPuDYiXsjXWFnYd11OWyrpjdVTAucrzW3pBLYt7FseEUvy\n9kPAzLz9ALBD3t6f1ED5Un69EekN/LLqRSNiAfCJFrc2n9S7sSFwfUQsyukvRcQteXsB8P68vb2k\na4BtgA3p+vbq+ly2v5d0J/BOUm/avIh4AtL8H2CfiJiuNCfoIEkPAxtExEMtYq2nWbkjaTNSg+/a\nXIfkuCE18o4GZgPHAN9vkb+uiBjfYNcFwCRJ95HqbiGwus37eoya57iwb3r+vQAY2eb5Wi3PtxPw\nduD2fN+DgN/VZmpyr608A7xF0usi4qWGuWYVtnfAw8/MzMzsNa9SqbTV4GungVX7aXy9OTfd3jRG\nxEpJuwIfBD4FHAl8rl7eLieSTiP10gRwQESsKOwbRHrD+xJwcxux1yq+oazGcTzwemC3iOhUWrRh\n4zr5OwuvO1lbdiL1NjzSi3jWiIi7ciPvQOAKSRdFxFTg5UK21YXrXgJ8JyJuzr0/xTfcxToSjedJ\nVdMnk3qfHqZrD01fGgS8kHvoat0AnCdpK2AMaTjf5k3y90hErKIw7yjX8WNtHlvvOT41764+D8V6\n+Ttde4a7DaltQcCDuae0N54Bti+8flNOAyAiHpO0FHhC0n4NG9P79vLq1n/cCC6lMs4tMNdbWbne\nymkg1VtHR0eXeCZOnFg3XztDBEdKKg5ju6tm/zxgrKSt8zCzY4HZkoaShlD9AvgaMCYiXgSeknQo\ngKSNJG1SPFlEXBYRu+WhUytq9nVGxA6koVZHN4j3TuBISVvna2zVIF+1gTUEeC43rvala09EO18G\ndBtw+poDCqu29YTSinXPRcRk4MekhkazGLZgbc/GSTX7Ds1lOxQYR+odgzR8c2RuqB4N3A0QEfNI\nb8qPBa5qEN/SFrfQtNxzI2e5pCMK59wl7/szqU4nATdF0jB/T0kaknsGkfRxYHZ+FlGaf7dNk2O7\nPceNsubfjwOjlWxP6j1sePo6acuAN0jaK19/A7Wx6mHBDcCJ+di9gJUR8WzhfnYhvRXftpc9lWZm\nZmbWRDsNrIeBT0taQpocX12VLAByI+hMoEIaejU/Im4kzfuoSFoI/DTngfTm73RJi4A5QHECfrt+\nA2xdb0ce0nceqZG3kLVzqhr1xE0jNTwWAScAS+vkqXd81TmkhUAWKy1pf3ZthmZzsAo6gEV5GNtR\nwHdbXHci8J+S5tN9MYrFpPq4Bzi70FC9F7iUNNzxt7nRUHUNMCciui3MkRsZTTUp96ITgFOUFux4\nkDQPqepqUm/izwtpxzfJ302TeUlvBR7MjcQPkueH5SF4o4A/NDlto+e47vMUEXNIjayHSHW4oDZP\ng9fV418GjgAulHQ/6d/Uu9u91zycdLmkR4EfUTNnEtgKeDwiOmuPtZLzHKxSKuPcAnO9lZXrrZzK\nWG+l/KLhPN9paESc2TLza4yk8cCqiLi4Jn0c8IWIqNtIkXQjcHFEzKqz70Bgx4i4dF3E3F8kvQ04\nOSK+2N+xvFokHQV8JCKObZLHXzRcRv6i4VIayF+gaY253srJ9VZOA7ne1OCLhsvawBoFXAG8WPNd\nWK95PW1gSRpCGua5MCKOefUitVeb0vL77wW+EhF3NMnnBpatexPcwDIzs3JbrxpYZrbuSPIfBVvn\nhm03jBVPr2id0czMbIBq1MBqZw6Wmb3GRIR/SvYza9asfo+hJz9uXCVlnFtgrreycr2VUxnrzQ0s\nMzMzMzOzPuIhgmbWhaTw3wUzMzOz5jxE0MzMzMzMbB1zA8vMbD1QxjHq5norK9dbObneyqmM9eYG\nlpmZmZmZWR/xHCwz68JzsMzMzMxaazQHa4P+CMbMBjap298KMzMzs/XesGEjWbHi8Vd0DvdgmVkX\n6YuG/XehfCpARz/HYD1XwfVWRhVcb2VUwfVWRhVe3XoT7baPerWKoKSRkh5osG+WpDFtXb2PSRoh\n6T5JtxbSlvdHLI1IGidpShv5BlTcRe3E1iqPpPGSzui7qLqeU9IUSWPbOOZ7kh6RdL+k0W3knyVp\nRIv9PXr+JR0haYmkO/Lrq3I8n833cXiL41veq6Tjupj/5QAAIABJREFUJC3KP3dL2qWw7yJJD0ka\n15O4zczMzKx97QwRHIgfZR8GzIiIMwtpAzHOdmIaiHFXlT1+ACR9GBgVEW+W9C7gh8Be/RDKKcCp\nEXGPpOHAHhHx5hxjy8Z4mx4DxkbEHyV9CLicfK8R8QVJ84CPAbP76Ho2YHT0dwDWKx39HYD1Skd/\nB2C90tHfAVivdPR3AD3WziqCG0qamj95v0bSxrUZJB0raXH+uSCnDcqfuC/On6Z/NqePknR7/uT+\nXkk79iLuLYHnatKeL8RzYr7mQklX5rQpkiZJmiPp0WpvgaTNJM3MsSySdEhOHylpaT5umaRpkj6Q\nj18maY+cb1NJkyXNlbRA0sE5jL8Bf2zjXp7P5xkuaXbumVssae+cvkrSubm87pH0hpx+UOGaMwrp\n4yX9JOddJunUnD4un/8mSQ9LukzJyZL+vVB2p0q6qLZMW8XfqNyLJP2jpFslzc+xvEXSFpIeL+TZ\nVNKTkgbXy1/n+itJZd3MocBPACLi18AQScNaHPN7YHWj5zg7StKvc3lW6+skSZcU7udGSWMlfR3Y\nB5gs6VvAbcB2ub73qSmnMZIq+b5vLcTa8l4jYm5EVJ+7ucB2NVlWkP79mJmZmdm6EBENf4CRQCew\nV349GTgjb88CxgDbAE8AW5MabHcAh+R9Mwrn2iL/ngsckrc3AjZuFkODuCYCn2uwb2fgYWCr/HrL\n/HsKcHXefivwSN4eDGyet4cW0keS3szunF/fC0zO24cA0/P2ecBxeXsIsAzYpCam3YHLW9zTGcBX\n8raAzfJ2J3BA3r4Q+Gr1WoVjTwG+nbfHAwtz2Q4FngSGA+OA/833JWAGcDiwGfAoMDgfPwd4Wy/q\npFG5jy88MzNJPUkA7wTuyNu/AMbl7aOqZdUk/5pz1nkuDqqTfiPwnsLrmcCYNu+r0XM8q1DmHwZu\nz9snAd+rufbYwjG7FZ6vxYV8U3J9bJDrYGihPCa3e681eb5Y+9wB7wVuanFcQPindD+zBkAM/nG9\nvVZ+XG/l/HG9lfPn1a43ol05L7U/7QwRfDIi5ubtqcBngIsL+/cEZkXEHwAkTQPGAucCO0qaBNwC\nzJC0ObBtRNxAiqhVz0M3kgTsmmOp533AtRHxQr7GysK+63LaUklvrJ4SOF9pbksnsG1h3/KIWJK3\nHyK9MQd4ANghb+8PHCzpS/n1RsAIUkOLfL0FwCda3Np8Uu/GhsD1EbEop78UEbfk7QXA+/P29pKu\nITVwNwSWF851fS7b30u6k9Q4+SMwLyKegDT/B9gnIqYrzQk6SNLDwAYR8VCLWOtpVu5I2gx4D3Bt\nrkNy3ADXAEeThq0dA3y/Rf66ImJ8L+Ju5TFqnuPCvun59wJSg6kdrZbn2wl4O3B7vu9BwO9qM7W6\nV0n7AieTes2KngHeIul1EfFS4zNMKGx3UMbueTMzM7O+VKlU2vri497Mwap9DXXeNEbESkm7Ah8E\nPgUcCXyuXt4uJ5JOAz6er3NARKwo7BtEesP7EnBzG7HXKr6hrMZxPPB6Us9Cp9KiDRvXyd9ZeN3J\n2rIT8NGIeKQX8awREXflRt6BwBWSLoqIqcDLhWyrC9e9BPhORNystGhB8Q13sY5E/Tor5psMfJXU\nAzXlldxHE4OAFyKi3sIQNwDnSdqK1GN0J7B5k/w99QywfeH1m3JaSw2e41Pz7urzUKyXv9N16G23\nIbUtCHgwIvbu4XFrT5AWtrgc+FC1wVsVEY9JWgo8IWm/xo3pCb29vPWbjv4OwHqlo78DsF7p6O8A\nrFc6+jsA65WO/g5gjY6ODjo6Ota8njhxYt187czBGqm0MADAccBdNfvnAWMlbS1pMHAsMFvSUNKw\ns18AXyMNyXoReErSoQCSNpK0SfFkEXFZROwWEWOKjau8rzMidiAN1zu6Qbx3AkdK2jpfY6sG+aoN\nrCHAc7lxtS9deyLa+TKg24DT1xzQxgp1dYNJK9Y9FxGTgR+TGhrNYtiCtT0bJ9XsOzSX7VDS0MD5\nOX1Ppbllg0jldzdARMwjNUCOBa5qEN/SFrfQtNwjYhWwXNIRhXPukvf9mVSnk0jD16JZ/l64ATgx\nn2MvYGVEPJtfz5S0TaMD6z3HjbLm348Do/P8tu1JvYcNT18nbRnwhhwnkjaQtHOTc9TGOwL4L+Cf\nI+K3dfbvAuxI6knuTU+lmZmZmTXRTgPrYeDTkpaQJsf/MKcHQG4EnUlapH4hMD8ibiRNrq9IWgj8\nNOeB9Eb3dEmLSHNNWi02UM9vSHO+uslD+s4jNfIWAtUFGxr1xE0jNTwWAScAS+vkqXd81TmkhUAW\nKy1pf3ZtBkm7S7q8yf1Aap4vknQfad7Nd1tcdyLwn5Lm030xisWk+rgHOLvQUL0XuJQ03PG3udFQ\ndQ0wJ9YukFCMf2iL2JuVe9EJwClKC3Y8SJrLVnU1qTfx54W045vk70bSREkH1YntFlJj7VHgR8Bp\nOb+AUcAfmpy20XNc93mKiDmkRtZDpDpcUJunwevq8S8DRwAXSrqf9G/q3e3eK/B10r+Ny5QWG5lX\ns38r4PGI6KxzrJVapb8DsF6p9HcA1iuV/g7AeqXS3wFYr1T6O4AeK+UXDef5TkOj6zLtRlpFEFgV\nERfXpI8DvhARdRspkm4ELo6IWXX2HQjsGBGXrouY+4uktwEnR8QX+zuWV4uko4CPRMSxTfJE43a9\nDVwVBtIwCmtXBddbGVVwvZVRBddbGVUo2xcNl7WBNQq4AngxIj7cz+EMKD1tYEkaQhrmuTAijnn1\nIrVXm9Ly++8lrVZ5R5N8bmCZmZnZa9RrtIFlZuuOG1hmZmb22vXKG1jtrCJoZq857azvYmZmZrZ+\nGTas3W/eacwNLDPrxj3b5VOpVLosHWvl4HorJ9dbObneyqmM9eYhgmbWhaTw3wUzMzOz5hoNEWxn\nmXYzMzMzMzNrgxtYZmbrgUql0t8hWC+43srJ9VZOrrdyKmO9uYFlZmZmZmbWRzwHy8y68BwsMzMz\ns9Y8B8vMzMzMzGwd8zLtZtaN5O/BMjMzs4Fj2HbDWPH0iv4Ooy0eImhmXUgKJvR3FNZjy4Ed+zsI\n6zHXWzm53srJ9VZO1XqbMPC+p7NXQwQljZT0QIN9sySN6asAe0LSCEn3Sbq1kLa8P2JpRNI4SVPa\nyDeg4i5qJ7ZWeSSNl3RG30XV9ZySpkga28Yx35P0iKT7JY1uI/8sSSNa7O/R8y/pCElLJN2RX1+V\n4/lsvo/DWxz/iu5V0kWSHpI0ridxW0n4TUM5ud7KyfVWTq63ciphvbUzB2tgNRWTw4AZEfHhQtpA\njLOdmAZi3FVljx8ASR8GRkXEm4FPAj/sp1BOAU6NiP0kDQf2iIjRETGpry7Q7F4j4gvA2cDH+up6\nZmZmZtZVOw2sDSVNzZ+8XyNp49oMko6VtDj/XJDTBuVP3BdLWiTpszl9lKTb86fr90rqTbt0S+C5\nmrTnC/GcmK+5UNKVOW2KpEmS5kh6tNpbIGkzSTNzLIskHZLTR0pamo9bJmmapA/k45dJ2iPn21TS\nZElzJS2QdHAO42/AH9u4l+fzeYZLmp175hZL2junr5J0bi6veyS9IacfVLjmjEL6eEk/yXmXSTo1\np4/L579J0sOSLlNysqR/L5TdqZIuqi3TVvE3KvciSf8o6VZJ83Msb5G0haTHC3k2lfSkpMH18te5\n/kpSWTdzKPATgIj4NTBE0rAWx/weWN3oOc6OkvTrXJ7V+jpJ0iWF+7lR0lhJXwf2ASZL+hZwG7Bd\nru99asppjKRKvu9bC7H2xb2uIP37sfXNgO0Lt6Zcb+Xkeisn11s5lbDe2mlg7QRcGhE7A6uA04o7\nJW0DXAB0AKOBPXMjZTSwXUTsEhG7AtXhctOASyJiNPAe4P/1Iu7BQGcxISLelePZGfgq0BERuwHF\nN8TDI2Jv4GDgwpz2V+CwiNgDeB9wUSH/KODbEbFTLodj8vFfytcAOAu4IyL2ysd/R9ImEfGriPh8\njml3SZfXu5Fq3MBxwC8jYgywK3B/Tt8MuCeX113Ax3P6XRGxV0TsDlwNfLlw2neQ6uM9wDeUeksA\n9gQ+DbwV+CfgI8A1wMGSBuc8JwP/URNbQ22We9XlwL9GxJ6kMvxBRPwJWKi1w9YOyuWwul7+Otf/\nfETMzTFMlHRQnetuBzxVeP1MTmt2X0dExDM0fo4BBuf7/zx0mbXUrVcvIs4B7gWOi4gvA4cAj0bE\nmIi4u5pP0gbAJcBH831PAb7Zh/faSfr3Y2ZmZmbrQDurCD5ZfVMHTAU+A1xc2L8nMCsi/gAgaRow\nFjgX2FHSJOAWYIakzYFtI+IGgIho9Wl8N5JEaoBMbZDlfcC1EfFCvsbKwr7rctpSSW+snhI4X2lu\nSyewbWHf8ohYkrcfAmbm7QeAHfL2/qQGypfy642AEcCy6kUjYgHwiRa3Np/Uu7EhcH1ELMrpL0XE\nLXl7AfD+vL29pGuAbYAN6dq+vz6X7e8l3Qm8k9SbNi8inoA0/wfYJyKmK80JOkjSw8AGEfFQi1jr\naVbuSNqM1OC7NtchOW5IjbyjgdnAMcD3W+SvKyLG9yLuVh6j5jku7Juefy8ARrZ5vlbL8+0EvB24\nPd/3IOB3tZlewb0+A7xF0usi4qWGuWYVtneglOOfX3NcR+Xkeisn11s5ud7KaQDVW6VSoVKptMzX\nTgOr9tP4enNuur1pjIiVknYFPgh8CjgS+Fy9vF1OJJ1G6qUJ4ICIWFHYN4j0hvcl4OY2Yq9VfENZ\njeN44PXAbhHRqbRow8Z18ncWXneytuxE6m14pBfxrBERd+VG3oHAFZIuioipwMuFbKsL170E+E5E\n3Jx7f4pvuIt1JBrPk6qmTyb1Pj1M1x6avjQIeCH30NW6AThP0lbAGOBOYPMm+XvqGWD7wus35bSW\nGjzHp+bd1eehWC9/p2vPcLchtS0IeDD3lPZG03uNiMckLQWekLRfw8b0vr28upmZmdl6qqOjg46O\njjWvJ06cWDdfO0MER0oqDmO7q2b/PGCspK3zMLNjgdmShpKGUP0C+BowJiJeBJ6SdCiApI0kbVI8\nWURcFhG75aFTK2r2dUbEDqShVkc3iPdO4EhJW+drbNUgX7WBNQR4Ljeu9qVrT0Q7XwZ0G3D6mgPa\nWKGubjBpxbrnImIy8GNSQ6NZDFuwtmfjpJp9h+ayHQqMI/WOQRq+OTI3VI8G7gaIiHmkN+XHAlc1\niG9pi1toWu4RsQpYLumIwjl3yfv+TKrTScBNkTTM3ws3ACfmc+wFrIyIZ/PrmXmYa131nuNGWfPv\nx4HRSrYn9R42PH2dtGXAG3KcSNogD79sV8N7zWm7kD4L2raXPZU2UJVwjLrheisr11s5ud7KqYT1\n1k4D62Hg05KWkCbHV1clC4DcCDoTqAALgfkRcSNp3kdF0kLgpzkPpDd/p0taBMwBWi02UM9vgK3r\n7chD+s4jNfIWsnZOVaOeuGmkhsci4ARgaZ089Y6vOoe0EMhipSXtz67N0GwOVkEHsEjSfcBRwHdb\nXHci8J+S5tN9MYrFpPq4Bzi70FC9F7iUNNzxt7nRUHUNMCciui3MkRsZTTUp96ITgFOUFux4kDQP\nqepqUm/izwtpxzfJ302jeUl5iOVySY8CPyLPI8xD8EYBf2hy2kbPcd3nKSLmkBpZD5HqcEFtngav\nq8e/DBwBXCjpftK/qXe/0nst2Ap4PCI6a481MzMzs1eulF80nOc7DY2IM1tmfo2RNB5YFREX16SP\nA74QEXUbKZJuBC6OiFl19h0I7BgRl66LmPuLpLcBJ0fEF/s7lleLpKOAj0TEsU3y+IuGzczMbGCZ\nsJ580fAANh3YW4UvGrbekTRE0jLgz/UaVwARcfP61rgCiIiHXmONq4uAL5KGoJqZmZnZOlDKHiwz\nW3ck+Y+CmZmZDSjDthvGiqdXtM74KmrUg9XOKoJm9hrjD17Kp1KpdFnZyMrB9VZOrrdycr2VUxnr\nzT1YZtaFpPDfBTMzM7Pm1rc5WGZmZmZmZgOOG1hmZuuBdr5Z3gYe11s5ud7KyfVWTmWsNzewzMzM\nzMzM+ojnYJlZF56DZWZmZtaa52CZmZmZmZmtY25gmVk3kgbEz/A3De/voiiNMo5RN9dbWbneysn1\nVk5lrDd/D5aZdTehvwNInp3wbH+HYGZmZtYjnoNl/U7Sqoj4h/6OA0DSGOBKYF5EnJLTlkfEjv0Q\ny67AthFxa359ErBDRExscdytwF7AXRFxSCH9WGA88KOI+Pcmx8dAaWAxwV96bGZmZgOT52DZQDaQ\n3kGfAHy/2rjKehSfpL76dzUaOKAmrZ1YvkW6j64HRlwFjAM+/8pDMzMzM7N63MCyAUPSREkLJd0n\n6WlJkyWNlLRU0hRJyyRNk/QBSXPy6z3ysXtKukfSAkl3S3pzL8PYEniuJu35fI1xkmZLuknSw5Iu\nK8S+StJ3JC0E9pI0RlJF0nxJt0oalvOdLukhSfdL+llO2zTf69wc/8GSNgTOBo7K5XEk8L/Ai61u\nICJmNcoXEc8CQ3pcKjbglXGMurneysr1Vk6ut3IqY715DpYNGBExHhgvaQjw38Aledco4KMRsUTS\nvcAxEbG3pEOAs4CPAEuBfSKiU9J+wPnAEb0IYzDQWRPXuwov9wTeCjwJ3Cbp8IiYDmwG/Coivihp\nA2A2cEhE/F7SUcA3gVOAfyMN83tZ0hb5nGcBd0TEKfne5wEzgW8Au0fE6bVBSjo475vQi3v0Bytm\nZmZm64gbWDYQTQUuioj7JY0ElkfEkrzvIVLjA+ABYGTe3hL4Se65CnrxbOeG0dtY27CrZ15EPJHz\nXwXsA0wHVuffADsBbwdulyRSg+Z3ed8i4GeSrgOuy2n7AwdL+lJ+vREwolmsEXEjcGP7d9fFHySN\niojfNswxq7C9A/Cqz0Cznuro6OjvEKwXXG/l5HorJ9dbOQ2keqtUKm31qLmBZQOKpAnAkxHxk0Ly\nS4XtzsLrTtY+w+cAd0bE4blRVmwiVM99LnAgEBExpmbfm0g9R49GxL1NQqydA1V9/ZfCt/MKeDAi\n9q5z/IHAWOAQ4CxJ78j5PxoRj9TEtFeTOF6JScD9kj4TEVfUzbHvOrqymZmZWUl1dHR0afBNnFh/\n3TEPFbKBQLBm2Nv7gc/W29/CEOCZvH1yvQwR8bWI2K22cZX3PQ1sl8JQR5PrvDPPCxsEHA3cVSfG\nZcAbqg0kSRtI2jnvGxERs4EzgS1IQwtvA9YMA5Q0Om+uynl6QzQut68C/9SwcWWlVMYx6uZ6KyvX\nWzm53sqpjPXmBpYNBNWen88D2wLz88IOE2r2124XfQu4QNICevlc5x6oR4Gtm2S7F7iUNFTxtxFR\nHea3Jq6IeJk0/+tCSfcDC4F35yGIUyUtAhYAkyLiT6Tetw0lLZb0AGlxC0i9cDsXFrlYIy+EMaFe\ngJL+G7gaeJ+kJyV9oCbLRnmxCzMzMzPrY/4eLLMCSd8HHoiIH9bZNw74QvG7pcpG0huBRRGxTZM8\n/h4sMzMzsxb8PVhm7fkJcLKkyf0dSF/LXzQ8g9TbZ2ZmZmbrgHuwzKwLSQPmj8Kw7Yax4ukV/R1G\nKVQqlQG10pK1x/VWTq63cnK9ldNArrdGPVheRdDMuvEHL2ZmZma94x4sM+tCUvjvgpmZmVlznoNl\nZmZmZma2jrmBZWa2Hijj94SY662sXG/l5HorpzLWmxtYZmZmZmZmfcRzsMysC8/BMjMzM2vNc7DM\nzMzMzMzWMTewzKwbSa/oZ/ibhvf3LbzmlHGMurneysr1Vk6ut3IqY735e7DMrLsJr+zwZyc82ydh\nmJmZmZWN52CZ9RNJI4GbIuIdbeb/FnAw8BLwW+DkiPhTD663EzAFGAN8NSIubpAvXmkDiwn+smIz\nMzNbv3kOltnA1JNWyAzgbRExGngE+EoPr/V74DPAt3t4nJmZmZm1yQ0ss/61oaSpkpZIukbSxpJ2\nl7RQ0n2SFktaDRARMyOiMx83F3hTTy4UEf8TEQuAv/fxPdgAUMYx6uZ6KyvXWzm53sqpjPXmBpZZ\n/9oJuDQidgZWAadFxIKI2C0ixgC/pH6P08eAW1/FOM3MzMysDZ6DZdZP8hys2RGxQ369L/CZiDg8\nvz4aOBXYv/jFVJLOAsZExEd7ed3xwKqmc7DGFRJ2AHbs4UUmeA6WmZmZrV8qlUqXHrWJEyfWnYPl\nVQTN+ldtKyQAJL0d+Abw3prG1b8ABwDvq3cySecCBwKRe8B6Z99eH2lmZv+/vTsPsqws7zj+/SGF\nCwIxUQczCGjU4IYM4qCBShor4BYQNSoGRVyyQQIVxXJLdKbKqMQykaASFzIhmIFCRUXKEhBoE1QE\nHAaQTRIWlzhoBQiSKBF48sd9G05PLzPT0829p/v7qZq657xne24/dXv6ue/7niNpURobG2NsbOz+\n9dWrV0+7n0MEpeHaLcm+bfkPgIuS7ASsBY6oqtsmdkzyQuBtwCFVdfd0J6uqv+wML5zNlG9b1G99\nHKMu89ZX5q2fzFs/9TFv9mBJw3UdcHSSNcB3gZOAVwG7Ap9KEh7ojToR2A44b9DMxVV11OZeKMky\n4DJgB+C+JMcCT6uqu+bzDUmSJC1lzsGSNInPwZIkSdo0n4MlSZIkSQvMHixJkyTZ6l8Ky5YvY8MP\nN8xHONpM4+Pjkybeqh/MWz+Zt34yb/00ynmbqQfLOViSpvCLF0mSpLmxB0vSJEnK3wuSJEmzcw6W\nJEmSJC0wCyxJWgT6+JwQmbe+Mm/9ZN76qY95s8CSJEmSpHniHCxJkzgHS5IkadOcgyVJkiRJC8wC\nS9IUSRb9v5132XnYP+Z51ccx6jJvfWXe+sm89VMf8+ZzsCRNtWrYASy8W1fdOuwQJEnSIuQcLGnE\nJLkJeHZV3ZbkoqraP8nvAMdV1cFbcd6Tgd8Dbq2qPWfZr5ZCgcUqH6gsSZLmzjlYUn/c/1d/Ve0/\nXfscrQFesJXnkCRJ0iwssKQhSfLHSS5Psi7JjUnOn9jU2ednnUN2SnJ2kuuSfHxLr1dVFwG3b2XY\nGlF9HKMu89ZX5q2fzFs/9TFvFljSkFTVJ6pqBbAS+AHw4el26yw/BzgaeCrwpCQvX/goJUmStCW8\nyYU0fH8PXFBVX9nEfpdU1S0ASU4D9gfOXJCILuws7w48YUGuonk0NjY27BA0B+atn8xbP5m3fhql\nvI2Pj29Wj5oFljRESY4EHl9VR23G7hvPwZq0nmQl8InW/p6qOnvOgR0w5yMlSZIWpbGxsUkF3+rV\nq6fdzyGC0pAkeTbwVuC1s+3WWd43yW5JtgFeDVzU3bGqLqmqFVW19yzFVTY6pxaJPo5Rl3nrK/PW\nT+atn/qYNwssaXiOBh4FXNhudPHJ1t7tmeouXwJ8FLga+I+q+sKWXCzJWuCbwFOSfD/JG+YeuiRJ\nkqbjc7AkTeJzsCRJkjbN52BJkiRJ0gKzwJKkRaCPY9Rl3vrKvPWTeeunPubNuwhKmmrVsANYeMuW\nLxt2CJIkaRFyDpakSZKUvxckSZJm5xwsSZIkSVpgFliStAj0cYy6zFtfmbd+Mm/91Me8WWBJkiRJ\n0jxxDpakSZyDJUmStGnOwZIkSZKkBWaBJUmLQB/HqMu89ZV56yfz1k99zJvPwZI0RTKlt1uSJPXM\nsuXL2PDDDcMOY8lxDpakSZLUUnjQsCRJi94q8G/9heMcLGkzJLkvyYc6629N8p4hxbJbi+foTtuJ\nSY4YRjySJEnaNAssabK7gZcn+dVhB9L8BDg2icN5Nbubhh2A5sS89ZN56yfz1kt9nINlgSVNdg/w\nSeAtG29oPUrnJ1mf5Lwku7T2NUlOSPKNJP+e5OWdY45Lckk75r1ziOenwPnAkdPEs1eSb7Vzfz7J\nTq39wiQfTPLtJNcl2a+1b5Pkb1r7+iR/OId4JEmSNAsLLGmyAj4GHJ5kh422nQisqaq9gLVtfcLO\nVbUfcDBwPECSA4EnV9VKYAWwT5L95xDP8cBxmXrniVOAt7V4vgt0C7iHVNW+wF/A/TOq3gTc0dpX\nAn+UZLctjEej6gnDDkBzYt76ybz1k3nrpbGxsWGHsMUcdiRtpKruSnIKcCzw886m5wEva8un0gqp\n5ovt2GuTPLa1HQQcmGQdEGB74MnARVsYz81JLgYOn2hLsiOwU1VNnOsU4IzOYWe21+8AE0XUQcAz\nk7yyre/Y4rllykUv7Czvjv8pSZKkJW98fHyzhixaYEnTOwFYB6zptM12G567O8vpvH6gqj4100FJ\nDmXQ81TAm6tq3Qy7fgD4HDA+zXVmi+deHvicB/jzqjpvluMGDtjkHho1N2Eh3EfmrZ/MWz+Zt14a\nHx8fmV6ssbGxSbGsXr162v0cIihNFoCqup1Bj9CbOtu+CbymLb8W+LfZzgGcA7wxyfYASX49yWO6\nO1bVF6tqRVXtPUNxNRHP9cA1wCFt/U7gton5VcDrgK9vRjxHTdwwI8mTkzx8hmMkSZI0B/ZgSZN1\ne6k+DBzdaTsGWJPkOAY3n3jDNMfcv15V5yXZA/hWmz71MwaF2U/nGM9fM+hVm3Ak8A+tSLpxU/EA\nn2Yw4G9dm8/1E+DQLYhFo8xvZfvJvPWTeesn89ZLo9J7tSV80LCkSXzQsCRJi8QqHzS8kHzQsCQt\nZj7fpZ/MWz+Zt34yb73Ux+dgOURQ0lSrhh2AJEnaWsuWLxt2CFtt/fr1vRsmaIElaQqHE/TPqlWr\nWLVq1bDD0BYyb/1k3vrJvPXTHXfcMewQtphDBCVJkiRpnlhgSdIicPPNNw87BM2Beesn89ZP5q2f\n+pg37yIoaZIk/lKQJEnaDNPdRdACS5IkSZLmiUMEJUmSJGmeWGBJkiRJ0jyxwJIkSZKkeWKBJQmA\nJC9Mcl2S7yV5+7DjWeqSnJzk1iRXdtoeleTcJNcnOSfJTp1t70xyQ5JrkxzUad87yZUtrx95sN/H\nUpNklyQXJLk6yVVJjmnt5m6EJXlokm8nubxC8BmbAAAGy0lEQVTl7v2t3bz1QJJtkqxLclZbN28j\nLsnNSa5on7lLWtuiyZsFliSSbAN8FHgB8HTgNUn2GG5US94aBvnoegfwtar6TeAC4J0ASZ4GvAp4\nKvAi4ONJJu5qdBLwpqp6CvCUJBufU/PrHuAtVfV04HnA0e2zZO5GWFXdDRxQVSuAPYHnJ9kP89YX\nxwLXdNbN2+i7DxirqhVVtbK1LZq8WWBJAlgJ3FBVt1TVL4HTgZcOOaYlraouAm7fqPmlwClt+RTg\n0LZ8CHB6Vd1TVTcDNwArk+wM7FBVl7b9/rlzjBZAVW2oqvVt+S7gWmAXzN3Iq6r/bYsPZfD30e2Y\nt5GXZBfgxcCnO83mbfSFqXXIosmbBZYkgOXADzrrP2xtGi2PrapbYfCHPPDY1r5x/n7U2pYzyOUE\n8/ogSrI7sBdwMbDM3I22NszscmADMF5V12De+uDvgLcB3ecOmbfRV8B5SS5N8ubWtmjytu2wA5Ak\nzZkPMhxRSR4JfA44tqrumuYB3uZuxFTVfcCKJDsC5yQZY2qezNsISfIS4NaqWt/yNRPzNnr2q6of\nJ3kMcG6S61lEnzd7sCTB4NugXTvru7Q2jZZbkywDaEMjftLafwQ8vrPfRP5matcCSrItg+Lq1Kr6\nUms2dz1RVXcCXwH2wbyNuv2AQ5LcCJzGYO7cqcAG8zbaqurH7fWnwBcZTFVYNJ83CyxJAJcCT0qy\nW5LtgMOAs4YckwZj1NNZPws4si2/HvhSp/2wJNsleQLwJOCSNsTiv5OsbBOCj+gco4Xzj8A1VXVC\np83cjbAkj564Y1mShwMHApdj3kZaVb2rqnatqicy+H/rgqp6HfBlzNvISvKI1stPku2Bg4CrWESf\nN4cISqKq7k3yZ8C5DL54Obmqrh1yWEtakrXAGPBrSb4PvBf4IPDZJG8EbmFwVyWq6pokZzC4i9Yv\ngaOqamJoxdHAPwEPA75SVV99MN/HUtPuPHc4cFWbz1PAu4DjgTPM3ch6HHBK+yNtGwa9j+e3HJq3\n/vkg5m2ULQO+0IZObwv8S1Wdm+QyFkne8kB8kiRJkqSt4RBBSZIkSZonFliSJEmSNE8ssCRJkiRp\nnlhgSZIkSdI8scCSJEmSpHligSVJkiRJ88QCS5IkjZQkFybZe4Ztpyd5Ylu+OcnXN9q+PsmVbfn1\nSU6c5TonJXneDNsOSfJXc38XkpYqCyxJktQLSX4D2L6qbmxNBeyQZHnbvkdr65rtgZ/7AhfPsO3L\nwCuSbLsVIUtagiywJEnSrJI8IsnZSS5PcmWSV7b2m5Ic39ou7vQsPTrJ55J8u/37rc55Tm77fifJ\nIa39YUlOS3J1kjOBh80QymEMCp+uM1o7wGuAtRtt37X1iF2f5D2d97QH8L2qqiTHtGuvT7IWoKoK\n+CZw0Bx/bJKWKAssSZK0KS8EflRVK6pqT+CrnW23t7aPASe0thOAv62qfYHfBz7d2t8NnF9VzwWe\nD3woycOBPwX+p6qeDrwX2GeGOPYHLuusF/B54GVt/WCmFmDPadufBbyyM/TwRZ338XZgr6raC/iT\nzrGXAr89QyySNC0LLEmStClXAQcm+UCS/avqZ51tp7fX04DntuXfBT6a5HLgLOCRSR7BoDfoHa19\nHNgO2JVBEfMZgKq6Crhihjh2A368Udt/AbcneTVwDfDzjbafV1V3VNUvgDMZFGkAL+CBAusKYG2S\nw4F7O8f+J7D7DLFI0rQcVyxJkmZVVTe0np8XA+9L8rWqet/E5u6u7XUbYN+q+mX3PEkAXlFVN0zT\nPqlpplBm2HYGgx60I2Y4ZtJ66zXbqao2tLaXMCjyDgHeneQZVXVfu9Zsc7gkaQp7sCRJ0qySPA74\neVWtBT4EdO/w9+r2ehjwrbZ8DnBs5/hnddqP6bTv1Rb/FTi8tT0D2HOGUG4Bdu6G1l6/ABwPnDvN\nMQcm+ZVWVB0KfAM4ALiwXS/ArlX1deAdwI7AI9uxj2vXlKTNZg+WJEnalGcymC91H/B/TJ6n9Kgk\nVwC/YHCTCRgUVx9r7Q9hUEAdBbwP+Ei7jXqAmxj0Gp0ErElyNXAtk+dZdV3EYH7WurZeAFV1F4PC\nb7resEsYDA1cDpxaVevards/27Y/BPhMkh1bTCdU1Z1t20rg7E3+dCSpI4Ob5EiSJG2ZJDcBz66q\n2x6k6z0ROLGqXrKV57mMwRDGe2fZJwwKuedU1T1bcz1JS4tDBCVJ0lw9qN/Studf3TlxO/itOM8+\nsxVXzcHA5y2uJG0pe7AkSZIkaZ7YgyVJkiRJ88QCS5IkSZLmiQWWJEmSJM0TCyxJkiRJmicWWJIk\nSZI0T/4f8YECY6U7o3gAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "co_t, de_t = compression_decompression_times()\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Decompression speed', fontsize=14, y=1.01)\n", - "\n", - "\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "xlim = (0, np.max((nbytes / 1000000) / np.array(de_t)) + 100)\n", - "ax.set_xlim(*xlim)\n", - "ax.set_ylim(0, len(de_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import cpuinfo" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Vendor ID: GenuineIntel\n", - "Hardware Raw: \n", - "Brand: Intel(R) Xeon(R) CPU E3-1505M v5 @ 2.80GHz\n", - "Hz Advertised: 2.8000 GHz\n", - "Hz Actual: 1.1000 GHz\n", - "Hz Advertised Raw: (2800000000, 0)\n", - "Hz Actual Raw: (1100000000, 0)\n", - "Arch: X86_64\n", - "Bits: 64\n", - "Count: 8\n", - "Raw Arch String: x86_64\n", - "L2 Cache Size: 8192 KB\n", - "L2 Cache Line Size: 0\n", - "L2 Cache Associativity: 0\n", - "Stepping: 3\n", - "Model: 94\n", - "Family: 6\n", - "Processor Type: 0\n", - "Extended Model: 0\n", - "Extended Family: 0\n", - "Flags: 3dnowprefetch, abm, acpi, adx, aes, aperfmperf, apic, arat, arch_perfmon, avx, avx2, bmi1, bmi2, bts, clflush, clflushopt, cmov, constant_tsc, cx16, cx8, de, ds_cpl, dtes64, dtherm, dts, eagerfpu, epb, ept, erms, est, f16c, flexpriority, fma, fpu, fsgsbase, fxsr, hle, ht, hwp, hwp_act_window, hwp_epp, hwp_noitfy, ida, invpcid, lahf_lm, lm, mca, mce, mmx, monitor, movbe, mpx, msr, mtrr, nonstop_tsc, nopl, nx, pae, pat, pbe, pcid, pclmulqdq, pdcm, pdpe1gb, pebs, pge, pln, pni, popcnt, pse, pse36, pts, rdrand, rdseed, rdtscp, rep_good, rtm, sep, smap, smep, smx, ss, sse, sse2, sse4_1, sse4_2, ssse3, syscall, tm, tm2, tpr_shadow, tsc, tsc_adjust, tsc_deadline_timer, vme, vmx, vnmi, vpid, x2apic, xgetbv1, xsave, xsavec, xsaveopt, xtopology, xtpr\n" - ] - } - ], - "source": [ - "cpuinfo.main()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/object_arrays.ipynb b/notebooks/object_arrays.ipynb deleted file mode 100644 index 714d024907..0000000000 --- a/notebooks/object_arrays.ipynb +++ /dev/null @@ -1,350 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Object arrays\n", - "\n", - "See [#212](https://github.com/alimanfoo/zarr/pull/212) for more information." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.2.0a2.dev82+dirty'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.5.0'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numcodecs\n", - "numcodecs.__version__" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## API changes in Zarr version 2.2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Creation of an object array requires providing new ``object_codec`` argument:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To maintain backwards compatibility with previously-created data, the object codec is treated as a filter and inserted as the first filter in the chain:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeobject
Shape(10,)
Chunk shape(5,)
OrderC
Read-onlyFalse
Filter [0]MsgPack(encoding='utf-8')
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes80
No. bytes stored396
Storage ratio0.2
Chunks initialized0/2
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : object\n", - "Shape : (10,)\n", - "Chunk shape : (5,)\n", - "Order : C\n", - "Read-only : False\n", - "Filter [0] : MsgPack(encoding='utf-8')\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 80\n", - "No. bytes stored : 396\n", - "Storage ratio : 0.2\n", - "Chunks initialized : 0/2" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['foo', 'bar', 1, list([2, 4, 6, 'baz']), {'a': 'b', 'c': 'd'}, None,\n", - " None, None, None, None], dtype=object)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z[0] = 'foo'\n", - "z[1] = b'bar' # msgpack doesn't support bytes objects correctly\n", - "z[2] = 1\n", - "z[3] = [2, 4, 6, 'baz']\n", - "z[4] = {'a': 'b', 'c': 'd'}\n", - "a = z[:]\n", - "a" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If no ``object_codec`` is provided, a ``ValueError`` is raised:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "missing object_codec for object array", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzarr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/creation.py\u001b[0m in \u001b[0;36mempty\u001b[0;34m(shape, **kwargs)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \"\"\"\n\u001b[0;32m--> 206\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/creation.py\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(shape, chunks, dtype, compressor, fill_value, order, store, synchronizer, overwrite, path, chunk_store, filters, cache_metadata, read_only, object_codec, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor,\n\u001b[1;32m 113\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m chunk_store=chunk_store, filters=filters, object_codec=object_codec)\n\u001b[0m\u001b[1;32m 115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0;31m# instantiate array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/storage.py\u001b[0m in \u001b[0;36minit_array\u001b[0;34m(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec)\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[0mchunk_store\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunk_store\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfilters\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m object_codec=object_codec)\n\u001b[0m\u001b[1;32m 292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/storage.py\u001b[0m in \u001b[0;36m_init_array_metadata\u001b[0;34m(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec)\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfilters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[0;31m# there are no filters so we can be sure there is no object codec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 348\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'missing object_codec for object array'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 349\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 350\u001b[0m \u001b[0;31m# one of the filters may be an object codec, issue a warning rather\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mValueError\u001b[0m: missing object_codec for object array" - ] - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For API backward-compatibility, if object codec is provided via filters, issue a warning but don't raise an error." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/aliman/src/github/alimanfoo/zarr/zarr/storage.py:353: FutureWarning: missing object_codec for object array; this will raise a ValueError in version 3.0\n", - " 'ValueError in version 3.0', FutureWarning)\n" - ] - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, filters=[numcodecs.MsgPack()])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If a user tries to subvert the system and create an object array with no object codec, a runtime check is added to ensure no object arrays are passed down to the compressor (which could lead to nasty errors and/or segfaults):" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z._filters = None # try to live dangerously, manually wipe filters" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "cannot write object array without object codec", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'foo'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, selection, value)\u001b[0m\n\u001b[1;32m 1094\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1095\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1096\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1097\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1098\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mset_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mset_basic_selection\u001b[0;34m(self, selection, value, fields)\u001b[0m\n\u001b[1;32m 1189\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1190\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1191\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_basic_selection_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mset_orthogonal_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_set_basic_selection_nd\u001b[0;34m(self, selection, value, fields)\u001b[0m\n\u001b[1;32m 1480\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1482\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1484\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_set_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_set_selection\u001b[0;34m(self, indexer, value, fields)\u001b[0m\n\u001b[1;32m 1528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1529\u001b[0m \u001b[0;31m# put data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1530\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_chunk_setitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_coords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_selection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1532\u001b[0m def _chunk_getitem(self, chunk_coords, chunk_selection, out, out_selection,\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_setitem\u001b[0;34m(self, chunk_coords, chunk_selection, value, fields)\u001b[0m\n\u001b[1;32m 1633\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1634\u001b[0m self._chunk_setitem_nosync(chunk_coords, chunk_selection, value,\n\u001b[0;32m-> 1635\u001b[0;31m fields=fields)\n\u001b[0m\u001b[1;32m 1636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1637\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_chunk_setitem_nosync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_coords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_selection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_setitem_nosync\u001b[0;34m(self, chunk_coords, chunk_selection, value, fields)\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1708\u001b[0m \u001b[0;31m# encode chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1709\u001b[0;31m \u001b[0mcdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_encode_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1711\u001b[0m \u001b[0;31m# store\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_encode_chunk\u001b[0;34m(self, chunk)\u001b[0m\n\u001b[1;32m 1753\u001b[0m \u001b[0;31m# check object encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1754\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1755\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cannot write object array without object codec'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1756\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1757\u001b[0m \u001b[0;31m# compress\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: cannot write object array without object codec" - ] - } - ], - "source": [ - "z[0] = 'foo'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here is another way to subvert the system, wiping filters **after** storing some data. To cover this case a runtime check is added to ensure no object arrays are handled inappropriately during decoding (which could lead to nasty errors and/or segfaults)." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', 'Hei maailma!',\n", - " 'Xin chào thế giới', 'Njatjeta Botë!', 'Γεια σου κόσμε!', 'こんにちは世界',\n", - " '世界，你好！', 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from numcodecs.tests.common import greetings\n", - "z = zarr.array(greetings, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "cannot read object array without object codec", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# try to live dangerously, manually wipe filters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 553\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 555\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEllipsis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mget_basic_selection\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 678\u001b[0m return self._get_basic_selection_nd(selection=selection, out=out,\n\u001b[0;32m--> 679\u001b[0;31m fields=fields)\n\u001b[0m\u001b[1;32m 680\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 681\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_basic_selection_nd\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 719\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 720\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 721\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 722\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 723\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_orthogonal_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_selection\u001b[0;34m(self, indexer, out, fields)\u001b[0m\n\u001b[1;32m 1007\u001b[0m \u001b[0;31m# load chunk selection into output array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1008\u001b[0m self._chunk_getitem(chunk_coords, chunk_selection, out, out_selection,\n\u001b[0;32m-> 1009\u001b[0;31m drop_axes=indexer.drop_axes, fields=fields)\n\u001b[0m\u001b[1;32m 1010\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1011\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_getitem\u001b[0;34m(self, chunk_coords, chunk_selection, out, out_selection, drop_axes, fields)\u001b[0m\n\u001b[1;32m 1597\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1598\u001b[0m \u001b[0;31m# decode chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1599\u001b[0;31m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_decode_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1601\u001b[0m \u001b[0;31m# select data from chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_decode_chunk\u001b[0;34m(self, cdata)\u001b[0m\n\u001b[1;32m 1733\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1734\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1735\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cannot read object array without object codec'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1736\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1737\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: cannot read object array without object codec" - ] - } - ], - "source": [ - "z._filters = [] # try to live dangerously, manually wipe filters\n", - "z[:]" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/repr_info.ipynb b/notebooks/repr_info.ipynb deleted file mode 100644 index 487a4175ba..0000000000 --- a/notebooks/repr_info.ipynb +++ /dev/null @@ -1,365 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import zarr" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root = zarr.group()\n", - "root" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members0
No. arrays0
No. groups0
" - ], - "text/plain": [ - "Name : /\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 0\n", - "No. arrays : 0\n", - "No. groups : 0" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root.info" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "z = root.zeros('foo/bar/baz', shape=1000000)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar/baz
Typezarr.core.Array
Data typefloat64
Shape(1000000,)
Chunk shape(15625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typezarr.storage.DictStore
No. bytes8000000 (7.6M)
No. bytes stored321
Storage ratio24922.1
Chunks initialized0/64
" - ], - "text/plain": [ - "Name : /foo/bar/baz\n", - "Type : zarr.core.Array\n", - "Data type : float64\n", - "Shape : (1000000,)\n", - "Chunk shape : (15625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : zarr.storage.DictStore\n", - "No. bytes : 8000000 (7.6M)\n", - "No. bytes stored : 321\n", - "Storage ratio : 24922.1\n", - "Chunks initialized : 0/64" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "z[:] = 42" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar/baz
Typezarr.core.Array
Data typefloat64
Shape(1000000,)
Chunk shape(15625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typezarr.storage.DictStore
No. bytes8000000 (7.6M)
No. bytes stored39553 (38.6K)
Storage ratio202.3
Chunks initialized64/64
" - ], - "text/plain": [ - "Name : /foo/bar/baz\n", - "Type : zarr.core.Array\n", - "Data type : float64\n", - "Shape : (1000000,)\n", - "Chunk shape : (15625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : zarr.storage.DictStore\n", - "No. bytes : 8000000 (7.6M)\n", - "No. bytes stored : 39553 (38.6K)\n", - "Storage ratio : 202.3\n", - "Chunks initialized : 64/64" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(1000):\n", - " root.create_group(i)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members1001
No. arrays0
No. groups1001
Groups0, 1, 10, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 11, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 12, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 13, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 14, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 15, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 16, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 17, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 18, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 19, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 2, 20, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 21, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 22, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 23, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 24, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 25, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 26, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 27, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 28, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 29, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 3, 30, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 31, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 32, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 33, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 34, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 35, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 36, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 37, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 38, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 39, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 4, 40, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 41, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 42, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 43, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 44, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 45, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 46, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 47, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 48, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 49, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 5, 50, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 51, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 52, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 53, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 54, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 55, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 56, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 57, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 58, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 59, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 6, 60, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 61, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 62, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 63, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 64, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 65, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 66, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 67, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 68, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 69, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 7, 70, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 71, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 72, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 73, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 74, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 75, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 76, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 77, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 78, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 79, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 8, 80, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 81, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 82, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 83, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 84, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 85, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 86, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 87, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 88, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 89, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 9, 90, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 91, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 92, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 93, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 94, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 95, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 96, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 97, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 98, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 99, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, foo
" - ], - "text/plain": [ - "Name : /\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 1001\n", - "No. arrays : 0\n", - "No. groups : 1001\n", - "Groups : 0, 1, 10, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 11,\n", - " : 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 12, 120, 121,\n", - " : 122, 123, 124, 125, 126, 127, 128, 129, 13, 130, 131, 132, 133,\n", - " : 134, 135, 136, 137, 138, 139, 14, 140, 141, 142, 143, 144, 145,\n", - " : 146, 147, 148, 149, 15, 150, 151, 152, 153, 154, 155, 156, 157,\n", - " : 158, 159, 16, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,\n", - " : 17, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 18, 180,\n", - " : 181, 182, 183, 184, 185, 186, 187, 188, 189, 19, 190, 191, 192,\n", - " : 193, 194, 195, 196, 197, 198, 199, 2, 20, 200, 201, 202, 203, 204,\n", - " : 205, 206, 207, 208, 209, 21, 210, 211, 212, 213, 214, 215, 216,\n", - " : 217, 218, 219, 22, 220, 221, 222, 223, 224, 225, 226, 227, 228,\n", - " : 229, 23, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 24,\n", - " : 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 25, 250, 251,\n", - " : 252, 253, 254, 255, 256, 257, 258, 259, 26, 260, 261, 262, 263,\n", - " : 264, 265, 266, 267, 268, 269, 27, 270, 271, 272, 273, 274, 275,\n", - " : 276, 277, 278, 279, 28, 280, 281, 282, 283, 284, 285, 286, 287,\n", - " : 288, 289, 29, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 3,\n", - " : 30, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 31, 310,\n", - " : 311, 312, 313, 314, 315, 316, 317, 318, 319, 32, 320, 321, 322,\n", - " : 323, 324, 325, 326, 327, 328, 329, 33, 330, 331, 332, 333, 334,\n", - " : 335, 336, 337, 338, 339, 34, 340, 341, 342, 343, 344, 345, 346,\n", - " : 347, 348, 349, 35, 350, 351, 352, 353, 354, 355, 356, 357, 358,\n", - " : 359, 36, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 37,\n", - " : 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 38, 380, 381,\n", - " : 382, 383, 384, 385, 386, 387, 388, 389, 39, 390, 391, 392, 393,\n", - " : 394, 395, 396, 397, 398, 399, 4, 40, 400, 401, 402, 403, 404, 405,\n", - " : 406, 407, 408, 409, 41, 410, 411, 412, 413, 414, 415, 416, 417,\n", - " : 418, 419, 42, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429,\n", - " : 43, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 44, 440,\n", - " : 441, 442, 443, 444, 445, 446, 447, 448, 449, 45, 450, 451, 452,\n", - " : 453, 454, 455, 456, 457, 458, 459, 46, 460, 461, 462, 463, 464,\n", - " : 465, 466, 467, 468, 469, 47, 470, 471, 472, 473, 474, 475, 476,\n", - " : 477, 478, 479, 48, 480, 481, 482, 483, 484, 485, 486, 487, 488,\n", - " : 489, 49, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 5, 50,\n", - " : 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 51, 510, 511,\n", - " : 512, 513, 514, 515, 516, 517, 518, 519, 52, 520, 521, 522, 523,\n", - " : 524, 525, 526, 527, 528, 529, 53, 530, 531, 532, 533, 534, 535,\n", - " : 536, 537, 538, 539, 54, 540, 541, 542, 543, 544, 545, 546, 547,\n", - " : 548, 549, 55, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559,\n", - " : 56, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 57, 570,\n", - " : 571, 572, 573, 574, 575, 576, 577, 578, 579, 58, 580, 581, 582,\n", - " : 583, 584, 585, 586, 587, 588, 589, 59, 590, 591, 592, 593, 594,\n", - " : 595, 596, 597, 598, 599, 6, 60, 600, 601, 602, 603, 604, 605, 606,\n", - " : 607, 608, 609, 61, 610, 611, 612, 613, 614, 615, 616, 617, 618,\n", - " : 619, 62, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 63,\n", - " : 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 64, 640, 641,\n", - " : 642, 643, 644, 645, 646, 647, 648, 649, 65, 650, 651, 652, 653,\n", - " : 654, 655, 656, 657, 658, 659, 66, 660, 661, 662, 663, 664, 665,\n", - " : 666, 667, 668, 669, 67, 670, 671, 672, 673, 674, 675, 676, 677,\n", - " : 678, 679, 68, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689,\n", - " : 69, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 7, 70, 700,\n", - " : 701, 702, 703, 704, 705, 706, 707, 708, 709, 71, 710, 711, 712,\n", - " : 713, 714, 715, 716, 717, 718, 719, 72, 720, 721, 722, 723, 724,\n", - " : 725, 726, 727, 728, 729, 73, 730, 731, 732, 733, 734, 735, 736,\n", - " : 737, 738, 739, 74, 740, 741, 742, 743, 744, 745, 746, 747, 748,\n", - " : 749, 75, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 76,\n", - " : 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 77, 770, 771,\n", - " : 772, 773, 774, 775, 776, 777, 778, 779, 78, 780, 781, 782, 783,\n", - " : 784, 785, 786, 787, 788, 789, 79, 790, 791, 792, 793, 794, 795,\n", - " : 796, 797, 798, 799, 8, 80, 800, 801, 802, 803, 804, 805, 806, 807,\n", - " : 808, 809, 81, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819,\n", - " : 82, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 83, 830,\n", - " : 831, 832, 833, 834, 835, 836, 837, 838, 839, 84, 840, 841, 842,\n", - " : 843, 844, 845, 846, 847, 848, 849, 85, 850, 851, 852, 853, 854,\n", - " : 855, 856, 857, 858, 859, 86, 860, 861, 862, 863, 864, 865, 866,\n", - " : 867, 868, 869, 87, 870, 871, 872, 873, 874, 875, 876, 877, 878,\n", - " : 879, 88, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 89,\n", - " : 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 9, 90, 900, 901,\n", - " : 902, 903, 904, 905, 906, 907, 908, 909, 91, 910, 911, 912, 913,\n", - " : 914, 915, 916, 917, 918, 919, 92, 920, 921, 922, 923, 924, 925,\n", - " : 926, 927, 928, 929, 93, 930, 931, 932, 933, 934, 935, 936, 937,\n", - " : 938, 939, 94, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949,\n", - " : 95, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 96, 960,\n", - " : 961, 962, 963, 964, 965, 966, 967, 968, 969, 97, 970, 971, 972,\n", - " : 973, 974, 975, 976, 977, 978, 979, 98, 980, 981, 982, 983, 984,\n", - " : 985, 986, 987, 988, 989, 99, 990, 991, 992, 993, 994, 995, 996,\n", - " : 997, 998, 999, foo" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root.info" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members1
No. arrays1
No. groups0
Arraysbaz
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 1\n", - "No. arrays : 1\n", - "No. groups : 0\n", - "Arrays : baz" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root['foo/bar'].info" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/store_benchmark.ipynb b/notebooks/store_benchmark.ipynb deleted file mode 100644 index 869e7df608..0000000000 --- a/notebooks/store_benchmark.ipynb +++ /dev/null @@ -1,1303 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are lies, damn lies and benchmarks..." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.2.0a2.dev22+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'6.2.5'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import bsddb3\n", - "bsddb3.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.93'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import lmdb\n", - "lmdb.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import dbm.gnu\n", - "import dbm.ndbm" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import shutil\n", - "bench_dir = '../data/bench'\n", - "\n", - "\n", - "def clean():\n", - " if os.path.isdir(bench_dir):\n", - " shutil.rmtree(bench_dir)\n", - " os.makedirs(bench_dir)\n", - "\n", - " \n", - "def setup(a, name='foo/bar'):\n", - " global fdict_z, hdict_z, lmdb_z, gdbm_z, ndbm_z, bdbm_btree_z, bdbm_hash_z, zip_z, dir_z\n", - " \n", - " clean()\n", - " fdict_root = zarr.group(store=dict())\n", - " hdict_root = zarr.group(store=zarr.DictStore())\n", - " lmdb_root = zarr.group(store=zarr.LMDBStore(os.path.join(bench_dir, 'lmdb')))\n", - " gdbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'gdbm'), open=dbm.gnu.open))\n", - " ndbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'ndbm'), open=dbm.ndbm.open))\n", - " bdbm_btree_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_btree'), open=bsddb3.btopen))\n", - " bdbm_hash_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_hash'), open=bsddb3.hashopen))\n", - " zip_root = zarr.group(store=zarr.ZipStore(os.path.join(bench_dir, 'zip'), mode='w'))\n", - " dir_root = zarr.group(store=zarr.DirectoryStore(os.path.join(bench_dir, 'dir')))\n", - "\n", - " fdict_z = fdict_root.empty_like(name, a)\n", - " hdict_z = hdict_root.empty_like(name, a)\n", - " lmdb_z = lmdb_root.empty_like(name, a)\n", - " gdbm_z = gdbm_root.empty_like(name, a)\n", - " ndbm_z = ndbm_root.empty_like(name, a)\n", - " bdbm_btree_z = bdbm_btree_root.empty_like(name, a)\n", - " bdbm_hash_z = bdbm_hash_root.empty_like(name, a)\n", - " zip_z = zip_root.empty_like(name, a)\n", - " dir_z = dir_root.empty_like(name, a)\n", - "\n", - " # check compression ratio\n", - " fdict_z[:] = a\n", - " return fdict_z.info\n", - " \n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Main benchmarks" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def save(a, z):\n", - " if isinstance(z.store, zarr.ZipStore):\n", - " # needed for zip benchmarks to avoid duplicate entries\n", - " z.store.clear()\n", - " z[:] = a\n", - " if hasattr(z.store, 'flush'):\n", - " z.store.flush()\n", - " \n", - " \n", - "def load(z, a):\n", - " z.get_basic_selection(out=a)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## arange" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.core.Array
Data typeint64
Shape(500000000,)
Chunk shape(488282,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes4000000000 (3.7G)
No. bytes stored59269657 (56.5M)
Storage ratio67.5
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (500000000,)\n", - "Chunk shape : (488282,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 4000000000 (3.7G)\n", - "No. bytes stored : 59269657 (56.5M)\n", - "Storage ratio : 67.5\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(500000000)\n", - "setup(a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### save" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "324 ms ± 60.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "302 ms ± 11.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "316 ms ± 12.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "938 ms ± 111 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "406 ms ± 8.93 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.43 s ± 156 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.24 s ± 260 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "519 ms ± 59.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "609 ms ± 48.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, dir_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### load" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "454 ms ± 56.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(fdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "428 ms ± 13.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(hdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "429 ms ± 19.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(lmdb_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "459 ms ± 10 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(gdbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "473 ms ± 5.71 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(ndbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "504 ms ± 8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(bdbm_btree_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "519 ms ± 9.59 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(bdbm_hash_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "575 ms ± 12.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(zip_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "494 ms ± 10.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(dir_z, a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## randint" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.core.Array
Data typeint64
Shape(500000000,)
Chunk shape(488282,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes4000000000 (3.7G)
No. bytes stored2020785466 (1.9G)
Storage ratio2.0
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (500000000,)\n", - "Chunk shape : (488282,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 4000000000 (3.7G)\n", - "No. bytes stored : 2020785466 (1.9G)\n", - "Storage ratio : 2.0\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.random.seed(42)\n", - "a = np.random.randint(0, 2**30, size=500000000)\n", - "setup(a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### save" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "670 ms ± 78.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "611 ms ± 6.11 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "846 ms ± 24 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.35 s ± 785 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4.62 s ± 1.09 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "7.84 s ± 1.66 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.49 s ± 808 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.68 s ± 441 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.55 s ± 1.24 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, dir_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### load" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "566 ms ± 72.8 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(fdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "521 ms ± 16.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(hdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "532 ms ± 16.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(lmdb_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.2 s ± 10.9 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(gdbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.18 s ± 13.2 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(ndbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.59 s ± 16.7 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(bdbm_btree_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.61 s ± 7.31 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(bdbm_hash_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2.33 s ± 19.8 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(zip_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "790 ms ± 56 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(dir_z, a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### dask" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "import dask.array as da" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [], - "source": [ - "def dask_op(source, sink, chunks=None):\n", - " if isinstance(sink.store, zarr.ZipStore):\n", - " sink.store.clear()\n", - " if chunks is None:\n", - " try:\n", - " chunks = sink.chunks\n", - " except AttributeError:\n", - " chunks = source.chunks\n", - " d = da.from_array(source, chunks=chunks, asarray=False, fancy=False, lock=False)\n", - " result = (d // 2) * 2\n", - " da.store(result, sink, lock=False)\n", - " if hasattr(sink.store, 'flush'):\n", - " sink.store.flush()\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Compare sources" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.6 s, sys: 1.8 s, total: 17.4 s\n", - "Wall time: 3.07 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.5 s, sys: 104 ms, total: 16.6 s\n", - "Wall time: 2.59 s\n" - ] - } - ], - "source": [ - "%time dask_op(hdict_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.1 s, sys: 524 ms, total: 15.6 s\n", - "Wall time: 3.02 s\n" - ] - } - ], - "source": [ - "%time dask_op(lmdb_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.5 s, sys: 712 ms, total: 17.2 s\n", - "Wall time: 3.13 s\n" - ] - } - ], - "source": [ - "%time dask_op(gdbm_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.3 s, sys: 604 ms, total: 16.9 s\n", - "Wall time: 3.22 s\n" - ] - } - ], - "source": [ - "%time dask_op(ndbm_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 19.6 s, sys: 1.24 s, total: 20.9 s\n", - "Wall time: 3.27 s\n" - ] - } - ], - "source": [ - "%time dask_op(bdbm_btree_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 20.3 s, sys: 1.08 s, total: 21.4 s\n", - "Wall time: 3.53 s\n" - ] - } - ], - "source": [ - "%time dask_op(bdbm_hash_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.7 s, sys: 700 ms, total: 16.4 s\n", - "Wall time: 3.12 s\n" - ] - } - ], - "source": [ - "%time dask_op(zip_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 17.4 s, sys: 1.08 s, total: 18.5 s\n", - "Wall time: 2.91 s\n" - ] - } - ], - "source": [ - "%time dask_op(dir_z, fdict_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Compare sinks" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.8 s, sys: 1.4 s, total: 17.2 s\n", - "Wall time: 3.04 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.2 s, sys: 1.6 s, total: 17.8 s\n", - "Wall time: 2.71 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.8 s, sys: 3.05 s, total: 19.8 s\n", - "Wall time: 8.01 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 17.9 s, sys: 3.01 s, total: 20.9 s\n", - "Wall time: 5.46 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.8 s, sys: 3.39 s, total: 17.2 s\n", - "Wall time: 7.87 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.9 s, sys: 3.27 s, total: 17.2 s\n", - "Wall time: 6.73 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.9 s, sys: 2.5 s, total: 16.4 s\n", - "Wall time: 3.8 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.7 s, sys: 3.72 s, total: 19.4 s\n", - "Wall time: 3.1 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, dir_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "lmdb_z.store.close()\n", - "gdbm_z.store.close()\n", - "ndbm_z.store.close()\n", - "bdbm_btree_z.store.close()\n", - "bdbm_hash_z.store.close()\n", - "zip_z.store.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/zip_benchmark.ipynb b/notebooks/zip_benchmark.ipynb deleted file mode 100644 index 6805552422..0000000000 --- a/notebooks/zip_benchmark.ipynb +++ /dev/null @@ -1,343 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.2.dev0+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3L/calldata/genotype, (7449486, 773, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 10.7G; nbytes_stored: 193.5M; ratio: 56.7; initialized: 11380/11380\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: ZipStore" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "store = zarr.ZipStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes.zip',\n", - " mode='r')\n", - "grp = zarr.Group(store)\n", - "z = grp['3L/calldata/genotype']\n", - "z" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 1832 function calls in 0.024 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 0.024 0.024 {built-in method builtins.exec}\n", - " 1 0.000 0.000 0.024 0.024 :1()\n", - " 1 0.000 0.000 0.024 0.024 core.py:292(__getitem__)\n", - " 20 0.000 0.000 0.023 0.001 core.py:539(_chunk_getitem)\n", - " 20 0.000 0.000 0.020 0.001 core.py:679(_decode_chunk)\n", - " 20 0.000 0.000 0.020 0.001 codecs.py:355(decode)\n", - " 20 0.020 0.001 0.020 0.001 {zarr.blosc.decompress}\n", - " 20 0.000 0.000 0.002 0.000 storage.py:766(__getitem__)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:1235(open)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:821(read)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:901(_read1)\n", - " 80 0.000 0.000 0.001 0.000 zipfile.py:660(read)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:854(_update_crc)\n", - " 40 0.000 0.000 0.000 0.000 {built-in method zlib.crc32}\n", - " 80 0.000 0.000 0.000 0.000 {method 'read' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:937(_read2)\n", - " 80 0.000 0.000 0.000 0.000 core.py:390()\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:953(close)\n", - " 20 0.000 0.000 0.000 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - " 20 0.000 0.000 0.000 0.000 util.py:106(is_total_slice)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:708(__init__)\n", - " 20 0.000 0.000 0.000 0.000 {method 'decode' of 'bytes' objects}\n", - " 20 0.000 0.000 0.000 0.000 core.py:676(_chunk_key)\n", - " 80 0.000 0.000 0.000 0.000 {method 'seek' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 {built-in method numpy.core.multiarray.frombuffer}\n", - " 80 0.000 0.000 0.000 0.000 core.py:398()\n", - " 20 0.000 0.000 0.000 0.000 {method 'join' of 'str' objects}\n", - " 20 0.000 0.000 0.000 0.000 core.py:386()\n", - " 20 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 40 0.000 0.000 0.000 0.000 util.py:121()\n", - " 231 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance}\n", - " 20 0.000 0.000 0.000 0.000 cp437.py:14(decode)\n", - " 80 0.000 0.000 0.000 0.000 {method 'tell' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:667(close)\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _struct.unpack}\n", - " 140 0.000 0.000 0.000 0.000 {built-in method builtins.max}\n", - " 20 0.000 0.000 0.000 0.000 {function ZipExtFile.close at 0x7f8cd5ca2048}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:1194(getinfo)\n", - " 140 0.000 0.000 0.000 0.000 {built-in method builtins.min}\n", - " 20 0.000 0.000 0.000 0.000 threading.py:1224(current_thread)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:654(__init__)\n", - " 1 0.000 0.000 0.000 0.000 util.py:195(get_chunk_range)\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _codecs.charmap_decode}\n", - " 1 0.000 0.000 0.000 0.000 util.py:166(normalize_array_selection)\n", - " 1 0.000 0.000 0.000 0.000 util.py:198()\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:1715(_fpclose)\n", - " 20 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", - " 63 0.000 0.000 0.000 0.000 {built-in method builtins.len}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method numpy.core.multiarray.empty}\n", - " 2 0.000 0.000 0.000 0.000 util.py:182()\n", - " 20 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _thread.get_ident}\n", - " 1 0.000 0.000 0.000 0.000 util.py:130(normalize_axis_selection)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:636(_get_decompressor)\n", - " 20 0.000 0.000 0.000 0.000 threading.py:1298(main_thread)\n", - " 4 0.000 0.000 0.000 0.000 core.py:373()\n", - " 3 0.000 0.000 0.000 0.000 util.py:187()\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "import cProfile\n", - "cProfile.run('z[:10]', sort='cumtime')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.11.0'" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import dask\n", - "import dask.array as da\n", - "dask.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d = da.from_array(z, chunks=z.chunks)\n", - "d" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3min 35s, sys: 4.36 s, total: 3min 40s\n", - "Wall time: 29.5 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[3, 0],\n", - " [1, 0],\n", - " [2, 0],\n", - " ..., \n", - " [2, 8],\n", - " [8, 8],\n", - " [0, 1]])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d.sum(axis=1).compute()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3L/calldata/genotype, (7449486, 773, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 10.7G; nbytes_stored: 193.5M; ratio: 56.7; initialized: 11380/11380\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: DirectoryStore" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# compare with same data via directory store\n", - "store_dir = zarr.DirectoryStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes')\n", - "grp_dir = zarr.Group(store_dir)\n", - "z_dir = grp_dir['3L/calldata/genotype']\n", - "z_dir" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d_dir = da.from_array(z_dir, chunks=z_dir.chunks)\n", - "d_dir" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3min 39s, sys: 4.91 s, total: 3min 44s\n", - "Wall time: 31.1 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[3, 0],\n", - " [1, 0],\n", - " [2, 0],\n", - " ..., \n", - " [2, 8],\n", - " [8, 8],\n", - " [0, 1]])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d_dir.sum(axis=1).compute()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} From 7a826e836311e85d22c052ec420f14461f5f1321 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 19 Jun 2025 11:21:38 +0100 Subject: [PATCH 156/160] Fix typing in test_v2 (#3143) Co-authored-by: Davis Bennett --- pyproject.toml | 1 - tests/test_v2.py | 44 ++++++++++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2680396e7c..7de27131c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -384,7 +384,6 @@ module = [ "tests.test_indexing", "tests.test_properties", "tests.test_sync", - "tests.test_v2", "tests.test_regression.scripts.*" ] ignore_errors = true diff --git a/tests/test_v2.py b/tests/test_v2.py index 66e5a1ecfb..29f031663f 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -1,7 +1,8 @@ import json -from collections.abc import Iterator +from pathlib import Path from typing import Any, Literal +import numcodecs.abc import numcodecs.vlen import numpy as np import pytest @@ -18,12 +19,13 @@ from zarr.core.dtype import FixedLengthUTF32, Structured, VariableLengthUTF8 from zarr.core.dtype.npy.bytes import NullTerminatedBytes from zarr.core.dtype.wrapper import ZDType +from zarr.core.group import Group from zarr.core.sync import sync from zarr.storage import MemoryStore, StorePath @pytest.fixture -async def store() -> Iterator[StorePath]: +async def store() -> StorePath: return StorePath(await MemoryStore.open()) @@ -68,7 +70,9 @@ def test_codec_pipeline() -> None: ("|V10", "|V10", b"X", "WAAAAAAAAAAAAA=="), ], ) -async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_json) -> None: +async def test_v2_encode_decode( + dtype: str, expected_dtype: str, fill_value: bytes, fill_value_json: str +) -> None: with config.set( { "array.v2_default_filters.bytes": [{"id": "vlen-bytes"}], @@ -99,8 +103,7 @@ async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_js assert serialized == expected data = zarr.open_array(store=store, path="foo")[:] - expected = np.full((3,), b"X", dtype=dtype) - np.testing.assert_equal(data, expected) + np.testing.assert_equal(data, np.full((3,), b"X", dtype=dtype)) @pytest.mark.parametrize( @@ -111,7 +114,7 @@ async def test_v2_encode_decode(dtype, expected_dtype, fill_value, fill_value_js (VariableLengthUTF8(), "Y"), ], ) -def test_v2_encode_decode_with_data(dtype: ZDType[Any, Any], value: str): +def test_v2_encode_decode_with_data(dtype: ZDType[Any, Any], value: str) -> None: expected = np.full((3,), value, dtype=dtype.to_native_dtype()) a = zarr.create( shape=(3,), @@ -136,12 +139,13 @@ def test_v2_filters_codecs(filters: Any, order: Literal["C", "F"]) -> None: @pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_create_array_defaults(store: Store): +def test_create_array_defaults(store: Store) -> None: """ Test that passing compressor=None results in no compressor. Also test that the default value of the compressor parameter does produce a compressor. """ g = zarr.open(store, mode="w", zarr_format=2) + assert isinstance(g, Group) arr = g.create_array("one", dtype="i8", shape=(1,), chunks=(1,), compressor=None) assert arr._async_array.compressor is None assert not (arr.filters) @@ -183,17 +187,19 @@ def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal[" arr[6:9, 3:6] = a[6:9, 3:6] # The slice on the RHS is important np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6]) + buf = sync(store.get("2.1", default_buffer_prototype())) + assert buf is not None np.testing.assert_array_equal( a[6:9, 3:6], - np.frombuffer( - sync(store.get("2.1", default_buffer_prototype())).to_bytes(), dtype="float64" - ).reshape((3, 3), order=zarr_order), + np.frombuffer(buf.to_bytes(), dtype="float64").reshape((3, 3), order=zarr_order), ) # After writing and reading from zarr array, order should be same as zarr order + sub_arr = arr[6:9, 3:6] + assert isinstance(sub_arr, np.ndarray) if zarr_order == "F": - assert (arr[6:9, 3:6]).flags.f_contiguous + assert (sub_arr).flags.f_contiguous else: - assert (arr[6:9, 3:6]).flags.c_contiguous + assert (sub_arr).flags.c_contiguous # Contiguous write store = MemoryStore() @@ -214,19 +220,21 @@ def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal[" arr[6:9, 3:6] = a np.testing.assert_array_equal(arr[6:9, 3:6], a) # After writing and reading from zarr array, order should be same as zarr order + sub_arr = arr[6:9, 3:6] + assert isinstance(sub_arr, np.ndarray) if zarr_order == "F": - assert (arr[6:9, 3:6]).flags.f_contiguous + assert (sub_arr).flags.f_contiguous else: - assert (arr[6:9, 3:6]).flags.c_contiguous + assert (sub_arr).flags.c_contiguous -def test_default_compressor_deprecation_warning(): +def test_default_compressor_deprecation_warning() -> None: with pytest.warns(DeprecationWarning, match="default_compressor is deprecated"): - zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" + zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" # type: ignore[attr-defined] @pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"]) -def test_structured_dtype_roundtrip(fill_value, tmp_path) -> None: +def test_structured_dtype_roundtrip(fill_value: float | bytes, tmp_path: Path) -> None: a = np.array( [(b"aaa", 1, 4.2), (b"bbb", 2, 8.4), (b"ccc", 3, 12.6)], dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], @@ -289,7 +297,7 @@ def test_parse_structured_fill_value_valid( @pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"]) -def test_other_dtype_roundtrip(fill_value, tmp_path) -> None: +def test_other_dtype_roundtrip(fill_value: None | bytes, tmp_path: Path) -> None: a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7") array_path = tmp_path / "data.zarr" za = zarr.create( From 3532d587abf650c662d5aad34cc155d5108635ad Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 19 Jun 2025 12:42:15 +0200 Subject: [PATCH 157/160] Clean up after getting rid of notebooks (#3152) --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7de27131c1..6c18563a1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,6 @@ exclude = [ "/.github", "/bench", "/docs", - "/notebooks" ] [project] @@ -282,7 +281,6 @@ extend-exclude = [ "buck-out", "build", "dist", - "notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6 "venv", "docs", "tests/test_regression/scripts/", # these are scripts that use a different version of python From 286bef86b040b76b458a2f4db2e3215fbd9457ac Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 19 Jun 2025 12:08:11 -0400 Subject: [PATCH 158/160] Add with_read_only() convenience method to store (#3138) * Add with_read_only() convenience method to store * Consolidate test * Add with_read_only to FsspecStore * Add with_read_only to LocalStore * Add with_read_only to ObjectStore * Add changelog * Include type in NotImplementedError * Apply suggestions from code review Co-authored-by: David Stansby * Lint * Apply suggestion to shorten try block --------- Co-authored-by: David Stansby --- changes/3138.feature.rst | 1 + src/zarr/abc/store.py | 21 +++++++++++++++ src/zarr/storage/_fsspec.py | 10 +++++++ src/zarr/storage/_local.py | 7 +++++ src/zarr/storage/_memory.py | 7 +++++ src/zarr/storage/_obstore.py | 7 +++++ src/zarr/testing/store.py | 52 ++++++++++++++++++++++++++++++++++++ 7 files changed, 105 insertions(+) create mode 100644 changes/3138.feature.rst diff --git a/changes/3138.feature.rst b/changes/3138.feature.rst new file mode 100644 index 0000000000..ecd339bf9c --- /dev/null +++ b/changes/3138.feature.rst @@ -0,0 +1 @@ +Adds a `with_read_only` convenience method to the `Store` abstract base class (raises `NotImplementedError`) and implementations to the `MemoryStore`, `ObjectStore`, `LocalStore`, and `FsspecStore` classes. \ No newline at end of file diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index db4dee8cdd..1fbdb3146c 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -83,6 +83,27 @@ async def open(cls, *args: Any, **kwargs: Any) -> Self: await store._open() return store + def with_read_only(self, read_only: bool = False) -> Store: + """ + Return a new store with a new read_only setting. + + The new store points to the same location with the specified new read_only state. + The returned Store is not automatically opened, and this store is + not automatically closed. + + Parameters + ---------- + read_only + If True, the store will be created in read-only mode. Defaults to False. + + Returns + ------- + A new store of the same type with the new read only attribute. + """ + raise NotImplementedError( + f"with_read_only is not implemented for the {type(self)} store type." + ) + def __enter__(self) -> Self: """Enter a context manager that will close the store upon exiting.""" return self diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index ba673056a3..4f6929456e 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -122,6 +122,7 @@ class FsspecStore(Store): fs: AsyncFileSystem allowed_exceptions: tuple[type[Exception], ...] + path: str def __init__( self, @@ -258,6 +259,15 @@ def from_url( return cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) + def with_read_only(self, read_only: bool = False) -> FsspecStore: + # docstring inherited + return type(self)( + fs=self.fs, + path=self.path, + allowed_exceptions=self.allowed_exceptions, + read_only=read_only, + ) + async def clear(self) -> None: # docstring inherited try: diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index 15b043b1dc..43e585415d 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -102,6 +102,13 @@ def __init__(self, root: Path | str, *, read_only: bool = False) -> None: ) self.root = root + def with_read_only(self, read_only: bool = False) -> LocalStore: + # docstring inherited + return type(self)( + root=self.root, + read_only=read_only, + ) + async def _open(self) -> None: if not self.read_only: self.root.mkdir(parents=True, exist_ok=True) diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index ea25f82a3b..0dc6f13236 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -54,6 +54,13 @@ def __init__( store_dict = {} self._store_dict = store_dict + def with_read_only(self, read_only: bool = False) -> MemoryStore: + # docstring inherited + return type(self)( + store_dict=self._store_dict, + read_only=read_only, + ) + async def clear(self) -> None: # docstring inherited self._store_dict.clear() diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index c048721cae..047ed07fbb 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -69,6 +69,13 @@ def __init__(self, store: _UpstreamObjectStore, *, read_only: bool = False) -> N super().__init__(read_only=read_only) self.store = store + def with_read_only(self, read_only: bool = False) -> ObjectStore: + # docstring inherited + return type(self)( + store=self.store, + read_only=read_only, + ) + def __str__(self) -> str: return f"object_store://{self.store}" diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 0e73599791..970329f393 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -149,6 +149,58 @@ async def test_read_only_store_raises(self, open_kwargs: dict[str, Any]) -> None ): await store.delete("foo") + async def test_with_read_only_store(self, open_kwargs: dict[str, Any]) -> None: + kwargs = {**open_kwargs, "read_only": True} + store = await self.store_cls.open(**kwargs) + assert store.read_only + + # Test that you cannot write to a read-only store + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) + + # Check if the store implements with_read_only + try: + writer = store.with_read_only(read_only=False) + except NotImplementedError: + # Test that stores that do not implement with_read_only raise NotImplementedError with the correct message + with pytest.raises( + NotImplementedError, + match=f"with_read_only is not implemented for the {type(store)} store type.", + ): + store.with_read_only(read_only=False) + return + + # Test that you can write to a new store copy + assert not writer._is_open + assert not writer.read_only + await writer.set("foo", self.buffer_cls.from_bytes(b"bar")) + await writer.delete("foo") + + # Test that you cannot write to the original store + assert store.read_only + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.delete("foo") + + # Test that you cannot write to a read-only store copy + reader = store.with_read_only(read_only=True) + assert reader.read_only + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await reader.set("foo", self.buffer_cls.from_bytes(b"bar")) + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await reader.delete("foo") + @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize( ("data", "byte_range"), From 4bf8a7e0353afd593da5fdd30526fd84f871b7af Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 20 Jun 2025 10:01:11 +0200 Subject: [PATCH 159/160] Appending to dictionary following its definition (#3159) --- src/zarr/core/dtype/npy/structured.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index d9e1ff55ae..07e3000826 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -162,8 +162,10 @@ def to_json( [f_name, f_dtype.to_json(zarr_format=zarr_format)] # type: ignore[list-item] for f_name, f_dtype in self.fields ] - base_dict = {"name": self._zarr_v3_name} - base_dict["configuration"] = {"fields": fields} # type: ignore[assignment] + base_dict = { + "name": self._zarr_v3_name, + "configuration": {"fields": fields}, + } return cast("DTypeSpec_V3", base_dict) raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover From 2911be8f20357c7f717eed410ccc99782d487808 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 20 Jun 2025 10:47:16 +0200 Subject: [PATCH 160/160] Apply ruff/flake8-pyi preview rule PYI059 (#3154) PYI059 `Generic[]` should always be the last base class --- src/zarr/core/dtype/wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/core/dtype/wrapper.py b/src/zarr/core/dtype/wrapper.py index e974712e38..7be97fa4b4 100644 --- a/src/zarr/core/dtype/wrapper.py +++ b/src/zarr/core/dtype/wrapper.py @@ -57,7 +57,7 @@ @dataclass(frozen=True, kw_only=True, slots=True) -class ZDType(Generic[TDType_co, TScalar_co], ABC): +class ZDType(ABC, Generic[TDType_co, TScalar_co]): """ Abstract base class for wrapping native array data types, e.g. numpy dtypes