diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 29281f5be9..9503f3df8a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,4 +1,4 @@ Contributing ============ -Please see the [project documentation](https://zarr.readthedocs.io/en/stable/contributing.html) for information about contributing to Zarr. +Please see the [project documentation](https://zarr.readthedocs.io/en/stable/developers/contributing.html) for information about contributing to Zarr. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 705cd31cb5..84bb89d82a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -57,7 +57,22 @@ body: id: reproduce attributes: label: Steps to reproduce - description: Minimal, reproducible code sample, a copy-pastable example if possible. + description: Minimal, reproducible code sample. Must list dependencies in [inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/#example). When put in a file named `issue.py` calling `uv run issue.py` should show the issue. + value: | + ```python + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "zarr@git+https://github.com/zarr-developers/zarr-python.git@main", + # ] + # /// + # + # This script automatically imports the development branch of zarr to check for issues + + import zarr + # your reproducer code + # zarr.print_debug_info() + ``` validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index edbd88eaf2..27239f5861 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,7 +5,7 @@ contact_links: about: A new major feature should be discussed in the Zarr specifications repository. - name: Discuss something on ZulipChat url: https://ossci.zulipchat.com/ - about: For questions like "How do I do X with Zarr?", you can move to our ZulipChat. + about: For questions like "How do I do X with Zarr?", consider posting your question to our developer chat. - name: Discuss something on GitHub Discussions url: https://github.com/zarr-developers/zarr-python/discussions about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a0d41f9841..9b64c97d0a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,7 +3,7 @@ TODO: * [ ] Add unit tests and/or doctests in docstrings * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions -* [ ] New/modified features documented in docs/tutorial.rst -* [ ] Changes documented in docs/release.rst +* [ ] New/modified features documented in `docs/user-guide/*.rst` +* [ ] Changes documented as a new file in `changes/` * [ ] GitHub Actions have all passed * [ ] Test coverage is 100% (Codecov passes) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a437a5c269..469b6a4d19 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,7 @@ --- version: 2 updates: - # Updates for v3 branch (the default branch) - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "daily" - groups: - actions: - patterns: - - "*" + # Updates for main - package-ecosystem: "github-actions" directory: "/" schedule: @@ -19,19 +11,19 @@ updates: patterns: - "*" - # Same updates, but for main branch + # Updates for support/v2 branch - package-ecosystem: "pip" directory: "/" - target-branch: "main" + target-branch: "support/v2" schedule: - interval: "daily" + interval: "weekly" groups: requirements: patterns: - "*" - package-ecosystem: "github-actions" directory: "/" - target-branch: "main" + target-branch: "support/v2" schedule: interval: "weekly" groups: diff --git a/.github/labeler.yml b/.github/labeler.yml index dbc3b95333..ede89c9d35 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,2 +1,4 @@ needs release notes: -- all: ['!docs/release.rst'] + - all: + - changed-files: + - all-globs-to-all-files: '!changes/*.rst' diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index b13da7d36f..752440719b 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: python-version: ['3.11'] - numpy-version: ['2.1'] + numpy-version: ['2.2'] dependency-set: ["minimal"] steps: @@ -64,3 +64,9 @@ jobs: - name: Run Tests run: | hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage + + - name: Upload coverage + uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 1029063ef4..776f859d6e 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -25,12 +25,19 @@ jobs: strategy: matrix: - python-version: ['3.11'] - numpy-version: ['2.1'] + python-version: ['3.12'] + numpy-version: ['2.2'] dependency-set: ["optional"] steps: - uses: actions/checkout@v4 + - name: Set HYPOTHESIS_PROFILE based on trigger + run: | + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "HYPOTHESIS_PROFILE=nightly" >> $GITHUB_ENV + else + echo "HYPOTHESIS_PROFILE=ci" >> $GITHUB_ENV + fi - name: Set up Python uses: actions/setup-python@v5 with: @@ -58,6 +65,7 @@ jobs: if: success() id: status run: | + echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-hypothesis # explicitly save the cache so it gets updated, also do this even if it fails. @@ -69,6 +77,12 @@ jobs: path: .hypothesis/ key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) + - name: Generate and publish the report if: | failure() diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml index d81ee0bdc4..7a6c5462b4 100644 --- a/.github/workflows/needs_release_notes.yml +++ b/.github/workflows/needs_release_notes.yml @@ -4,11 +4,14 @@ on: - pull_request_target jobs: - triage: + labeler: if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }} && ${{ github.event.pull_request.user.login != 'pre-commit-ci[bot]' }} + permissions: + contents: read + pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@main + - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: true diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 1b23260c2e..c8903aa779 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -55,7 +55,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.12.3 + - uses: pypa/gh-action-pypi-publish@v1.12.4 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5309ea4565..7cfce41312 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: python-version: ['3.11', '3.12', '3.13'] - numpy-version: ['1.25', '2.1'] + numpy-version: ['1.25', '2.2'] dependency-set: ["minimal", "optional"] os: ["ubuntu-latest"] include: @@ -30,7 +30,7 @@ jobs: dependency-set: 'optional' os: 'macos-latest' - python-version: '3.13' - numpy-version: '2.1' + numpy-version: '2.2' dependency-set: 'optional' os: 'macos-latest' - python-version: '3.11' @@ -38,13 +38,15 @@ jobs: dependency-set: 'optional' os: 'windows-latest' - python-version: '3.13' - numpy-version: '2.1' + numpy-version: '2.2' dependency-set: 'optional' os: 'windows-latest' runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 # grab all branches and tags - name: Set up Python uses: actions/setup-python@v5 with: @@ -59,8 +61,16 @@ jobs: hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env - name: Run Tests + env: + HYPOTHESIS_PROFILE: ci run: | - hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run + hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage + - name: Upload coverage + if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) test-upstream-and-min-deps: name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }} @@ -77,6 +87,8 @@ jobs: dependency-set: upstream steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5 with: @@ -92,7 +104,12 @@ jobs: hatch env run -e ${{ matrix.dependency-set }} list-env - name: Run Tests run: | - hatch env run --env ${{ matrix.dependency-set }} run + hatch env run --env ${{ matrix.dependency-set }} run-coverage + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) doctests: name: doctests diff --git a/.gitignore b/.gitignore index 5663f62d04..1b2b63e651 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ src/zarr/_version.py data/* src/fixture/ fixture/ +junit.xml .DS_Store tests/.hypothesis diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea1cd4dbab..fd50366a1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,23 +6,23 @@ ci: default_stages: [pre-commit, pre-push] repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.2 + rev: v0.11.9 hooks: - - id: ruff - args: ["--fix", "--show-fixes"] - - id: ruff-format + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell args: ["-L", "fo,ihs,kake,te", "-S", "fixture"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - - id: check-yaml - - id: trailing-whitespace + - id: check-yaml + - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.13.0 + rev: v1.15.0 hooks: - id: mypy files: src|tests @@ -31,13 +31,16 @@ repos: - packaging - donfig - numcodecs[crc32c] - - numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved + - numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved - typing_extensions - universal-pathlib + - obstore>=0.5.1 # Tests - pytest + - hypothesis + - s3fs - repo: https://github.com/scientific-python/cookie - rev: 2024.08.19 + rev: 2025.05.02 hooks: - id: sp-repo-review - repo: https://github.com/pre-commit/pygrep-hooks @@ -49,3 +52,7 @@ repos: rev: v1.8.0 hooks: - id: numpydoc-validation + - repo: https://github.com/twisted/towncrier + rev: 24.8.0 + hooks: + - id: towncrier-check diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 32a3f0e4e1..6253a7196f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,6 +4,13 @@ build: os: ubuntu-22.04 tools: python: "3.12" + jobs: + pre_build: + - | + if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; + then + towncrier build --version Unreleased --yes; + fi sphinx: configuration: docs/conf.py diff --git a/README.md b/README.md index 5ee6748ada..97f5617934 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ - Zulip + Developer Chat @@ -101,13 +101,13 @@ Zarr is a Python package providing an implementation of compressed, chunked, N-d ## Main Features -- [**Create**](https://zarr.readthedocs.io/en/stable/tutorial.html#creating-an-array) N-dimensional arrays with any NumPy `dtype`. -- [**Chunk arrays**](https://zarr.readthedocs.io/en/stable/tutorial.html#chunk-optimizations) along any dimension. -- [**Compress**](https://zarr.readthedocs.io/en/stable/tutorial.html#compressors) and/or filter chunks using any NumCodecs codec. -- [**Store arrays**](https://zarr.readthedocs.io/en/stable/tutorial.html#tutorial-storage) in memory, on disk, inside a zip file, on S3, etc... -- [**Read**](https://zarr.readthedocs.io/en/stable/tutorial.html#reading-and-writing-data) an array [**concurrently**](https://zarr.readthedocs.io/en/stable/tutorial.html#parallel-computing-and-synchronization) from multiple threads or processes. -- Write to an array concurrently from multiple threads or processes. -- Organize arrays into hierarchies via [**groups**](https://zarr.readthedocs.io/en/stable/tutorial.html#groups). +- [**Create**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#creating-an-array) N-dimensional arrays with any NumPy `dtype`. +- [**Chunk arrays**](https://zarr.readthedocs.io/en/stable/user-guide/performance.html#chunk-optimizations) along any dimension. +- [**Compress**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#compressors) and/or filter chunks using any NumCodecs codec. +- [**Store arrays**](https://zarr.readthedocs.io/en/stable/user-guide/storage.html) in memory, on disk, inside a zip file, on S3, etc... +- [**Read**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#reading-and-writing-data) an array [**concurrently**](https://zarr.readthedocs.io/en/stable/user-guide/performance.html#parallel-computing-and-synchronization) from multiple threads or processes. +- [**Write**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#reading-and-writing-data) to an array concurrently from multiple threads or processes. +- Organize arrays into hierarchies via [**groups**](https://zarr.readthedocs.io/en/stable/quickstart.html#hierarchical-groups). ## Where to get it diff --git a/changes/.gitignore b/changes/.gitignore new file mode 100644 index 0000000000..f935021a8f --- /dev/null +++ b/changes/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/changes/2774.feature.rst b/changes/2774.feature.rst new file mode 100644 index 0000000000..4df83f54ec --- /dev/null +++ b/changes/2774.feature.rst @@ -0,0 +1 @@ +Add `zarr.storage.FsspecStore.from_mapper()` so that `zarr.open()` supports stores of type `fsspec.mapping.FSMap`. \ No newline at end of file diff --git a/changes/2871.feature.rst b/changes/2871.feature.rst new file mode 100644 index 0000000000..a39f30c558 --- /dev/null +++ b/changes/2871.feature.rst @@ -0,0 +1,8 @@ +Added public API for Buffer ABCs and implementations. + +Use :mod:`zarr.buffer` to access buffer implementations, and +:mod:`zarr.abc.buffer` for the interface to implement new buffer types. + +Users previously importing buffer from ``zarr.core.buffer`` should update their +imports to use :mod:`zarr.buffer`. As a reminder, all of ``zarr.core`` is +considered a private API that's not covered by zarr-python's versioning policy. \ No newline at end of file diff --git a/changes/2874.feature.rst b/changes/2874.feature.rst new file mode 100644 index 0000000000..4c50532ae0 --- /dev/null +++ b/changes/2874.feature.rst @@ -0,0 +1,9 @@ +Adds zarr-specific data type classes. This replaces the internal use of numpy data types for zarr +v2 and a fixed set of string enums for zarr v3. This change is largely internal, but it does +change the type of the ``dtype`` and ``data_type`` fields on the ``ArrayV2Metadata`` and +``ArrayV3Metadata`` classes. It also changes the JSON metadata representation of the +variable-length string data type, but the old metadata representation can still be +used when reading arrays. The logic for automatically choosing the chunk encoding for a given data +type has also changed, and this necessitated changes to the ``config`` API. + +For more on this new feature, see the `documentation `_ \ No newline at end of file diff --git a/changes/2921.bugfix.rst b/changes/2921.bugfix.rst new file mode 100644 index 0000000000..65db48654f --- /dev/null +++ b/changes/2921.bugfix.rst @@ -0,0 +1 @@ +Ignore stale child metadata when reconsolidating metadata. diff --git a/changes/3021.feature.rst b/changes/3021.feature.rst new file mode 100644 index 0000000000..8805797ce3 --- /dev/null +++ b/changes/3021.feature.rst @@ -0,0 +1 @@ +Implemented ``move`` for ``LocalStore`` and ``ZipStore``. This allows users to move the store to a different root path. \ No newline at end of file diff --git a/changes/3066.feature.rst b/changes/3066.feature.rst new file mode 100644 index 0000000000..89d5ddb1c6 --- /dev/null +++ b/changes/3066.feature.rst @@ -0,0 +1 @@ +Added `~zarr.errors.GroupNotFoundError`, which is raised when attempting to open a group that does not exist. diff --git a/changes/3068.bugfix.rst b/changes/3068.bugfix.rst new file mode 100644 index 0000000000..9ada322c13 --- /dev/null +++ b/changes/3068.bugfix.rst @@ -0,0 +1 @@ +Trying to open an array with ``mode='r'`` when the store is not read-only now raises an error. diff --git a/changes/3081.feature.rst b/changes/3081.feature.rst new file mode 100644 index 0000000000..8cf83ea7c2 --- /dev/null +++ b/changes/3081.feature.rst @@ -0,0 +1 @@ +Adds ``fill_value`` to the list of attributes displayed in the output of the ``AsyncArray.info()`` method. \ No newline at end of file diff --git a/changes/3082.feature.rst b/changes/3082.feature.rst new file mode 100644 index 0000000000..e990d1f3a0 --- /dev/null +++ b/changes/3082.feature.rst @@ -0,0 +1 @@ +Use :py:func:`numpy.zeros` instead of :py:func:`np.full` for a performance speedup when creating a `zarr.core.buffer.NDBuffer` with `fill_value=0`. \ No newline at end of file diff --git a/changes/3100.bugfix.rst b/changes/3100.bugfix.rst new file mode 100644 index 0000000000..11f06628c0 --- /dev/null +++ b/changes/3100.bugfix.rst @@ -0,0 +1,3 @@ +For Zarr format 2, allow fixed-length string arrays to be created without automatically inserting a +``Vlen-UT8`` codec in the array of filters. Fixed-length string arrays do not need this codec. This +change fixes a regression where fixed-length string arrays created with Zarr Python 3 could not be read with Zarr Python 2.18. \ No newline at end of file diff --git a/changes/3103.bugfix.rst b/changes/3103.bugfix.rst new file mode 100644 index 0000000000..93aecce908 --- /dev/null +++ b/changes/3103.bugfix.rst @@ -0,0 +1,7 @@ +When creating arrays without explicitly specifying a chunk size using `zarr.create` and other +array creation routines, the chunk size will now set automatically instead of defaulting to the data shape. +For large arrays this will result in smaller default chunk sizes. +To retain previous behaviour, explicitly set the chunk shape to the data shape. + +This fix matches the existing chunking behaviour of +`zarr.save_array` and `zarr.api.asynchronous.AsyncArray.create`. diff --git a/changes/3127.bugfix.rst b/changes/3127.bugfix.rst new file mode 100644 index 0000000000..35d7f5d329 --- /dev/null +++ b/changes/3127.bugfix.rst @@ -0,0 +1,2 @@ +When `zarr.save` has an argument `path=some/path/` and multiple arrays in `args`, the path resulted in `some/path/some/path` due to using the `path` +argument twice while building the array path. This is now fixed. \ No newline at end of file diff --git a/changes/3128.bugfix.rst b/changes/3128.bugfix.rst new file mode 100644 index 0000000000..b93416070e --- /dev/null +++ b/changes/3128.bugfix.rst @@ -0,0 +1 @@ +Fix `zarr.open` default for argument `mode` when `store` is `read_only` \ No newline at end of file diff --git a/changes/3130.feature.rst b/changes/3130.feature.rst new file mode 100644 index 0000000000..7a64582f06 --- /dev/null +++ b/changes/3130.feature.rst @@ -0,0 +1 @@ +Port more stateful testing actions from `Icechunk `_. diff --git a/changes/3138.feature.rst b/changes/3138.feature.rst new file mode 100644 index 0000000000..ecd339bf9c --- /dev/null +++ b/changes/3138.feature.rst @@ -0,0 +1 @@ +Adds a `with_read_only` convenience method to the `Store` abstract base class (raises `NotImplementedError`) and implementations to the `MemoryStore`, `ObjectStore`, `LocalStore`, and `FsspecStore` classes. \ No newline at end of file diff --git a/changes/README.md b/changes/README.md new file mode 100644 index 0000000000..74ed9f94a9 --- /dev/null +++ b/changes/README.md @@ -0,0 +1,14 @@ +Writing a changelog entry +------------------------- + +Please put a new file in this directory named `xxxx..rst`, where + +- `xxxx` is the pull request number associated with this entry +- `` is one of: + - feature + - bugfix + - doc + - removal + - misc + +Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-python`. diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000000..83274aedec --- /dev/null +++ b/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + patch: + default: + target: auto + project: + default: + target: auto + threshold: 0.1 +comment: false diff --git a/docs/conf.py b/docs/conf.py index 2a93e61d3e..68bf003ad5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,15 +15,12 @@ import os import sys +from importlib.metadata import version as get_version from typing import Any import sphinx import sphinx.application -from importlib.metadata import version as get_version - -import sphinx - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -71,7 +68,7 @@ def skip_submodules( ) -> bool: # Skip documenting zarr.codecs submodules # codecs are documented in the main zarr.codecs namespace - if what == "module" and name.startswith("zarr.codecs."): + if what == "module" and name.startswith("zarr.codecs.") or name.startswith("zarr.core"): skip = True return skip @@ -91,7 +88,7 @@ def skip_submodules( # General information about the project. project = "zarr" -copyright = "2024, Zarr Developers" +copyright = "2025, Zarr Developers" author = "Zarr Developers" version = get_version("zarr") @@ -105,10 +102,10 @@ def skip_submodules( "license": "https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt", "tutorial": "user-guide", "getting-started": "quickstart", - "release": "developers/release.html", "roadmap": "developers/roadmap.html", "installation": "user-guide/installation.html", - "api": "api/zarr/index" + "api": "api/zarr/index", + "release": "release-notes.html", } # The language for content autogenerated by Sphinx. Refer to documentation @@ -181,6 +178,7 @@ def skip_submodules( ], "collapse_navigation": True, "navigation_with_keys": False, + "announcement": "Zarr-Python 3 is here! Check out the release announcement here.", } # Add any paths that contain custom themes here, relative to this directory. @@ -371,6 +369,7 @@ def setup(app: sphinx.application.Sphinx) -> None: "python": ("https://docs.python.org/3/", None), "numpy": ("https://numpy.org/doc/stable/", None), "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None), + "obstore": ("https://developmentseed.org/obstore/latest/", None), } diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst index 4358230eff..03388e1544 100644 --- a/docs/developers/contributing.rst +++ b/docs/developers/contributing.rst @@ -98,7 +98,7 @@ you can do something like the following:: To verify that your development environment is working, you can run the unit tests for one of the test environments, e.g.:: - $ hatch env run --env test.py3.12-2.1-optional run + $ hatch env run --env test.py3.12-2.1-optional run-pytest Creating a branch ~~~~~~~~~~~~~~~~~ @@ -140,7 +140,7 @@ Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see `creating a development environment`_ above) and invoke:: - $ hatch env run --env test.py3.12-2.1-optional run + $ hatch env run --env test.py3.12-2.1-optional run-pytest All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is @@ -190,9 +190,13 @@ Both unit tests and docstring doctests are included when computing coverage. Run $ hatch env run --env test.py3.12-2.1-optional run-coverage -will automatically run the test suite with coverage and produce a coverage report. +will automatically run the test suite with coverage and produce a XML coverage report. This should be 100% before code can be accepted into the main code base. +You can also generate an HTML coverage report by running:: + + $ hatch env run --env test.py3.12-2.1-optional run-coverage-html + When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. @@ -212,8 +216,8 @@ The documentation consists both of prose and API documentation. All user-facing and functions are included in the API documentation, under the ``docs/api`` folder using the `autodoc `_ extension to sphinx. Any new features or important usage information should be included in the -user-guide (``docs/user-guide``). Any changes should also be included in the release -notes (``docs/developers/release.rst``). +user-guide (``docs/user-guide``). Any changes should also be included as a new file in the +:file:`changes` directory. The documentation can be built locally by running:: @@ -226,128 +230,135 @@ during development at `http://0.0.0.0:8000/ `_. This can b $ hatch --env docs run serve -Development best practices, policies and procedures ---------------------------------------------------- +.. _changelog: + +Changelog +~~~~~~~~~ + +zarr-python uses `towncrier`_ to manage release notes. Most pull requests should +include at least one news fragment describing the changes. To add a release +note, you'll need the GitHub issue or pull request number and the type of your +change (``feature``, ``bugfix``, ``doc``, ``removal``, ``misc``). With that, run +```towncrier create``` with your development environment, which will prompt you +for the issue number, change type, and the news text:: + + towncrier create + +Alternatively, you can manually create the files in the ``changes`` directory +using the naming convention ``{issue-number}.{change-type}.rst``. + +See the `towncrier`_ docs for more. + +.. _towncrier: https://towncrier.readthedocs.io/en/stable/tutorial.html The following information is mainly for core developers, but may also be of interest to contributors. Merging pull requests -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Pull requests submitted by an external contributor should be reviewed and approved by at least -one core developers before being merged. Ideally, pull requests submitted by a core developer -should be reviewed and approved by at least one other core developers before being merged. +one core developer before being merged. Ideally, pull requests submitted by a core developer +should be reviewed and approved by at least one other core developer before being merged. Pull requests should not be merged until all CI checks have passed (GitHub Actions Codecov) against code that has had the latest main merged in. Compatibility and versioning policies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Because Zarr is a data storage library, there are two types of compatibility to -consider: API compatibility and data format compatibility. - -API compatibility -""""""""""""""""" - -All functions, classes and methods that are included in the API -documentation (files under ``docs/api/*.rst``) are considered as part of the Zarr **public API**, -except if they have been documented as an experimental feature, in which case they are part of -the **experimental API**. - -Any change to the public API that does **not** break existing third party -code importing Zarr, or cause third party code to behave in a different way, is a -**backwards-compatible API change**. For example, adding a new function, class or method is usually -a backwards-compatible change. However, removing a function, class or method; removing an argument -to a function or method; adding a required argument to a function or method; or changing the -behaviour of a function or method, are examples of **backwards-incompatible API changes**. - -If a release contains no changes to the public API (e.g., contains only bug fixes or -other maintenance work), then the micro version number should be incremented (e.g., -2.2.0 -> 2.2.1). If a release contains public API changes, but all changes are -backwards-compatible, then the minor version number should be incremented -(e.g., 2.2.1 -> 2.3.0). If a release contains any backwards-incompatible public API changes, -the major version number should be incremented (e.g., 2.3.0 -> 3.0.0). - -Backwards-incompatible changes to the experimental API can be included in a minor release, -although this should be minimised if possible. I.e., it would be preferable to save up -backwards-incompatible changes to the experimental API to be included in a major release, and to -stabilise those features at the same time (i.e., move from experimental to public API), rather than -frequently tinkering with the experimental API in minor releases. +------------------------------------- -Data format compatibility -""""""""""""""""""""""""" +Versioning +~~~~~~~~~~ +Versions of this library are identified by a triplet of integers with the form +``..``, for example ``3.0.4``. A release of ``zarr-python`` is associated with a new +version identifier. That new identifier is generated by incrementing exactly one of the components of +the previous version identifier by 1. When incrementing the ``major`` component of the version identifier, +the ``minor`` and ``patch`` components is reset to 0. When incrementing the minor component, +the patch component is reset to 0. -The data format used by Zarr is defined by a specification document, which should be -platform-independent and contain sufficient detail to construct an interoperable -software library to read and/or write Zarr data using any programming language. The -latest version of the specification document is available on the -`Zarr specifications website `_. - -Here, **data format compatibility** means that all software libraries that implement a -particular version of the Zarr storage specification are interoperable, in the sense -that data written by any one library can be read by all others. It is obviously -desirable to maintain data format compatibility wherever possible. However, if a change -is needed to the storage specification, and that change would break data format -compatibility in any way, then the storage specification version number should be -incremented (e.g., 2 -> 3). - -The versioning of the Zarr software library is related to the versioning of the storage -specification as follows. A particular version of the Zarr library will -implement a particular version of the storage specification. For example, Zarr version -2.2.0 implements the Zarr storage specification version 2. If a release of the Zarr -library implements a different version of the storage specification, then the major -version number of the Zarr library should be incremented. E.g., if Zarr version 2.2.0 -implements the storage spec version 2, and the next release of the Zarr library -implements storage spec version 3, then the next library release should have version -number 3.0.0. Note however that the major version number of the Zarr library may not -always correspond to the spec version number. For example, Zarr versions 2.x, 3.x, and -4.x might all implement the same version of the storage spec and thus maintain data -format compatibility, although they will not maintain API compatibility. - -When to make a release -~~~~~~~~~~~~~~~~~~~~~~ +Releases are classified by the library changes contained in that release. This classification +determines which component of the version identifier is incremented on release. -Ideally, any bug fixes that don't change the public API should be released as soon as -possible. It is fine for a micro release to contain only a single bug fix. +* ``major`` releases (for example, ``2.18.0`` -> ``3.0.0``) are for changes that will + require extensive adaptation efforts from many users and downstream projects. + For example, breaking changes to widely-used user-facing APIs should only be applied in a major release. -When to make a minor release is at the discretion of the core developers. There are no -hard-and-fast rules, e.g., it is fine to make a minor release to make a single new -feature available; equally, it is fine to make a minor release that includes a number of -changes. -Major releases obviously need to be given careful consideration, and should be done as -infrequently as possible, as they will break existing code and/or affect data -compatibility in some way. + Users and downstream projects should carefully consider the impact of a major release before + adopting it. + In advance of a major release, developers should communicate the scope of the upcoming changes, + and help users prepare for them. -Release procedure -~~~~~~~~~~~~~~~~~ +* ``minor`` releases (or example, ``3.0.0`` -> ``3.1.0``) are for changes that do not require + significant effort from most users or downstream downstream projects to respond to. API changes + are possible in minor releases if the burden on users imposed by those changes is sufficiently small. -.. note:: + For example, a recently released API may need fixes or refinements that are breaking, but low impact + due to the recency of the feature. Such API changes are permitted in a minor release. - Most of the release process is now handled by GitHub workflow which should - automatically push a release to PyPI if a tag is pushed. -Before releasing, make sure that all pull requests which will be -included in the release have been properly documented in -`docs/release.rst`. + Minor releases are safe for most users and downstream projects to adopt. -To make a new release, go to -https://github.com/zarr-developers/zarr-python/releases and -click "Draft a new release". Choose a version number prefixed -with a `v` (e.g. `v0.0.0`). For pre-releases, include the -appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`). +* ``patch`` releases (for example, ``3.1.0`` -> ``3.1.1``) are for changes that contain no breaking + or behaviour changes for downstream projects or users. Examples of changes suitable for a patch release are + bugfixes and documentation improvements. -Set the description of the release to:: - See release notes https://zarr.readthedocs.io/en/stable/release.html#release-0-0-0 + Users should always feel safe upgrading to a the latest patch release. -replacing the correct version numbers. For pre-release versions, -the URL should omit the pre-release suffix, e.g. "a1" or "rc1". +Note that this versioning scheme is not consistent with `Semantic Versioning `_. +Contrary to SemVer, the Zarr library may release breaking changes in ``minor`` releases, or even +``patch`` releases under exceptional circumstances. But we should strive to avoid doing so. -Click on "Generate release notes" to auto-file the description. +A better model for our versioning scheme is `Intended Effort Versioning `_, +or "EffVer". The guiding principle off EffVer is to categorize releases based on the *expected effort +required to upgrade to that release*. + +Zarr developers should make changes as smooth as possible for users. This means making +backwards-compatible changes wherever possible. When a backwards-incompatible change is necessary, +users should be notified well in advance, e.g. via informative deprecation warnings. + +Data format compatibility +""""""""""""""""""""""""" + +The Zarr library is an implementation of a file format standard defined externally -- +see the `Zarr specifications website `_ for the list of +Zarr file format specifications. + + +If an existing Zarr format version changes, or a new version of the Zarr format is released, then +the Zarr library will generally require changes. It is very likely that a new Zarr format will +require extensive breaking changes to the Zarr library, and so support for a new Zarr format in the +Zarr library will almost certainly come in new ``major`` release. +When the Zarr library adds support for a new Zarr format, there may be a period of accelerated +changes as developers refine newly added APIs and deprecate old APIs. In such a transitional phase +breaking changes may be more frequent than usual. + + +Release procedure +----------------- + +Pre-release +~~~~~~~~~~~ +1. Make sure that all pull requests which will be included in the release + have been properly documented as changelog files in the :file:`changes/` directory. +2. Run ``towncrier build --version x.y.z`` to create the changelog, and commit the result + to the main branch. + +Releasing +~~~~~~~~~ +1. Go to https://github.com/zarr-developers/zarr-python/releases +2. Click "Draft a new release". +3. Choose a version number prefixed with a `v` (e.g. `v0.0.0`). + For pre-releases, include the appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`). +4. Set the description of the release to:: + + See release notes https://zarr.readthedocs.io/en/stable/release-notes.html#release-0-0-0 + + replacing the correct version numbers. For pre-release versions, + the URL should omit the pre-release suffix, e.g. "a1" or "rc1". +5. Click on "Generate release notes" to auto-fill the description. After creating the release, the documentation will be built on https://readthedocs.io. Full releases will be available under @@ -355,5 +366,9 @@ https://readthedocs.io. Full releases will be available under pre-releases will be available under `/latest `_. -Also review and merge the https://github.com/conda-forge/zarr-feedstock -pull request that will be automatically generated. +Post-release +~~~~~~~~~~~~ + +- Review and merge the pull request on the + `conda-forge feedstock `_ that will be + automatically generated. diff --git a/docs/developers/index.rst b/docs/developers/index.rst index 3feb0aff71..4bccb3a469 100644 --- a/docs/developers/index.rst +++ b/docs/developers/index.rst @@ -6,5 +6,4 @@ Developer's Guide :maxdepth: 1 contributing - release roadmap diff --git a/docs/developers/release.rst b/docs/developers/release.rst deleted file mode 100644 index ce15c68f4a..0000000000 --- a/docs/developers/release.rst +++ /dev/null @@ -1,2334 +0,0 @@ -Release notes -============= - -.. - # Copy the warning statement _under_ the latest release version - # and unindent for pre-releases. - - .. warning:: - Pre-release! Use :command:`pip install --pre zarr` to evaluate this release. - -.. - # Unindent the section between releases in order - # to document your changes. On releases it will be - # re-indented so that it does not show up in the notes. - -.. note:: - Zarr-Python 2.18.* is expected be the final release in the 2.* series. Work on Zarr-Python 3.0 is underway. - See `GH1777 `_ for more details on the upcoming - 3.0 release. - -.. release_3.0.0-beta: - -3.0.0-beta series ------------------ - -.. warning:: - Zarr-Python 3.0.0-beta is a pre-release of the upcoming 3.0 release. This release is not feature complete or - expected to be ready for production applications. - -.. note:: - The complete release notes for 3.0 have not been added to this document yet. See the - `3.0.0-beta `_ release on GitHub - for a record of changes included in this release. - -Dependency Changes -~~~~~~~~~~~~~~~~~~ - -* fsspec was moved from a required dependency to an optional one. Users should install - fsspec and any relevant implementations (e.g. s3fs) before using the ``RemoteStore``. - By :user:`Joe Hamman ` :issue:`2391`. - -* ``RemoteStore`` was renamed to ``FsspecStore``. - By :user:`Joe Hamman ` :issue:`2557`. - -.. release_3.0.0-alpha: - -3.0.0-alpha series ------------------- - -.. warning:: - Zarr-Python 3.0.0-alpha is a pre-release of the upcoming 3.0 release. This release is not feature complete or - expected to be ready for production applications. - -.. note:: - The complete release notes for 3.0 have not been added to this document yet. See the - `3.0.0-alpha `_ release on GitHub - for a record of changes included in this release. - -Enhancements -~~~~~~~~~~~~ - -* Implement listing of the sub-arrays and sub-groups for a V3 ``Group``. - By :user:`Davis Bennett ` :issue:`1726`. - -* Bootstrap v3 branch with zarrita. - By :user:`Joe Hamman ` :issue:`1584`. - -* Extensible codecs for V3. - By :user:`Norman Rzepka ` :issue:`1588`. - -* Don't import from tests. - By :user:`Davis Bennett ` :issue:`1601`. - -* Listable V3 Stores. - By :user:`Joe Hamman ` :issue:`1634`. - -* Codecs without array metadata. - By :user:`Norman Rzepka ` :issue:`1632`. - -* fix sync group class methods. - By :user:`Joe Hamman ` :issue:`1652`. - -* implement eq for LocalStore. - By :user:`Charoula Kyriakides ` :issue:`1792`. - -* V3 reorg. - By :user:`Joe Hamman ` :issue:`1809`. - -* [v3] Sync with futures. - By :user:`Davis Bennett ` :issue:`1804`. - -* implement group.members. - By :user:`Davis Bennett ` :issue:`1726`. - -* Remove implicit groups. - By :user:`Joe Hamman ` :issue:`1827`. - -* feature(store): ``list_*`` -> AsyncGenerators. - By :user:`Joe Hamman ` :issue:`1844`. - -* Test codec entrypoints. - By :user:`Norman Rzepka ` :issue:`1835`. - -* Remove extra v3 sync module. - By :user:`Max Jones ` :issue:`1856`. - -* Use donfig for V3 configuration. - By :user:`Max Jones ` :issue:`1655`. - -* groundwork for V3 group tests. - By :user:`Davis Bennett ` :issue:`1743`. - -* [v3] First step to generalizes ndarray and bytes. - By :user:`Mads R. B. Kristensen ` :issue:`1826`. - -* Reworked codec pipelines. - By :user:`Norman Rzepka ` :issue:`1670`. - -* Followup on codecs. - By :user:`Norman Rzepka ` :issue:`1889`. - -* Protocols for Buffer and NDBuffer. - By :user:`Mads R. B. Kristensen ` :issue:`1899`. - -* [V3] Expand store tests. - By :user:`Davis Bennett ` :issue:`1900`. - -* [v3] Feature: Store open mode. - By :user:`Joe Hamman ` :issue:`1911`. - -* fix(types): Group.info -> NotImplementedError. - By :user:`Joe Hamman ` :issue:`1936`. - -* feature(typing): add py.typed file to package root. - By :user:`Joe Hamman ` :issue:`1935`. - -* Support all indexing variants. - By :user:`Norman Rzepka ` :issue:`1917`. - -* Feature: group and array name properties. - By :user:`Joe Hamman ` :issue:`1940`. - -* implement .chunks on v3 arrays. - By :user:`Ryan Abernathey ` :issue:`1929`. - -* Fixes bug in transpose. - By :user:`Norman Rzepka ` :issue:`1949`. - -* Buffer Prototype Argument. - By :user:`Mads R. B. Kristensen ` :issue:`1910`. - -* Feature: Top level V3 API. - By :user:`Joe Hamman ` :issue:`1884`. - -* Basic working FsspecStore. - By :user:`Martin Durant `; :issue:`1785`. - -Typing -~~~~~~ - -* Resolve Mypy errors in v3 branch. - By :user:`Daniel Jahn ` :issue:`1692`. - -* Allow dmypy to be run on v3 branch. - By :user:`David Stansby ` :issue:`1780`. - -* Remove unused typing ignore comments. - By :user:`David Stansby ` :issue:`1781`. - -* Check untyped defs on v3. - By :user:`David Stansby ` :issue:`1784`. - -* [v3] Enable some more strict mypy options. - By :user:`David Stansby ` :issue:`1793`. - -* [v3] Disallow generic Any typing. - By :user:`David Stansby ` :issue:`1794`. - -* Disallow incomplete type definitions. - By :user:`David Stansby ` :issue:`1814`. - -* Disallow untyped calls. - By :user:`David Stansby ` :issue:`1811`. - -* Fix some untyped calls. - By :user:`David Stansby ` :issue:`1865`. - -* Disallow untyped defs. - By :user:`David Stansby ` :issue:`1834`. - -* Add more typing to zarr.group. - By :user:`David Stansby ` :issue:`1870`. - -* Fix any generics in zarr.array. - By :user:`David Stansby ` :issue:`1861`. - -* Remove some unused mypy overrides. - By :user:`David Stansby ` :issue:`1894`. - -* Finish typing zarr.metadata. - By :user:`David Stansby ` :issue:`1880`. - -* Disallow implicit re-exports. - By :user:`David Stansby ` :issue:`1908`. - -* Make typing strict. - By :user:`David Stansby ` :issue:`1879`. - -* Enable extra mypy error codes. - By :user:`David Stansby ` :issue:`1909`. - -* Enable warn_unreachable for mypy. - By :user:`David Stansby ` :issue:`1937`. - -* Fix final typing errors. - By :user:`David Stansby ` :issue:`1939`. - -Maintenance -~~~~~~~~~~~ - -* Remedy a situation where ``zarr-python`` was importing ``DummyStorageTransformer`` from the test suite. - The dependency relationship is now reversed: the test suite imports this class from ``zarr-python``. - By :user:`Davis Bennett ` :issue:`1601`. - -* [V3] Update minimum supported Python and Numpy versions. - By :user:`Joe Hamman ` :issue:`1638` - -* use src layout and use hatch for packaging. - By :user:`Davis Bennett ` :issue:`1592`. - -* temporarily disable mypy in v3 directory. - By :user:`Joe Hamman ` :issue:`1649`. - -* create hatch test env. - By :user:`Ryan Abernathey ` :issue:`1650`. - -* removed unused environments and workflows. - By :user:`Ryan Abernathey ` :issue:`1651`. - -* Add env variables to sprint setup instructions. - By :user:`Max Jones ` :issue:`1654`. - -* Add test matrix for V3. - By :user:`Max Jones ` :issue:`1656`. - -* Remove attrs. - By :user:`Davis Bennett ` :issue:`1660`. - -* Specify hatch envs using GitHub actions matrix for v3 tests. - By :user:`Max Jones ` :issue:`1728`. - -* black -> ruff format + cleanup. - By :user:`Saransh Chopra ` :issue:`1639`. - -* Remove old v3. - By :user:`Davis Bennett ` :issue:`1742`. - -* V3 update pre commit. - By :user:`Joe Hamman ` :issue:`1808`. - -* remove windows testing on v3 branch. - By :user:`Joe Hamman ` :issue:`1817`. - -* fix: add mypy to test dependencies. - By :user:`Davis Bennett ` :issue:`1789`. - -* chore(ci): add numpy 2 release candidate to test matrix. - By :user:`Joe Hamman ` :issue:`1828`. - -* fix dependencies. - By :user:`Norman Rzepka ` :issue:`1840`. - -* Add pytest to mypy dependencies. - By :user:`David Stansby ` :issue:`1846`. - -* chore(pre-commit): update pre-commit versions and remove attrs dep mypy section. - By :user:`Joe Hamman ` :issue:`1848`. - -* Enable some ruff rules (RUF) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1869`. - -* Configure Ruff to apply flake8-bugbear/isort/pyupgrade. - By :user:`Norman Rzepka ` :issue:`1890`. - -* chore(ci): remove mypy from test action in favor of pre-commit action. - By :user:`Joe Hamman ` :issue:`1887`. - -* Enable ruff/flake8-raise rules (RSE) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1872`. - -* Apply assorted ruff/refurb rules (FURB). - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1873`. - -* Enable ruff/flake8-implicit-str-concat rules (ISC) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1868`. - -* Add numpy to mypy pre-commit check env. - By :user:`David Stansby ` :issue:`1893`. - -* remove fixture files from src. - By :user:`Davis Bennett ` :issue:`1897`. - -* Fix list of packages in mypy pre-commit environment. - By :user:`David Stansby ` :issue:`1907`. - -* Run sphinx directly on readthedocs. - By :user:`David Stansby ` :issue:`1919`. - -* Apply preview ruff rules. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1942`. - -* Enable and apply ruff rule RUF009. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1941`. - -Documentation -~~~~~~~~~~~~~ - -* Specify docs hatch env for v3 branch. - By :user:`Max Jones ` :issue:`1655`. - -* Development installation/contributing docs updates. - By :user:`Alden Keefe Sampson ` :issue:`1643`. - -* chore: update project settings per scientific python repo-review. - By :user:`Joe Hamman ` :issue:`1863`. - -* doc: update release notes for 3.0.0.alpha. - By :user:`Joe Hamman ` :issue:`1959`. - -.. _release_2.18.3: - -2.18.3 ------- - -Enhancements -~~~~~~~~~~~~ -* Added support for creating a copy of data when converting a `zarr.Array` - to a numpy array. - By :user:`David Stansby ` (:issue:`2106`) and - :user:`Joe Hamman ` (:issue:`2123`). - -Maintenance -~~~~~~~~~~~ -* Removed support for Python 3.9. - By :user:`David Stansby ` (:issue:`2074`). - -* Fix a regression when using orthogonal indexing with a scalar. - By :user:`Deepak Cherian ` :issue:`1931` - -* Added compatibility with NumPy 2.1. - By :user:`David Stansby ` - -* Bump minimum NumPy version to 1.24. - :user:`Joe Hamman ` (:issue:`2127`). - -Deprecations -~~~~~~~~~~~~ - -* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These - stores are slated to be removed in Zarr Python 3.0. - By :user:`Joe Hamman ` :issue:`2085`. - -.. _release_2.18.2: - -2.18.2 ------- - -Enhancements -~~~~~~~~~~~~ - -* Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` - -.. _release_2.18.1: - -2.18.1 ------- - -Maintenance -~~~~~~~~~~~ -* Fix a regression when getting or setting a single value from arrays with size-1 chunks. - By :user:`Deepak Cherian ` :issue:`1874` - -.. _release_2.18.0: - -2.18.0 ------- - -Enhancements -~~~~~~~~~~~~ -* Performance improvement for reading and writing chunks if any of the dimensions is size 1. - By :user:`Deepak Cherian ` :issue:`1730`. - -Maintenance -~~~~~~~~~~~ -* Enable ruff/bugbear rules (B) and fix issues. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. - -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. - By :user:`Joe Hamman ` :issue:`1842`. - -Deprecations -~~~~~~~~~~~~ - -* Deprecate experimental v3 support by issuing a `FutureWarning`. - Also updated docs to warn about using the experimental v3 version. - By :user:`Joe Hamman ` :issue:`1802` and :issue:`1807`. - -* Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, - :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, - and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. - By :user:`Joe Hamman ` :issue:`1801`. - -.. _release_2.17.2: - -2.17.2 ------- - -Enhancements -~~~~~~~~~~~~ - -* [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` - and handling any error that may arise. - By :user:`Deepak Cherian ` :issue:`1741`. - -* [v3] Reuse the downloaded array metadata when creating an ``Array``. - By :user:`Deepak Cherian ` :issue:`1734`. - -* Optimize ``Array.info`` so that it calls `getsize` only once. - By :user:`Deepak Cherian ` :issue:`1733`. - -* Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. - By :user:`Deepak Cherian ` :issue:`1716`. - -* FSStore now raises rather than return bad data. - By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. - -* Avoid redundant ``__contains__``. - By :user:`Deepak Cherian ` :issue:`1739`. - -Docs -~~~~ - -* Fix link to GCSMap in ``tutorial.rst``. - By :user:`Daniel Jahn ` :issue:`1689`. - -* Endorse `SPEC0000 `_ and state version support policy in ``installation.rst``. - By :user:`Sanket Verma ` :issue:`1665`. - -* Migrate v1 and v2 specification to `Zarr-Specs `_. - By :user:`Sanket Verma ` :issue:`1582`. - -Maintenance -~~~~~~~~~~~ - -* Add CI test environment for Python 3.12 - By :user:`Joe Hamman ` :issue:`1719`. - -* Bump minimum supported NumPy version to 1.23 (per spec 0000) - By :user:`Joe Hamman ` :issue:`1719`. - -* Minor fixes: Using ``is`` instead of ``type`` and removing unnecessary ``None``. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1737`. - -* Fix tests failure related to Pytest 8. - By :user:`David Stansby ` :issue:`1714`. - -.. _release_2.17.1: - -2.17.1 ------- - -Enhancements -~~~~~~~~~~~~ - -* Change occurrences of % and format() to f-strings. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1423`. - -* Proper argument for numpy.reshape. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1425`. - -* Add typing to dimension separator arguments. - By :user:`David Stansby ` :issue:`1620`. - -Docs -~~~~ - -* ZIP related tweaks. - By :user:`Davis Bennett ` :issue:`1641`. - -Maintenance -~~~~~~~~~~~ - -* Update config.yml with Zulip. - By :user:`Josh Moore `. - -* Replace Gitter with the new Zulip Chat link. - By :user:`Sanket Verma ` :issue:`1685`. - -* Fix RTD build. - By :user:`Sanket Verma ` :issue:`1694`. - -.. _release_2.17.0: - -2.17.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Added type hints to ``zarr.creation.create()``. - By :user:`David Stansby ` :issue:`1536`. - -* Pyodide support: Don't require fasteners on Emscripten. - By :user:`Hood Chatham ` :issue:`1663`. - -Docs -~~~~ - -* Minor correction and changes in documentation. - By :user:`Sanket Verma ` :issue:`1509`. - -* Fix typo in documentation. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1554` - -* The documentation build now fails if there are any warnings. - By :user:`David Stansby ` :issue:`1548`. - -* Add links to ``numcodecs`` docs in the tutorial. - By :user:`David Stansby ` :issue:`1535`. - -* Enable offline formats for documentation builds. - By :user:`Sanket Verma ` :issue:`1551`. - -* Minor tweak to advanced indexing tutorial examples. - By :user:`Ross Barnowski ` :issue:`1550`. - -* Automatically document array members using sphinx-automodapi. - By :user:`David Stansby ` :issue:`1547`. - -* Add a markdown file documenting the current and former core-developer team. - By :user:`Joe Hamman ` :issue:`1628`. - -* Add Norman Rzepka to core-dev team. - By :user:`Joe Hamman ` :issue:`1630`. - -* Added section about accessing ZIP archives on s3. - By :user:`Jeff Peck ` :issue:`1613`, :issue:`1615`, and :user:`Davis Bennett ` :issue:`1641`. - -* Add V3 roadmap and design document. - By :user:`Joe Hamman ` :issue:`1583`. - -Maintenance -~~~~~~~~~~~ - -* Drop Python 3.8 and NumPy 1.20 - By :user:`Josh Moore `; :issue:`1557`. - -* Cache result of ``FSStore._fsspec_installed()``. - By :user:`Janick Martinez Esturo ` :issue:`1581`. - -* Extend copyright notice to 2023. - By :user:`Jack Kelly ` :issue:`1528`. - -* Change occurrence of ``io.open()`` into ``open()``. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1421`. - -* Preserve ``dimension_separator`` when resizing arrays. - By :user:`Ziwen Liu ` :issue:`1533`. - -* Initialise some sets in tests with set literals instead of list literals. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1534`. - -* Allow ``black`` code formatter to be run with any Python version. - By :user:`David Stansby ` :issue:`1549`. - -* Remove ``sphinx-rtd-theme`` dependency from ``pyproject.toml``. - By :user:`Sanket Verma ` :issue:`1563`. - -* Remove ``CODE_OF_CONDUCT.md`` file from the Zarr-Python repository. - By :user:`Sanket Verma ` :issue:`1572`. - -* Bump version of black in pre-commit. - By :user:`David Stansby ` :issue:`1559`. - -* Use list comprehension where applicable. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1555`. - -* Use format specification mini-language to format string. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1558`. - -* Single startswith() call instead of multiple ones. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1556`. - -* Move codespell options around. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1196`. - -* Remove unused mypy ignore comments. - By :user:`David Stansby ` :issue:`1602`. - -.. _release_2.16.1: - -2.16.1 ------- - -Maintenance -~~~~~~~~~~~ - -* Require ``setuptools_scm`` version ``1.5.4``\+ - By :user:`John A. Kirkham ` :issue:`1477`. - -* Add ``docs`` requirements to ``pyproject.toml`` - By :user:`John A. Kirkham ` :issue:`1494`. - -* Fixed caching issue in ``LRUStoreCache``. - By :user:`Mads R. B. Kristensen ` :issue:`1499`. - -.. _release_2.16.0: - -2.16.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Allow for partial codec specification in V3 array metadata. - By :user:`Joe Hamman ` :issue:`1443`. - -* Add ``__contains__`` method to ``KVStore``. - By :user:`Christoph Gohlke ` :issue:`1454`. - -* **Block Indexing**: Implemented blockwise (chunk blocks) indexing to ``zarr.Array``. - By :user:`Altay Sansal ` :issue:`1428` - -Maintenance -~~~~~~~~~~~ - -* Refactor the core array tests to reduce code duplication. - By :user:`Davis Bennett ` :issue:`1462`. - -* Style the codebase with ``ruff`` and ``black``. - By :user:`Davis Bennett ` :issue:`1459` - -* Ensure that chunks is tuple of ints upon array creation. - By :user:`Philipp Hanslovsky ` :issue:`1461` - -.. _release_2.15.0: - -2.15.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Implement more extensive fallback of getitem/setitem for orthogonal indexing. - By :user:`Andreas Albert ` :issue:`1029`. - -* Getitems supports ``meta_array``. - By :user:`Mads R. B. Kristensen ` :issue:`1131`. - -* ``open_array()`` now takes the ``meta_array`` argument. - By :user:`Mads R. B. Kristensen ` :issue:`1396`. - -Maintenance -~~~~~~~~~~~ - -* Remove ``codecov`` from GitHub actions. - By :user:`John A. Kirkham ` :issue:`1391`. - -* Replace ``np.product`` with ``np.prod`` due to deprecation. - By :user:`James Bourbeau ` :issue:`1405`. - -* Activate Py 3.11 builds. - By :user:`Joe Hamman ` :issue:`1415`. - -Documentation -~~~~~~~~~~~~~ - -* Add API reference for V3 Implementation in the docs. - By :user:`Sanket Verma ` :issue:`1345`. - -Bug fixes -~~~~~~~~~ - -* Fix the conda-forge error. Read :issue:`1347` for detailed info. - By :user:`Josh Moore ` :issue:`1364` and :issue:`1367`. - -* Fix ``ReadOnlyError`` when opening V3 store via fsspec reference file system. - By :user:`Joe Hamman ` :issue:`1383`. - -* Fix ``normalize_fill_value`` for structured arrays. - By :user:`Alan Du ` :issue:`1397`. - -.. _release_2.14.2: - -2.14.2 ------- - -Bug fixes -~~~~~~~~~ - -* Ensure ``zarr.group`` uses writeable mode to fix issue with :issue:`1304`. - By :user:`Brandur Thorgrimsson ` :issue:`1354`. - -.. _release_2.14.1: - -2.14.1 ------- - -Documentation -~~~~~~~~~~~~~ - -* Fix API links. - By :user:`Josh Moore ` :issue:`1346`. - -* Fix unit tests which prevented the conda-forge release. - By :user:`Josh Moore ` :issue:`1348`. - -.. _release_2.14.0: - -2.14.0 ------- - -Major changes -~~~~~~~~~~~~~ - -* Improve Zarr V3 support, adding partial store read/write and storage transformers. - Add new features from the `v3 spec `_: - - * storage transformers - * `get_partial_values` and `set_partial_values` - * efficient `get_partial_values` implementation for `FSStoreV3` - * sharding storage transformer - - By :user:`Jonathan Striebel `; :issue:`1096`, :issue:`1111`. - -* N5 nows supports Blosc. - Remove warnings emitted when using N5Store or N5FSStore with a blosc-compressed array. - By :user:`Davis Bennett `; :issue:`1331`. - -Bug fixes -~~~~~~~~~ - -* Allow reading utf-8 encoded json files - By :user:`Nathan Zimmerberg ` :issue:`1308`. - -* Ensure contiguous data is give to ``FSStore``. Only copying if needed. - By :user:`Mads R. B. Kristensen ` :issue:`1285`. - -* NestedDirectoryStore.listdir now returns chunk keys with the correct '/' dimension_separator. - By :user:`Brett Graham ` :issue:`1334`. - -* N5Store/N5FSStore dtype returns zarr Stores readable dtype. - By :user:`Marwan Zouinkhi ` :issue:`1339`. - -.. _release_2.13.6: - -2.13.6 ------- - -Maintenance -~~~~~~~~~~~ - -* Bump gh-action-pypi-publish to 1.6.4. - By :user:`Josh Moore ` :issue:`1320`. - -.. _release_2.13.5: - -2.13.5 ------- - -Bug fixes -~~~~~~~~~ - -* Ensure ``zarr.create`` uses writeable mode to fix issue with :issue:`1304`. - By :user:`James Bourbeau ` :issue:`1309`. - -.. _release_2.13.4: - -2.13.4 ------- - -Appreciation -~~~~~~~~~~~~~ - -Special thanks to Outreachy participants for contributing to most of the -maintenance PRs. Please read the blog post summarising the contribution phase -and welcoming new Outreachy interns: -https://zarr.dev/blog/welcoming-outreachy-2022-interns/ - - -Enhancements -~~~~~~~~~~~~ - -* Handle fsspec.FSMap using FSStore store. - By :user:`Rafal Wojdyla ` :issue:`1304`. - -Bug fixes -~~~~~~~~~ - -* Fix bug that caused double counting of groups in ``groups()`` and ``group_keys()`` methods with V3 stores. - By :user:`Ryan Abernathey ` :issue:`1228`. - -* Remove unnecessary calling of `contains_array` for key that ended in `.array.json`. - By :user:`Joe Hamman ` :issue:`1149`. - -* Fix bug that caused double counting of groups in ``groups()`` and ``group_keys()`` - methods with V3 stores. - By :user:`Ryan Abernathey ` :issue:`1228`. - -Documentation -~~~~~~~~~~~~~ - -* Fix minor indexing errors in tutorial and specification examples of documentation. - By :user:`Kola Babalola ` :issue:`1277`. - -* Add `requirements_rtfd.txt` in `contributing.rst`. - By :user:`AWA BRANDON AWA ` :issue:`1243`. - -* Add documentation for find/findall using visit. - By :user:`Weddy Gikunda ` :issue:`1241`. - -* Refresh of the main landing page. - By :user:`Josh Moore ` :issue:`1173`. - -Maintenance -~~~~~~~~~~~ - -* Migrate to ``pyproject.toml`` and remove redundant infrastructure. - By :user:`Saransh Chopra ` :issue:`1158`. - -* Require ``setuptools`` 64.0.0+ - By :user:`Saransh Chopra ` :issue:`1193`. - -* Pin action versions (pypi-publish, setup-miniconda) for dependabot - By :user:`Saransh Chopra ` :issue:`1205`. - -* Remove ``tox`` support - By :user:`Saransh Chopra ` :issue:`1219`. - -* Add workflow to label PRs with "needs release notes". - By :user:`Saransh Chopra ` :issue:`1239`. - -* Simplify if/else statement. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1227`. - -* Get coverage up to 100%. - By :user:`John Kirkham ` :issue:`1264`. - -* Migrate coverage to ``pyproject.toml``. - By :user:`John Kirkham ` :issue:`1250`. - -* Use ``conda-incubator/setup-miniconda@v2.2.0``. - By :user:`John Kirkham ` :issue:`1263`. - -* Delete unused files. - By :user:`John Kirkham ` :issue:`1251`. - -* Skip labeller for bot PRs. - By :user:`Saransh Chopra ` :issue:`1271`. - -* Restore Flake8 configuration. - By :user:`John Kirkham ` :issue:`1249`. - -* Add missing newline at EOF. - By :user:`Dimitri Papadopoulos` :issue:`1253`. - -* Add `license_files` to `pyproject.toml`. - By :user:`John Kirkham ` :issue:`1247`. - -* Adding `pyupgrade` suggestions. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1225`. - -* Fixed some linting errors. - By :user:`Weddy Gikunda ` :issue:`1226`. - -* Added the link to main website in readthedocs sidebar. - By :user:`Stephanie_nkwatoh ` :issue:`1216`. - -* Remove redundant wheel dependency in `pyproject.toml`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1233`. - -* Turned on `isloated_build` in `tox.ini` file. - By :user:`AWA BRANDON AWA ` :issue:`1210`. - -* Fixed `flake8` alert and avoid duplication of `Zarr Developers`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1203`. - -* Bump to NumPy 1.20+ in `environment.yml`. - By :user:`John Kirkham ` :issue:`1201`. - -* Bump to NumPy 1.20 in `pyproject.toml`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1192`. - -* Remove LGTM (`.lgtm.yml`) configuration file. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1191`. - -* Codespell will skip `fixture` in pre-commit. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1197`. - -* Add msgpack in `requirements_rtfd.txt`. - By :user:`Emmanuel Bolarinwa ` :issue:`1188`. - -* Added license to docs fixed a typo from `_spec_v2` to `_spec_v3`. - By :user:`AWA BRANDON AWA ` :issue:`1182`. - -* Fixed installation link in `README.md`. - By :user:`AWA BRANDON AWA ` :issue:`1177`. - -* Fixed typos in `installation.rst` and `release.rst`. - By :user:`Chizoba Nweke ` :issue:`1178`. - -* Set `docs/conf.py` language to `en`. - By :user:`AWA BRANDON AWA ` :issue:`1174`. - -* Added `installation.rst` to the docs. - By :user:`AWA BRANDON AWA ` :issue:`1170`. - -* Adjustment of year to `2015-2018` to `2015-2022` in the docs. - By :user:`Emmanuel Bolarinwa ` :issue:`1165`. - -* Updated `Forking the repository` section in `contributing.rst`. - By :user:`AWA BRANDON AWA ` :issue:`1171`. - -* Updated GitHub actions. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1134`. - -* Update web links: `http:// → https://`. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1313`. - -.. _release_2.13.3: - -2.13.3 ------- - -* Improve performance of slice selections with steps by omitting chunks with no relevant - data. - By :user:`Richard Shaw ` :issue:`843`. - -.. _release_2.13.2: - -2.13.2 ------- - -* Fix test failure on conda-forge builds (again). - By :user:`Josh Moore `; see - `zarr-feedstock#65 `_. - -.. _release_2.13.1: - -2.13.1 ------- - -* Fix test failure on conda-forge builds. - By :user:`Josh Moore `; see - `zarr-feedstock#65 `_. - -.. _release_2.13.0: - -2.13.0 ------- - -Major changes -~~~~~~~~~~~~~ - -* **Support of alternative array classes** by introducing a new argument, - meta_array, that specifies the type/class of the underlying array. The - meta_array argument can be any class instance that can be used as the like - argument in NumPy (see `NEP 35 - `_). - enabling support for CuPy through, for example, the creation of a CuPy CPU - compressor. - By :user:`Mads R. B. Kristensen ` :issue:`934`. - -* **Remove support for Python 3.7** in concert with NumPy dependency. - By :user:`Davis Bennett ` :issue:`1067`. - -* **Zarr v3: add support for the default root path** rather than requiring - that all API users pass an explicit path. - By :user:`Gregory R. Lee ` :issue:`1085`, :issue:`1142`. - - -Bug fixes -~~~~~~~~~ - -* Remove/relax erroneous "meta" path check (**regression**). - By :user:`Gregory R. Lee ` :issue:`1123`. - -* Cast all attribute keys to strings (and issue deprecation warning). - By :user:`Mattia Almansi ` :issue:`1066`. - -* Fix bug in N5 storage that prevented arrays located in the root of the hierarchy from - bearing the `n5` keyword. Along with fixing this bug, new tests were added for N5 routines - that had previously been excluded from testing, and type annotations were added to the N5 codebase. - By :user:`Davis Bennett ` :issue:`1092`. - -* Fix bug in LRUEStoreCache in which the current size wasn't reset on invalidation. - By :user:`BGCMHou ` and :user:`Josh Moore ` :issue:`1076`, :issue:`1077`. - -* Remove erroneous check that disallowed array keys starting with "meta". - By :user:`Gregory R. Lee ` :issue:`1105`. - -Documentation -~~~~~~~~~~~~~ - -* Typo fixes to close quotes. By :user:`Pavithra Eswaramoorthy ` - -* Added copy button to documentation. - By :user:`Altay Sansal ` :issue:`1124`. - -Maintenance -~~~~~~~~~~~ - -* Simplify release docs. - By :user:`Josh Moore ` :issue:`1119`. - -* Pin werkzeug to prevent test hangs. - By :user:`Davis Bennett ` :issue:`1098`. - -* Fix a few DeepSource.io alerts - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1080`. - -* Fix URLs. - By :user:`Dimitri Papadopoulos Orfanos `, :issue:`1074`. - -* Fix spelling. - By :user:`Dimitri Papadopoulos Orfanos `, :issue:`1073`. - -* Update GitHub issue templates with `YAML` format. - By :user:`Saransh Chopra ` :issue:`1079`. - -* Remove option to return None from _ensure_store. - By :user:`Gregory Lee ` :issue:`1068`. - -* Fix a typo of "integers". - By :user:`Richard Scott ` :issue:`1056`. - -.. _release_2.12.0: - -2.12.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* **Add support for reading and writing Zarr V3.** The new `zarr._store.v3` - package has the necessary classes and functions for evaluating Zarr V3. - Since the format is not yet finalized, the classes and functions are not - automatically imported into the regular `zarr` name space. Setting the - `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them. - By :user:`Gregory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` - as well as by :user:`Josh Moore ` :issue:`1032`. - -* **Create FSStore from an existing fsspec filesystem**. If you have created - an fsspec filesystem outside of Zarr, you can now pass it as a keyword - argument to ``FSStore``. - By :user:`Ryan Abernathey `; :issue:`911`. - -* Add numpy encoder class for json.dumps - By :user:`Eric Prestat `; :issue:`933`. - -* Appending performance improvement to Zarr arrays, e.g., when writing to S3. - By :user:`hailiangzhang `; :issue:`1014`. - -* Add number encoder for ``json.dumps`` to support numpy integers in - ``chunks`` arguments. By :user:`Eric Prestat ` :issue:`697`. - -Bug fixes -~~~~~~~~~ - -* Fix bug that made it impossible to create an ``FSStore`` on unlistable filesystems - (e.g. some HTTP servers). - By :user:`Ryan Abernathey `; :issue:`993`. - - -Documentation -~~~~~~~~~~~~~ - -* Update resize doc to clarify surprising behavior. - By :user:`hailiangzhang `; :issue:`1022`. - -Maintenance -~~~~~~~~~~~ - -* Added Pre-commit configuration, incl. Yaml Check. - By :user:`Shivank Chaudhary `; :issue:`1015`, :issue:`1016`. - -* Fix URL to renamed file in Blosc repo. - By :user:`Andrew Thomas ` :issue:`1028`. - -* Activate Py 3.10 builds. - By :user:`Josh Moore ` :issue:`1027`. - -* Make all unignored zarr warnings errors. - By :user:`Josh Moore ` :issue:`1021`. - - -.. _release_2.11.3: - -2.11.3 ------- - -Bug fixes -~~~~~~~~~ - -* Fix missing case to fully revert change to default write_empty_chunks. - By :user:`Tom White `; :issue:`1005`. - - -.. _release_2.11.2: - -2.11.2 ------- - -Bug fixes -~~~~~~~~~ - -* Changes the default value of ``write_empty_chunks`` to ``True`` to prevent - unanticipated data losses when the data types do not have a proper default - value when empty chunks are read back in. - By :user:`Vyas Ramasubramani `; :issue:`965`, :issue:`1001`. - -.. _release_2.11.1: - -2.11.1 ------- - -Bug fixes -~~~~~~~~~ - -* Fix bug where indexing with a scalar numpy value returned a single-value array. - By :user:`Ben Jeffery ` :issue:`967`. - -* Removed `clobber` argument from `normalize_store_arg`. This enables to change - data within an opened consolidated group using mode `"r+"` (i.e region write). - By :user:`Tobias Kölling ` :issue:`975`. - -.. _release_2.11.0: - -2.11.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* **Sparse changes with performance impact!** One of the advantages of the Zarr - format is that it is sparse, which means that chunks with no data (more - precisely, with data equal to the fill value, which is usually 0) don't need - to be written to disk at all. They will simply be assumed to be empty at read - time. However, until this release, the Zarr library would write these empty - chunks to disk anyway. This changes in this version: a small performance - penalty at write time leads to significant speedups at read time and in - filesystem operations in the case of sparse arrays. To revert to the old - behavior, pass the argument ``write_empty_chunks=True`` to the array creation - function. By :user:`Juan Nunez-Iglesias `; :issue:`853` and - :user:`Davis Bennett `; :issue:`738`. - -* **Fancy indexing**. Zarr arrays now support NumPy-style fancy indexing with - arrays of integer coordinates. This is equivalent to using zarr.Array.vindex. - Mixing slices and integer arrays is not supported. - By :user:`Juan Nunez-Iglesias `; :issue:`725`. - -* **New base class**. This release of Zarr Python introduces a new - ``BaseStore`` class that all provided store classes implemented in Zarr - Python now inherit from. This is done as part of refactoring to enable future - support of the Zarr version 3 spec. Existing third-party stores that are a - MutableMapping (e.g. dict) can be converted to a new-style key/value store - inheriting from ``BaseStore`` by passing them as the argument to the new - ``zarr.storage.KVStore`` class. For backwards compatibility, various - higher-level array creation and convenience functions still accept plain - Python dicts or other mutable mappings for the ``store`` argument, but will - internally convert these to a ``KVStore``. - By :user:`Gregory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. - -* Allow to assign array ``fill_values`` and update metadata accordingly. - By :user:`Ryan Abernathey `, :issue:`662`. - -* Allow to update array fill_values - By :user:`Matthias Bussonnier ` :issue:`665`. - -Bug fixes -~~~~~~~~~ - -* Fix bug where the checksum of zipfiles is wrong - By :user:`Oren Watson ` :issue:`930`. - -* Fix consolidate_metadata with FSStore. - By :user:`Joe Hamman ` :issue:`916`. - -* Unguarded next inside generator. - By :user:`Dimitri Papadopoulos Orfanos ` :issue:`889`. - -Documentation -~~~~~~~~~~~~~ - -* Update docs creation of dev env. - By :user:`Ray Bell ` :issue:`921`. - -* Update docs to use ``python -m pytest``. - By :user:`Ray Bell ` :issue:`923`. - -* Fix versionadded tag in zarr.Array docstring. - By :user:`Juan Nunez-Iglesias ` :issue:`852`. - -* Doctest seem to be stricter now, updating tostring() to tobytes(). - By :user:`John Kirkham ` :issue:`907`. - -* Minor doc fix. - By :user:`Mads R. B. Kristensen ` :issue:`937`. - -Maintenance -~~~~~~~~~~~ - -* Upgrade MongoDB in test env. - By :user:`Joe Hamman ` :issue:`939`. - -* Pass dimension_separator on fixture generation. - By :user:`Josh Moore ` :issue:`858`. - -* Activate Python 3.9 in GitHub Actions. - By :user:`Josh Moore ` :issue:`859`. - -* Drop shortcut ``fsspec[s3]`` for dependency. - By :user:`Josh Moore ` :issue:`920`. - -* and a swath of code-linting improvements by :user:`Dimitri Papadopoulos Orfanos `: - - - Unnecessary comprehension (:issue:`899`) - - - Unnecessary ``None`` provided as default (:issue:`900`) - - - use an if ``expression`` instead of `and`/`or` (:issue:`888`) - - - Remove unnecessary literal (:issue:`891`) - - - Decorate a few method with `@staticmethod` (:issue:`885`) - - - Drop unneeded ``return`` (:issue:`884`) - - - Drop explicit ``object`` inheritance from ``class``-es (:issue:`886`) - - - Unnecessary comprehension (:issue:`883`) - - - Codespell configuration (:issue:`882`) - - - Fix typos found by codespell (:issue:`880`) - - - Proper C-style formatting for integer (:issue:`913`) - - - Add LGTM.com / DeepSource.io configuration files (:issue:`909`) - -.. _release_2.10.3: - -2.10.3 ------- - -Bug fixes -~~~~~~~~~ - -* N5 keywords now emit UserWarning instead of raising a ValueError. - By :user:`Boaz Mohar `; :issue:`860`. - -* blocks_to_decompress not used in read_part function. - By :user:`Boaz Mohar `; :issue:`861`. - -* defines blocksize for array, updates hexdigest values. - By :user:`Andrew Fulton `; :issue:`867`. - -* Fix test failure on Debian and conda-forge builds. - By :user:`Josh Moore `; :issue:`871`. - -.. _release_2.10.2: - -2.10.2 ------- - -Bug fixes -~~~~~~~~~ - -* Fix NestedDirectoryStore datasets without dimension_separator metadata. - By :user:`Josh Moore `; :issue:`850`. - -.. _release_2.10.1: - -2.10.1 ------- - -Bug fixes -~~~~~~~~~ - -* Fix regression by setting normalize_keys=False in fsstore constructor. - By :user:`Davis Bennett `; :issue:`842`. - -.. _release_2.10.0: - -2.10.0 ------- - -Enhancements -~~~~~~~~~~~~ - -* Add N5FSStore. - By :user:`Davis Bennett `; :issue:`793`. - -Bug fixes -~~~~~~~~~ - -* Ignore None dim_separators in save_array. - By :user:`Josh Moore `; :issue:`831`. - -.. _release_2.9.5: - -2.9.5 ------ - -Bug fixes -~~~~~~~~~ - -* Fix FSStore.listdir behavior for nested directories. - By :user:`Gregory Lee `; :issue:`802`. - -.. _release_2.9.4: - -2.9.4 ------ - -Bug fixes -~~~~~~~~~ - -* Fix structured arrays that contain objects - By :user: `Attila Bergou `; :issue: `806` - -.. _release_2.9.3: - -2.9.3 ------ - -Maintenance -~~~~~~~~~~~ - -* Mark the fact that some tests that require ``fsspec``, without compromising the code coverage score. - By :user:`Ben Williams `; :issue:`823`. - -* Only inspect alternate node type if desired isn't present. - By :user:`Trevor Manz `; :issue:`696`. - -.. _release_2.9.2: - -2.9.2 ------ - -Maintenance -~~~~~~~~~~~ - -* Correct conda-forge deployment of Zarr by fixing some Zarr tests. - By :user:`Ben Williams `; :issue:`821`. - -.. _release_2.9.1: - -2.9.1 ------ - -Maintenance -~~~~~~~~~~~ - -* Correct conda-forge deployment of Zarr. - By :user:`Josh Moore `; :issue:`819`. - -.. _release_2.9.0: - -2.9.0 ------ - -This release of Zarr Python is the first release of Zarr to not support Python 3.6. - -Enhancements -~~~~~~~~~~~~ - -* Update ABSStore for compatibility with newer `azure.storage.blob`. - By :user:`Tom Augspurger `; :issue:`759`. - -* Pathlib support. - By :user:`Chris Barnes `; :issue:`768`. - -Documentation -~~~~~~~~~~~~~ - -* Clarify that arbitrary key/value pairs are OK for attributes. - By :user:`Stephan Hoyer `; :issue:`751`. - -* Clarify how to manually convert a DirectoryStore to a ZipStore. - By :user:`pmav99 `; :issue:`763`. - -Bug fixes -~~~~~~~~~ - -* Fix dimension_separator support. - By :user:`Josh Moore `; :issue:`775`. - -* Extract ABSStore to zarr._storage.absstore. - By :user:`Josh Moore `; :issue:`781`. - -* avoid NumPy 1.21.0 due to https://github.com/numpy/numpy/issues/19325 - By :user:`Gregory Lee `; :issue:`791`. - -Maintenance -~~~~~~~~~~~ - -* Drop 3.6 builds. - By :user:`Josh Moore `; :issue:`774`, :issue:`778`. - -* Fix build with Sphinx 4. - By :user:`Elliott Sales de Andrade `; :issue:`799`. - -* TST: add missing assert in test_hexdigest. - By :user:`Gregory Lee `; :issue:`801`. - -.. _release_2.8.3: - -2.8.3 ------ - -Bug fixes -~~~~~~~~~ - -* FSStore: default to normalize_keys=False - By :user:`Josh Moore `; :issue:`755`. -* ABSStore: compatibility with ``azure.storage.python>=12`` - By :user:`Tom Augspurger `; :issue:`618` - - -.. _release_2.8.2: - -2.8.2 ------ - -Documentation -~~~~~~~~~~~~~ - -* Add section on rechunking to tutorial - By :user:`David Baddeley `; :issue:`730`. - -Bug fixes -~~~~~~~~~ - -* Expand FSStore tests and fix implementation issues - By :user:`Davis Bennett `; :issue:`709`. - -Maintenance -~~~~~~~~~~~ - -* Updated ipytree warning for jlab3 - By :user:`Ian Hunt-Isaak `; :issue:`721`. - -* b170a48a - (issue-728, copy-nested) Updated ipytree warning for jlab3 (#721) (3 weeks ago) -* Activate dependabot - By :user:`Josh Moore `; :issue:`734`. - -* Update Python classifiers (Zarr is stable!) - By :user:`Josh Moore `; :issue:`731`. - -.. _release_2.8.1: - -2.8.1 ------ - -Bug fixes -~~~~~~~~~ - -* raise an error if create_dataset's dimension_separator is inconsistent - By :user:`Gregory R. Lee `; :issue:`724`. - -.. _release_2.8.0: - -2.8.0 ------ - -V2 Specification Update -~~~~~~~~~~~~~~~~~~~~~~~ - -* Introduce optional dimension_separator .zarray key for nested chunks. - By :user:`Josh Moore `; :issue:`715`, :issue:`716`. - -.. _release_2.7.1: - -2.7.1 ------ - -Bug fixes -~~~~~~~~~ - -* Update Array to respect FSStore's key_separator (#718) - By :user:`Gregory R. Lee `; :issue:`718`. - -.. _release_2.7.0: - -2.7.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* Start stop for iterator (`islice()`) - By :user:`Sebastian Grill `; :issue:`621`. - -* Add capability to partially read and decompress chunks - By :user:`Andrew Fulton `; :issue:`667`. - -Bug fixes -~~~~~~~~~ - -* Make DirectoryStore __setitem__ resilient against antivirus file locking - By :user:`Eric Younkin `; :issue:`698`. - -* Compare test data's content generally - By :user:`John Kirkham `; :issue:`436`. - -* Fix dtype usage in zarr/meta.py - By :user:`Josh Moore `; :issue:`700`. - -* Fix FSStore key_seperator usage - By :user:`Josh Moore `; :issue:`669`. - -* Simplify text handling in DB Store - By :user:`John Kirkham `; :issue:`670`. - -* GitHub Actions migration - By :user:`Matthias Bussonnier `; - :issue:`641`, :issue:`671`, :issue:`674`, :issue:`676`, :issue:`677`, :issue:`678`, - :issue:`679`, :issue:`680`, :issue:`682`, :issue:`684`, :issue:`685`, :issue:`686`, - :issue:`687`, :issue:`695`, :issue:`706`. - -.. _release_2.6.1: - -2.6.1 ------ - -* Minor build fix - By :user:`Matthias Bussonnier `; :issue:`666`. - -.. _release_2.6.0: - -2.6.0 ------ - -This release of Zarr Python is the first release of Zarr to not support Python 3.5. - -* End Python 3.5 support. - By :user:`Chris Barnes `; :issue:`602`. - -* Fix ``open_group/open_array`` to allow opening of read-only store with - ``mode='r'`` :issue:`269` - -* Add `Array` tests for FSStore. - By :user:`Andrew Fulton `; :issue: `644`. - -* fix a bug in which ``attrs`` would not be copied on the root when using ``copy_all``; :issue:`613` - -* Fix ``FileNotFoundError`` with dask/s3fs :issue:`649` - -* Fix flaky fixture in test_storage.py :issue:`652` - -* Fix FSStore getitems fails with arrays that have a 0 length shape dimension :issue:`644` - -* Use async to fetch/write result concurrently when possible. :issue:`536`, See `this comment - `_ for some performance analysis - showing order of magnitude faster response in some benchmark. - -See `this link `_ -for the full list of closed and merged PR tagged with the 2.6 milestone. - -* Add ability to partially read and decompress arrays, see :issue:`667`. It is - only available to chunks stored using fsspec and using Blosc as a compressor. - - For certain analysis case when only a small portion of chunks is needed it can - be advantageous to only access and decompress part of the chunks. Doing - partial read and decompression add high latency to many of the operation so - should be used only when the subset of the data is small compared to the full - chunks and is stored contiguously (that is to say either last dimensions for C - layout, firsts for F). Pass ``partial_decompress=True`` as argument when - creating an ``Array``, or when using ``open_array``. No option exists yet to - apply partial read and decompress on a per-operation basis. - -.. _release_2.5.0: - -2.5.0 ------ - -This release will be the last to support Python 3.5, next version of Zarr will be Python 3.6+. - -* `DirectoryStore` now uses `os.scandir`, which should make listing large store - faster, :issue:`563` - -* Remove a few remaining Python 2-isms. - By :user:`Poruri Sai Rahul `; :issue:`393`. - -* Fix minor bug in `N5Store`. - By :user:`gsakkis`, :issue:`550`. - -* Improve error message in Jupyter when trying to use the ``ipytree`` widget - without ``ipytree`` installed. - By :user:`Zain Patel `; :issue:`537` - -* Add typing information to many of the core functions :issue:`589` - -* Explicitly close stores during testing. - By :user:`Elliott Sales de Andrade `; :issue:`442` - -* Many of the convenience functions to emit errors (``err_*`` from - ``zarr.errors`` have been replaced by ``ValueError`` subclasses. The corresponding - ``err_*`` function have been removed. :issue:`590`, :issue:`614`) - -* Improve consistency of terminology regarding arrays and datasets in the - documentation. - By :user:`Josh Moore `; :issue:`571`. - -* Added support for generic URL opening by ``fsspec``, where the URLs have the - form "protocol://[server]/path" or can be chained URls with "::" separators. - The additional argument ``storage_options`` is passed to the backend, see - the ``fsspec`` docs. - By :user:`Martin Durant `; :issue:`546` - -* Added support for fetching multiple items via ``getitems`` method of a - store, if it exists. This allows for concurrent fetching of data blocks - from stores that implement this; presently HTTP, S3, GCS. Currently only - applies to reading. - By :user:`Martin Durant `; :issue:`606` - -* Efficient iteration expanded with option to pass start and stop index via - ``array.islice``. - By :user:`Sebastian Grill `, :issue:`615`. - -.. _release_2.4.0: - -2.4.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* Add key normalization option for ``DirectoryStore``, ``NestedDirectoryStore``, - ``TempStore``, and ``N5Store``. - By :user:`James Bourbeau `; :issue:`459`. - -* Add ``recurse`` keyword to ``Group.array_keys`` and ``Group.arrays`` methods. - By :user:`James Bourbeau `; :issue:`458`. - -* Use uniform chunking for all dimensions when specifying ``chunks`` as an integer. - Also adds support for specifying ``-1`` to chunk across an entire dimension. - By :user:`James Bourbeau `; :issue:`456`. - -* Rename ``DictStore`` to ``MemoryStore``. - By :user:`James Bourbeau `; :issue:`455`. - -* Rewrite ``.tree()`` pretty representation to use ``ipytree``. - Allows it to work in both the Jupyter Notebook and JupyterLab. - By :user:`John Kirkham `; :issue:`450`. - -* Do not rename Blosc parameters in n5 backend and add `blocksize` parameter, - compatible with n5-blosc. By :user:`axtimwalde`, :issue:`485`. - -* Update ``DirectoryStore`` to create files with more permissive permissions. - By :user:`Eduardo Gonzalez ` and :user:`James Bourbeau `; :issue:`493` - -* Use ``math.ceil`` for scalars. - By :user:`John Kirkham `; :issue:`500`. - -* Ensure contiguous data using ``astype``. - By :user:`John Kirkham `; :issue:`513`. - -* Refactor out ``_tofile``/``_fromfile`` from ``DirectoryStore``. - By :user:`John Kirkham `; :issue:`503`. - -* Add ``__enter__``/``__exit__`` methods to ``Group`` for ``h5py.File`` compatibility. - By :user:`Chris Barnes `; :issue:`509`. - -Bug fixes -~~~~~~~~~ - -* Fix Sqlite Store Wrong Modification. - By :user:`Tommy Tran `; :issue:`440`. - -* Add intermediate step (using ``zipfile.ZipInfo`` object) to write - inside ``ZipStore`` to solve too restrictive permission issue. - By :user:`Raphael Dussin `; :issue:`505`. - -* Fix '/' prepend bug in ``ABSStore``. - By :user:`Shikhar Goenka `; :issue:`525`. - -Documentation -~~~~~~~~~~~~~ -* Fix hyperlink in ``README.md``. - By :user:`Anderson Banihirwe `; :issue:`531`. - -* Replace "nuimber" with "number". - By :user:`John Kirkham `; :issue:`512`. - -* Fix azure link rendering in tutorial. - By :user:`James Bourbeau `; :issue:`507`. - -* Update ``README`` file to be more detailed. - By :user:`Zain Patel `; :issue:`495`. - -* Import blosc from numcodecs in tutorial. - By :user:`James Bourbeau `; :issue:`491`. - -* Adds logo to docs. - By :user:`James Bourbeau `; :issue:`462`. - -* Fix N5 link in tutorial. - By :user:`James Bourbeau `; :issue:`480`. - -* Fix typo in code snippet. - By :user:`Joe Jevnik `; :issue:`461`. - -* Fix URLs to point to zarr-python - By :user:`John Kirkham `; :issue:`453`. - -Maintenance -~~~~~~~~~~~ - -* Add documentation build to CI. - By :user:`James Bourbeau `; :issue:`516`. - -* Use ``ensure_ndarray`` in a few more places. - By :user:`John Kirkham `; :issue:`506`. - -* Support Python 3.8. - By :user:`John Kirkham `; :issue:`499`. - -* Require Numcodecs 0.6.4+ to use text handling functionality from it. - By :user:`John Kirkham `; :issue:`497`. - -* Updates tests to use ``pytest.importorskip``. - By :user:`James Bourbeau `; :issue:`492` - -* Removed support for Python 2. - By :user:`jhamman`; :issue:`393`, :issue:`470`. - -* Upgrade dependencies in the test matrices and resolve a - compatibility issue with testing against the Azure Storage - Emulator. By :user:`alimanfoo`; :issue:`468`, :issue:`467`. - -* Use ``unittest.mock`` on Python 3. - By :user:`Elliott Sales de Andrade `; :issue:`426`. - -* Drop ``decode`` from ``ConsolidatedMetadataStore``. - By :user:`John Kirkham `; :issue:`452`. - - -.. _release_2.3.2: - -2.3.2 ------ - -Enhancements -~~~~~~~~~~~~ - -* Use ``scandir`` in ``DirectoryStore``'s ``getsize`` method. - By :user:`John Kirkham `; :issue:`431`. - -Bug fixes -~~~~~~~~~ - -* Add and use utility functions to simplify reading and writing JSON. - By :user:`John Kirkham `; :issue:`429`, :issue:`430`. - -* Fix ``collections``'s ``DeprecationWarning``\ s. - By :user:`John Kirkham `; :issue:`432`. - -* Fix tests on big endian machines. - By :user:`Elliott Sales de Andrade `; :issue:`427`. - - -.. _release_2.3.1: - -2.3.1 ------ - -Bug fixes -~~~~~~~~~ - -* Makes ``azure-storage-blob`` optional for testing. - By :user:`John Kirkham `; :issue:`419`, :issue:`420`. - - -.. _release_2.3.0: - -2.3.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* New storage backend, backed by Azure Blob Storage, class :class:`zarr.storage.ABSStore`. - All data is stored as block blobs. By :user:`Shikhar Goenka `, - :user:`Tim Crone ` and :user:`Zain Patel `; :issue:`345`. - -* Add "consolidated" metadata as an experimental feature: use - :func:`zarr.convenience.consolidate_metadata` to copy all metadata from the various - metadata keys within a dataset hierarchy under a single key, and - :func:`zarr.convenience.open_consolidated` to use this single key. This can greatly - cut down the number of calls to the storage backend, and so remove a lot of overhead - for reading remote data. - By :user:`Martin Durant `, :user:`Alistair Miles `, - :user:`Ryan Abernathey `, :issue:`268`, :issue:`332`, :issue:`338`. - -* Support has been added for structured arrays with sub-array shape and/or nested fields. By - :user:`Tarik Onalan `, :issue:`111`, :issue:`296`. - -* Adds the SQLite-backed :class:`zarr.storage.SQLiteStore` class enabling an - SQLite database to be used as the backing store for an array or group. - By :user:`John Kirkham `, :issue:`368`, :issue:`365`. - -* Efficient iteration over arrays by decompressing chunkwise. - By :user:`Jerome Kelleher `, :issue:`398`, :issue:`399`. - -* Adds the Redis-backed :class:`zarr.storage.RedisStore` class enabling a - Redis database to be used as the backing store for an array or group. - By :user:`Joe Hamman `, :issue:`299`, :issue:`372`. - -* Adds the MongoDB-backed :class:`zarr.storage.MongoDBStore` class enabling a - MongoDB database to be used as the backing store for an array or group. - By :user:`Noah D Brenowitz `, :user:`Joe Hamman `, - :issue:`299`, :issue:`372`, :issue:`401`. - -* **New storage class for N5 containers**. The :class:`zarr.n5.N5Store` has been - added, which uses :class:`zarr.storage.NestedDirectoryStore` to support - reading and writing from and to N5 containers. - By :user:`Jan Funke ` and :user:`John Kirkham `. - -Bug fixes -~~~~~~~~~ - -* The implementation of the :class:`zarr.storage.DirectoryStore` class has been modified to - ensure that writes are atomic and there are no race conditions where a chunk might appear - transiently missing during a write operation. By :user:`sbalmer `, :issue:`327`, - :issue:`263`. - -* Avoid raising in :class:`zarr.storage.DirectoryStore`'s ``__setitem__`` when file already exists. - By :user:`Justin Swaney `, :issue:`272`, :issue:`318`. - -* The required version of the `Numcodecs`_ package has been upgraded - to 0.6.2, which has enabled some code simplification and fixes a failing test involving - msgpack encoding. By :user:`John Kirkham `, :issue:`361`, :issue:`360`, :issue:`352`, - :issue:`355`, :issue:`324`. - -* Failing tests related to pickling/unpickling have been fixed. By :user:`Ryan Williams `, - :issue:`273`, :issue:`308`. - -* Corrects handling of ``NaT`` in ``datetime64`` and ``timedelta64`` in various - compressors (by :user:`John Kirkham `; :issue:`344`). - -* Ensure ``DictStore`` contains only ``bytes`` to facilitate comparisons and protect against writes. - By :user:`John Kirkham `, :issue:`350`. - -* Test and fix an issue (w.r.t. fill values) when storing complex data to ``Array``. - By :user:`John Kirkham `, :issue:`363`. - -* Always use a ``tuple`` when indexing a NumPy ``ndarray``. - By :user:`John Kirkham `, :issue:`376`. - -* Ensure when ``Array`` uses a ``dict``-based chunk store that it only contains - ``bytes`` to facilitate comparisons and protect against writes. Drop the copy - for the no filter/compressor case as this handles that case. - By :user:`John Kirkham `, :issue:`359`. - -Maintenance -~~~~~~~~~~~ - -* Simplify directory creation and removal in ``DirectoryStore.rename``. - By :user:`John Kirkham `, :issue:`249`. - -* CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and - upgrade all pinned package requirements. :user:`Alistair Miles `, :issue:`308`. - -* Start using pyup.io to maintain dependencies. - :user:`Alistair Miles `, :issue:`326`. - -* Configure flake8 line limit generally. - :user:`John Kirkham `, :issue:`335`. - -* Add missing coverage pragmas. - :user:`John Kirkham `, :issue:`343`, :issue:`355`. - -* Fix missing backslash in docs. - :user:`John Kirkham `, :issue:`254`, :issue:`353`. - -* Include tests for stores' ``popitem`` and ``pop`` methods. - By :user:`John Kirkham `, :issue:`378`, :issue:`380`. - -* Include tests for different compressors, endianness, and attributes. - By :user:`John Kirkham `, :issue:`378`, :issue:`380`. - -* Test validity of stores' contents. - By :user:`John Kirkham `, :issue:`359`, :issue:`408`. - - -.. _release_2.2.0: - -2.2.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* **Advanced indexing**. The ``Array`` class has several new methods and - properties that enable a selection of items in an array to be retrieved or - updated. See the :ref:`user-guide-indexing` tutorial section for more - information. There is also a `notebook - `_ - with extended examples and performance benchmarks. :issue:`78`, :issue:`89`, - :issue:`112`, :issue:`172`. - -* **New package for compressor and filter codecs**. The classes previously - defined in the :mod:`zarr.codecs` module have been factored out into a - separate package called `Numcodecs`_. The `Numcodecs`_ package also includes - several new codec classes not previously available in Zarr, including - compressor codecs for Zstd and LZ4. This change is backwards-compatible with - existing code, as all codec classes defined by Numcodecs are imported into the - :mod:`zarr.codecs` namespace. However, it is recommended to import codecs from - the new package, see the tutorial sections on :ref:`user-guide-compress` and - :ref:`user-guide-filters` for examples. With contributions by - :user:`John Kirkham `; :issue:`74`, :issue:`102`, :issue:`120`, - :issue:`123`, :issue:`139`. - -* **New storage class for DBM-style databases**. The - :class:`zarr.storage.DBMStore` class enables any DBM-style database such as gdbm, - ndbm or Berkeley DB, to be used as the backing store for an array or group. See the - tutorial section on :ref:`user-guide-storage` for some examples. :issue:`133`, - :issue:`186`. - -* **New storage class for LMDB databases**. The :class:`zarr.storage.LMDBStore` class - enables an LMDB "Lightning" database to be used as the backing store for an array or - group. :issue:`192`. - -* **New storage class using a nested directory structure for chunk files**. The - :class:`zarr.storage.NestedDirectoryStore` has been added, which is similar to - the existing :class:`zarr.storage.DirectoryStore` class but nests chunk files - for multidimensional arrays into sub-directories. :issue:`155`, :issue:`177`. - -* **New tree() method for printing hierarchies**. The ``Group`` class has a new - :func:`zarr.hierarchy.Group.tree` method which enables a tree representation of - a group hierarchy to be printed. Also provides an interactive tree - representation when used within a Jupyter notebook. See the - :ref:`user-guide-diagnostics` tutorial section for examples. By - :user:`John Kirkham `; :issue:`82`, :issue:`140`, :issue:`184`. - -* **Visitor API**. The ``Group`` class now implements the h5py visitor API, see - docs for the :func:`zarr.hierarchy.Group.visit`, - :func:`zarr.hierarchy.Group.visititems` and - :func:`zarr.hierarchy.Group.visitvalues` methods. By - :user:`John Kirkham `, :issue:`92`, :issue:`122`. - -* **Viewing an array as a different dtype**. The ``Array`` class has a new - :func:`zarr.Array.astype` method, which is a convenience that enables an - array to be viewed as a different dtype. By :user:`John Kirkham `, - :issue:`94`, :issue:`96`. - -* **New open(), save(), load() convenience functions**. The function - :func:`zarr.convenience.open` provides a convenient way to open a persistent - array or group, using either a ``DirectoryStore`` or ``ZipStore`` as the backing - store. The functions :func:`zarr.convenience.save` and - :func:`zarr.convenience.load` are also available and provide a convenient way to - save an entire NumPy array to disk and load back into memory later. See the - tutorial section :ref:`user-guide-persist` for examples. :issue:`104`, - :issue:`105`, :issue:`141`, :issue:`181`. - -* **IPython completions**. The ``Group`` class now implements ``__dir__()`` and - ``_ipython_key_completions_()`` which enables tab-completion for group members - to be used in any IPython interactive environment. :issue:`170`. - -* **New info property; changes to __repr__**. The ``Group`` and - ``Array`` classes have a new ``info`` property which can be used to print - diagnostic information, including compression ratio where available. See the - tutorial section on :ref:`user-guide-diagnostics` for examples. The string - representation (``__repr__``) of these classes has been simplified to ensure - it is cheap and quick to compute in all circumstances. :issue:`83`, - :issue:`115`, :issue:`132`, :issue:`148`. - -* **Chunk options**. When creating an array, ``chunks=False`` can be specified, - which will result in an array with a single chunk only. Alternatively, - ``chunks=True`` will trigger an automatic chunk shape guess. See - :ref:`user-guide-chunks` for more on the ``chunks`` parameter. :issue:`106`, - :issue:`107`, :issue:`183`. - -* **Zero-dimensional arrays** and are now supported; by - :user:`Prakhar Goel `, :issue:`154`, :issue:`161`. - -* **Arrays with one or more zero-length dimensions** are now fully supported; by - :user:`Prakhar Goel `, :issue:`150`, :issue:`154`, :issue:`160`. - -* **The .zattrs key is now optional** and will now only be created when the first - custom attribute is set; :issue:`121`, :issue:`200`. - -* **New Group.move() method** supports moving a sub-group or array to a different - location within the same hierarchy. By :user:`John Kirkham `, - :issue:`191`, :issue:`193`, :issue:`196`. - -* **ZipStore is now thread-safe**; :issue:`194`, :issue:`192`. - -* **New Array.hexdigest() method** computes an ``Array``'s hash with ``hashlib``. - By :user:`John Kirkham `, :issue:`98`, :issue:`203`. - -* **Improved support for object arrays**. In previous versions of Zarr, - creating an array with ``dtype=object`` was possible but could under certain - circumstances lead to unexpected errors and/or segmentation faults. To make it easier - to properly configure an object array, a new ``object_codec`` parameter has been - added to array creation functions. See the tutorial section on :ref:`user-guide-objects` - for more information and examples. Also, runtime checks have been added in both Zarr - and Numcodecs so that segmentation faults are no longer possible, even with a badly - configured array. This API change is backwards compatible and previous code that created - an object array and provided an object codec via the ``filters`` parameter will - continue to work, however a warning will be raised to encourage use of the - ``object_codec`` parameter. :issue:`208`, :issue:`212`. - -* **Added support for datetime64 and timedelta64 data types**; - :issue:`85`, :issue:`215`. - -* **Array and group attributes are now cached by default** to improve performance with - slow stores, e.g., stores accessing data via the network; :issue:`220`, :issue:`218`, - :issue:`204`. - -* **New LRUStoreCache class**. The class :class:`zarr.storage.LRUStoreCache` has been - added and provides a means to locally cache data in memory from a store that may be - slow, e.g., a store that retrieves data from a remote server via the network; - :issue:`223`. - -* **New copy functions**. The new functions :func:`zarr.convenience.copy` and - :func:`zarr.convenience.copy_all` provide a way to copy groups and/or arrays - between HDF5 and Zarr, or between two Zarr groups. The - :func:`zarr.convenience.copy_store` provides a more efficient way to copy - data directly between two Zarr stores. :issue:`87`, :issue:`113`, - :issue:`137`, :issue:`217`. - -Bug fixes -~~~~~~~~~ - -* Fixed bug where ``read_only`` keyword argument was ignored when creating an - array; :issue:`151`, :issue:`179`. - -* Fixed bugs when using a ``ZipStore`` opened in 'w' mode; :issue:`158`, - :issue:`182`. - -* Fill values can now be provided for fixed-length string arrays; :issue:`165`, - :issue:`176`. - -* Fixed a bug where the number of chunks initialized could be counted - incorrectly; :issue:`97`, :issue:`174`. - -* Fixed a bug related to the use of an ellipsis (...) in indexing statements; - :issue:`93`, :issue:`168`, :issue:`172`. - -* Fixed a bug preventing use of other integer types for indexing; :issue:`143`, - :issue:`147`. - -Documentation -~~~~~~~~~~~~~ - -* Some changes have been made to the Zarr Specification v2 document to clarify - ambiguities and add some missing information. These changes do not break compatibility - with any of the material as previously implemented, and so the changes have been made - in-place in the document without incrementing the document version number. See the - section on changes in the specification document for more information. -* A new :ref:`user-guide-indexing` section has been added to the tutorial. -* A new :ref:`user-guide-strings` section has been added to the tutorial - (:issue:`135`, :issue:`175`). -* The :ref:`user-guide-chunks` tutorial section has been reorganised and updated. -* The :ref:`user-guide-persist` and :ref:`user-guide-storage` tutorial sections have - been updated with new examples (:issue:`100`, :issue:`101`, :issue:`103`). -* A new tutorial section on :ref:`user-guide-pickle` has been added (:issue:`91`). -* A new tutorial section on :ref:`user-guide-datetime` has been added. -* A new tutorial section on :ref:`user-guide-diagnostics` has been added. -* The tutorial sections on :ref:`user-guide-sync` and :ref:`user-guide-tips-blosc` have been - updated to provide information about how to avoid program hangs when using the Blosc - compressor with multiple processes (:issue:`199`, :issue:`201`). - -Maintenance -~~~~~~~~~~~ - -* A data fixture has been included in the test suite to ensure data format - compatibility is maintained; :issue:`83`, :issue:`146`. -* The test suite has been migrated from nosetests to pytest; :issue:`189`, :issue:`225`. -* Various continuous integration updates and improvements; :issue:`118`, :issue:`124`, - :issue:`125`, :issue:`126`, :issue:`109`, :issue:`114`, :issue:`171`. -* Bump numcodecs dependency to 0.5.3, completely remove nose dependency, :issue:`237`. -* Fix compatibility issues with NumPy 1.14 regarding fill values for structured arrays, - :issue:`222`, :issue:`238`, :issue:`239`. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Code was contributed to this release by :user:`Alistair Miles `, :user:`John -Kirkham ` and :user:`Prakhar Goel `. - -Documentation was contributed to this release by :user:`Mamy Ratsimbazafy ` -and :user:`Charles Noyes `. - -Thank you to :user:`John Kirkham `, :user:`Stephan Hoyer `, -:user:`Francesc Alted `, and :user:`Matthew Rocklin ` for code -reviews and/or comments on pull requests. - -.. _release_2.1.4: - -2.1.4 ------ - -* Resolved an issue where calling ``hasattr`` on a ``Group`` object erroneously - returned a ``KeyError``. By :user:`Vincent Schut `; :issue:`88`, - :issue:`95`. - -.. _release_2.1.3: - -2.1.3 ------ - -* Resolved an issue with :func:`zarr.creation.array` where dtype was given as - None (:issue:`80`). - -.. _release_2.1.2: - -2.1.2 ------ - -* Resolved an issue when no compression is used and chunks are stored in memory - (:issue:`79`). - -.. _release_2.1.1: - -2.1.1 ------ - -Various minor improvements, including: ``Group`` objects support member access -via dot notation (``__getattr__``); fixed metadata caching for ``Array.shape`` -property and derivatives; added ``Array.ndim`` property; fixed -``Array.__array__`` method arguments; fixed bug in pickling ``Array`` state; -fixed bug in pickling ``ThreadSynchronizer``. - -.. _release_2.1.0: - -2.1.0 ------ - -* Group objects now support member deletion via ``del`` statement - (:issue:`65`). -* Added :class:`zarr.storage.TempStore` class for convenience to provide - storage via a temporary directory - (:issue:`59`). -* Fixed performance issues with :class:`zarr.storage.ZipStore` class - (:issue:`66`). -* The Blosc extension has been modified to return bytes instead of array - objects from compress and decompress function calls. This should - improve compatibility and also provides a small performance increase for - compressing high compression ratio data - (:issue:`55`). -* Added ``overwrite`` keyword argument to array and group creation methods - on the :class:`zarr.hierarchy.Group` class - (:issue:`71`). -* Added ``cache_metadata`` keyword argument to array creation methods. -* The functions :func:`zarr.creation.open_array` and - :func:`zarr.hierarchy.open_group` now accept any store as first argument - (:issue:`56`). - -.. _release_2.0.1: - -2.0.1 ------ - -The bundled Blosc library has been upgraded to version 1.11.1. - -.. _release_2.0.0: - -2.0.0 ------ - -Hierarchies -~~~~~~~~~~~ - -Support has been added for organizing arrays into hierarchies via groups. See -the tutorial section on :ref:`user-guide-groups` and the :mod:`zarr.hierarchy` -API docs for more information. - -Filters -~~~~~~~ - -Support has been added for configuring filters to preprocess chunk data prior -to compression. See the tutorial section on :ref:`user-guide-filters` and the -:mod:`zarr.codecs` API docs for more information. - -Other changes -~~~~~~~~~~~~~ - -To accommodate support for hierarchies and filters, the Zarr metadata format -has been modified. See the ``spec_v2`` for more information. To migrate an -array stored using Zarr version 1.x, use the :func:`zarr.storage.migrate_1to2` -function. - -The bundled Blosc library has been upgraded to version 1.11.0. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Thanks to :user:`Matthew Rocklin `, :user:`Stephan Hoyer ` and -:user:`Francesc Alted ` for contributions and comments. - -.. _release_1.1.0: - -1.1.0 ------ - -* The bundled Blosc library has been upgraded to version 1.10.0. The 'zstd' - internal compression library is now available within Blosc. See the tutorial - section on :ref:`user-guide-compress` for an example. -* When using the Blosc compressor, the default internal compression library - is now 'lz4'. -* The default number of internal threads for the Blosc compressor has been - increased to a maximum of 8 (previously 4). -* Added convenience functions :func:`zarr.blosc.list_compressors` and - :func:`zarr.blosc.get_nthreads`. - -.. _release_1.0.0: - -1.0.0 ------ - -This release includes a complete re-organization of the code base. The -major version number has been bumped to indicate that there have been -backwards-incompatible changes to the API and the on-disk storage -format. However, Zarr is still in an early stage of development, so -please do not take the version number as an indicator of maturity. - -Storage -~~~~~~~ - -The main motivation for re-organizing the code was to create an -abstraction layer between the core array logic and data storage (:issue:`21`). -In this release, any -object that implements the ``MutableMapping`` interface can be used as -an array store. See the tutorial sections on :ref:`user-guide-persist` -and :ref:`user-guide-storage`, the ``spec_v1``, and the -:mod:`zarr.storage` module documentation for more information. - -Please note also that the file organization and file name conventions -used when storing a Zarr array in a directory on the file system have -changed. Persistent Zarr arrays created using previous versions of the -software will not be compatible with this version. See the -:mod:`zarr.storage` API docs and the ``spec_v1`` for more -information. - -Compression -~~~~~~~~~~~ - -An abstraction layer has also been created between the core array -logic and the code for compressing and decompressing array -chunks. This release still bundles the c-blosc library and uses Blosc -as the default compressor, however other compressors including zlib, -BZ2 and LZMA are also now supported via the Python standard -library. New compressors can also be dynamically registered for use -with Zarr. See the tutorial sections on :ref:`user-guide-compress` and -:ref:`user-guide-tips-blosc`, the ``spec_v1``, and the -:mod:`zarr.compressors` module documentation for more information. - -Synchronization -~~~~~~~~~~~~~~~ - -The synchronization code has also been refactored to create a layer of -abstraction, enabling Zarr arrays to be used in parallel computations -with a number of alternative synchronization methods. For more -information see the tutorial section on :ref:`user-guide-sync` and the -:mod:`zarr.sync` module documentation. - -Changes to the Blosc extension -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -NumPy is no longer a build dependency for the :mod:`zarr.blosc` Cython -extension, so setup.py will run even if NumPy is not already -installed, and should automatically install NumPy as a runtime -dependency. Manual installation of NumPy prior to installing Zarr is -still recommended, however, as the automatic installation of NumPy may -fail or be sub-optimal on some platforms. - -Some optimizations have been made within the :mod:`zarr.blosc` -extension to avoid unnecessary memory copies, giving a ~10-20% -performance improvement for multi-threaded compression operations. - -The :mod:`zarr.blosc` extension now automatically detects whether it -is running within a single-threaded or multi-threaded program and -adapts its internal behaviour accordingly (:issue:`27`). There is no need for -the user to make any API calls to switch Blosc between contextual and -non-contextual (global lock) mode. See also the tutorial section on -:ref:`user-guide-tips-blosc`. - -Other changes -~~~~~~~~~~~~~ - -The internal code for managing chunks has been rewritten to be more -efficient. Now no state is maintained for chunks outside of the array -store, meaning that chunks do not carry any extra memory overhead not -accounted for by the store. This negates the need for the "lazy" -option present in the previous release, and this has been removed. - -The memory layout within chunks can now be set as either "C" -(row-major) or "F" (column-major), which can help to provide better -compression for some data (:issue:`7`). See the tutorial -section on :ref:`user-guide-chunks-order` for more information. - -A bug has been fixed within the ``__getitem__`` and ``__setitem__`` -machinery for slicing arrays, to properly handle getting and setting -partial slices. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Thanks to :user:`Matthew Rocklin `, :user:`Stephan Hoyer `, -:user:`Francesc Alted `, :user:`Anthony Scopatz ` and -:user:`Martin Durant ` for contributions and comments. - -.. _release_0.4.0: - -0.4.0 ------ - -See `v0.4.0 release notes on GitHub -`_. - -.. _release_0.3.0: - -0.3.0 ------ - -See `v0.3.0 release notes on GitHub -`_. - -.. _Numcodecs: https://numcodecs.readthedocs.io/ diff --git a/docs/index.rst b/docs/index.rst index 0dcfd7f90f..83d427e290 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,8 +11,8 @@ Zarr-Python quickstart user-guide/index API reference + release-notes developers/index - developers/release about **Version**: |version| @@ -20,7 +20,7 @@ Zarr-Python **Useful links**: `Source Repository `_ | `Issue Tracker `_ | -`Zulip Chat `_ | +`Developer Chat `_ | `Zarr specifications `_ Zarr-Python is a Python library for reading and writing Zarr groups and arrays. Highlights include: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 2d0e8ecef8..66bdae2a2e 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -74,7 +74,7 @@ Zarr supports data compression and filters. For example, to use Blosc compressio ... "data/example-3.zarr", ... mode="w", shape=(100, 100), ... chunks=(10, 10), dtype="f4", - ... compressor=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.SHUFFLE) + ... compressors=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle) ... ) >>> z[:, :] = np.random.random((100, 100)) >>> @@ -101,7 +101,7 @@ Zarr allows you to create hierarchical groups, similar to directories:: >>> root = zarr.group("data/example-2.zarr") >>> foo = root.create_group(name="foo") >>> bar = root.create_array( - ... name="bar", shape=(100, 10), chunks=(10, 10) + ... name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" ... ) >>> spam = foo.create_array(name="spam", shape=(10,), dtype="i4") >>> @@ -112,12 +112,35 @@ Zarr allows you to create hierarchical groups, similar to directories:: >>> # print the hierarchy >>> root.tree() / + ├── bar (100, 10) float32 └── foo └── spam (10,) int32 This creates a group with two datasets: ``foo`` and ``bar``. +Batch Hierarchy Creation +~~~~~~~~~~~~~~~~~~~~~~~~ + +Zarr provides tools for creating a collection of arrays and groups with a single function call. +Suppose we want to copy existing groups and arrays into a new storage backend: + + >>> # Create nested groups and add arrays + >>> root = zarr.group("data/example-3.zarr", attributes={'name': 'root'}) + >>> foo = root.create_group(name="foo") + >>> bar = root.create_array( + ... name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" + ... ) + >>> nodes = {'': root.metadata} | {k: v.metadata for k,v in root.members()} + >>> print(nodes) + >>> from zarr.storage import MemoryStore + >>> new_nodes = dict(zarr.create_hierarchy(store=MemoryStore(), nodes=nodes)) + >>> new_root = new_nodes[''] + >>> assert new_root.attrs == root.attrs + +Note that :func:`zarr.create_hierarchy` will only initialize arrays and groups -- copying array data must +be done in a separate step. + Persistent Storage ------------------ @@ -130,7 +153,7 @@ using external libraries like `s3fs `_ or >>> import s3fs # doctest: +SKIP >>> - >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10)) # doctest: +SKIP + >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10), dtype="f4") # doctest: +SKIP >>> z[:, :] = np.random.random((100, 100)) # doctest: +SKIP A single-file store can also be created using the the :class:`zarr.storage.ZipStore`:: diff --git a/docs/release-notes.rst b/docs/release-notes.rst new file mode 100644 index 0000000000..a89046dd6d --- /dev/null +++ b/docs/release-notes.rst @@ -0,0 +1,269 @@ +Release notes +============= + +.. towncrier release notes start + +3.0.8 (2025-05-19) +------------------ + +.. warning:: + + In versions 3.0.0 to 3.0.7 opening arrays or groups with ``mode='a'`` (the default for many builtin functions) + would cause any existing paths in the store to be deleted. This is fixed in 3.0.8, and + we recommend all users upgrade to avoid this bug that could cause unintentional data loss. + +Features +~~~~~~~~ + +- Added a `print_debug_info` function for bug reports. (:issue:`2913`) + + +Bugfixes +~~~~~~~~ + +- Fix a bug that prevented the number of initialized chunks being counted properly. (:issue:`2862`) +- Fixed sharding with GPU buffers. (:issue:`2978`) +- Fix structured `dtype` fill value serialization for consolidated metadata (:issue:`2998`) +- It is now possible to specify no compressor when creating a zarr format 2 array. + This can be done by passing ``compressor=None`` to the various array creation routines. + + The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. + To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. (:issue:`3039`) +- Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. (:issue:`3045`) +- Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. (:issue:`3062`) +- Internally use `typesize` constructor parameter for :class:`numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. (:issue:`2962`) +- Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed. (:issue:`2950`) + + +Misc +~~~~ + +- :issue:`2972`, :issue:`3027`, :issue:`3049` + + +3.0.7 (2025-04-22) +------------------ + +Features +~~~~~~~~ + +- Add experimental ObjectStore storage class based on obstore. (:issue:`1661`) +- Add ``zarr.from_array`` using concurrent streaming of source data (:issue:`2622`) + + +Bugfixes +~~~~~~~~ + +- 0-dimensional arrays are now returning a scalar. Therefore, the return type of ``__getitem__`` changed + to NDArrayLikeOrScalar. This change is to make the behavior of 0-dimensional arrays consistent with + ``numpy`` scalars. (:issue:`2718`) +- Fix `fill_value` serialization for `NaN` in `ArrayV2Metadata` and add property-based testing of round-trip serialization (:issue:`2802`) +- Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be + consistent with the behavior of `ArrayMetadata`. (:issue:`2996`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Updated the 3.0 migration guide to include the removal of "." syntax for getting group members. (:issue:`2991`, :issue:`2997`) + + +Misc +~~~~ +- Define a new versioning policy based on Effective Effort Versioning. This replaces the old Semantic + Versioning-based policy. (:issue:`2924`, :issue:`2910`) +- Make warning filters in the tests more specific, so warnings emitted by tests added in the future + are more likely to be caught instead of ignored. (:issue:`2714`) +- Avoid an unnecessary memory copy when writing Zarr to a local file (:issue:`2944`) + + +3.0.6 (2025-03-20) +------------------ + +Bugfixes +~~~~~~~~ + +- Restore functionality of `del z.attrs['key']` to actually delete the key. (:issue:`2908`) + + +3.0.5 (2025-03-07) +------------------ + +Bugfixes +~~~~~~~~ + +- Fixed a bug where ``StorePath`` creation would not apply standard path normalization to the ``path`` parameter, + which led to the creation of arrays and groups with invalid keys. (:issue:`2850`) +- Prevent update_attributes calls from deleting old attributes (:issue:`2870`) + + +Misc +~~~~ + +- :issue:`2796` + +3.0.4 (2025-02-23) +------------------ + +Features +~~~~~~~~ + +- Adds functions for concurrently creating multiple arrays and groups. (:issue:`2665`) + +Bugfixes +~~~~~~~~ + +- Fixed a bug where ``ArrayV2Metadata`` could save ``filters`` as an empty array. (:issue:`2847`) +- Fix a bug when setting values of a smaller last chunk. (:issue:`2851`) + +Misc +~~~~ + +- :issue:`2828` + + +3.0.3 (2025-02-14) +------------------ + +Features +~~~~~~~~ + +- Improves performance of FsspecStore.delete_dir for remote filesystems supporting concurrent/batched deletes, e.g., s3fs. (:issue:`2661`) +- Added :meth:`zarr.config.enable_gpu` to update Zarr's configuration to use GPUs. (:issue:`2751`) +- Avoid reading chunks during writes where possible. :issue:`757` (:issue:`2784`) +- :py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient. (:issue:`2804`) +- Add `zarr.testing.strategies.array_metadata` to generate ArrayV2Metadata and ArrayV3Metadata instances. (:issue:`2813`) +- Add arbitrary `shards` to Hypothesis strategy for generating arrays. (:issue:`2822`) + + +Bugfixes +~~~~~~~~ + +- Fixed bug with Zarr using device memory, instead of host memory, for storing metadata when using GPUs. (:issue:`2751`) +- The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the + specified fill value, or with zeros if no fill value is provided. + This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes. (:issue:`2755`) +- Fix zip-store path checking for stores with directories listed as files. (:issue:`2758`) +- Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list` (:issue:`2778`) +- Enable automatic removal of `needs release notes` with labeler action (:issue:`2781`) +- Use the proper label config (:issue:`2785`) +- Alters the behavior of ``create_array`` to ensure that any groups implied by the array's name are created if they do not already exist. Also simplifies the type signature for any function that takes an ArrayConfig-like object. (:issue:`2795`) +- Enitialise empty chunks to the default fill value during writing and add default fill values for datetime, timedelta, structured, and other (void* fixed size) data types (:issue:`2799`) +- Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests (:issue:`2801`) +- Fix pickling for ZipStore (:issue:`2807`) +- Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`. (:issue:`2811`) +- Fix fancy indexing (e.g. arr[5, [0, 1]]) with the sharding codec (:issue:`2817`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Added new user guide on :ref:`user-guide-gpu`. (:issue:`2751`) + + +3.0.2 (2025-01-31) +------------------ + +Features +~~~~~~~~ + +- Test ``getsize()`` and ``getsize_prefix()`` in ``StoreTests``. (:issue:`2693`) +- Test that a ``ValueError`` is raised for invalid byte range syntax in ``StoreTests``. (:issue:`2693`) +- Separate instantiating and opening a store in ``StoreTests``. (:issue:`2693`) +- Add a test for using Stores as a context managers in ``StoreTests``. (:issue:`2693`) +- Implemented ``LogingStore.open()``. (:issue:`2693`) +- ``LoggingStore`` is now a generic class. (:issue:`2693`) +- Change StoreTest's ``test_store_repr``, ``test_store_supports_writes``, + ``test_store_supports_partial_writes``, and ``test_store_supports_listing`` + to to be implemented using ``@abstractmethod``, rather raising ``NotImplementedError``. (:issue:`2693`) +- Test the error raised for invalid buffer arguments in ``StoreTests``. (:issue:`2693`) +- Test that data can be written to a store that's not yet open using the store.set method in ``StoreTests``. (:issue:`2693`) +- Adds a new function ``init_array`` for initializing an array in storage, and refactors ``create_array`` + to use ``init_array``. ``create_array`` takes two new parameters: ``data``, an optional array-like object, and ``write_data``, a bool which defaults to ``True``. + If ``data`` is given to ``create_array``, then the ``dtype`` and ``shape`` attributes of ``data`` are used to define the + corresponding attributes of the resulting Zarr array. Additionally, if ``data`` given and ``write_data`` is ``True``, + then the values in ``data`` will be written to the newly created array. (:issue:`2761`) + + +Bugfixes +~~~~~~~~ + +- Wrap sync fsspec filesystems with ``AsyncFileSystemWrapper``. (:issue:`2533`) +- Added backwards compatibility for Zarr format 2 structured arrays. (:issue:`2681`) +- Update equality for ``LoggingStore`` and ``WrapperStore`` such that 'other' must also be a ``LoggingStore`` or ``WrapperStore`` respectively, rather than only checking the types of the stores they wrap. (:issue:`2693`) +- Ensure that ``ZipStore`` is open before getting or setting any values. (:issue:`2693`) +- Use stdout rather than stderr as the default stream for ``LoggingStore``. (:issue:`2693`) +- Match the errors raised by read only stores in ``StoreTests``. (:issue:`2693`) +- Fixed ``ZipStore`` to make sure the correct attributes are saved when instances are pickled. + This fixes a previous bug that prevent using ``ZipStore`` with a ``ProcessPoolExecutor``. (:issue:`2762`) +- Updated the optional test dependencies to include ``botocore`` and ``fsspec``. (:issue:`2768`) +- Fixed the fsspec tests to skip if ``botocore`` is not installed. + Previously they would have failed with an import error. (:issue:`2768`) +- Optimize full chunk writes. (:issue:`2782`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Changed the machinery for creating changelog entries. + Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. (:issue:`2736`) + +Other +~~~~~ + +- Created a type alias ``ChunkKeyEncodingLike`` to model the union of ``ChunkKeyEncoding`` instances and the dict form of the + parameters of those instances. ``ChunkKeyEncodingLike`` should be used by high-level functions to provide a convenient + way for creating ``ChunkKeyEncoding`` objects. (:issue:`2763`) + + +3.0.1 (Jan. 17, 2025) +--------------------- + +* Implement ``zarr.from_array`` using concurrent streaming (:issue:`2622`). + +Bug fixes +~~~~~~~~~ +* Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`). + +* Fixes a bug that prevented reading Zarr format 2 data with consolidated + metadata written using ``zarr-python`` version 2 (:issue:`2694`). + +* Ensure that compressor=None results in no compression when writing Zarr + format 2 data (:issue:`2708`). + +* Fix for empty consolidated metadata dataset: backwards compatibility with + Zarr-Python 2 (:issue:`2695`). + +Documentation +~~~~~~~~~~~~~ +* Add v3.0.0 release announcement banner (:issue:`2677`). + +* Quickstart guide alignment with V3 API (:issue:`2697`). + +* Fix doctest failures related to numcodecs 0.15 (:issue:`2727`). + +Other +~~~~~ +* Removed some unnecessary files from the source distribution + to reduce its size. (:issue:`2686`). + +* Enable codecov in GitHub actions (:issue:`2682`). + +* Speed up hypothesis tests (:issue:`2650`). + +* Remove multiple imports for an import name (:issue:`2723`). + + +.. _release_3.0.0: + +3.0.0 (Jan. 9, 2025) +-------------------- + +3.0.0 is a new major release of Zarr-Python, with many breaking changes. +See the :ref:`v3 migration guide` for a listing of what's changed. + +Normal release note service will resume with further releases in the 3.0.0 +series. + +Release notes for the zarr-python 2.x and 1.x releases can be found here: +https://zarr.readthedocs.io/en/support-v2/release.html diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst index ba85ce1cda..c27f1296b9 100644 --- a/docs/user-guide/arrays.rst +++ b/docs/user-guide/arrays.rst @@ -182,7 +182,8 @@ which can be used to print useful diagnostics, e.g.:: >>> z.info Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C @@ -199,7 +200,8 @@ prints additional diagnostics, e.g.:: >>> z.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C @@ -209,8 +211,8 @@ prints additional diagnostics, e.g.:: Serializer : BytesCodec(endian=) Compressors : (BloscCodec(typesize=4, cname=, clevel=3, shuffle=, blocksize=0),) No. bytes : 400000000 (381.5M) - No. bytes stored : 9696302 - Storage ratio : 41.3 + No. bytes stored : 3558573 + Storage ratio : 112.4 Chunks Initialized : 100 .. note:: @@ -241,12 +243,12 @@ built-in delta filter:: >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000) >>> z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) >>> z.compressors - (_make_bytes_bytes_codec.._Codec(codec_name='numcodecs.lzma', codec_config={'id': 'lzma', 'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),) + (LZMA(codec_name='numcodecs.lzma', codec_config={'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),) The default compressor can be changed by setting the value of the using Zarr's :ref:`user-guide-config`, e.g.:: - >>> with zarr.config.set({'array.v2_default_compressor.numeric': {'id': 'blosc'}}): + >>> with zarr.config.set({'array.v2_default_compressor.default': {'id': 'blosc'}}): ... z = zarr.create_array(store={}, shape=(100000000,), chunks=(1000000,), dtype='int32', zarr_format=2) >>> z.filters () @@ -286,13 +288,14 @@ Here is an example using a delta filter with the Blosc compressor:: >>> z.info Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C Read-only : False Store type : LocalStore - Filters : (_make_array_array_codec.._Codec(codec_name='numcodecs.delta', codec_config={'id': 'delta', 'dtype': 'int32'}),) + Filters : (Delta(codec_name='numcodecs.delta', codec_config={'dtype': 'int32'}),) Serializer : BytesCodec(endian=) Compressors : (BloscCodec(typesize=4, cname=, clevel=1, shuffle=, blocksize=0),) No. bytes : 400000000 (381.5M) @@ -600,7 +603,8 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za >>> a.info_complete() Type : Array Zarr format : 3 - Data type : DataType.uint8 + Data type : UInt8() + Fill value : 0 Shape : (10000, 10000) Shard shape : (1000, 1000) Chunk shape : (100, 100) @@ -608,10 +612,10 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za Read-only : False Store type : LocalStore Filters : () - Serializer : BytesCodec(endian=) + Serializer : BytesCodec(endian=None) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 100000000 (95.4M) - No. bytes stored : 3981060 + No. bytes stored : 3981473 Storage ratio : 25.1 Shards Initialized : 100 diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst index a17bce9d99..5a9d26f2b9 100644 --- a/docs/user-guide/config.rst +++ b/docs/user-guide/config.rst @@ -3,7 +3,7 @@ Runtime configuration ===================== -:mod:`zarr.config ` is responsible for managing the configuration of zarr and +``zarr.config`` is responsible for managing the configuration of zarr and is based on the `donfig `_ Python library. Configuration values can be set using code like the following:: @@ -32,6 +32,7 @@ Configuration options include the following: - Whether empty chunks are written to storage ``array.write_empty_chunks`` - Async and threading options, e.g. ``async.concurrency`` and ``threading.max_workers`` - Selections of implementations of codecs, codec pipelines and buffers +- Enabling GPU support with ``zarr.config.enable_gpu()``. See :ref:`user-guide-gpu` for more. For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations in the registry and then select them in the config. @@ -42,38 +43,30 @@ This is the current default configuration:: >>> zarr.config.pprint() {'array': {'order': 'C', - 'v2_default_compressor': {'bytes': {'checksum': False, - 'id': 'zstd', - 'level': 0}, - 'numeric': {'checksum': False, - 'id': 'zstd', - 'level': 0}, - 'string': {'checksum': False, + 'v2_default_compressor': {'default': {'checksum': False, 'id': 'zstd', - 'level': 0}}, - 'v2_default_filters': {'bytes': [{'id': 'vlen-bytes'}], - 'numeric': None, - 'string': [{'id': 'vlen-utf8'}]}, - 'v3_default_compressors': {'bytes': [{'configuration': {'checksum': False, - 'level': 0}, - 'name': 'zstd'}], - 'numeric': [{'configuration': {'checksum': False, + 'level': 0}, + 'variable-length-string': {'checksum': False, + 'id': 'zstd', + 'level': 0}}, + 'v2_default_filters': {'default': None, + 'variable-length-string': [{'id': 'vlen-utf8'}]}, + 'v3_default_compressors': {'default': [{'configuration': {'checksum': False, 'level': 0}, 'name': 'zstd'}], - 'string': [{'configuration': {'checksum': False, - 'level': 0}, - 'name': 'zstd'}]}, - 'v3_default_filters': {'bytes': [], 'numeric': [], 'string': []}, - 'v3_default_serializer': {'bytes': {'name': 'vlen-bytes'}, - 'numeric': {'configuration': {'endian': 'little'}, - 'name': 'bytes'}, - 'string': {'name': 'vlen-utf8'}}, - 'write_empty_chunks': False}, - 'async': {'concurrency': 10, 'timeout': None}, - 'buffer': 'zarr.core.buffer.cpu.Buffer', - 'codec_pipeline': {'batch_size': 1, - 'path': 'zarr.core.codec_pipeline.BatchedCodecPipeline'}, - 'codecs': {'blosc': 'zarr.codecs.blosc.BloscCodec', + 'variable-length-string': [{'configuration': {'checksum': False, + 'level': 0}, + 'name': 'zstd'}]}, + 'v3_default_filters': {'default': [], 'variable-length-string': []}, + 'v3_default_serializer': {'default': {'configuration': {'endian': 'little'}, + 'name': 'bytes'}, + 'variable-length-string': {'name': 'vlen-utf8'}}, + 'write_empty_chunks': False}, + 'async': {'concurrency': 10, 'timeout': None}, + 'buffer': 'zarr.buffer.cpu.Buffer', + 'codec_pipeline': {'batch_size': 1, + 'path': 'zarr.core.codec_pipeline.BatchedCodecPipeline'}, + 'codecs': {'blosc': 'zarr.codecs.blosc.BloscCodec', 'bytes': 'zarr.codecs.bytes.BytesCodec', 'crc32c': 'zarr.codecs.crc32c_.Crc32cCodec', 'endian': 'zarr.codecs.bytes.BytesCodec', @@ -83,7 +76,7 @@ This is the current default configuration:: 'vlen-bytes': 'zarr.codecs.vlen_utf8.VLenBytesCodec', 'vlen-utf8': 'zarr.codecs.vlen_utf8.VLenUTF8Codec', 'zstd': 'zarr.codecs.zstd.ZstdCodec'}, - 'default_zarr_format': 3, - 'json_indent': 2, - 'ndbuffer': 'zarr.core.buffer.cpu.NDBuffer', - 'threading': {'max_workers': None}} + 'default_zarr_format': 3, + 'json_indent': 2, + 'ndbuffer': 'zarr.buffer.cpu.NDBuffer', + 'threading': {'max_workers': None}} diff --git a/docs/user-guide/consolidated_metadata.rst b/docs/user-guide/consolidated_metadata.rst index 3c015dcfca..4cd72dbc74 100644 --- a/docs/user-guide/consolidated_metadata.rst +++ b/docs/user-guide/consolidated_metadata.rst @@ -47,7 +47,7 @@ that can be used.: >>> from pprint import pprint >>> pprint(dict(sorted(consolidated_metadata.items()))) {'a': ArrayV3Metadata(shape=(1,), - data_type=, + data_type=Float64(endianness='little'), chunk_grid=RegularChunkGrid(chunk_shape=(1,)), chunk_key_encoding=DefaultChunkKeyEncoding(name='default', separator='/'), @@ -60,7 +60,7 @@ that can be used.: node_type='array', storage_transformers=()), 'b': ArrayV3Metadata(shape=(2, 2), - data_type=, + data_type=Float64(endianness='little'), chunk_grid=RegularChunkGrid(chunk_shape=(2, 2)), chunk_key_encoding=DefaultChunkKeyEncoding(name='default', separator='/'), @@ -73,7 +73,7 @@ that can be used.: node_type='array', storage_transformers=()), 'c': ArrayV3Metadata(shape=(3, 3, 3), - data_type=, + data_type=Float64(endianness='little'), chunk_grid=RegularChunkGrid(chunk_shape=(3, 3, 3)), chunk_key_encoding=DefaultChunkKeyEncoding(name='default', separator='/'), @@ -114,3 +114,23 @@ removed, or modified, consolidated metadata may not be desirable. metadata. .. _Consolidated Metadata: https://github.com/zarr-developers/zarr-specs/pull/309 + +Stores Without Support for Consolidated Metadata +------------------------------------------------ + +Some stores may want to opt out of the consolidated metadata mechanism. This +may be for several reasons like: + +* They want to maintain read-write consistency, which is challenging with + consolidated metadata. +* They have their own consolidated metadata mechanism. +* They offer good enough performance without need for consolidation. + +This type of store can declare it doesn't want consolidation by implementing +`Store.supports_consolidated_metadata` and returning `False`. For stores that don't support +consolidation, Zarr will: + +* Raise an error on `consolidate_metadata` calls, maintaining the store in + its unconsolidated state. +* Raise an error in `AsyncGroup.open(..., use_consolidated=True)` +* Not use consolidated metadata in `AsyncGroup.open(..., use_consolidated=None)` diff --git a/docs/user-guide/data_types.rst b/docs/user-guide/data_types.rst new file mode 100644 index 0000000000..87c8efc1f5 --- /dev/null +++ b/docs/user-guide/data_types.rst @@ -0,0 +1,172 @@ +Data types +========== + +Zarr's data type model +---------------------- + +Every Zarr array has a "data type", which defines the meaning and physical layout of the +array's elements. As Zarr Python is tightly integrated with `NumPy `_, +it's easy to create arrays with NumPy data types: + +.. code-block:: python + + >>> import zarr + >>> import numpy as np + >>> z = zarr.create_array(store={}, shape=(10,), dtype=np.dtype('uint8')) + >>> z + + +Unlike NumPy arrays, Zarr arrays are designed to accessed by Zarr +implementations in different programming languages. This means Zarr data types must be interpreted +correctly when clients read an array. Each Zarr data type defines procedures for +encoding and decoding both the data type itself, and scalars from that data type to and from Zarr array metadata. And these serialization procedures +depend on the Zarr format. + +Data types in Zarr version 2 +----------------------------- + +Version 2 of the Zarr format defined its data types relative to +`NumPy's data types `_, +and added a few non-NumPy data types as well. Thus the JSON identifier for a NumPy-compatible data +type is just the NumPy ``str`` attribute of that data type: + +.. code-block:: python + + >>> import zarr + >>> import numpy as np + >>> import json + >>> + >>> store = {} + >>> np_dtype = np.dtype('int64') + >>> z = zarr.create_array(store=store, shape=(1,), dtype=np_dtype, zarr_format=2) + >>> dtype_meta = json.loads(store['.zarray'].to_bytes())["dtype"] + >>> dtype_meta + '>> assert dtype_meta == np_dtype.str + +.. note:: + The ``<`` character in the data type metadata encodes the + `endianness `_, + or "byte order", of the data type. Following NumPy's example, + in Zarr version 2 each data type has an endianness where applicable. + However, Zarr version 3 data types do not store endianness information. + +In addition to defining a representation of the data type itself (which in the example above was +just a simple string ``"M[10s]"`` in + Zarr V2. This is more compact, but can be harder to parse. + +For more about data types in Zarr V3, see the +`V3 specification `_. + +Data types in Zarr Python +------------------------- + +The two Zarr formats that Zarr Python supports specify data types in two different ways: +data types in Zarr version 2 are encoded as NumPy-compatible strings, while data types in Zarr version +3 are encoded as either strings or ``JSON`` objects, +and the Zarr V3 data types don't have any associated endianness information, unlike Zarr V2 data types. + +To abstract over these syntactical and semantic differences, Zarr Python uses a class called +`ZDType <../api/zarr/dtype/index.html#zarr.dtype.ZDType>`_ provide Zarr V2 and Zarr V3 compatibility +routines for ""native" data types. In this context, a "native" data type is a Python class, +typically defined in another library, that models an array's data type. For example, ``np.uint8`` is a native +data type defined in NumPy, which Zarr Python wraps with a ``ZDType`` instance called +`UInt8 <../api/zarr/dtype/index.html#zarr.dtype.ZDType>`_. + +Each data type supported by Zarr Python is modeled by ``ZDType`` subclass, which provides an +API for the following operations: + +- Wrapping / unwrapping a native data type +- Encoding / decoding a data type to / from Zarr V2 and Zarr V3 array metadata. +- Encoding / decoding a scalar value to / from Zarr V2 and Zarr V3 array metadata. + + +Example Usage +~~~~~~~~~~~~~ + +Create a ``ZDType`` from a native data type: + +.. code-block:: python + + >>> from zarr.core.dtype import Int8 + >>> import numpy as np + >>> int8 = Int8.from_native_dtype(np.dtype('int8')) + +Convert back to native data type: + +.. code-block:: python + + >>> native_dtype = int8.to_native_dtype() + >>> assert native_dtype == np.dtype('int8') + +Get the default scalar value for the data type: + +.. code-block:: python + + >>> default_value = int8.default_scalar() + >>> assert default_value == np.int8(0) + + +Serialize to JSON for Zarr V2 and V3 + +.. code-block:: python + + >>> json_v2 = int8.to_json(zarr_format=2) + >>> json_v2 + {'name': '|i1', 'object_codec_id': None} + >>> json_v3 = int8.to_json(zarr_format=3) + >>> json_v3 + 'int8' + +Serialize a scalar value to JSON: + +.. code-block:: python + + >>> json_value = int8.to_json_scalar(42, zarr_format=3) + >>> json_value + 42 + +Deserialize a scalar value from JSON: + +.. code-block:: python + + >>> scalar_value = int8.from_json_scalar(42, zarr_format=3) + >>> assert scalar_value == np.int8(42) diff --git a/docs/user-guide/extending.rst b/docs/user-guide/extending.rst index 7647703fbb..4487e07ddf 100644 --- a/docs/user-guide/extending.rst +++ b/docs/user-guide/extending.rst @@ -83,7 +83,10 @@ Coming soon. Custom array buffers -------------------- -Coming soon. +Zarr-python provides control over where and how arrays stored in memory through +:mod:`zarr.buffer`. Currently both CPU (the default) and GPU implementations are +provided (see :ref:`user-guide-gpu` for more). You can implement your own buffer +classes by implementing the interface defined in :mod:`zarr.abc.buffer`. Other extensions ---------------- diff --git a/docs/user-guide/gpu.rst b/docs/user-guide/gpu.rst new file mode 100644 index 0000000000..4d3492f8bd --- /dev/null +++ b/docs/user-guide/gpu.rst @@ -0,0 +1,37 @@ +.. _user-guide-gpu: + +Using GPUs with Zarr +==================== + +Zarr can use GPUs to accelerate your workload by running +:meth:`zarr.config.enable_gpu`. + +.. note:: + + `zarr-python` currently supports reading the ndarray data into device (GPU) + memory as the final stage of the codec pipeline. Data will still be read into + or copied to host (CPU) memory for encoding and decoding. + + In the future, codecs will be available compressing and decompressing data on + the GPU, avoiding the need to move data between the host and device for + compression and decompression. + +Reading data into device memory +------------------------------- + +:meth:`zarr.config.enable_gpu` configures Zarr to use GPU memory for the data +buffers used internally by Zarr. + +.. code-block:: python + + >>> import zarr + >>> import cupy as cp # doctest: +SKIP + >>> zarr.config.enable_gpu() # doctest: +SKIP + >>> store = zarr.storage.MemoryStore() # doctest: +SKIP + >>> z = zarr.create_array( # doctest: +SKIP + ... store=store, shape=(100, 100), chunks=(10, 10), dtype="float32", + ... ) + >>> type(z[:10, :10]) # doctest: +SKIP + cupy.ndarray + +Note that the output type is a ``cupy.ndarray`` rather than a NumPy array. diff --git a/docs/user-guide/groups.rst b/docs/user-guide/groups.rst index da5f393246..4237a9df50 100644 --- a/docs/user-guide/groups.rst +++ b/docs/user-guide/groups.rst @@ -75,6 +75,31 @@ For more information on groups see the :class:`zarr.Group` API docs. .. _user-guide-diagnostics: +Batch Group Creation +-------------------- + +You can also create multiple groups concurrently with a single function call. :func:`zarr.create_hierarchy` takes +a :class:`zarr.storage.Store` instance and a dict of ``key : metadata`` pairs, parses that dict, and +writes metadata documents to storage: + + >>> from zarr import create_hierarchy + >>> from zarr.core.group import GroupMetadata + >>> from zarr.storage import LocalStore + >>> node_spec = {'a/b/c': GroupMetadata()} + >>> nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec)) + >>> print(sorted(nodes_created.items(), key=lambda kv: len(kv[0]))) + [('', ), ('a', ), ('a/b', ), ('a/b/c', )] + +Note that we only specified a single group named ``a/b/c``, but 4 groups were created. These additional groups +were created to ensure that the desired node ``a/b/c`` is connected to the root group ``''`` by a sequence +of intermediate groups. :func:`zarr.create_hierarchy` normalizes the ``nodes`` keyword argument to +ensure that the resulting hierarchy is complete, i.e. all groups or arrays are connected to the root +of the hierarchy via intermediate groups. + +Because :func:`zarr.create_hierarchy` concurrently creates metadata documents, it's more efficient +than repeated calls to :func:`create_group` or :func:`create_array`, provided you can statically define +the metadata for the groups and arrays you want to create. + Array and group diagnostics --------------------------- @@ -103,7 +128,8 @@ property. E.g.:: >>> bar.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int64 + Data type : Int64(endianness='little') + Fill value : 0 Shape : (1000000,) Chunk shape : (100000,) Order : C @@ -113,13 +139,14 @@ property. E.g.:: Serializer : BytesCodec(endian=) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 8000000 (7.6M) - No. bytes stored : 1432 - Storage ratio : 5586.6 - Chunks Initialized : 0 + No. bytes stored : 1614 + Storage ratio : 4956.6 + Chunks Initialized : 10 >>> baz.info Type : Array Zarr format : 3 - Data type : DataType.float32 + Data type : Float32(endianness='little') + Fill value : 0.0 Shape : (1000, 1000) Chunk shape : (100, 100) Order : C diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index a7bbd12453..ea34ac2561 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -8,6 +8,7 @@ User guide installation arrays + data_types groups attributes storage @@ -23,6 +24,7 @@ Advanced Topics performance consolidated_metadata extending + gpu .. Coming soon diff --git a/docs/user-guide/performance.rst b/docs/user-guide/performance.rst index 265bef8efe..7d24c87373 100644 --- a/docs/user-guide/performance.rst +++ b/docs/user-guide/performance.rst @@ -91,7 +91,8 @@ To use sharding, you need to specify the ``shards`` parameter when creating the >>> z6.info Type : Array Zarr format : 3 - Data type : DataType.uint8 + Data type : UInt8() + Fill value : 0 Shape : (10000, 10000, 1000) Shard shape : (1000, 1000, 1000) Chunk shape : (100, 100, 100) @@ -99,7 +100,7 @@ To use sharding, you need to specify the ``shards`` parameter when creating the Read-only : False Store type : MemoryStore Filters : () - Serializer : BytesCodec(endian=) + Serializer : BytesCodec(endian=None) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 100000000000 (93.1G) @@ -121,7 +122,8 @@ ratios, depending on the correlation structure within the data. E.g.:: >>> c.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : C @@ -131,7 +133,7 @@ ratios, depending on the correlation structure within the data. E.g.:: Serializer : BytesCodec(endian=) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 400000000 (381.5M) - No. bytes stored : 342588717 + No. bytes stored : 342588911 Storage ratio : 1.2 Chunks Initialized : 100 >>> with zarr.config.set({'array.order': 'F'}): @@ -140,7 +142,8 @@ ratios, depending on the correlation structure within the data. E.g.:: >>> f.info_complete() Type : Array Zarr format : 3 - Data type : DataType.int32 + Data type : Int32(endianness='little') + Fill value : 0 Shape : (10000, 10000) Chunk shape : (1000, 1000) Order : F @@ -150,7 +153,7 @@ ratios, depending on the correlation structure within the data. E.g.:: Serializer : BytesCodec(endian=) Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 400000000 (381.5M) - No. bytes stored : 342588717 + No. bytes stored : 342588911 Storage ratio : 1.2 Chunks Initialized : 100 diff --git a/docs/user-guide/storage.rst b/docs/user-guide/storage.rst index 46505271b4..4215cbaf20 100644 --- a/docs/user-guide/storage.rst +++ b/docs/user-guide/storage.rst @@ -47,7 +47,7 @@ Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers four built-in store: :class:`zarr.storage.LocalStore`, :class:`zarr.storage.FsspecStore`, -:class:`zarr.storage.ZipStore`, and :class:`zarr.storage.MemoryStore`. +:class:`zarr.storage.ZipStore`, :class:`zarr.storage.MemoryStore`, and :class:`zarr.storage.ObjectStore`. Local Store ~~~~~~~~~~~ @@ -99,6 +99,42 @@ Zarr data (metadata and chunks) to a dictionary.: >>> zarr.create_array(store=store, shape=(2,), dtype='float64') +Object Store +~~~~~~~~~~~~ + +:class:`zarr.storage.ObjectStore` stores the contents of the Zarr hierarchy using any ObjectStore +`storage implementation `_, including AWS S3 (:class:`obstore.store.S3Store`), Google Cloud Storage (:class:`obstore.store.GCSStore`), and Azure Blob Storage (:class:`obstore.store.AzureStore`). This store is backed by `obstore `_, which +builds on the production quality Rust library `object_store `_. + + + >>> from zarr.storage import ObjectStore + >>> from obstore.store import MemoryStore + >>> + >>> store = ObjectStore(MemoryStore()) + >>> zarr.create_array(store=store, shape=(2,), dtype='float64') + + +Here's an example of using ObjectStore for accessing remote data: + + >>> from zarr.storage import ObjectStore + >>> from obstore.store import S3Store + >>> + >>> s3_store = S3Store('noaa-nwm-retro-v2-zarr-pds', skip_signature=True, region="us-west-2") + >>> store = zarr.storage.ObjectStore(store=s3_store, read_only=True) + >>> group = zarr.open_group(store=store, mode='r') + >>> group.info + Name : + Type : Group + Zarr format : 2 + Read-only : True + Store type : ObjectStore + No. members : 12 + No. arrays : 12 + No. groups : 0 + +.. warning:: + The :class:`zarr.storage.ObjectStore` class is experimental. + .. _user-guide-custom-stores: Developing custom stores diff --git a/docs/user-guide/v3_migration.rst b/docs/user-guide/v3_migration.rst index d90b87a897..a6258534e4 100644 --- a/docs/user-guide/v3_migration.rst +++ b/docs/user-guide/v3_migration.rst @@ -1,3 +1,5 @@ +.. _v3 migration guide: + 3.0 Migration Guide =================== @@ -115,6 +117,8 @@ The Group class - Use :func:`zarr.Group.create_array` in place of :func:`zarr.Group.create_dataset` - Use :func:`zarr.Group.require_array` in place of :func:`zarr.Group.require_dataset` +3. Disallow "." syntax for getting group members. To get a member of a group named ``foo``, + use ``group["foo"]`` in place of ``group.foo``. The Store class ~~~~~~~~~~~~~~~ @@ -122,6 +126,30 @@ The Store class The Store API has changed significant in Zarr-Python 3. The most notable changes to the Store API are: +Store Import Paths +^^^^^^^^^^^^^^^^^^ +Several store implementations have moved from the top-level module to ``zarr.storage``: + +.. code-block:: diff + :caption: Store import changes from v2 to v3 + + # Before (v2) + - from zarr import MemoryStore, DirectoryStore + + from zarr.storage import MemoryStore, LocalStore # LocalStore replaces DirectoryStore + +Common replacements: + ++-------------------------+------------------------------------+ +| v2 Import | v3 Import | ++=========================+====================================+ +| ``zarr.MemoryStore`` | ``zarr.storage.MemoryStore`` | ++-------------------------+------------------------------------+ +| ``zarr.DirectoryStore`` | ``zarr.storage.LocalStore`` | ++-------------------------+------------------------------------+ +| ``zarr.TempStore`` | Use ``tempfile.TemporaryDirectory``| +| | with ``LocalStore`` | ++-------------------------+------------------------------------+ + 1. Replaced the ``MutableMapping`` base class in favor of a custom abstract base class (:class:`zarr.abc.store.Store`). 2. Switched to an asynchronous interface for all store methods that result in IO. This @@ -206,3 +234,5 @@ of Zarr-Python, please open (or comment on) a * Object dtypes (:issue:`2617`) * Ragged arrays (:issue:`2618`) * Groups and Arrays do not implement ``__enter__`` and ``__exit__`` protocols (:issue:`2619`) + * Big Endian dtypes (:issue:`2324`) + * Default filters for object dtypes for Zarr format 2 arrays (:issue:`2627`) diff --git a/notebooks/advanced_indexing.ipynb b/notebooks/advanced_indexing.ipynb deleted file mode 100644 index eba6b5880b..0000000000 --- a/notebooks/advanced_indexing.ipynb +++ /dev/null @@ -1,2798 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Advanced indexing" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.1.5.dev144'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "import numpy as np\n", - "np.random.seed(42)\n", - "import cProfile\n", - "zarr.__version__" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Functionality and API" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing a 1D array with a Boolean (mask) array\n", - "\n", - "Supported via ``get/set_mask_selection()`` and ``.vindex[]``. Also supported via ``get/set_orthogonal_selection()`` and ``.oindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)\n", - "ix = [False, True, False, True, False, True, False, True, False, True]" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.oindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 2, 30, 4, 50, 6, 70, 8, 90])" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.vindex[ix] = a[ix] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 100, 2, 300, 4, 500, 6, 700, 8, 900])" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.oindex[ix] = a[ix] * 100\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# if using .oindex, indexing array can be any array-like, e.g., Zarr array\n", - "zix = zarr.array(ix, chunks=2)\n", - "za = zarr.array(a, chunks=2)\n", - "za.oindex[zix] # will not load all zix into memory" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing a 1D array with a 1D integer (coordinate) array\n", - "\n", - "Supported via ``get/set_coordinate_selection()`` and ``.vindex[]``. Also supported via ``get/set_orthogonal_selection()`` and ``.oindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)\n", - "ix = [1, 3, 5, 7, 9]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.oindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 2, 30, 4, 50, 6, 70, 8, 90])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.vindex[ix] = a[ix] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 100, 2, 300, 4, 500, 6, 700, 8, 900])" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.oindex[ix] = a[ix] * 100\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing a 1D array with a multi-dimensional integer (coordinate) array\n", - "\n", - "Supported via ``get/set_coordinate_selection()`` and ``.vindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)\n", - "ix = np.array([[1, 3, 5], [2, 4, 6]])" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[1, 3, 5],\n", - " [2, 4, 6]])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 20, 30, 40, 50, 60, 7, 8, 9])" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.vindex[ix] = a[ix] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Slicing a 1D array with step > 1\n", - "\n", - "Slices with step > 1 are supported via ``get/set_basic_selection()``, ``get/set_orthogonal_selection()``, ``__getitem__`` and ``.oindex[]``. Negative steps are not supported." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za[1::2]" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 2, 30, 4, 50, 6, 70, 8, 90])" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.oindex[1::2] = a[1::2] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Orthogonal (outer) indexing of multi-dimensional arrays\n", - "\n", - "Orthogonal (a.k.a. outer) indexing is supported with either Boolean or integer arrays, in combination with integers and slices. This functionality is provided via the ``get/set_orthogonal_selection()`` methods. For convenience, this functionality is also available via the ``.oindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# orthogonal indexing with Boolean arrays\n", - "ix0 = [False, True, False, True, False]\n", - "ix1 = [True, False, True]\n", - "za.get_orthogonal_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# orthogonal indexing with integer arrays\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "za.get_orthogonal_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 4, 5],\n", - " [ 9, 10, 11]])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# combine with slice\n", - "za.oindex[[1, 3], :]" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 2],\n", - " [ 3, 5],\n", - " [ 6, 8],\n", - " [ 9, 11],\n", - " [12, 14]])" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# combine with slice\n", - "za.oindex[:, [0, 2]]" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 42],\n", - " [ 6, 7, 8],\n", - " [42, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items via Boolean selection\n", - "ix0 = [False, True, False, True, False]\n", - "ix1 = [True, False, True]\n", - "selection = ix0, ix1\n", - "value = 42\n", - "za.set_orthogonal_selection(selection, value)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 44],\n", - " [ 6, 7, 8],\n", - " [44, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [46, 4, 46],\n", - " [ 6, 7, 8],\n", - " [46, 10, 46],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items via integer selection\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "selection = ix0, ix1\n", - "value = 46\n", - "za.set_orthogonal_selection(selection, value)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [48, 4, 48],\n", - " [ 6, 7, 8],\n", - " [48, 10, 48],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1] = 48\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Coordinate indexing of multi-dimensional arrays\n", - "\n", - "Selecting arbitrary points from a multi-dimensional array by indexing with integer (coordinate) arrays is supported. This functionality is provided via the ``get/set_coordinate_selection()`` methods. For convenience, this functionality is also available via the ``.vindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "za.get_coordinate_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.vindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.set_coordinate_selection((ix0, ix1), 42)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.vindex[ix0, ix1] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Mask indexing of multi-dimensional arrays\n", - "\n", - "Selecting arbitrary points from a multi-dimensional array by a Boolean array is supported. This functionality is provided via the ``get/set_mask_selection()`` methods. For convenience, this functionality is also available via the ``.vindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix = np.zeros_like(a, dtype=bool)\n", - "ix[1, 0] = True\n", - "ix[3, 2] = True\n", - "za.get_mask_selection(ix)" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.set_mask_selection(ix, 42)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.vindex[ix] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Selecting fields from arrays with a structured dtype\n", - "\n", - "All ``get/set_selection_...()`` methods support a ``fields`` argument which allows retrieving/replacing data for a specific field or fields. Also h5py-like API is supported where fields can be provided within ``__getitem__``, ``.oindex[]`` and ``.vindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([(b'aaa', 1, 4.2), (b'bbb', 2, 8.4), (b'ccc', 3, 12.6)],\n", - " dtype=[('foo', 'S3'), ('bar', '\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'foo'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'baz'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices" - ] - } - ], - "source": [ - "a['foo', 'baz']" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([(b'aaa', 4.2), (b'bbb', 8.4), (b'ccc', 12.6)],\n", - " dtype=[('foo', 'S3'), ('baz', '", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mza\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'foo'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'baz'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 537\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 538\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 539\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEllipsis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mget_basic_selection\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 661\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 663\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_basic_selection_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 664\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 665\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_basic_selection_nd\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 702\u001b[0m \u001b[0;31m# setup indexer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 703\u001b[0;31m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 704\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 705\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/indexing.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, selection, array)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 276\u001b[0m raise IndexError('unsupported selection item for basic indexing; expected integer '\n\u001b[0;32m--> 277\u001b[0;31m 'or slice, got {!r}'.format(type(dim_sel)))\n\u001b[0m\u001b[1;32m 278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 279\u001b[0m \u001b[0mdim_indexers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim_indexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mIndexError\u001b[0m: unsupported selection item for basic indexing; expected integer or slice, got " - ] - } - ], - "source": [ - "za[['foo', 'baz']]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1D Benchmarking" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "800000000" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c = np.arange(100000000)\n", - "c.nbytes" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 480 ms, sys: 16 ms, total: 496 ms\n", - "Wall time: 141 ms\n" - ] - }, - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeint64
Shape(100000000,)
Chunk shape(97657,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes800000000 (762.9M)
No. bytes stored11854081 (11.3M)
Storage ratio67.5
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (100000000,)\n", - "Chunk shape : (97657,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 800000000 (762.9M)\n", - "No. bytes stored : 11854081 (11.3M)\n", - "Storage ratio : 67.5\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time zc = zarr.array(c)\n", - "zc.info" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "121 ms ± 1.49 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c.copy()" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "254 ms ± 942 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool dense selection" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9997476" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# relatively dense selection - 10%\n", - "ix_dense_bool = np.random.binomial(1, 0.1, size=c.shape[0]).astype(bool)\n", - "np.count_nonzero(ix_dense_bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "243 ms ± 5.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "433 ms ± 6.49 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "548 ms ± 5.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [], - "source": [ - "import tempfile\n", - "import cProfile\n", - "import pstats\n", - "\n", - "def profile(statement, sort='time', restrictions=(7,)):\n", - " with tempfile.NamedTemporaryFile() as f:\n", - " cProfile.run(statement, filename=f.name)\n", - " pstats.Stats(f.name).sort_stats(sort).print_stats(*restrictions)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:17:48 2017 /tmp/tmpruua2rs_\n", - "\n", - " 98386 function calls in 0.483 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 83 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1025 0.197 0.000 0.197 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.149 0.000 0.159 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.044 0.000 0.231 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1024 0.009 0.000 0.009 0.000 {built-in method numpy.core.multiarray.count_nonzero}\n", - " 1025 0.007 0.000 0.238 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.006 0.000 0.207 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/core.py:337()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Method ``nonzero`` is being called internally within numpy to convert bool to int selections, no way to avoid." - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:18:06 2017 /tmp/tmp7_bautep\n", - "\n", - " 52382 function calls in 0.592 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 2 0.219 0.110 0.219 0.110 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.096 0.000 0.101 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.094 0.047 0.094 0.047 ../zarr/indexing.py:630()\n", - " 1024 0.044 0.000 0.167 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.029 0.029 0.029 0.029 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 1 0.021 0.021 0.181 0.181 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "``.vindex[]`` is a bit slower, possibly because internally it converts to a coordinate array first." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int dense selection" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000000" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix_dense_int = np.random.choice(c.shape[0], size=c.shape[0]//10, replace=True)\n", - "ix_dense_int_sorted = ix_dense_int.copy()\n", - "ix_dense_int_sorted.sort()\n", - "len(ix_dense_int)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "62.2 ms ± 2.36 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "355 ms ± 3.53 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "351 ms ± 3.51 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "128 ms ± 137 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.71 s ± 5.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.68 s ± 3.87 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:09 2017 /tmp/tmpgmu5btr_\n", - "\n", - " 95338 function calls in 0.424 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 89 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.141 0.141 0.184 0.184 ../zarr/indexing.py:369(__init__)\n", - " 1024 0.099 0.000 0.106 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.046 0.000 0.175 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.027 0.000 0.027 0.000 ../zarr/indexing.py:424(__iter__)\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 1 0.010 0.010 0.010 0.010 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/function_base.py:1848(diff)\n", - " 1025 0.006 0.000 0.059 0.000 ../zarr/indexing.py:541(__iter__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_int_sorted]')" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:13 2017 /tmp/tmpay1gvnx8\n", - "\n", - " 52362 function calls in 0.398 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 85 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 2 0.107 0.054 0.107 0.054 ../zarr/indexing.py:630()\n", - " 1024 0.091 0.000 0.096 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.041 0.000 0.160 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.040 0.040 0.213 0.213 ../zarr/indexing.py:603(__init__)\n", - " 1 0.029 0.029 0.029 0.029 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 2048 0.011 0.000 0.011 0.000 ../zarr/indexing.py:695()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_int_sorted]')" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:20 2017 /tmp/tmpngsf6zpp\n", - "\n", - " 120946 function calls in 1.793 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 92 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.128 1.128 1.128 1.128 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 1024 0.139 0.000 0.285 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.132 0.132 1.422 1.422 ../zarr/indexing.py:369(__init__)\n", - " 1 0.120 0.120 0.120 0.120 {method 'take' of 'numpy.ndarray' objects}\n", - " 1024 0.116 0.000 0.123 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.034 0.000 0.034 0.000 ../zarr/indexing.py:424(__iter__)\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_int]')" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:22 2017 /tmp/tmpbskhj8de\n", - "\n", - " 50320 function calls in 1.730 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 86 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.116 1.116 1.116 1.116 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 1024 0.133 0.000 0.275 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2 0.121 0.060 0.121 0.060 ../zarr/indexing.py:654()\n", - " 1024 0.113 0.000 0.119 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.100 0.050 0.100 0.050 ../zarr/indexing.py:630()\n", - " 1 0.030 0.030 0.030 0.030 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.024 0.024 1.427 1.427 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_int]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When indices are not sorted, zarr needs to partially sort them so the occur in chunk order, so we only have to visit each chunk once. This sorting dominates the processing time and is unavoidable AFAIK." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool sparse selection" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9932" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# relatively sparse selection\n", - "ix_sparse_bool = np.random.binomial(1, 0.0001, size=c.shape[0]).astype(bool)\n", - "np.count_nonzero(ix_sparse_bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "15.7 ms ± 38.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "156 ms ± 2.1 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "133 ms ± 2.76 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:20:09 2017 /tmp/tmpb7nqc9ax\n", - "\n", - " 98386 function calls in 0.191 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 83 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.093 0.000 0.098 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.017 0.000 0.017 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.007 0.000 0.007 0.000 {built-in method numpy.core.multiarray.count_nonzero}\n", - " 1024 0.007 0.000 0.129 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.005 0.000 0.052 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.005 0.000 0.025 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_sparse_bool]')" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:20:09 2017 /tmp/tmphsko8nvh\n", - "\n", - " 52382 function calls in 0.160 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.093 0.000 0.098 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.017 0.008 0.017 0.008 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1025 0.008 0.000 0.014 0.000 ../zarr/indexing.py:674(__iter__)\n", - " 1024 0.006 0.000 0.127 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/indexing.py:695()\n", - " 2054 0.003 0.000 0.003 0.000 ../zarr/core.py:337()\n", - " 1024 0.002 0.000 0.005 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/core/arrayprint.py:381(wrapper)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_sparse_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int sparse selection" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000" - ] - }, - "execution_count": 81, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix_sparse_int = np.random.choice(c.shape[0], size=c.shape[0]//10000, replace=True)\n", - "ix_sparse_int_sorted = ix_sparse_int.copy()\n", - "ix_sparse_int_sorted.sort()\n", - "len(ix_sparse_int)" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "18.9 µs ± 392 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "20.3 µs ± 155 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "125 ms ± 296 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "109 ms ± 428 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "132 ms ± 489 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "108 ms ± 579 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:21:12 2017 /tmp/tmp0b0o2quo\n", - "\n", - " 120946 function calls in 0.196 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 92 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.105 0.000 0.111 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2048 0.006 0.000 0.013 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 1025 0.006 0.000 0.051 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.006 0.000 0.141 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/core.py:337()\n", - " 15373 0.004 0.000 0.010 0.000 {built-in method builtins.isinstance}\n", - " 1025 0.004 0.000 0.005 0.000 ../zarr/indexing.py:424(__iter__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_sparse_int]')" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:21:19 2017 /tmp/tmpdwju98kn\n", - "\n", - " 50320 function calls in 0.167 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 86 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.105 0.000 0.111 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.009 0.000 0.017 0.000 ../zarr/indexing.py:674(__iter__)\n", - " 1024 0.006 0.000 0.142 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/indexing.py:695()\n", - " 2054 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - " 1 0.003 0.003 0.162 0.162 ../zarr/core.py:591(_get_selection)\n", - " 1027 0.003 0.000 0.003 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_sparse_int]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For sparse selections, processing time is dominated by decompression, so we can't do any better." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### sparse bool selection as zarr array" - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typebool
Shape(100000000,)
Chunk shape(390625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes100000000 (95.4M)
No. bytes stored507131 (495.2K)
Storage ratio197.2
Chunks initialized256/256
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : bool\n", - "Shape : (100000000,)\n", - "Chunk shape : (390625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 100000000 (95.4M)\n", - "No. bytes stored : 507131 (495.2K)\n", - "Storage ratio : 197.2\n", - "Chunks initialized : 256/256" - ] - }, - "execution_count": 90, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zix_sparse_bool = zarr.array(ix_sparse_bool)\n", - "zix_sparse_bool.info" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "387 ms ± 5.47 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[zix_sparse_bool]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### slice with step" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "80.3 ms ± 377 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit np.array(c[::2])" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "168 ms ± 837 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::2]" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "136 ms ± 1.56 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::10]" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "104 ms ± 1.86 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::100]" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "100 ms ± 1.47 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::1000]" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:22:44 2017 /tmp/tmpg9dxqcpg\n", - "\n", - " 49193 function calls in 0.211 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 55 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.104 0.000 0.110 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.067 0.000 0.195 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.005 0.000 0.013 0.000 ../zarr/indexing.py:278(__iter__)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - " 2050 0.003 0.000 0.003 0.000 ../zarr/indexing.py:90(ceildiv)\n", - " 1025 0.003 0.000 0.006 0.000 ../zarr/indexing.py:109(__iter__)\n", - " 1024 0.003 0.000 0.003 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc[::2]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2D Benchmarking" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100000000,)" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100000, 1000)" - ] - }, - "execution_count": 100, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d = c.reshape(-1, 1000)\n", - "d.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeint64
Shape(100000, 1000)
Chunk shape(3125, 32)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes800000000 (762.9M)
No. bytes stored39228864 (37.4M)
Storage ratio20.4
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (100000, 1000)\n", - "Chunk shape : (3125, 32)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 800000000 (762.9M)\n", - "No. bytes stored : 39228864 (37.4M)\n", - "Storage ratio : 20.4\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 101, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zd = zarr.array(d)\n", - "zd.info" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool orthogonal selection" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [], - "source": [ - "ix0 = np.random.binomial(1, 0.5, size=d.shape[0]).astype(bool)\n", - "ix1 = np.random.binomial(1, 0.5, size=d.shape[1]).astype(bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "101 ms ± 577 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit d[np.ix_(ix0, ix1)]" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "373 ms ± 5.45 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int orthogonal selection" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "ix0 = np.random.choice(d.shape[0], size=int(d.shape[0] * .5), replace=True)\n", - "ix1 = np.random.choice(d.shape[1], size=int(d.shape[1] * .5), replace=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "174 ms ± 4.13 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit d[np.ix_(ix0, ix1)]" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "566 ms ± 12.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### coordinate (point) selection" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000000" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "n = int(d.size * .1)\n", - "ix0 = np.random.choice(d.shape[0], size=n, replace=True)\n", - "ix1 = np.random.choice(d.shape[1], size=n, replace=True)\n", - "n" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "243 ms ± 3.37 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit d[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2.03 s ± 17 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.vindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:24:31 2017 /tmp/tmp7c68z70p\n", - "\n", - " 62673 function calls in 2.065 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.112 1.112 1.112 1.112 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 3 0.244 0.081 0.244 0.081 ../zarr/indexing.py:654()\n", - " 3 0.193 0.064 0.193 0.064 ../zarr/indexing.py:630()\n", - " 1024 0.170 0.000 0.350 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1024 0.142 0.000 0.151 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1 0.044 0.044 0.044 0.044 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.043 0.043 1.676 1.676 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zd.vindex[ix0, ix1]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Points need to be partially sorted so all points in the same chunk are grouped and processed together. This requires ``argsort`` which dominates time." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## h5py comparison\n", - "\n", - "N.B., not really fair because using slower compressor, but for interest..." - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "import h5py\n", - "import tempfile" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [], - "source": [ - "h5f = h5py.File(tempfile.mktemp(), driver='core', backing_store=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hc = h5f.create_dataset('c', data=c, compression='gzip', compression_opts=1, chunks=zc.chunks, shuffle=True)\n", - "hc" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.16 s, sys: 172 ms, total: 1.33 s\n", - "Wall time: 1.32 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 0, 1, 2, ..., 99999997, 99999998, 99999999])" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time hc[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.11 s, sys: 0 ns, total: 1.11 s\n", - "Wall time: 1.11 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 1063, 28396, 37229, ..., 99955875, 99979354, 99995791])" - ] - }, - "execution_count": 81, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time hc[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [], - "source": [ - "# # this is pathological, takes minutes \n", - "# %time hc[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 38.3 s, sys: 136 ms, total: 38.4 s\n", - "Wall time: 38.1 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 0, 1000, 2000, ..., 99997000, 99998000, 99999000])" - ] - }, - "execution_count": 83, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# this is pretty slow\n", - "%time hc[::1000]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/blosc_microbench.ipynb b/notebooks/blosc_microbench.ipynb deleted file mode 100644 index 9361d8e95b..0000000000 --- a/notebooks/blosc_microbench.ipynb +++ /dev/null @@ -1,200 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.1'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10 loops, best of 3: 110 ms per loop\n", - "1 loop, best of 3: 235 ms per loop\n", - "Array((100000000,), int64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 11.2M; ratio: 67.8; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 loop, best of 3: 331 ms per loop\n", - "1 loop, best of 3: 246 ms per loop\n", - "Array((100000000,), float64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 724.8M; ratio: 1.1; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", - "data = np.random.normal(size=100000000)\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.2.dev0+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10 loops, best of 3: 92.7 ms per loop\n", - "1 loop, best of 3: 230 ms per loop\n", - "Array((100000000,), int64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 11.2M; ratio: 67.8; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 loop, best of 3: 338 ms per loop\n", - "1 loop, best of 3: 253 ms per loop\n", - "Array((100000000,), float64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 724.8M; ratio: 1.1; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", - "data = np.random.normal(size=100000000)\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/notebooks/dask_2d_subset.ipynb b/notebooks/dask_2d_subset.ipynb deleted file mode 100644 index 6e88b510d5..0000000000 --- a/notebooks/dask_2d_subset.ipynb +++ /dev/null @@ -1,869 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook has some profiling of Dask used to make a selection along both first and second axes of a large-ish multidimensional array. The use case is making selections of genotype data, e.g., as required for making a web-browser for genotype data as in www.malariagen.net/apps/ag1000g." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 2.1.1\n", - "dask 0.11.0\n" - ] - } - ], - "source": [ - "import zarr; print('zarr', zarr.__version__)\n", - "import dask; print('dask', dask.__version__)\n", - "import dask.array as da\n", - "import numpy as np" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Real data" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Group(/, 8)\n", - " arrays: 1; samples\n", - " groups: 7; 2L, 2R, 3L, 3R, UNKN, X, Y_unplaced\n", - " store: DirectoryStore" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# here's the real data\n", - "callset = zarr.open_group('/kwiat/2/coluzzi/ag1000g/data/phase1/release/AR3.1/variation/main/zarr2/zstd/ag1000g.phase1.ar3',\n", - " mode='r')\n", - "callset" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3R/calldata/genotype, (22632425, 765, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 32.2G; nbytes_stored: 1.0G; ratio: 31.8; initialized: 34540/34540\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: DirectoryStore" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# here's the array we're going to work with\n", - "g = callset['3R/calldata/genotype']\n", - "g" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 4 ms, sys: 0 ns, total: 4 ms\n", - "Wall time: 5.13 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# wrap as dask array with very simple chunking of first dim only\n", - "%time gd = da.from_array(g, chunks=(g.chunks[0], None, None))\n", - "gd" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "((22632425,), dtype('bool'), 13167162)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# load condition used to make selection on first axis\n", - "dim0_condition = callset['3R/variants/FILTER_PASS'][:]\n", - "dim0_condition.shape, dim0_condition.dtype, np.count_nonzero(dim0_condition)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "# invent a random selection for second axis\n", - "dim1_indices = sorted(np.random.choice(765, size=100, replace=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.3 s, sys: 256 ms, total: 15.5 s\n", - "Wall time: 15.5 s\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# setup the 2D selection - this is the slow bit\n", - "%time gd_sel = gd[dim0_condition][:, dim1_indices]\n", - "gd_sel" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.21 s, sys: 152 ms, total: 1.36 s\n", - "Wall time: 316 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " ..., \n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 1],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]]], dtype=int8)" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# now load a slice from this new selection - quick!\n", - "%time gd_sel[1000000:1100000].compute(optimize_graph=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 105406881 function calls (79072145 primitive calls) in 26.182 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "13167268/6 6.807 0.000 9.038 1.506 slicing.py:623(check_index)\n", - " 2 4.713 2.356 5.831 2.916 slicing.py:398(partition_by_size)\n", - "13167270/2 4.470 0.000 8.763 4.382 slicing.py:540(posify_index)\n", - " 52669338 4.118 0.000 4.119 0.000 {built-in method builtins.isinstance}\n", - " 2 2.406 1.203 8.763 4.382 slicing.py:563()\n", - " 1 0.875 0.875 0.875 0.875 slicing.py:44()\n", - " 13182474 0.600 0.000 0.600 0.000 {built-in method builtins.len}\n", - " 2 0.527 0.264 0.527 0.264 slicing.py:420(issorted)\n", - " 13189168 0.520 0.000 0.520 0.000 {method 'append' of 'list' objects}\n", - " 2 0.271 0.136 0.271 0.136 slicing.py:479()\n", - " 2 0.220 0.110 0.220 0.110 {built-in method builtins.sorted}\n", - " 1 0.162 0.162 0.162 0.162 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.113 0.056 26.071 13.035 core.py:1024(__getitem__)\n", - " 2 0.112 0.056 6.435 3.217 slicing.py:441(take_sorted)\n", - " 1 0.111 0.111 26.182 26.182 :1()\n", - " 2 0.060 0.030 24.843 12.422 slicing.py:142(slice_with_newaxes)\n", - " 106/3 0.039 0.000 1.077 0.359 slicing.py:15(sanitize_index)\n", - " 3 0.037 0.012 0.037 0.012 {built-in method _hashlib.openssl_md5}\n", - " 6726 0.012 0.000 0.017 0.000 slicing.py:567(insert_many)\n", - " 3364 0.004 0.000 0.021 0.000 slicing.py:156()\n", - " 20178 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 8 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 2 0.000 0.000 25.920 12.960 slicing.py:60(slice_array)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 106/4 0.000 0.000 0.037 0.009 utils.py:502(__call__)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 2 0.000 0.000 0.037 0.019 base.py:343(tokenize)\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 2 0.000 0.000 24.763 12.381 slicing.py:170(slice_wrap_lists)\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 6.962 3.481 slicing.py:487(take)\n", - " 1 0.000 0.000 26.182 26.182 {built-in method builtins.exec}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 1 0.000 0.000 0.037 0.037 base.py:314(normalize_array)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 2 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 4 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "# what's taking so long?\n", - "import cProfile\n", - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='time')" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 105406881 function calls (79072145 primitive calls) in 25.630 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 25.630 25.630 {built-in method builtins.exec}\n", - " 1 0.107 0.107 25.630 25.630 :1()\n", - " 2 0.102 0.051 25.523 12.761 core.py:1024(__getitem__)\n", - " 2 0.001 0.000 25.381 12.691 slicing.py:60(slice_array)\n", - " 2 0.049 0.024 24.214 12.107 slicing.py:142(slice_with_newaxes)\n", - " 2 0.000 0.000 24.147 12.073 slicing.py:170(slice_wrap_lists)\n", - "13167268/6 6.664 0.000 8.855 1.476 slicing.py:623(check_index)\n", - "13167270/2 4.354 0.000 8.466 4.233 slicing.py:540(posify_index)\n", - " 2 2.277 1.139 8.465 4.233 slicing.py:563()\n", - " 2 0.000 0.000 6.826 3.413 slicing.py:487(take)\n", - " 2 0.111 0.056 6.331 3.165 slicing.py:441(take_sorted)\n", - " 2 4.628 2.314 5.704 2.852 slicing.py:398(partition_by_size)\n", - " 52669338 4.026 0.000 4.026 0.000 {built-in method builtins.isinstance}\n", - " 106/3 0.071 0.001 1.167 0.389 slicing.py:15(sanitize_index)\n", - " 1 0.943 0.943 0.943 0.943 slicing.py:44()\n", - " 13182474 0.581 0.000 0.581 0.000 {built-in method builtins.len}\n", - " 13189168 0.497 0.000 0.497 0.000 {method 'append' of 'list' objects}\n", - " 2 0.495 0.248 0.495 0.248 slicing.py:420(issorted)\n", - " 2 0.281 0.141 0.281 0.141 slicing.py:479()\n", - " 2 0.234 0.117 0.234 0.117 {built-in method builtins.sorted}\n", - " 1 0.152 0.152 0.152 0.152 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.039 0.020 base.py:343(tokenize)\n", - " 106/4 0.000 0.000 0.039 0.010 utils.py:502(__call__)\n", - " 1 0.000 0.000 0.039 0.039 base.py:314(normalize_array)\n", - " 3 0.039 0.013 0.039 0.013 {built-in method _hashlib.openssl_md5}\n", - " 3364 0.003 0.000 0.019 0.000 slicing.py:156()\n", - " 6726 0.012 0.000 0.016 0.000 slicing.py:567(insert_many)\n", - " 20178 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 8 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 4 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 2 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='cumtime')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Synthetic data" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array((20000000, 200, 2), int8, chunks=(10000, 100, 2), order=C)\n", - " nbytes: 7.5G; nbytes_stored: 2.7G; ratio: 2.8; initialized: 4000/4000\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: dict" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# create a synthetic dataset for profiling\n", - "a = zarr.array(np.random.randint(-1, 4, size=(20000000, 200, 2), dtype='i1'),\n", - " chunks=(10000, 100, 2), compressor=zarr.Blosc(cname='zstd', clevel=1, shuffle=2))\n", - "a" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# create a synthetic selection for first axis\n", - "c = np.random.randint(0, 2, size=a.shape[0], dtype=bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# create a synthetic selection for second axis\n", - "s = sorted(np.random.choice(a.shape[1], size=100, replace=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 208 ms, sys: 0 ns, total: 208 ms\n", - "Wall time: 206 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d = da.from_array(a, chunks=(a.chunks[0], None, None))\n", - "d" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 12 s, sys: 200 ms, total: 12.2 s\n", - "Wall time: 12.2 s\n" - ] - } - ], - "source": [ - "%time ds = d[c][:, s]" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 80095589 function calls (60091843 primitive calls) in 19.467 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "10001773/6 4.872 0.000 6.456 1.076 slicing.py:623(check_index)\n", - " 2 3.517 1.758 4.357 2.179 slicing.py:398(partition_by_size)\n", - "10001775/2 3.354 0.000 6.484 3.242 slicing.py:540(posify_index)\n", - " 40007358 2.965 0.000 2.965 0.000 {built-in method builtins.isinstance}\n", - " 2 1.749 0.875 6.484 3.242 slicing.py:563()\n", - " 1 0.878 0.878 0.878 0.878 slicing.py:44()\n", - " 10019804 0.451 0.000 0.451 0.000 {built-in method builtins.len}\n", - " 10027774 0.392 0.000 0.392 0.000 {method 'append' of 'list' objects}\n", - " 2 0.363 0.181 0.363 0.181 slicing.py:420(issorted)\n", - " 2 0.270 0.135 4.786 2.393 slicing.py:441(take_sorted)\n", - " 1 0.207 0.207 0.207 0.207 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.158 0.079 0.158 0.079 {built-in method builtins.sorted}\n", - " 1 0.094 0.094 19.467 19.467 :1()\n", - " 2 0.079 0.040 19.373 9.686 core.py:1024(__getitem__)\n", - " 2 0.035 0.017 18.147 9.074 slicing.py:142(slice_with_newaxes)\n", - " 3 0.033 0.011 0.033 0.011 {built-in method _hashlib.openssl_md5}\n", - " 106/3 0.028 0.000 1.112 0.371 slicing.py:15(sanitize_index)\n", - " 8002 0.015 0.000 0.020 0.000 slicing.py:567(insert_many)\n", - " 4002 0.004 0.000 0.023 0.000 slicing.py:156()\n", - " 24006 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 8 0.001 0.000 0.001 0.000 {method 'update' of 'dict' objects}\n", - " 2 0.001 0.000 0.001 0.000 slicing.py:479()\n", - " 2 0.000 0.000 19.259 9.630 slicing.py:60(slice_array)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 106/4 0.000 0.000 0.034 0.008 utils.py:502(__call__)\n", - " 2 0.000 0.000 18.089 9.044 slicing.py:170(slice_wrap_lists)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 5.149 2.574 slicing.py:487(take)\n", - " 2 0.000 0.000 0.034 0.017 base.py:343(tokenize)\n", - " 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 1 0.000 0.000 19.467 19.467 {built-in method builtins.exec}\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 4 0.000 0.000 0.001 0.000 dicttoolz.py:19(merge)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 2 0.000 0.000 0.001 0.000 exceptions.py:15(merge)\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 4 0.000 0.000 0.001 0.000 functoolz.py:217(__call__)\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "cProfile.run('d[c][:, s]', sort='time')" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 452 ms, sys: 8 ms, total: 460 ms\n", - "Wall time: 148 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[[ 2, -1],\n", - " [ 2, 3],\n", - " [ 3, 0],\n", - " ..., \n", - " [ 1, 3],\n", - " [-1, -1],\n", - " [ 1, 1]],\n", - "\n", - " [[ 1, -1],\n", - " [ 2, 2],\n", - " [-1, 2],\n", - " ..., \n", - " [ 2, -1],\n", - " [ 1, 3],\n", - " [-1, -1]],\n", - "\n", - " [[ 1, -1],\n", - " [ 2, 0],\n", - " [ 0, 3],\n", - " ..., \n", - " [ 2, 2],\n", - " [ 3, 2],\n", - " [ 0, 2]],\n", - "\n", - " ..., \n", - " [[ 1, 2],\n", - " [ 3, -1],\n", - " [ 2, 1],\n", - " ..., \n", - " [ 1, 2],\n", - " [ 1, 0],\n", - " [ 2, 0]],\n", - "\n", - " [[ 1, 2],\n", - " [ 1, 0],\n", - " [ 2, 3],\n", - " ..., \n", - " [-1, 2],\n", - " [ 3, 3],\n", - " [ 1, -1]],\n", - "\n", - " [[-1, 3],\n", - " [ 2, 2],\n", - " [ 1, 1],\n", - " ..., \n", - " [ 3, 3],\n", - " [ 0, 0],\n", - " [ 0, 2]]], dtype=int8)" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time ds[1000000:1100000].compute(optimize_graph=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 80055494 function calls (60052157 primitive calls) in 19.425 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "10001670/3 5.032 0.000 6.671 2.224 slicing.py:623(check_index)\n", - " 1 3.459 3.459 4.272 4.272 slicing.py:398(partition_by_size)\n", - "10001671/1 3.287 0.000 6.378 6.378 slicing.py:540(posify_index)\n", - " 40006704 2.999 0.000 2.999 0.000 {built-in method builtins.isinstance}\n", - " 1 1.731 1.731 6.378 6.378 slicing.py:563()\n", - " 1 0.849 0.849 0.849 0.849 slicing.py:44()\n", - " 10011685 0.433 0.000 0.433 0.000 {built-in method builtins.len}\n", - " 10015670 0.381 0.000 0.381 0.000 {method 'append' of 'list' objects}\n", - " 1 0.355 0.355 0.355 0.355 slicing.py:420(issorted)\n", - " 1 0.196 0.196 0.196 0.196 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 1 0.193 0.193 0.193 0.193 slicing.py:479()\n", - " 1 0.157 0.157 0.157 0.157 {built-in method builtins.sorted}\n", - " 1 0.085 0.085 4.707 4.707 slicing.py:441(take_sorted)\n", - " 1 0.085 0.085 19.425 19.425 :1()\n", - " 1 0.079 0.079 19.341 19.341 core.py:1024(__getitem__)\n", - " 1 0.034 0.034 18.157 18.157 slicing.py:142(slice_with_newaxes)\n", - " 2 0.033 0.017 0.033 0.017 {built-in method _hashlib.openssl_md5}\n", - " 1 0.026 0.026 1.071 1.071 slicing.py:15(sanitize_index)\n", - " 4001 0.007 0.000 0.009 0.000 slicing.py:567(insert_many)\n", - " 2001 0.002 0.000 0.011 0.000 slicing.py:156()\n", - " 12003 0.001 0.000 0.001 0.000 {method 'pop' of 'list' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 1 0.000 0.000 19.228 19.228 slicing.py:60(slice_array)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n", - " 1 0.000 0.000 18.111 18.111 slicing.py:170(slice_wrap_lists)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 1 0.000 0.000 5.062 5.062 slicing.py:487(take)\n", - " 1 0.000 0.000 0.033 0.033 base.py:343(tokenize)\n", - " 1 0.000 0.000 19.425 19.425 {built-in method builtins.exec}\n", - " 2 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 3 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 2 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 1 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 2 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 4 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 1 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 1 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 4 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 2 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 2 0.000 0.000 0.033 0.017 utils.py:502(__call__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 1 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 1 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 2 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 1 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "# problem is in fact just the dim0 selection\n", - "cProfile.run('d[c]', sort='time')" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/notebooks/dask_copy.ipynb b/notebooks/dask_copy.ipynb deleted file mode 100644 index ba4391737a..0000000000 --- a/notebooks/dask_copy.ipynb +++ /dev/null @@ -1,1518 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Profile array copy via dask threaded scheduler" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook profiles a very simple array copy operation, using synthetic data." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.0.1.dev18+dirty\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "\n", - "(function(global) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " if (typeof (window._bokeh_onload_callbacks) === \"undefined\") {\n", - " window._bokeh_onload_callbacks = [];\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", - " delete window._bokeh_onload_callbacks\n", - " console.info(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(js_urls, callback) {\n", - " window._bokeh_onload_callbacks.push(callback);\n", - " if (window._bokeh_is_loading > 0) {\n", - " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " window._bokeh_is_loading = js_urls.length;\n", - " for (var i = 0; i < js_urls.length; i++) {\n", - " var url = js_urls[i];\n", - " var s = document.createElement('script');\n", - " s.src = url;\n", - " s.async = false;\n", - " s.onreadystatechange = s.onload = function() {\n", - " window._bokeh_is_loading--;\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", - " run_callbacks()\n", - " }\n", - " };\n", - " s.onerror = function() {\n", - " console.warn(\"failed to load library \" + url);\n", - " };\n", - " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", - " }\n", - " };\n", - "\n", - " var js_urls = ['https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-compiler-0.12.0.min.js'];\n", - "\n", - " var inline_js = [\n", - " function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - " \n", - " function(Bokeh) {\n", - " Bokeh.$(\"#d4821cb3-378c-411d-a941-d0708c0c532b\").text(\"BokehJS successfully loaded\");\n", - " },\n", - " function(Bokeh) {\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " for (var i = 0; i < inline_js.length; i++) {\n", - " inline_js[i](window.Bokeh);\n", - " }\n", - " }\n", - "\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(js_urls, function() {\n", - " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(this));" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", - "import bcolz\n", - "# don't let bcolz use multiple threads internally, we want to \n", - "# see whether dask can make good use of multiple CPUs\n", - "bcolz.set_nthreads(1)\n", - "import multiprocessing\n", - "import dask\n", - "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", - "from bokeh.io import output_notebook\n", - "output_notebook()" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import tempfile\n", - "import operator\n", - "from functools import reduce\n", - "from zarr.util import human_readable_size\n", - "\n", - "\n", - "def h5fmem(**kwargs):\n", - " \"\"\"Convenience function to create an in-memory HDF5 file.\"\"\"\n", - "\n", - " # need a file name even tho nothing is ever written\n", - " fn = tempfile.mktemp()\n", - "\n", - " # file creation args\n", - " kwargs['mode'] = 'w'\n", - " kwargs['driver'] = 'core'\n", - " kwargs['backing_store'] = False\n", - "\n", - " # open HDF5 file\n", - " h5f = h5py.File(fn, **kwargs)\n", - "\n", - " return h5f\n", - "\n", - "\n", - "def h5d_diagnostics(d):\n", - " \"\"\"Print some diagnostics on an HDF5 dataset.\"\"\"\n", - " \n", - " print(d)\n", - " nbytes = reduce(operator.mul, d.shape) * d.dtype.itemsize\n", - " cbytes = d._id.get_storage_size()\n", - " if cbytes > 0:\n", - " ratio = nbytes / cbytes\n", - " else:\n", - " ratio = np.inf\n", - " r = ' compression: %s' % d.compression\n", - " r += '; compression_opts: %s' % d.compression_opts\n", - " r += '; shuffle: %s' % d.shuffle\n", - " r += '\\n nbytes: %s' % human_readable_size(nbytes)\n", - " r += '; nbytes_stored: %s' % human_readable_size(cbytes)\n", - " r += '; ratio: %.1f' % ratio\n", - " r += '; chunks: %s' % str(d.chunks)\n", - " print(r)\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "def profile_dask_copy(src, dst, chunks, num_workers=multiprocessing.cpu_count(), dt=0.1, lock=True):\n", - " dsrc = da.from_array(src, chunks=chunks)\n", - " with Profiler() as prof, ResourceProfiler(dt=dt) as rprof:\n", - " da.store(dsrc, dst, num_workers=num_workers, lock=lock)\n", - " visualize([prof, rprof], min_border_top=60, min_border_bottom=60)\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## NumPy arrays" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1314, 2727, 2905, ..., 1958, 1325, 1971], dtype=uint16)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# a1 = np.arange(400000000, dtype='i4')\n", - "a1 = np.random.normal(2000, 1000, size=200000000).astype('u2')\n", - "a1" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'381.5M'" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "human_readable_size(a1.nbytes)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "a2 = np.empty_like(a1)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "chunks = 2**20, # 4M" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 56 ms, sys: 36 ms, total: 92 ms\n", - "Wall time: 91.7 ms\n" - ] - } - ], - "source": [ - "%time a2[:] = a1" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(a1, a2, chunks, lock=True, dt=.01)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(a1, a2, chunks, lock=False, dt=.01)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Zarr arrays (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((200000000,), uint16, chunks=(1048576,), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 381.5M; nbytes_stored: 318.2M; ratio: 1.2; initialized: 191/191\n", - " store: builtins.dict" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z1 = zarr.array(a1, chunks=chunks, compression='blosc', \n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", - "z1" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((200000000,), uint16, chunks=(1048576,), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 381.5M; nbytes_stored: 294; ratio: 1360544.2; initialized: 0/191\n", - " store: builtins.dict" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z2 = zarr.empty_like(z1)\n", - "z2" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(z1, z2, chunks, lock=True, dt=.02)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(z1, z2, chunks, lock=False, dt=0.02)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 251 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison, using blosc internal threads\n", - "%timeit -n3 -r5 z2[:] = z1" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " " - ] - } - ], - "source": [ - "%prun z2[:] = z1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Without the dask lock, we get better CPU utilisation. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## HDF5 datasets (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "h5f = h5fmem()\n", - "h5f" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " compression: lzf; compression_opts: None; shuffle: True\n", - " nbytes: 381.5M; nbytes_stored: 357.4M; ratio: 1.1; chunks: (1048576,)\n" - ] - } - ], - "source": [ - "h1 = h5f.create_dataset('h1', data=a1, chunks=chunks, compression='lzf', shuffle=True)\n", - "h5d_diagnostics(h1)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " compression: lzf; compression_opts: None; shuffle: True\n", - " nbytes: 762.9M; nbytes_stored: 0; ratio: inf; chunks: (1048576,)\n" - ] - } - ], - "source": [ - "h2 = h5f.create_dataset('h2', shape=h1.shape, chunks=h1.chunks, \n", - " compression=h1.compression, compression_opts=h1.compression_opts, \n", - " shuffle=h1.shuffle)\n", - "h5d_diagnostics(h2)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(h1, h2, chunks, lock=True, dt=0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(h1, h2, chunks, lock=False, dt=0.1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bcolz carrays (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "carray((200000000,), uint16)\n", - " nbytes := 381.47 MB; cbytes := 318.98 MB; ratio: 1.20\n", - " cparams := cparams(clevel=1, shuffle=2, cname='lz4', quantize=0)\n", - " chunklen := 1048576; chunksize: 2097152; blocksize: 16384\n", - "[1314 2727 2905 ..., 1958 1325 1971]" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c1 = bcolz.carray(a1, chunklen=chunks[0],\n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", - "c1" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "carray((200000000,), uint16)\n", - " nbytes := 381.47 MB; cbytes := 2.00 MB; ratio: 190.73\n", - " cparams := cparams(clevel=1, shuffle=2, cname='lz4', quantize=0)\n", - " chunklen := 1048576; chunksize: 2097152; blocksize: 4096\n", - "[0 0 0 ..., 0 0 0]" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c2 = bcolz.zeros(a1.shape, chunklen=chunks[0], dtype=a1.dtype, \n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", - "c2" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(c1, c2, chunks, lock=True, dt=0.05)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# not sure it's safe to use bcolz without a lock, but what the heck...\n", - "profile_dask_copy(c1, c2, chunks, lock=False, dt=0.05)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 649 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison\n", - "%timeit -n3 -r5 c2[:] = c1" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 557 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison\n", - "%timeit -n3 -r5 c1.copy()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/dask_count_alleles.ipynb b/notebooks/dask_count_alleles.ipynb deleted file mode 100644 index 8b9b7cec6e..0000000000 --- a/notebooks/dask_count_alleles.ipynb +++ /dev/null @@ -1,648 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Profile allele count from genotype data via dask.array" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.0.1.dev18+dirty\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "\n", - "(function(global) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " if (typeof (window._bokeh_onload_callbacks) === \"undefined\") {\n", - " window._bokeh_onload_callbacks = [];\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", - " delete window._bokeh_onload_callbacks\n", - " console.info(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(js_urls, callback) {\n", - " window._bokeh_onload_callbacks.push(callback);\n", - " if (window._bokeh_is_loading > 0) {\n", - " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " window._bokeh_is_loading = js_urls.length;\n", - " for (var i = 0; i < js_urls.length; i++) {\n", - " var url = js_urls[i];\n", - " var s = document.createElement('script');\n", - " s.src = url;\n", - " s.async = false;\n", - " s.onreadystatechange = s.onload = function() {\n", - " window._bokeh_is_loading--;\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", - " run_callbacks()\n", - " }\n", - " };\n", - " s.onerror = function() {\n", - " console.warn(\"failed to load library \" + url);\n", - " };\n", - " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", - " }\n", - " };\n", - "\n", - " var js_urls = ['https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-compiler-0.12.0.min.js'];\n", - "\n", - " var inline_js = [\n", - " function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - " \n", - " function(Bokeh) {\n", - " Bokeh.$(\"#b153ad5f-436a-4afb-945c-87790add89c8\").text(\"BokehJS successfully loaded\");\n", - " },\n", - " function(Bokeh) {\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " for (var i = 0; i < inline_js.length; i++) {\n", - " inline_js[i](window.Bokeh);\n", - " }\n", - " }\n", - "\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(js_urls, function() {\n", - " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(this));" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", - "import multiprocessing\n", - "import dask\n", - "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", - "from bokeh.io import output_notebook\n", - "output_notebook()\n", - "from functools import reduce\n", - "import operator\n", - "import allel" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", - "genotype" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((13167162, 765, 2), int8, chunks=(6553, 200, 2), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 18.8G; nbytes_stored: 683.2M; ratio: 28.1; initialized: 8040/8040\n", - " store: builtins.dict" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# copy into a zarr array\n", - "# N.B., chunks in HDF5 are too small really, use something bigger\n", - "chunks = (genotype.chunks[0], genotype.chunks[1] * 20, genotype.chunks[2])\n", - "genotype_zarr = zarr.array(genotype, chunks=chunks, compression='blosc',\n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", - "genotype_zarr" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We want to perform an allele count. Compare serial and parallel implementations, and compare working direct from HDF5 versus from Zarr." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1min 50s, sys: 512 ms, total: 1min 51s\n", - "Wall time: 1min 50s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))
nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;
compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);
data: bcolz.carray_ext.carray
0123
01523500
11527100
21527100
31527100
41527100
\n", - "

...

" - ], - "text/plain": [ - "AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))\n", - " nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;\n", - " compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);\n", - " data: bcolz.carray_ext.carray" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "# linear implementation from HDF5 on disk\n", - "allel.GenotypeChunkedArray(genotype).count_alleles()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 2min 27s, sys: 2.14 s, total: 2min 29s\n", - "Wall time: 1min 23s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))
nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;
compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);
data: bcolz.carray_ext.carray
0123
01523500
11527100
21527100
31527100
41527100
\n", - "

...

" - ], - "text/plain": [ - "AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))\n", - " nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;\n", - " compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);\n", - " data: bcolz.carray_ext.carray" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "# linear implementation from zarr in memory\n", - "# (although blosc can use multiple threads internally)\n", - "allel.GenotypeChunkedArray(genotype_zarr).count_alleles()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# multi-threaded implementation from HDF5 on disk\n", - "gd = allel.model.dask.GenotypeDaskArray.from_array(genotype, chunks=chunks)\n", - "ac = gd.count_alleles(max_allele=3)\n", - "with Profiler() as prof, ResourceProfiler(dt=1) as rprof:\n", - " ac.compute(num_workers=8)\n", - "visualize([prof, rprof], min_border_bottom=60, min_border_top=60);" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# multi-threaded implementation from zarr in memory\n", - "gdz = allel.model.dask.GenotypeDaskArray.from_array(genotype_zarr, chunks=chunks)\n", - "acz = gdz.count_alleles(max_allele=3)\n", - "with Profiler() as prof, ResourceProfiler(dt=1) as rprof:\n", - " acz.compute(num_workers=8)\n", - "visualize([prof, rprof], min_border_bottom=60, min_border_top=60);" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/genotype_benchmark_compressors.ipynb b/notebooks/genotype_benchmark_compressors.ipynb deleted file mode 100644 index b262e63fa0..0000000000 --- a/notebooks/genotype_benchmark_compressors.ipynb +++ /dev/null @@ -1,548 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.1.1.dev7+dirty\n", - "blosc ('1.10.0.dev', '$Date:: 2016-07-20 #$')\n" - ] - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import functools\n", - "import timeit\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "print('blosc', blosc.version())\n", - "import numpy as np\n", - "import h5py\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", - "genotype" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "n_variants = 500000" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(500000, 765, 2)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "genotype_sample = genotype[1000000:1000000+n_variants, ...]\n", - "genotype_sample.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "765000000" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nbytes = genotype_sample.nbytes\n", - "nbytes" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(685, 765, 2)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# 1M chunks of first dimension\n", - "chunks = (int(2**20 / (genotype_sample.shape[1] * genotype_sample.shape[2])), \n", - " genotype_sample.shape[1], \n", - " genotype_sample.shape[2])\n", - "chunks" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "8" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "blosc.get_nthreads()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((500000, 765, 2), int8, chunks=(685, 765, 2), order=C)\n", - " compression: blosc; compression_opts: {'cname': 'lz4', 'clevel': 1, 'shuffle': 2}\n", - " nbytes: 729.6M; nbytes_stored: 23.0M; ratio: 31.7; initialized: 730/730\n", - " store: builtins.dict" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='lz4', clevel=1, shuffle=2))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((500000, 765, 2), int8, chunks=(685, 765, 2), order=C)\n", - " compression: blosc; compression_opts: {'cname': 'zstd', 'clevel': 1, 'shuffle': 2}\n", - " nbytes: 729.6M; nbytes_stored: 12.0M; ratio: 60.7; initialized: 730/730\n", - " store: builtins.dict" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='zstd', clevel=1, shuffle=2))" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "compression_configs = (\n", - " (None, None),\n", - " ('zlib', 1),\n", - " ('bz2', 1),\n", - " ('lzma', dict(preset=1)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=2)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=2)),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def log(*msg):\n", - " print(*msg, file=sys.stdout)\n", - " sys.stdout.flush()" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "@functools.lru_cache(maxsize=None)\n", - "def compression_ratios():\n", - " x = list()\n", - " for compression, compression_opts in compression_configs:\n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " ratio = z.nbytes / z.nbytes_stored\n", - " x.append(ratio)\n", - " log(compression, compression_opts, ratio)\n", - " return x\n" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcFNW99/HPFwQJKLKYsCkDEjUuERk3FAKjqNEECdcg\ngrhcgxqveSIuSR6vG+CCmohPTKIm5iGokSTiDXpRQxSExmhERDYFxIVNUBTZrugjRub3/FGnsaan\nl+phZnpGfu/Xq19TXXWqzq9+XWKdPudUy8xwzjnnnHPOObfrmpQ6AOecc84555z7svAGlnPOOeec\nc87VEm9gOeecc84551wt8QaWc84555xzztUSb2A555xzzjnnXC3xBpZzzjnnnHPO1RJvYDnnnGv0\nJM2S9KtSx1EsSZWSzix1HM4552qP/HewnHOucZH0NeA64LvAfsAGYDHwGzObVsrYSkVSG+BfZvZx\nqWPJRtJEoL2ZDcpY/zVgs5n9qzSROeecq217lDoA55xzyUkqA/4JbAX+N1HDqglwMnAf0K1kweUg\nqVldNyDMbEtdHj+XXT03M/ugNuNxzjlXej5E0DnnGpf7gErgKDP7q5m9aWbLzewe4Ih0IUn7S3pM\n0v+E118ldYltHy3pVUnnS1opaZukCZKaSfqxpHckfSjpF/HKQ9nRkv4o6SNJ70m6OqNMpaTLQp3b\ngFvD+kMlPRnieV/SnyR1iO13uKQZkraGYy+Q1D9s20PSryStk/SppNWSxsX2rTJEUFIbSQ9K2iTp\nE0nTJR0a235BqOOkkIdtkmaGBmxO2c5NUhNJ/1fSilDXG5J+Gs81cAHw3bD/Dkn9Ysc7M1b28BDr\nJ5I2SpooqXW+mJxzzjUs3sByzrlGQlJb4NtEQwH/X+Z2M/ufUE7AVOCrQH+gAugMPJaxSzdgENFQ\nw38DhgJPAUcS9YiNBEZJ+l7GflcCS4BewI3AOEmDM8rcGI51OHCPpI7AbKIet6OBAUAr4L9j+/wJ\neDds7wmMAT4N20YB3wsxfh04G1hePUs7PQgcA5wR/n4C/F3SnrEyewLXAP8O9AbaAL/Nc8ys50b0\n/9K1wBDgG8C1wH9KujCUvxOYDMwAOgCdiHohq5DUEnga+J+Qg8HACcCEBDE555xrIHyIoHPONR5f\nBwS8XqDcyUQ3/weY2TsAks4B3pJ0kpnNDOWaAP9uZtuApZL+DvQDvmNmnwPLJb1A1BiKN4ReMrPb\nw/Jbko4FrgIej5X5i5n9If1G0lhgoZldG1v378BGSUeb2TygDPiFmb0ZiqyIHa8r8IaZvRDerwXm\nZDt5SQcSNay+lS4v6TxgDTACSMfVFLjMzN4KZe4kWWOmyrkFY2LLayQdBQwHJprZx5L+H9DSzDbk\nOe4IoCVwnpl9EmK6BJgl6QAzW5FnX+eccw2E92A551zjoYTlvgG8m25cAZjZSqLeoUNj5daExlXa\n+0SNmM8z1n0t4/gvZnl/aMa6VzLeHwX0D8PyPpL0EVGDx4AeocxdwARJz0q6VtLBsf0fAHqF4Xe/\nkfSd0FOXzTeAHcQaYKF379WMOLenG1fBu0Dz8MCMfDLPDUmXSnpZ0gfh3K4kahQW4xvA4nTjKvgn\n0ZDQzPw655xroLyB5ZxzjcebRA2SQ3bhGPFHx2Y+nMFyrKvJ/ysyn+bXBHiSaJ5Yz9jrwLAeMxtL\ndG6PEQ2NWxx6uTCzBUQ9XNcQNTQfBJ6pQVzx8/88x7ZC51vl3CSdDfwfop6xU4nO616geQ3iy8Uf\n+eucc42EN7Ccc66RMLPNRHN0/leYr1OFpH3C4jKgs6SusW0HEM3DWlILofTOeH98qDOf+cBhRL1m\nKzJeOxssZva2mf3GzAYSDde7KLbtYzObYmY/Ipo3NkDS17PUtYzo/2/Hp1eEB0V8k9o5/0x9gDlm\ndp+ZLQxD+TLj+oxoSGI+y4BvSmqVcWxROL/OOecaCG9gOedc4/IjohvueZKGSDpI0sGS/gNYBGBm\nM4iGw02SdJSko4GHgXlmlqqFGHpL+t+Svi7pYuBcouF9+dwD7ANMlnSspO6STpb0O0mtJLUIQ//6\nSyqTdBzQl9AgknSlpGGSvhEaVSOIHlW/NrOiMOxvKvA7SX0lfTOc/1bgzwXiTDoMM+4NoFzSaSEn\nNxDNZYtbBRwePq/2krI1tiYRPYzjofA0wX5ED934q8+/cs65xsMbWM4514iEuVTlwHTgdqJG1bNE\nT9i7IlZ0ENEPEM8M298lelJgbbiLaKjfAuAm4AYziz+hsNpwNjN7j6g3ZgcwDXgN+DXRUwK3h/Vt\ngYlED/H4K/ACkH4E/EfAT4GXgHmh/tPNLP2Uwcw6/x2YS/RwjjlETww8zcy2Fzi3QkPxsm3/HdFT\nAieFOrsSPTkw7vdEvVDzgA+IclHleOHJkN8GWhOd52NEORhZICbnnHMNiMx8WLdzzrlkJK0Efm1m\nhXqsnHPOud2S92A555xzzjnnXC3xBpZzzrli+LAH55xzLg8fIuicc84555xztcR7sJxzzjnnnHOu\nlngDyznnnHPOOedqiTewnHPOOeecc66W7FHqAJxzDYskn5jpnHPOOZeAmVX7gXrvwXLOVWNm/irw\nuuCCC0oeQ2N4eZ48T54nz1NDfXmePE+7+srFG1jOOVcD3bp1K3UIjYLnKRnPUzKep2Q8T8l4npLx\nPBXPG1jOOeecc845V0v8d7Ccc1X4HCznnHP5dOhQxvr1q0odxi775S9/yRVXXFHqMBo8z1NukrAs\nc7D8IRfOuSy8jVVYCqgocQyNQQrPUxIpPE9JpPA8JZGiLvP0/vvV7icbpSOPPLLUITQKnqfieQ+W\nc66KqAfL/11wzjmXi/JO8HfF69atG6tXry51GC6HsrIyVq1aVW19rh6sepuDJalM0qs5ts2SVF5f\nsWTU3VXSfEnTYutWliKWXCT1lzQxQbm8cUv6KPztJGlyWL5A0q9rcryEdY6WdFWh4xQjfkxJEyX1\nK1C+v6Qt4XOeL+n6BHXMktS1wPairllJQyQtlfRseP9nSQsljQrncWaB/ZOc6zmSFoXX85KOiG0b\nL2mJpP7FxO2cc865urV69eqSPxHPX7lfxTZ+6/shFw3x647BwDNmdnpsXUOMM0lMhcoYgJm9Z2ZD\nE+xXG3U2FM+ZWXl43VKiGEYCF5nZAEkdgaPN7Egzu7sW61gB9DOznsAtwP3pDWZ2NXAT8INarG83\nlip1AI1EqtQBNBKpUgfQSKRKHUAjkSp1AI1CKpUqdQjuS6q+G1jNJD0cvsWfLKlFZgFJwyUtDq/b\nw7om4dv7xeGb+VFhfQ9J00MvwDxJ3WsQUxvgg4x1G2LxnB/qXCDpwbBuoqS7Jb0g6a10z4OkVpJm\nhFgWSRoU1pdJWhb2Wy5pkqRTwv7LJR0dyrWUNEHSHEmvSDojhPEZsDXBuWwIxxkb4p0vaa2kCenT\nicUT703sGnpklku6MVseCtWZK1dxkg6QNE3Sy5JmSzpIUmtJq2JlWkpaI6lptvJZ6t9ClJ9Cih0w\nvhHYkevaC4ZKeknS65L6hPir9AhKekJSP0k3AH2BCZJ+DjwNdAmfUd8qgUrlklLhvKdJ6pD0XM1s\njpmlr5U5QJeMIuuJrnnnnHPOOVcH6vshFwcDF5rZnHDTfxlwV3qjpE7A7UAvopvJ6aGRshboYmZH\nhHKtwy6TgHFmNlVSc2rWYGwKVMZXmNlxoZ5DgWuB481ss6T4jWlHM+sj6RBgKjAF+BQYbGbbJLUn\nusGdGsr3AL5vZkslzQOGhf0HhTrOBK4DnjWzkZL2AeZKmmFmLwIvhpiOAn5oZpdknkg6bjMbDYwO\nx3gOSN/wx3ub4svHAIeF+F+W9KSZzU8fL5+EuUq7P8T+tqRjgftCb84CSf3NbDYwEPi7me2QVK08\nMCCj/ivTy5LGAi+b2ZNZ6j5e0kJgHfBTM1ta4LyGhGOWk/3aA2hqZsdJOh0YA5yS3j3L8W6WdBJw\nlZktkHQP8ISZlYfjjgx/9yD6vAaZ2UZJQ4FxwMgizjXtImBaxrpKomu+gDGx5Qp8Unk2FaUOoJGo\nKHUAjURFqQNoJCpKHUAjUVHqABqFioqKUofgGplUKpWo57O+G1hrzGxOWH4Y+DGxBhbRjf4sM9sE\nIGkS0I9oqFN3SXcDfwOekbQX0NnMpgKYWZJejCokCegZYsnmJOBRM9sc6tgS2/Z4WLdM0tfShwRu\nUzRPphLoHNu2MnZTvwSYEZZfBbqF5VOBMyT9NLxvDnQFlqcrNbNXgGqNqxweBu4ys4UFyk1Pn5uk\nKUQ9LfMT1pGWL1dIagWcADwa8g7QLPydDJwNzAaGAfcUKJ9VaFhm8wrQ1cw+CY2hx4FsvWHZrCDj\n2ottmxI7flnC4xXqSTsYOJzoywURfWnwbmahPOcaVSKdCFxI9FnGrQMOkrSnmW3PfYQxBcJ0zjnn\nnNu9VFRUVGmYjx07Nmu5Us/ByjZ/p9oNaLhZ70k0qPhS4Pe5ylY5kHRZbKhcx4xtTYCVwCHAU4mi\nryp+c5qOYwSwL9DLzHoRDT1skaV8Zex9JV80dEXUy9UrvLqb2XJqQNIYogZttaF6WST5XHZVE2Bz\nmAOVPr/Dw7apwGmS2gLlwMwC5YtiZtvM7JOwPI1oqGq7hPvmuvbgi89wB198hp9T9b+rasNgCxDw\nWuy8e2bMDyx8gOjBFvcT9YJtjm8zsxXAMmC1pMOKjM1VkSp1AI1EqtQBNBKpUgfQSKRKHUAjkSp1\nAI1CQ5+D1bFjNyTV2atjx26J4ujevTszZ87Muu3555/nkEMOqZXzzVdPIZ9++ilnnHEGbdq04eyz\nzwbg+uuv56tf/SqdO3dm9erVNGnShMrKygJHqh313cAqk5QednYO8I+M7XOBfpLaSWoKDAdmh+F2\nTc3sMeB6oNzMtgHvSPoegKTmkr4SP5iZ3RtuUsvNbH3Gtkoz6wbMI+o9yWYmcFb6Zjw0ALJJN7D2\nAT4ws8rQg1CWpUw+TwOX79xBqtEPDyiau3UyMCpzU45dTpHUJuRvMPBClmMuK1Bt3lyZ2UfASklD\nYsc8Imz7mOhzuBt40iI5yxcrNoeJMNRQsV7SGWFoaq59q117uYqGv6uAIxXZHzg2X2hZ1i0Hviqp\nd6h/jzD8MhFFTz78K3Cemb2dZfsRQHei3t8lSY/rnHPOufr1/vurib7zrptXdPxd07dvX5Yt++IW\ncVcaSbviv/7rv9iwYQObN2/mkUce4Z133uGuu+7i9ddf5913o4FAXwyIqnv13cB6HfiRpKVEE+1/\nG9ann263HriG6KuXBURzTJ4gmqifkrQA+GMoA3A+cLmkRUSNgp030kV4A8jamxGG9N1K1MhbAIyP\nxxsvGv5OAo4J8ZxL1FOQWSbb/mk3E/WuLFb0EIqbMgtIOirMTcrnSqAz0Xyq+aE3K1+9c4mGuy0k\nGuZXZXhgaGTklSdXcecCIxU9lOQ1YFBs2yNEPYB/ia0bkad8NYoe7jEwy6Yhkl4Lcf2SaBhieoho\nD2BTnsPmuvayXgNm9gJRI2tJqOuVzDI53qf3/xcwBLhD0ZyxBcDxRZzrDUTX872h93Zuxva2wCoz\nq5+vcL7UKkodQCNRUeoAGomKUgfQSFSUOoBGoqLUATQKPgfry2P16tUcdNBBOxtRq1evZt9996V9\n+4K3sHUj3zPfd4cX8FPg9lLH0ZBfwHeB/1XqOOrgvA4D7ix1HPV8zkOBPxcoY2D+8pe//OUvf+V4\nYa52Zctp3f//ONnn2K1bN7vtttvs0EMPtXbt2tkPfvAD2759u5mZpVIp22+//czM7LzzzrMmTZpY\ny5Ytbe+997Zf/OIX1Y714Ycf2sCBA61NmzbWrl0769evX5V67rzzTjviiCOsTZs2NmzYsJ31PPDA\nA9a3b98qx5Jkb7/9to0ePdqaN29uzZo1s7333tt+97vf2Ve+8hVr2rSp7b333nbhhRfaqlWrrEmT\nJrZjxw4zM9u6dauNHDnSOnXqZPvtt59df/31VllZWdTnE1tP5qu+H3LREE0BHpA0zYqc67K7MLOa\nzFFr8CwaIveTUsdRXySNB74F/GeC0nUdjnPOuUaqQ4eyUodQK1KplPdiJfSnP/2J6dOn07JlSwYO\nHMgtt9zCTTdFA63SvUYPPfQQ//jHP/jDH/7AiSeemPU448ePZ//992fjxo2YGXPmzKmy/dFHH+WZ\nZ55hzz335IQTTuCBBx7gkksuqVJPWvr9mDFjkMTbb7/NQw89BMDBBx/Meeedx5o1awCq/VDwBRdc\nQKdOnVixYgXbtm1j4MCBdO3alYsvvnhX0rTTbt/AsmieyrdKHYdzdc2iHxpOWrYuQ/lS8P8xJ+N5\nSsbzlIznKRnPk6ttP/7xj+ncuTMA1113HZdffvnOBlamfPcQzZo147333mPlypX06NGDPn36VNk+\natQoOnSIZvycccYZLFyY+0HYNb1Xef/995k2bRpbt25lzz33pEWLFlxxxRXcf//9tdbAqu85WM45\n96XgNy/JeJ6S8Twl43lKxvOUjOcpuf3222/ncllZ2c4HRxTrZz/7GT169ODUU0/l61//OnfccUeV\n7enGFUDLli3Ztm1bzQLOY82aNfzrX/+iU6dOtGvXjrZt23LppZfy4Ycf1lodu30PlnPOOeeccy63\nd955Z+fy6tWrd/ZmZSr0pL5WrVpx5513cuedd7J06VJOPPFEjj322JxDCuP7ffLJJzvfr1+/vsZP\nBdx///1p0aIFGzdurLMnC3oPlnPO1UBD//2UhsLzlIznKRnPUzKep2Q8T8ndc889rFu3jk2bNjFu\n3DiGDRuWtVzHjh1ZsWJFzuM89dRTvP129Csye++9N3vssQdNmzYtWH/Pnj1ZsmQJixcvZvv27Tl/\n4Def9JDCjh07cuqpp3LllVfy0UcfYWasWLGC5557ruhj5uINLOecc8455xqY6GEiqrNX0oeVSOKc\nc87ZOazvwAMP5Lrrrsta9pprruHmm2+mXbt23HXXXdW2v/nmm5x88snsvffe9OnThx/96Ef069dv\nZz25HHjggdx4440MGDCAgw46iG99q/jHJ8SP/9BDD/HZZ59x6KGH0q5dO8466yzWr1+fZ+8i6/LJ\n7M65OEnm/y4455xz9UeSP2CqAcv1+YT11VqG3oPlnHPOOeecc7XEH3LhnKumriZ9Oufc7q5Dlw6s\nX1t7Q5Gy8ce0J+N5cnXFG1jOuerGlDqARmAl0L3UQTQCnqdkPE/JfAny9P6Y90sdgnOujvkcLOdc\nFZLMG1jOOVdHxviPubvqfA5Ww9Zg52BJKpP0ao5tsySV11csGXV3lTRf0rTYupWliCUXSf0lTUxQ\nLm/ckj4KfztJmhyWL5D065ocL2GdoyVdVeg4xYgfU9JESf0KlO8vaUv4nOdLuj5BHbMkdS2wvahr\nVtIQSUslPRve/1nSQkmjwnmcWWD/gucayv1K0pvh2EfG1o+XtERS/2Lids4555xzydX3Qy4aYtN8\nMPCMmZ0eW9cQ40wSU6EyBmBm75nZ0AT71UadDcVzZlYeXreUKIaRwEVmNkBSR+BoMzvSzO6urQok\nnQ70MLMDgR8Cv01vM7OrgZuAH9RWfbu1BvU1TAPmeUrG85SM5ykR/32nZDxPrq7UdwOrmaSHw7f4\nkyW1yCwgabikxeF1e1jXJHx7v1jSIkmjwvoekqaHb+rnSarJyOw2wAcZ6zbE4jk/1LlA0oNh3URJ\nd0t6QdJb6Z4HSa0kzQixLJI0KKwvk7Qs7Ldc0iRJp4T9l0s6OpRrKWmCpDmSXpF0RgjjM2BrgnPZ\nEI4zNsQ7X9JaSRPSpxOLJ96b2DX0yCyXdGO2PBSqM1eu4iQdIGmapJclzZZ0kKTWklbFyrSUtEZS\n02zls9S/hSg/hRT71IaNwI5c114wVNJLkl6X1CfEX6VHUNITkvpJugHoC0yQ9HPgaaBL+Iz6VglU\nKpeUCuc9TVKHIs71e8BDAGb2ErBPbH+A9UTXvHPOOeecqwP1/ZCLg4ELzWxOuOm/DNj5K2SSOgG3\nA72Ibianh0bKWqCLmR0RyrUOu0wCxpnZVEnNqVmDsSlQGV9hZseFeg4FrgWON7PNkuI3ph3NrI+k\nQ4CpwBTgU2CwmW2T1B6YE7YB9AC+b2ZLJc0DhoX9B4U6zgSuA541s5GS9gHmSpphZi8CL4aYjgJ+\naGaXZJ5IOm4zGw2MDsd4Dkjf8Md7m+LLxwCHhfhflvSkmc1PHy+fhLlKuz/E/rakY4H7Qm/OAkn9\nzWw2MBD4u5ntkFStPDAgo/4r08uSxgIvm9mTWeo+XtJCYB3wUzNbWuC8hoRjlpP92gNoambHhV6j\nMcAp6d2zHO9mSScBV5nZAkn3AE+YWXk47sjwdw+iz2uQmW2UNBQYB4xMeK5dgHdi79eFdelZ1ZVE\n13x+s2LL3Wj0k8rrhOckGc9TMp6nZDxPifiT8ZLxPLlipVKpRD2f9d2DtcbM5oTlh4m+0Y87Bphl\nZpvMrJKoAdUPWAF0D71G3wY+krQX0NnMpgKY2Wdm9mkxwUgS0JOoAZfNScCjZrY51LEltu3xsG4Z\n8LX0IYHbJC0CZgCdJaW3rYzd1C8J2wFeJbqFBTgVuEbSAiAFNAeqzAMys1eyNa5yeBi4y8wWFig3\n3cy2hPxNofrnkkS+XCGpFXAC8Gg4v98B6Z6VycDZYXkY8EiB8lmZ2egcjatXgK5mdiTwG8Jnl1C1\nay+2bUrs+Ml+Dr1wT9rBwOFEXy4sIGp0d84slOdcC1kHHCRpz7ylToy9/IbGOeecq3cd9+uIpDp7\nddyvY6lPcacTTzyRP/zhDzXe/8ILL6Rdu3b07t0bgPvuu4+OHTvSunVrNm3aRJMmTVixYsUux1lR\nUcGYMWN2vnKp7x6szG/2s83fqXYDamZbJPUEvg1cCpwFXJGtbJUDSZcBF4d6vmNm62PbmhDdPG8H\nniriHNK2Z4l5BLAv0MvMKhU9AKJFlvKVsfeVfPE5iKiX680axFOFpDFEDdpqQ/WySPK57KomwOZ0\nj02GqcCtktoC5cBMYK885YtiZttiy9Mk3SupnZltSrBvtmvvorA5/Rnu4IvP8HOqfnFRbRhsAQJe\nM7M+Re6Xtg7YP/Z+v7AOADNbIWkZsFrSADNbUsN63JfgcdH1wvOUjOcpGc9TIv77Tsk09Dy9v+79\nOv3ZlC/LTwY8//zzPPvss7z77ru0aNGCzz//nKuvvpq5c+dy+OGHA/X/+5713YNVJik97Owc4B8Z\n2+cC/SS1k9QUGA7MDsPtmprZY8D1QHm4aX5H0vcAJDWX9JX4wczsXjPrFR5ssD5jW6WZdQPm8UXv\nSaaZwFmS2oU62uYol/7U9gE+CI2rE6naq5Hkk30auHznDrEnwBVD0dytk4FRmZty7HKKpDYhf4OB\nF7Icc1mBavPmysw+AlZKGhI75hFh28dEn8PdwJMWyVm+WPE5SGGoodKNK0Vz5jrl2bfatZeraPi7\nCjhSkf2BY/OFlmXdcuCrknqH+vcIwy+TmgqcH/btDWwxs53/goYcdifq/fXGlXPOOecatVWrVtGt\nWzdatIi+016/fj3bt2/nkEMO2Vmmvh+BX98NrNeBH0laSjTRPv2Es/TT7dYD1xANj1tANMfkCaI5\nJKkwZOqPoQxEN5KXhyF5L1BgCFkObwDtsm0IQ/puJWrkLQDGx+ONFw1/JwHHhHjOBZZlKZNt/7Sb\niR4EsljRQyhuyiwg6agwNymfK4mGlb2s6CEKYwrUO5douNtComF+8zPqbF+gvny5ijsXGKnooSSv\nAYNi2x4h6gH8S2zdiDzlq1H0cI+BWTYNkfRaiOuXRMMQ00NEewD5erJyXXtZrwEze4GokbUk1PVK\nZpkc79P7/wsYAtyhaM7YAuD4pOdqZn8japi+RTSs8rKMIm2BVWEIrtsV/i16Mp6nZDxPyXieEmnI\nvTINiecpme7duzN+/Hh69uxJ27ZtGT58OJ999sUzt37/+99z4IEHsu+++zJ48GDee++9rMfZvn07\n5513Hvvuuy9t27bluOOOY8OGL56ntmrVKvr27Uvr1q057bTT2LQpuj2bPXs2+++/f5Vjde/enZkz\nZ/KHP/yBiy++mBdffJHWrVszYsQIvvGNbwDQtm1bTj755GpxfPbZZ/zkJz+hrKyMTp06cdlll7F9\n+/Zq5XbFbv9Dw5J+CrQ3s2sKFt5NSfou0N3MflPqWGqTpMOIHrryk1LHUl8UPTTj38xseJ4y/kPD\nzjlXV8b4Dw276pTlh2wl1ekQwaTXYvfu3enQoQP//d//zZ577skJJ5zAFVdcwSWXXMLMmTM5++yz\nmTFjBoceeihXX301ixYtYvbs2dWOc//99/PUU08xefJkmjdvzsKFCznwwAPZa6+9OPHEE1m7di1/\n//vf2W+//TjttNM4/vjjGTduHLNnz+a8885jzZo1VWKaMGECJ510Eg8++CATJkzgueeeA2D16tUc\ncMABfP755zuHBjZp0oS33nqLAw44gCuvvJKVK1fy4IMPsscee3DOOedw+OGHc+utt+bMQbbPJ7a+\n2oik+p6D1RBNAR6QNC3jt7BcYGY1maPW4IUhcrtT42o88C3gPwsWHlPX0Tjn3O6pQ5eaDLYpTkOf\nW9RQeJ6SGzVqFB06RNfuGWecwcKF0fPT/vSnPzFy5Eh69uwJwG233Ubbtm1Zs2YNXbtWeU4bzZo1\nY+PGjbzxxht885vfpFevXlW2X3jhhfTo0QOAoUOH8sQTT+xSzGaWde7V73//e1599VX22WcfAK65\n5hpGjBiRt4FVrN2+gWVmbxPddDr3pRZ+aDhp2boM5UvB/8ecjOcpGc9TMp4n50oj3bgCaNmy5c5h\ngO+++y5HHXXUzm2tWrWiffv2rFu3rloD6/zzz2ft2rUMGzaMrVu3MmLECMaNG0fTptGvx3Ts2LFK\nHdu2baO2bdiwgU8++aRKzJWVlbV+31Pfc7Ccc+5LwW/ykvE8JeN5SsbzlIznKRnP067r3Lkzq1ev\n3vn+448/ZuPGjXTp0qVa2aZNm3LDDTewZMkS/vnPf/Lkk0/y0EMPFayjVatWfPLJJzvf79ixo8rc\nrWLsu+87K3VnAAAgAElEQVS+tGzZkiVLlrBp0yY2bdrEli1b2Lp1a42Ol4s3sJxzzjnnnHNFGz58\nOBMnTmTx4sVs376da6+9lt69e1frvYKoB/q1116jsrKSvfbai2bNmu3svcrnoIMO4tNPP2XatGl8\n/vnn3HLLLVUespFNrh4pSVx88cVcccUVOxtp69at45lnnklwtsnt9kMEnXOuJnyoUjKep2Q8T8l4\nnpLxPCXT0PPUoUuHOv2tqqTzAfP9htSAAQO4+eabOfPMM9myZQsnnHACf/nLX7KWXb9+PZdeeinr\n1q1jr732YtiwYZx77rkF62jdujX33nsvI0eOpLKykp/97Gfst99+RcUcf3/HHXcwduxYevfuvbO3\n7T/+4z849dRT8x6zGLv9UwSdc1VJMv93obCG/j/mhsLzlIznKRnPUzKep2QaUp5yPaXONQzFPkXQ\nG1jOuSq8geWcc87VL29gNWzFNrB8DpZzzjnnnHPO1RKfg+WcqybfWGjnnGvsOnQoY/36VaUOo840\npKFvDZnnydUVb2A557LwYQqFpYCKEsfQGKTwPCWRwvOURIrayNP77/uXSM65uuNzsJxzVUgyb2A5\n577cfL6La1h8DlbD1mDnYEkqk/Rqjm2zJJXXVywZdXeVNF/StNi6laWIJRdJ/SVNTFAub9ySPgp/\nO0maHJYvkPTrmhwvYZ2jJV1V6DjFiB9T0kRJ/QqUHyRpkaQFkuZJOilBHbMkVf8Rh6rbi7pmJQ2R\ntFTSs+H9nyUtlDQqnMeZBfZPcq7nhHNdJOl5SUfEto2XtERS/2Lids4551zdKisrQ5K/GuirrKys\nqM+zvh9y0RCb5oOBZ8zs9Ni6hhhnkpgKlTEAM3vPzIYm2K826mwIZphZTzPrBVwI3F+iOEYCF5nZ\nAEkdgaPN7Egzu7sW61gB9DOznsAtxM7VzK4GbgJ+UIv17cZSpQ6gkUiVOoBGIlXqABqJVKkDaBRS\nqVSpQ2gUGlKeVq1ahZk1yNesWbNKHkOpX6tWrSrq86zvBlYzSQ+Hb/EnS2qRWUDScEmLw+v2sK5J\n+PZ+cfhmflRY30PSdEW9APMkda9BTG2ADzLWbYjFc76+6P14MKybKOluSS9Iekuh50FSK0kzQiyL\nJA0K68skLQv7LZc0SdIpYf/lko4O5VpKmiBpjqRXJJ0RwvgM2JrgXDaE44wN8c6XtFbShPTpxOKJ\n9yZ2VdQjs1zSjdnyUKjOXLmKk3SApGmSXpY0W9JBklpLWhUr01LSGklNs5XPUv8WovzkZGafxN7u\nBXyY4Lw2AjtyXXvBUEkvSXpdUp8Qf5UeQUlPSOon6QagLzBB0s+Bp4Eu4TPqm5GnckmpcN7TJKV/\nCTDJuc4xs/S1MgfoklFkPdE175xzzjnn6kJ9tfyAMqAS6B3eTwCuCsuzgHKgE7AaaEfU+HsWGBS2\nPRM7Vuvwdw4wKCw3B1rUIK6xwBU5th0KvA60De/bhL8TgUfC8iHAm2G5KbBXWG4fW19GdGN8aHg/\nD5gQlgcBU8LyrcA5YXkfYDnwlYyYjgLuT3hu+wCLgCPD+/+JxbM4LF8ArCO66W4BvAqU1yCPuXI1\nOvY5zwB6hOVjgWfD8mNA/7A8NH1+ecrvPGaWz3JgjvgGA8uAzcCxRZxXrmtvFvCLsHw6MD2Wz1/F\nyj9B1KOU3qdX5mcQu6bOJHrwzAtA+1g+JhRzrrEyP8m8VoBvAU8W2M9gdOw1y8D85S9/+etL9MKc\nc65Ys2bNstGjR+98hX9LyHzV91ME15jZnLD8MPBj4K7Y9mOAWWa2CUDSJKAf0VCn7pLuBv4GPCNp\nL6CzmU0FMLO83+xnI0lAzxBLNicBj5rZ5lDHlti2x8O6ZZK+lj4kcJuieTKVQOfYtpVmtjQsLyFq\nPEDUoOkWlk8FzpD00/C+OdCVqKFFqO8V4JKEp/gwcJeZLSxQbnr63CRNIeppmZ+wjrR8uUJSK+AE\n4NGQd4Bm4e9k4GxgNjAMuKdA+azMbHSebY8Dj4feoj8CByc8rxVkXHuxbVPC31eIGkxJFHp01cHA\n4cD0cN5NgHczC+U7VwBJJxINh+ybsWkdcJCkPc1se+4jjCkQpnPOOefc7qWioqLKo/3Hjh2btVyp\n52BlvocsN6DhZr0n0eDrS4Hf5ypb5UDSZbGhch0ztjUBVhL1QD2VKPqq4jen6ThGAPsS9VL0Ihp6\n2CJL+crY+0q+eFy+gO+bWa/w6m5my6kBSWOIGrTVhuplkeRz2VVNgM1mVh47v8PDtqnAaZLaEvUY\nzSxQvsbM7HlgD0ntE5bPde3BF5/hDr74DD+n6n9X1YbBFiDgtdh597Sq8wMLHyB6sMX9RL27m+Pb\nzGwFUU/eakmHFRmbqyJV6gAaiVSpA2gkUqUOoJFIlTqARqEhzS1qyDxPyXieilffDawySceF5XOA\nf2Rsnwv0k9ROUlNgODA73Aw3NbPHgOuJhrBtA96R9D0ASc0lfSV+MDO7N9yklpvZ+oxtlWbWjWi4\n3tk54p0JnCWpXaijbY5y6QbWPsAHZlYZehDKspTJ52ng8p07SEcm2Kd6MNHcrZOBUZmbcuxyiqQ2\nIX+DiYaoZR5zWYFq8+bKzD4CVkoaEjvmEWHbx0Sfw91Ew9csX/liSeoRWy4PdW4M72dI6pRn32rX\nXq6i4e8q4EhF9ica2pjz8FnWLQe+Kql3qH8PSYfmOUZmvF2BvwLnmdnbWbYfAXQn6v1dkvS4zjnn\nnHMumfpuYL0O/EjSUqI5P78N6w0gNIKuIfqKagHwspk9QTRRPyVpAdHwrmvCfucDl0taRNQoSD8M\noBhvEM35qiYM6buVqJG3ABgfjzdeNPydBBwT4jmXqKcgs0y2/dNuJnoQyOLwEIqbMgtIOkpSoafg\nXQl0Bl4OvXdjCtQ7l2i420KiYX5Vhgcm6e3Jk6u4c4GRih5K8hrR/LO0R4h6AP8SWzciT/lqFD3c\nY2CWTd+X9Jqk+USNuGGhvIAewKY8h8117WW9BszsBaJG1hLgl0TDB8m3T8b+/wKGAHdIWkj038Hx\nRZzrDUTX872h93Zuxva2wCozq8yyrytKRakDaCQqSh1AI1FR6gAaiYpSB9AoxIcwudw8T8l4noq3\n2//QcJjv1N7MrilYeDcl6btAdzP7TaljqU1hiNyFZvaTUsdSXyQNBf7NzIbnKWN1M0rUOecaCv9R\nV+fcrlOpf2i4AZsC9FHsh4ZdVWb21JetcQVgZkt2s8bVeKInC/7fUsfy5ZAqdQCNRKrUATQSqVIH\n0EikSh1Ao+BzZpLxPCXjeSpefT9FsMEJ81S+Veo4nKtrFv3QcEJJpgw651zj1KFD0ge/Oudc8Xb7\nIYLOuaokmf+74JxzzjmXnw8RdM4555xzzrk65g0s55yrAR+TnoznKRnPUzKep2Q8T8l4npLxPBXP\nG1jOOeecc845V0t8DpZzrgqfg+Wcc845V5jPwXLOOeecc865OuYNLOecqwEfk56M5ykZz1Mynqdk\nPE/JeJ6S8TwVb7f/HSznXHWS/w6Wc7u7Dl06sH7t+lKH4ZxzjY7PwXLOVSHJGFPqKJxzJTcG/B7B\nOedy8zlYzjnnnHPOOVfH6q2BJalM0qs5ts2SVF5fsWTU3VXSfEnTYutWliKWXCT1lzQxQbm8cUv6\nKPztJGlyWL5A0q9rcryEdY6WdFWh4xQjfkxJEyX1K1B+kKRFkhZImifppAR1zJLUtcD2oq5ZSUMk\nLZX0bHj/Z0kLJY0K53Fmgf0Lnmso9ytJb4ZjHxlbP17SEkn9i4nb5dCg/pVowDxPyXieEvG5IMl4\nnpLxPCXjeSpefc/BaohjDQYDz5jZNbF1DTHOJDEVKmMAZvYeMDTBfrVRZ0Mww8ymAkj6JvAY8PUS\nxDESuMjM/impI3C0mR0Y4irYgE5C0ulADzM7UNJxwG+B3gBmdrWkucAPgNm1UZ9zzjnnnKuqvocI\nNpP0cPgWf7KkFpkFJA2XtDi8bg/rmoRv7xeHnohRYX0PSdPDN/XzJHWvQUxtgA8y1m2IxXN+rPfj\nwbBuoqS7Jb0g6a10z4OkVpJmhFgWSRoU1pdJWhb2Wy5pkqRTwv7LJR0dyrWUNEHSHEmvSDojhPEZ\nsDXBuWwIxxkb4p0vaa2kCenTicUT703sGnpklku6MVseCtWZK1dxkg6QNE3Sy5JmSzpIUmtJq2Jl\nWkpaI6lptvJZ6t9ClJ+czOyT2Nu9gA8TnNdGYEeuay8YKuklSa9L6hPir9IjKOkJSf0k3QD0BSZI\n+jnwNNAlfEZ9M/JULikVznuapA5JzxX4HvBQOO+XgH1i+wOsJ7rm3a6qyb82uyPPUzKep0QqKipK\nHUKj4HlKxvOUjOepePXdg3UwcKGZzQk3/ZcBd6U3SuoE3A70IrqZnB4aKWuBLmZ2RCjXOuwyCRhn\nZlMlNadmDcamQGV8hZkdF+o5FLgWON7MNkuK35h2NLM+kg4BpgJTgE+BwWa2TVJ7YE7YBtAD+L6Z\nLZU0DxgW9h8U6jgTuA541sxGStoHmCtphpm9CLwYYjoK+KGZXZJ5Ium4zWw0MDoc4zkgfcMf722K\nLx8DHBbif1nSk2Y2P328fBLmKu3+EPvbko4F7jOzAaFB1t/MZgMDgb+b2Q5J1coDAzLqvzK9LGks\n8LKZPZlZsaTBwG1AR+DbCc5rSNivnOzXHkBTMztOUa/RGOCU9O5ZjnezoqGJV5nZAkn3AE+YWXk4\n7sjwdw+iz2uQmW2UNBQYB4xMeK5dgHdi79eFde+H95VE13x+s2LL3fCbP+ecc87t9lKpVKIhk/Xd\ng7XGzOaE5YeJvtGPOwaYZWabzKySqAHVD1gBdA+9Rt8GPpK0F9A5PfTLzD4zs0+LCUaSgJ5EDbhs\nTgIeNbPNoY4tsW2Ph3XLgK+lDwncJmkRMAPoLCm9baWZLQ3LS8J2gFeJbmEBTgWukbQASAHNgSrz\ngMzslWyNqxweBu4ys4UFyk03sy0hf1Oo/rkkkS9XSGoFnAA8Gs7vd0C6Z2UycHZYHgY8UqB8VmY2\nOlvjKmx73MwOAc4A/ljEeVW79mLbpoS/rwBlCY9X6PnnBwOHE325sICo0d05s1C+cy1gHXCQpD3z\nljox9vLGVXY+ZyYZz1MynqdEfC5IMp6nZDxPyXievlBRUcGYMWN2vnIp9RysbPN3qt2AmtkWST2J\neh4uBc4CrshWtsqBpMuAi0M93zGz9bFtTYhunrcDTxVxDmnbs8Q8AtgX6GVmlYoeANEiS/nK2PtK\nvvgcRNTL9WYN4qlC0hiiBm21oXpZJPlcdlUTYHO6xybDVOBWSW2BcmAm0VC+XOVrzMyel7SHpPZm\ntjFB+WzX3kVhc/oz3MEXn+HnVP3iotow2AIEvGZmfYrcL20dsH/s/X5hHQBmtkLSMmC1pAFmtqSG\n9TjnnHPOuSzquwerTNHEe4BzgH9kbJ8L9JPUTlJTYDgwOwy3a2pmjwHXA+Vmtg14R9L3ACQ1l/SV\n+MHM7F4z62Vm5fHGVdhWaWbdgHl80XuSaSZwlqR2oY62OcqlG1j7AB+ExtWJVO3VSPLLrU8Dl+/c\nIfYEuGIomrt1MjAqc1OOXU6R1CbkbzDwQpZjLitQbd5cmdlHwEpJQ2LHPCJs+5joc7gbeNIiOcsX\nS1KP2HJ5qHNjeD8jDE3NtW+1ay9X0fB3FXCkIvsDx+YLLcu65cBXJfUO9e8Rhl8mNRU4P+zbG9hi\nZunhgekcdifq/fXG1a7wnr1kPE/JeJ4S8bkgyXiekvE8JeN5Kl59N7BeB34kaSnRRPvfhvXpp9ut\nB64hGh63gGiOyRNEc0hSYcjUH0MZiG4kLw9D8l6gwBCyHN4A2mXbEIb03UrUyFsAjI/HGy8a/k4C\njgnxnAssy1Im2/5pNxM9CGSxoodQ3JRZQNJRYW5SPlcSDSt7OTxEYUyBeucSDXdbSDTMb35Gne0L\n1JcvV3HnAiMVPZTkNWBQbNsjRD2Af4mtG5GnfDWKHu4xMMum70t6TdJ8okbcsFBeRHPjNuU5bK5r\nL+s1YGYvEDWylgC/JBo+SL59Mvb/FzAEuEPSQqL/Do5Peq5m9jeihulbRMMqL8so0hZYFYbgOuec\nc865Wqbd/VfaJf0UaJ/xmHYXI+m7QHcz+02pY6lNkg4jeujKT0odS30JD834NzMbnqeMMab+Ymq0\nVuK9Dkl4npJpiHkaAw3tHiGVSvm36Ql4npLxPCXjecpNEmZWbURSfc/BaoimAA9ImmZmp5c6mIbI\nzGoyR63BC0PkdqfG1XjgW8B/Fiw8pq6jcc41dB261GRQiHPOud2+B8s5V5Uk838XnHPOOefyy9WD\nVd9zsJxzzjnnnHPuS8sbWM45VwP+uyDJeJ6S8Twl43lKxvOUjOcpGc9T8byB5ZxzzjnnnHO1xOdg\nOeeq8DlYzjnnnHOF+Rws55xzzjnnnKtj3sByzrka8DHpyXiekvE8JeN5SsbzlIznKRnPU/H8d7Cc\nc9VI1Xq7nXOuQejQoYz161eVOgznnMvJ52A556qQZOD/LjjnGirh9y7OuYbA52A555xzzjnnXB2r\ntwaWpDJJr+bYNktSeX3FklF3V0nzJU2LrVtZilhykdRf0sQE5fLGLemj8LeTpMlh+QJJv67J8RLW\nOVrSVYWOU4z4MSVNlNSvQPmDJf1T0qdJYwnXZNcC24u6ZiUNkbRU0rPh/Z8lLZQ0KpzHmQX2T3Ku\n50haFF7PSzoitm28pCWS+hcTt8slVeoAGolUqQNoJFKlDqCRSJU6gEbB58wk43lKxvNUvPqeg9UQ\n+/QHA8+Y2TWxdQ0xziQxFSpjAGb2HjA0wX61UWdDsBH4MdFnXUojgYvM7J+SOgJHm9mBEDWeaqmO\nFUA/M9sq6TTgfqA3gJldLWku8ANgdi3V55xzzjnnYup7iGAzSQ+Hb/EnS2qRWUDScEmLw+v2sK5J\n+PZ+cfhmflRY30PS9NALME9S9xrE1Ab4IGPdhlg854c6F0h6MKybKOluSS9Ieivd8yCplaQZIZZF\nkgaF9WWSloX9lkuaJOmUsP9ySUeHci0lTZA0R9Irks4IYXwGbE1wLhvCccaGeOdLWitpQvp0YvHE\nexO7hh6Z5ZJuzJaHQnXmylWcpAMkTZP0sqTZkg6S1FrSqliZlpLWSGqarXyW+rcQ5ScnM/vQzF4B\nPk9wPmkbgR25rr1gqKSXJL0uqU+Iv0qPoKQnJPWTdAPQF5gg6efA00CX8Bn1zchTuaRUOO9pkjoU\nca5zzCx9rcwBumQUWU90zbtdVlHqABqJilIH0EhUlDqARqKi1AE0ChUVFaUOoVHwPCXjeSpeffdg\nHQxcaGZzwk3/ZcBd6Y2SOgG3A72Ibianh0bKWqCLmR0RyrUOu0wCxpnZVEnNqVmDsSlQGV9hZseF\neg4FrgWON7PNkuI3ph3NrI+kQ4CpwBTgU2CwmW2T1J7oBndqKN8D+L6ZLZU0DxgW9h8U6jgTuA54\n1sxGStoHmCtphpm9CLwYYjoK+KGZXZJ5Ium4zWw0MDoc4zkgfcMf722KLx8DHBbif1nSk2Y2P328\nfBLmKu3+EPvbko4F7jOzAaFB1t/MZgMDgb+b2Q5J1coDAzLqvzK9LGks8LKZPVko7gTnNSQcs5zs\n1x5AUzM7TtLpwBjglPTuWY53s6STgKvMbIGke4AnzKw8HHdk+LsH0ec1yMw2ShoKjANG1uBcLwKm\nZayrJLrmCxgTW67Ab2qcc845t7tLpVKJhkzWdwNrjZnNCcsPEw3buiu2/RhglpltApA0CegH3AJ0\nl3Q38DfgGUl7AZ3NbCqAmeX9Zj8bSQJ6hliyOQl41Mw2hzq2xLY9HtYtk/S19CGB2xTNk6kEOse2\nrTSzpWF5CTAjLL8KdAvLpwJnSPppeN8c6AosT1caemKqNa5yeBi4y8wWFig3PX1ukqYQ9bTMT1hH\nWr5cIakVcALwaMg7QLPwdzJwNtGwtWHAPQXKZxUalrVtBRnXXmzblPD3FaAs4fEKPf/8YOBwoi8X\nRPSlwbuZhQqdq6QTgQuJPsu4dcBBkvY0s+25jzCmQJgumgtSUeIYGoMUnqckUniekkiVOoBGIZVK\nea9DAp6nZDxPX6ioqKiSi7Fjx2YtV+o5WNnm71S7ATWzLZJ6At8GLgXOAq7IVrbKgaTLgItDPd8x\ns/WxbU2Ibp63A08VcQ5p8ZvTdBwjgH2BXmZWqegBEC2ylK+Mva/ki89BRL1cb9YgniokjSFq0FYb\nqpdFks9lVzUBNqd7bDJMBW6V1BYoB2YCe+UpX29yXHsXhc3pz3AHX3yGn1O1J7XaMNgCBLxmZn1q\nFjEoerDF/cBp6QZvmpmtkLQMWC1pgJktqWk9zjnnnHOuuvqeg1UmKT3s7BzgHxnb5wL9JLWT1BQY\nDswOw+2amtljwPVAuZltA96R9D0ASc0lfSV+MDO718x6mVl5vHEVtlWaWTdgHlHvSTYzgbMktQt1\ntM1RLt3A2gf4IDSuTqRqr0aSX259Grh85w7SkQn2qR5MNHfrZGBU5qYcu5wiqU3I32DghSzHXFag\n2ry5MrOPgJWShsSOeUTY9jHR53A38KRFcpbfRVVyoGjOXKechbNcewWOuwo4UpH9gWOTxhIsB74q\nqXeof48w/DIRRU8+/Ctwnpm9nWX7EUB3ot5fb1ztkopSB9BIVJQ6gEaiotQBNBIVpQ6gUfDehmQ8\nT8l4nopX3w2s14EfSVpKNNH+t2F9+ul264FriMYALCCaY/IE0UT9lKQFwB9DGYDzgcslLSJqFKQf\nBlCMN4B22TaEIX23EjXyFgDj4/HGi4a/k4BjQjznAsuylMm2f9rNRA8CWazoIRQ3ZRaQdFSYm5TP\nlUBnovlU80NvVr565xINd1tINMyvyvDA0MjIK0+u4s4FRip6KMlrwKDYtkeIegD/Els3Ik/5ahQ9\n3GNglvUdJL1DlJfrFD1EY68wBK8HsCnPYXNde1mvATN7gaiRtQT4JdHwQfLtk7H/v4AhwB2SFhL9\nd3B80nMFbiC6nu8Nc9vmZmxvC6wys8rquzrnnHPOuV2l3f3X0MN8p/YZj2l3MZK+C3Q3s9+UOpba\nJOkwooeu/KTUsdSX8NCMfzOz4XnKWON4+n6ppfBv05NI4XlKIoXnKYkUcCK7+71LIT5nJhnPUzKe\np9wkYWbVRiTV9xyshmgK8ICkaWZ2eqmDaYjMrCZz1Bq8MERud2pcjQe+BfxngtJ1HY5zztVIhw5J\nnynknHOlsdv3YDnnqpJk/u+Cc84551x+uXqw6nsOlnPOOeecc859aXkDyznnaiDJDw06z1NSnqdk\nPE/JeJ6S8Twl43kqnjewnHPOOeecc66W+Bws51wVPgfLOeecc64wn4PlnHPOOeecc3XMG1jOOVcD\nPiY9Gc9TMp6nZDxPyXiekvE8JeN5Kp7/DpZzrhrJfwfLuYakQ5cOrF+7vtRhOOecS8DnYDnnqpBk\njCl1FM65KsaA///aOecaFp+D5ZxzzjnnnHN1rN4aWJLKJL2aY9ssSeX1FUtG3V0lzZc0LbZuZSli\nyUVSf0kTE5TLG7ekj8LfTpImh+ULJP26JsdLWOdoSVcVOk4x4seUNFFSvwLlD5b0T0mfJo0lXJNd\nC2wv6pqVNETSUknPhvd/lrRQ0qhwHmcW2L/guYZyv5L0Zjj2kbH14yUtkdS/mLhdDg3qX4kGzPOU\njOcpEZ8LkoznKRnPUzKep+LV9xyshji+YTDwjJldE1vXEONMElOhMgZgZu8BQxPsVxt1NgQbgR8T\nfdalNBK4yMz+KakjcLSZHQhR46k2KpB0OtDDzA6UdBzwW6A3gJldLWku8ANgdm3U55xzzjnnqqrv\nIYLNJD0cvsWfLKlFZgFJwyUtDq/bw7om4dv7xZIWSRoV1veQND18Uz9PUvcaxNQG+CBj3YZYPOeH\nOhdIejCsmyjpbkkvSHor3fMgqZWkGSGWRZIGhfVlkpaF/ZZLmiTplLD/cklHh3ItJU2QNEfSK5LO\nCGF8BmxNcC4bwnHGhnjnS1oraUL6dGLxxHsTu4YemeWSbsyWh0J15spVnKQDJE2T9LKk2ZIOktRa\n0qpYmZaS1khqmq18lvq3EOUnJzP70MxeAT5PcD5pG4Edua69YKiklyS9LqlPiL9Kj6CkJyT1k3QD\n0BeYIOnnwNNAl/AZ9c3IU7mkVDjvaZI6JD1X4HvAQ+G8XwL2ie0PsJ7omne7qib/2uyOPE/JeJ4S\nqaioKHUIjYLnKRnPUzKep+LVdw/WwcCFZjYn3PRfBtyV3iipE3A70IvoZnJ6aKSsBbqY2RGhXOuw\nyyRgnJlNldScmjUYmwKV8RVmdlyo51DgWuB4M9ssKX5j2tHM+kg6BJgKTAE+BQab2TZJ7YE5YRtA\nD+D7ZrZU0jxgWNh/UKjjTOA64FkzGylpH2CupBlm9iLwYojpKOCHZnZJ5omk4zaz0cDocIzngPQN\nf7y3Kb58DHBYiP9lSU+a2fz08fJJmKu0+0Psb0s6FrjPzAaEBll/M5sNDAT+bmY7JFUrDwzIqP/K\n9LKkscDLZvZkobgTnNeQcMxysl97AE3N7DhFvUZjgFPSu2c53s2STgKuMrMFku4BnjCz8nDckeHv\nHkSf1yAz2yhpKDAOGJnwXLsA78Terwvr3g/vK4mu+fxmxZa74Td/zjnnnNvtpVKpREMm67uBtcbM\n5oTlh4mGbd0V234MMMvMNgFImgT0A24Buku6G/gb8IykvYDOZjYVwMwKfbNfjSQBPUMs2ZwEPGpm\nm0MdW2LbHg/rlkn6WvqQwG2K5slUAp1j21aa2dKwvASYEZZfJbqFBTiV/8/e/UdJVd353n9/QBlF\nUcEf/DBja1wa442jYvwVvdr6RExMNF6UJGoGV+JosnCpmJi1fDJ50uAvjI5JdObJGI1DouIa5F6S\nQY0iYpejKILQgAqiTtCIT1Bv/MksRq/yff7Y34LT1XW6ThXQRdHf11q1+tQ5+5z9Pd9zGmr33vsU\nnDQBvhkAACAASURBVC7ph/5+ELAPsLJcqffE9Ghc5bgb+JmZLalRbk753CTNJPW0LC5YR1lvuULS\nTsAXgBmed4Dt/ee9wDdIw9a+Cfy/NcpX5Q3Lze2PVNx7mW0z/ecioK3g8Wo9//wzwOdIf1wQ6Y8G\n/19loU0419eBAyX9lZl9mFvqpAaP3p+sIhqeRUSeiok8FVIqleKv6QVEnoqJPBUTedqovb29Wy4m\nT55ctVyz52BVm7/T4wOomb0r6VDgVOB7wDhgYrWy3Q4kTQAu9HpOM7M1mW0DSB+ePwQeqOMcyrIf\nTstxnAfsARxuZuuVHgCxQ5Xy6zPv17PxOojUy/VSA/F0I2kSqUHbY6heFUWuy6YaALxT7rGpMAu4\nVtJQYDTwKLBzL+X7TM6993e+uXwNP2HjNfyY7j2pPYbB1iDgOTM7rrGIeR3468z7T/k6AMzsj5JW\nAK9K+r/M7PkG6wkhhBBCCFX09RysNqWJ9wDnAo9XbF8AnCBpmKSBwDnAYz7cbqCZ/Q74MTDazNYC\nr0n6GoCkQZJ2zB7MzH5pZoeb2ehs48q3rTezfYFnSL0n1TwKjJM0zOsYmlOu3MDaFXjTG1cn0b1X\no8g3t84GLt2wQ+YJcPVQmrv1ReCyyk05u5wiaTfP35nAvCrHXFGj2l5zZWYfAKsknZ055t/4tv8k\nXYebgfstyS2/ibrlQGnO3MjcwlXuvRrHfQU4TMlfA0cVjcWtBPaUdIzXv50PvyxqFjDe9z0GeNfM\nysMDyzncj9T7G42rTRG9DcVEnoqJPBUSf0UvJvJUTOSpmMhT/fq6gfUCcLGk5aSJ9rf6+vLT7dYA\nVwIloIs0x+Q+0hySkqQu4C4vA+mD5KWSlpIaBdnJ/EW9CAyrtsGH9F1LauR1ATdl480W9Z/TgCM9\nnm8BK6qUqbZ/2dWkB4EsU3oIxVWVBSQd4XOTenM5MIo0n2qx92b1Vu8C0nC3JaRhft2GB3ojo1e9\n5CrrW8AFSg8leQ44I7NtOqkH8F8z687rpXwPSg/3+GqV9cMlvUbKy98rPURjZx+Ctz/wdi+Hzbv3\nqt4DZjaP1Mh6HvgFafggve1Tsf//Ac4GfippCen34Nii52pmfyA1TF8GfkWa55g1FHjFzNZX7htC\nCCGEEDad+vs3w/t8p90rHtMeMiR9BdjPzP6p2bFsTpL+G+mhK1c0O5a+4g/N+B9mdk4vZYxJfRdT\ny4o5M8VEnoqpladJ0N//v4aYC1JU5KmYyFMxkad8kjCzHiOS+noO1tZoJvAbSQ+a2ZebHczWyMwa\nmaO21fMhcv2pcXUT8N+B/7tm4UlbOpoQQj2G793IAI0QQgjN0O97sEII3Umy+HchhBBCCKF3eT1Y\nfT0HK4QQQgghhBC2WdHACiGEBhT5osEQeSoq8lRM5KmYyFMxkadiIk/1iwZWCCGEEEIIIWwmMQcr\nhNBNzMEKIYQQQqgt5mCFEEIIIYQQwhYWDawQQmhAjEkvJvJUTOSpmMhTMZGnYiJPxUSe6hffgxVC\n6EHq0dsdQgghhD40fHgba9a80uwwQgNiDlYIoRtJBvHvQgghhNBcIj6nb91iDlYIIYQQQgghbGFb\ntIElqU3SsznbOiWN3pL155G0j6TFkh7MrFvVjFjySDpR0tQC5eqOW9JlknbI2Xa+pFt8uUPS+BrH\nOl9SR40yH9QbYy3lY/o91lmgfKekFyR1+bXfo0b5XvPv2++rM+ZBkuZ4/eMkHS/pOX9/UN7vSmb/\nmucqaUdJ90taIelZSddlth3o9U2vJ+6Qp9TsAFpEqdkBtIhSswNoEaVmB9AiSs0OoEWUmh1AS4g5\nWPXrix6srbFv80zgYTP7cmbd1hhnkZgaiXsiMLiB/RqNYUvk1nKWe3OOmR1uZqPN7H/XWUcj2yuN\nBszrnwGcB1xnZqOBdQWPV6TMjWb2WeBw4HhJp5IqftHMPgccImm/OmMPIYQQQggF9EUDa3tJd0ta\nLuneaj0nks6RtMxf1/u6AZKm+rqlki7z9ft7L8ASSc80+EFxN+DNinVvZeIZ73V2Sfqtr5sq6WZJ\n8yS9LGmsr99J0iMey1JJZ/j6Nu9FmCpppaRpkk7x/VdK+ryXGyzpDknzJS2SdLqH8RHwXoFzecuP\nMznTO7PajznYezO6PI/jJF0CjAI6Jc31fb/tMc0Hjsscey3pg39v1nk5JO0laaZfmy5Jx5RTmsnt\nFZIWeJkOXzdF0oRMmQ5J388rX+ET4O0CeYL67vcN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0d4\n79mewF3AkX6ci4CvA1dn9/V9Bki6QdLTft4XFj1XM1tnZo/58sfAYuBTFcXeIP0OhE3S3uwAWkR7\nswNoEe3NDqBFtDc7gBbR3uwAWkR7swNoCe3t7c0OofWY2RZ7AW3AeuAYf38H8H1f7iT9RX8k8Cow\njPQBeC5whm97OHOsXfznfOAMXx4E7NBAXJOBiTnbDgZeAIb6+93851Rgui9/FnjJlwcCO/vy7pn1\nbaQP6Qf7+2eAO3z5DGCmL18LnOvLuwIrgR0rYjoCuK3gue0KLCX1XowFfpXZNsR//jFzfiMy+d8O\neAK4pcHr/a/Apb6sTH3v+89TyvH49vuA44HDgFLmOM8De+eV9/cfVKl/JHB/TmydwLOkBseP6zyv\nWcCxvjzY79MTgXe8TgFPAl/I5HdY5to96ssnArMyx50KjM3cL8t8+ULgR5l7fCHQVvRcM2V2A/4D\n2Ldi/Vzg873sZ9CReXUaWLziFa94xSte8erTFxa2Lp2dndbR0bHh5deIyldf9GD9yczm+/LdpA/U\nWUcCnWb2tpmtB6YBJ5A+pO7nvUanAh9I2hkYZWazSGf0kZn9Vz3BSBJwKLA6p8jJwAwze8freDez\n7fe+bgWwV/mQwBRJS4FHgFGSyttWmdlyX37et0P6oL+vL48BrpTURRoMPAjYJxuQmS0ys4sKnuLd\nwE1m1uX1nOI9RMebWXkulNjYq3Q0G/P/MbAp83NOBv7ZY7ZMfWVjPJ7FpIbOZ4ADzGwJsKekEZL+\nBnjbzF7PK59XuZn92cy+mrP5XDM7BPjvwH+X9K06zmse8HPv/Rvq9ynAAq/TgCVsvKab+ozzMcB4\nvyeeJjV+u513jXNF0kDgHuAXZvZKxebVpN+BXkzKvNqLR96vlJodQIsoNTuAFlFqdgAtotTsAFpE\nqdkBtIhSswNoCTEHa6P29nYmTZq04ZWnL74Hy2q8hyofSM3sXUmHAqcC3wPGkeYO9frh1YeaXej1\nnGZmazLbBpAabh8CD9RxDmUfVon5PGAP4HAzW6/00IkdqpRfn3m/no25F3CWmb3UQDzdSJpEatDe\nCWBmLyk9SOQ04BpJj5jZNdV23dS6XbVrW1nPFDO7vcq2GaRrPIKNjbzeyteqq3thsz/7z/+UdA9w\nFKkxWmTfn0q6H/gKME/SGN+Uvb6fsPGafszG4YhVHyZSg4BLzGxOA/uW3QasNLN/rLLtV8BsSUeZ\n2Xc3oY4QQgghhFChL3qw2iQd7cvnAo9XbF8AnCBpmP/V/RzgMUm7AwPN7HfAj4HRZrYWeE3S12DD\nU9l2zB7MzH5pGx9ksKZi23oz25c0XO8bOfE+CozLzKEZmlOu3CjZFXjTG1cnkYZ6VZbpzWzg0g07\nSIcV2KdnMGnu1heByzLrRgLrzOwe4EbSsEuA94FdfPlpUv6HStqe1MipdvyLs/OkcswFJnj5AZKG\nlHf3n7OB75TnMEka5XOTAO4FvgmcRWps5ZXfo+KYNUka6PcTfo5fBZ7z92cq86S9nP0/bWbPm9kN\npOF6B9WochVpaCB+PvWaDUyQtJ3Xf0DlfV4j3mtIQ2ovzylyBXBBNK42VXuzA2gR7c0OoEW0NzuA\nFtHe7ABaRHuzA2gR7c0OoCXEHKz69UUD6wXgYknLSXNCbvX1BuCNoCtJ/bRdwEIzu480B6fkw6Tu\n8jIA44FLfUjePGB4AzG9SBp21YMP6buW1MjrAm7Kxpst6j+nkR5csBT4FrCiSplq+5ddTXoQyDKl\nx3RfVVnAH5RwWy/nA3A56eEVC/0hCpOAQ4AFfh4/Acq9V7cDD0ma6/mfTJrb9jiwvMeRk4OAv9SI\nYSJwkqRlpEbswb6+fK3nkIatPeVlZgA7+7blwBBgtZm90Uv5IdljZkka6T1Nlf6K1GOzhDTUcLXn\nAGB/aj9MZKLSI8+XkubVPVilTDaeq4BbJC0g9Wblybsnfk26Dov9nriVit7mvHOVtDfwI+DgzIM5\nvlNRbCjwci9xhRBCCCGEBilNH+lfJP0Q2N3MrqxZOAAgaRbpgQy9NRhajqQ7gcvNrFbjcZvgcxCX\nAWeb2cqcMlbnCMx+qkT89bOIEpGnIkpEnoooEXkqokTkqYgSW3eexNbwOb1UKkUvVg5JmFmPUVV9\n0YO1NZoJHKfMFw2H3pnZGdta4wrAzMb3o8bVgaRe4i5SL24IIYQQQtjM+mUPVgghX+rBCiGEEEIz\nDR/expo1rzQ7jNCLvB6svniKYAihxcQfXkIIIYQQGtNfhwiGEMImie8FKSbyVEzkqZjIUzGRp2Ii\nT8VEnuoXDawQQgghhBBC2ExiDlYIoRtJFv8uhBBCCCH0Lp4iGEIIIYQQQghbWDSwQgihATEmvZjI\nUzGRp2IiT8VEnoqJPBUTeapfNLBCCCGEEEIIYTOJOVghhG7ie7BCCCFUM3zv4axZvabZYYSw1cib\ngxUNrBBCN5KMSc2OIoQQwlZnUnxPYghZ8ZCLEELYnFY1O4AWEXkqJvJUTOSpmMhTITG3qJjIU/22\naANLUpukZ3O2dUoavSXrzyNpH0mLJT2YWbdV/XMk6URJUwuUqztuSZdJ2iFn2/mSbvHlDknjaxzr\nfEkdNcp8UG+MtZSP6fdYZ4HynZJekNTl136PGuV7zb9vv6/OmAdJmuP1j5N0vKTn/P1Beb8rmf2L\nnutoScskvSjpF5n1B3p90+uJO4QQQgghFNcXPVhbY1/ymcDDZvblzLqtMc4iMTUS90RgcAP7NRrD\nlsit5Sz35hwzO9zMRpvZ/66zjka2VxoNmNc/AzgPuM7MRgPrCh6vSJl/Bi4wswOBAyWdSqr4RTP7\nHHCIpP3qjD1UigwWE3kqJvJUTOSpmMhTIe3t7c0OoSVEnurXFw2s7SXdLWm5pHur9ZxIOsf/4r5M\n0vW+boCkqb5uqaTLfP3+3guwRNIzDX5Q3A14s2LdW5l4xnudXZJ+6+umSrpZ0jxJL0sa6+t3kvSI\nx7JU0hm+vk3SCt9vpaRpkk7x/VdK+ryXGyzpDknzJS2SdLqH8RHwXoFzecuPMznTO7PajzlY0v2+\nfpn3mlwCjAI6Jc31fb/tMc0Hjsscey3pg39v1nk5JO0laaZfmy5Jx5RTmsntFZIWeJkOXzdF0oRM\nmQ5J388rX+ET4O0CeYL67vcN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0d479mewF3AkX6ci4Cv\nA1dn9/V9Bki6QdLTft4XFj1XSSOAIWa20FfdSfqDQtYbpN+BEEIIIYSwmW3XB3V8Bvi2mc2XdAcw\nAfhZeaOkkcD1wOHAu8Acb6SsBvY2s7/xcrv4LtNIf/WfJWkQjTUSBwLrsyvM7Giv52DgR8CxZvaO\npOwH0RFmdpykzwKzgJnAfwFnmtlaSbsD830bwP7AWWa2XNIzwDd9/zO8jrHA3wNzzewCSbsCCyQ9\nYmZPAU95TEcA3zWziypPpBy3mXUAHX6Mfwf+CfgS8LqZfdWPM8TMPpB0OdDu5zcCmETK//tACVjs\nx7ypViLN7N7M21uAkpmNlSRg53Ixr/8U4AAzO8q3z5J0PDAd+AXwSy//dWBMXnkzewJvtJnZauBs\nP/5I4Pby+VbxG0n/B5hpZtfUOK8N+Qd+AEwws6ckDSZdc4DDgIOBNcA8SV8wsyfp2ctkZvaWpL8D\nfmBm5Ub4scB9ZjZTUlum/AXAu2Z2tN/j8yQ9bGavFjjXvUm/O2WrfV3WetLvQL7sQMR9ib+GVrOK\nyEsRkadiIk/FRJ6KiTwVUiqVonemgMjTRqVSqdCctL5oYP3JzOb78t3AJWQaWMCRQKeZvQ0gaRpw\nAnANsJ+km4E/AA9L2hkYZWazAMzso3qD8Q/qh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0gf\nXEdltq0ys+W+/DzwiC8/S/rYCjAGOF3SD/39IGAfYGW5UjNbBPRoXOW4G7jJzLokrQX+QdIU4AFv\nmJRjLvcqHU33/E8HDihYV6WTgb/1mA2onHs1BjhF0mKvfydSA2qqpD29sbcX8LaZvS5pYrXywBNU\nYWZ/BvIaV+ea2Z+992mmpG+ZWd49UGke8HO/N2d6bAALvE4kLSFd0yfJ9Ng1aAxpGN84f78L6bxf\nLReoca61rCb9DjyTW+KkBo8cQgghhLCNam9v79bYnDx5ctVyfdHA6vHX/CplenwgNbN3JR0KnAp8\nDxhHmjvU64dXH2p2oddzmpmtyWwbAPwR+BB4oI5zKPuwSsznAXsAh5vZeqWHTuxQpfz6zPv1bMy9\nSL1cLzUQTzeSJpEatHcCmNlLSg8SOQ24xnvGqvXcbGqDoKzW/CABU8zs9irbZpCu8QhSj1at8nXN\nfyo3hMzsPyXdAxxFfiO7ct+fSrof+AqpN2mMb8pe30/YeE0/ZmPPatWHidQg4BIzm9PAvq8Df515\n/ylfl/UrYLako8zsuw3UESD+OlxU5KmYyFMxkadiIk+FRK9MMZGn+vXFHKw2SUf78rnA4xXbFwAn\nSBomaSBwDvCYD7cbaGa/A34MjDaztcBrkr4GG57KtmP2YGb2y8yDDNZUbFtvZvuS/nL/jZx4HwXG\nZebQDM0pV26U7Aq86Y2rk4C2KmV6Mxu4dMMO0mEF9ukZTJq79UXgssy6kcA6M7sHuJH0kAVIQwHL\nQy6fJuV/qKTtSY2case/ODtPKsdc0hDQ8jyiIeXd/eds4DvlOUySRvncJIB7gW8CZ5EaW3nl96g4\nZk2SBvr9hJ/jV4Hn/P2Zkq6rsf+nzex5M7sBWAgcVKPKVcARvnxW0TgzZgMTJG3n9R9QeZ/n8Xv+\nPUnlYZXjgX+rKHYF6SEY0bgKIYQQQtjM+qKB9QJwsaTlpIn1t/p6gw0fCK8kzf3pAhaa2X2keSMl\nSV2khwNc6fuNBy6VtJQ0dGt4AzG9CAyrtsGH9F1LauR1AeV5SHk9cdNIDy5YCnwLWFGlTLX9y64m\nPQhkmdJjuq+qLKD0oITbejkfgMtJD69YqPQQhUnAIaQ5XV3AT0jDLgFuBx6SNNfzP5k0d+xxYHmP\nIycHAX+pEcNE4CRJy0iN2IN9fflazwHuAZ7yMjPweVqe9yHAajN7o5fyQ7LHzJI00nuaKv0Vqcdm\nCWl+2WrPAaR5crUeJjJR0rN+jT8CHqxSJhvPVcAtkhaQerPy5N0TvyZdh8V+T9xKRW9zL+cKcDFw\nB+k+f8nMHqrYPhR4uZe4QhFb1Rc7bMUiT8VEnoqJPBUTeSokvt+pmMhT/dQfv5Hb5zvtbmZX1iwc\nAJA0CxhrZr01GFqOpDuBy82sVuNxm+C9WsuAs81sZU4ZY1KfhtWaYhJ5MZGnYiJPxUSeitlSeZoE\n29Lnxnh4QzGRp3ySMLMeo6r6awNrf+A3wNqK78IKYZsl6UDSUMxlwPmW88svqf/9oxBCCKGm4XsP\nZ83qNbULhtBPRAMrhFCIpLy2VwghhBBCcHkNrL6YgxVCCNucGJNeTOSpmMhTMZGnYiJPxUSeiok8\n1S8aWCGEEEIIIYSwmcQQwRBCNzFEMIQQQgihthgiGEIIIYQQQghbWDSwQgihATEmvZjIUzGRp2Ii\nT8VEnoqJPBUTeapfNLBCCCGEEEIIYTOJOVghhG7ie7BCCCH0d8OHt7FmzSvNDiNs5eJ7sEIIhaQG\nVvy7EEIIoT8T8Rk51BIPuQghhM2q1OwAWkSp2QG0iFKzA2gRpWYH0CJKzQ6gRZSaHUBLiDlY9dui\nDSxJbZKezdnWKWn0lqw/j6R9JC2W9GBm3apmxJJH0omSphYoV3fcki6TtEPOtvMl3eLLHZLG1zjW\n+ZI6apT5oN4Yaykf0++xzgLlH5TUJek5Sb+WtF2N8r3m37ffV2fMgyTN8XtvnKTjPZ7Fkg7K+13J\n7F/zXCXtKOl+SSskPSvpusy2A72+6fXEHUIIIYQQiuuLHqytsX/1TOBhM/tyZt3WGGeRmBqJeyIw\nuIH9Go1hS+TWcpbzjDOzw83sc8BuwDfqrKOR7ZVGA2Zmo81sBnAecJ2ZjQbWFTxekTI3mtlngcOB\n4yWdSqr4RT//QyTtV2fsoYf2ZgfQItqbHUCLaG92AC2ivdkBtIj2ZgfQItqbHUBLaG9vb3YILacv\nGljbS7pb0nJJ91brOZF0jqRl/rre1w2QNNXXLZV0ma/f33sBlkh6psEPirsBb1aseysTz3ivs0vS\nb33dVEk3S5on6WVJY339TpIe8ViWSjrD17d5L8JUSSslTZN0iu+/UtLnvdxgSXdImi9pkaTTPYyP\ngPcKnMtbfpzJHu9iSav9mIO9N6PL8zhO0iXAKKBT0lzf99se03zguMyx15I++PdmnZdD0l6SZvq1\n6ZJ0TDmlmdxeIWmBl+nwdVMkTciU6ZD0/bzyFT4B3q6VJDMrx7g9MAj4S41dNuTfe6vKuV0kaScv\nM0TSDL/Od2XiXyVpmC8fodRbuydwF3CkH+ci4OvA1dl9fZ8Bkm6Q9LSf94VFz9XM1pnZY778MbAY\n+FRFsTdIvwMhhBBCCGFzM7Mt9gLagPXAMf7+DuD7vtxJ+ov+SOBVYBipwTcXOMO3PZw51i7+cz5w\nhi8PAnZoIK7JwMScbQcDLwBD/f1u/nMqMN2XPwu85MsDgZ19effM+jbSh/SD/f0zwB2+fAYw05ev\nBc715V2BlcCOFTEdAdxW8Nx2BZaSei/GAr/KbBviP/+YOb8RmfxvBzwB3NLg9f5X4FJfVqa+9/3n\nKeV4fPt9wPHAYUApc5zngb3zyvv7D6rUPxK4v5f4HiI1rKbXeV6zgGN9ebDfpycC73idAp4EvpDJ\n77DMtXvUl08EZmWOOxUYm7lflvnyhcCPMvf4QqCtnnMt37vAfwD7VqyfC3y+l/0MOjKvTgOLV49X\n5CXyFHmKPG2tr8jTpucJC0lnZ2ezQ9hqdHZ2WkdHx4aX3ydUvnqdh7KZ/MnM5vvy3cAlwM8y248E\nOs3sbQBJ04ATgGuA/STdDPwBeFjSzsAoM5tFOqOP6g1GkoBDPZZqTgZmmNk7Xse7mW2/93UrJO1V\nPiQwRdIJpMbkqMy2VWa23JefBx7x5WeBfX15DHC6pB/6+0HAPqSGFl7fIuCigqd4N3CTmXVJWgv8\ng6QpwANm9kQm5nKv0tF0z/904ICCdVU6Gfhbj9mAyrlXY4BTJC32+ncCDjCzqZL2lDQC2At428xe\nlzSxWnlSI7AHM/sz8NW84MzsS5IGAfdKGm9mdxY8r3nAz/3enOmxASzwOpG0hHRNnyTTY9egMaRh\nfOP8/S6k8341cy69nqukgcA9wC/M7JWKzatJvwPP5Icwqf6oQwghhBC2Ye3t7d2GTE6ePLlqub5o\nYFmN91DlA6mZvSvpUOBU4HvAONLcoV4/vPpQswu9ntPMbE1m2wBS78KHwAN1nEPZh1ViPg/YAzjc\nzNYrPXRihyrl12fer2dj7gWcZWYvNRBPN5ImkRq0dwKY2UtKDxI5DbhG0iNmdk21XTe1blft2lbW\nM8XMbq+ybQbpGo8AphcoX6uu6gGafSTpfwFHAYUaWGb2U0n3A18B5kka45uy1/cTNl7Tj9k4/Lbq\nw0RqEHCJmc1pYN+y24CVZvaPVbb9Cpgt6Sgz++4m1NHPtTc7gBbR3uwAWkR7swNoEe3NDqBFtDc7\ngBbR3uwAWkLMwapfX8zBapN0tC+fCzxesX0BcIKkYf5X93OAxyTtDgw0s98BPwZGW5pH85qkr8GG\np7LtmD2Ymf3S0sMMRmcbV75tvZntS/rLfd5DDh4FxmXm0AzNKVdulOwKvOmNq5NIQ70qy/RmNnDp\nhh2kwwrs0zOYNHfri8BlmXUjgXVmdg9wI2nYJcD7pF4RgKdJ+R/q85PGUYWki7PzpHLMBSZ4+QGS\nhpR395+zge+U5zBJGuVzkwDuBb4JnEVqbOWV36PimDUpzZMb4cvbkRpKS/z9mco8aS9n/0+b2fNm\ndgNpuN5BNapcRRoaiJ9PvWYDEzxWJB1QeZ/XiPca0pDay3OKXAFcEI2rEEIIIYTNry8aWC8AF0ta\nTpoTcquvNwBvBF1J+jKCLmChmd1HmoNTktRFejjAlb7feOBSSUtJQ7eGNxDTi6Q5Rz34kL5rSY28\nLuCmbLzZov5zGunBBUuBbwErqpSptn/Z1aQHgSxTekz3VZUF/EEJt/VyPgCXkx5esdAfojAJOARY\n4OfxE9KwS4DbgYckzfX8TybNbXscWN7jyMlB1H4wxETgJEnLSI3Yg319+VrPIQ1be8rLzAB29m3L\ngSHAajN7o5fyQ7LHzJI00nuaKu0EzPJhfIuA14B/8W37U/thIhOVHnm+lDSv7sEqZbLxXAXcImkB\nqTcrT9498WvSdVjs98StVPQ2552rpL2BHwEHZx7M8Z2KYkOBl3uJKxRSanYALaLU7ABaRKnZAbSI\nUrMDaBGlZgfQIkrNDqAlxPdg1U9pqkz/4vOddjezK2sWDgBImkV6IENvDYaWI+lO4HIzq9V43Cb4\nHMRlwNlmtjKnjDU4ArOfKRHDS4ooEXkqokTkqYgSkaciSkSeiiiRnyfRHz8jV1MqlWKYYA5JmFmP\nUVX9tYG1P/AbYK11/y6sELZZkg4kDcVcBpxvOb/8qYEVQggh9F/Dh7exZs0rzQ4jbOWigRVCKERS\nXtsrhBBCCCG4vAZWX8zBCiGEbU6MSS8m8lRM5KmYyFMxkadiIk/FRJ7qFw2sEEIIIYQQQthMbNT0\nHgAAIABJREFUYohgCKGbGCIYQgghhFBbDBEMIYQQQgghhC0sGlghhNCAGJNeTOSpmMhTMZGnYiJP\nxUSeiok81S8aWCGEEEIIIYSwmcQcrBBCN/E9WCGE0DeG7z2cNavXNDuMEEKD4nuwQgiFSDImNTuK\nEELoByZBfA4LoXXFQy5CCGFzWtXsAFpE5KmYyFMxkadCYs5MMZGnYiJP9duiDSxJbZKezdnWKWn0\nlqw/j6R9JC2W9GBm3Vb1z7akEyVNLVCu7rglXSZph5xt50u6xZc7JI2vcazzJXXUKPNBvTHWUj6m\n32OdBco/KKlL0nOSfi1puxrle82/b7+vzpgHSZrj9944Scd7PIslHZT3u5LZv+i5jpa0TNKLkn6R\nWX+g1ze9nrhDCCGEEEJxfdGDtTX2fZ8JPGxmX86s2xrjLBJTI3FPBAY3sF+jMWyJ3FrOcp5xZna4\nmX0O2A34Rp11NLK90mjAzGy0mc0AzgOuM7PRwLqCxytS5p+BC8zsQOBASaeSKn7Rz/8QSfvVGXuo\nFBksJvJUTOSpmMhTIe3t7c0OoSVEnoqJPNWvLxpY20u6W9JySfdW6zmRdI7/xX2ZpOt93QBJU33d\nUkmX+fr9vRdgiaRnGvyguBvwZsW6tzLxjPc6uyT91tdNlXSzpHmSXpY01tfvJOkRj2WppDN8fZuk\nFb7fSknTJJ3i+6+U9HkvN1jSHZLmS1ok6XQP4yPgvQLn8pYfZ7LHu1jSaj/mYEn3+/pl3mtyCTAK\n6JQ01/f9tsc0Hzguc+y1pA/+vVnn5ZC0l6SZfm26JB1TTmkmt1dIWuBlOnzdFEkTMmU6JH0/r3yF\nT4C3ayXJzMoxbg8MAv5SY5cN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0co9dbuCdwFHOnHuQj4\nOnB1dl/fZ4CkGyQ97ed9YdFzlTQCGGJmC33VnaQ/KGS9QfodCCGEEEIIm1mvw6Q2k88A3zaz+ZLu\nACYAPytvlDQSuB44HHgXmOONlNXA3mb2N15uF99lGumv/rMkDaKxRuJAYH12hZkd7fUcDPwIONbM\n3pGU/SA6wsyOk/RZYBYwE/gv4EwzWytpd2C+bwPYHzjLzJZLegb4pu9/htcxFvh7YK6ZXSBpV2CB\npEfM7CngKY/pCOC7ZnZR5YmU4zazDqDDj/HvwD8BXwJeN7Ov+nGGmNkHki4H2v38RgCTSPl/HygB\ni/2YN9VKpJndm3l7C1Ays7GSBOxcLub1nwIcYGZH+fZZko4HpgO/AH7p5b8OjMkrb2ZP4I02M1sN\nnO3HHwncXj7fSpIeAo4EHjGzh2qc14b8Az8AJpjZU5IGk645wGHAwcAaYJ6kL5jZk/TsZTIze0vS\n3wE/MLNyI/xY4D4zmympLVP+AuBdMzva7/F5kh42s1cLnOvepN+dstW+Lms96XcgX3Yg4r7EX42r\nWUXkpYjIUzGRp2IiT4WUSqXodSgg8lRM5GmjUqlUaE5aXzSw/mRm8335buASMg0s0gfeTjN7G0DS\nNOAE4BpgP0k3A38AHpa0MzDKzGYBmNlH9QbjH9QP9ViqORmYYWbveB3vZrb93tetkLRX+ZDAFEkn\nkD64jspsW2Vmy335eeARX36W9LEVYAxwuqQf+vtBwD7AynKlZrYI6NG4ynE3cJOZdUlaC/yDpCnA\nA94wKcdc7lU6mu75nw4cULCuSicDf+sxG1A592oMcIqkxV7/TqQG1FRJe3pjby/gbTN7XdLEauWB\nJ6jCzP4MVG1c+fYveYPlXknjzezOguc1D/i535szPTaABV4nkpaQrumTZHrsGjSGNIxvnL/fhXTe\nr2bOpddzrWE16XfgmdwSJzV45BBCCCGEbVR7e3u3xubkyZOrluuLBlaPv+ZXKdPjA6mZvSvpUOBU\n4HvAONLcoV4/vPpQswu9ntPMbE1m2wDgj8CHwAN1nEPZh1ViPg/YAzjczNYrPXRihyrl12fer2dj\n7kXq5XqpgXi6kTSJ1KC9E8DMXlJ6kMhpwDXeM3ZNtV03tW5Xa36QgClmdnuVbTNI13gEqUerVvmG\n5nWZ2UeS/hdwFGn4XJF9firpfuArpN6kMb4pe30/YeM1/ZiNPatVHyZSg4BLzGxOA/u+Dvx15v2n\nfF3Wr4DZko4ys+82UEeA+Ct6UZGnYiJPxUSeConehmIiT8VEnurXF3Ow2iQd7cvnAo9XbF8AnCBp\nmKSBwDnAYz7cbqCZ/Q74MTDa59G8JulrsOGpbDtmD2Zmv/SHGYzONq5823oz25f0l/u8hxw8CozL\nzKEZmlOu3CjZFXjTG1cnAW1VyvRmNnDphh2kwwrs0zOYNHfri8BlmXUjgXVmdg9wI+khC5CGApaH\nXD5Nyv9Qn580jiokXZydJ5VjLmkIaHke0ZDy7v5zNvCd8hwmSaN8bhLAvcA3gbNIja288ntUHLMm\npXlyI3x5O1JDaYm/P1PSdTX2/7SZPW9mNwALgYNqVLkKOMKXzyoaZ8ZsYILHiqQDKu/zPH7Pvyep\nPKxyPPBvFcWuID0EIxpXIYQQQgibWV80sF4ALpa0nDSx/lZfb7DhA+GVpLk/XcBCM7uPNG+kJKmL\n9HCAK32/8cClkpaShm4NbyCmF4Fh1Tb4kL5rSY28LqA8DymvJ24a6cEFS4FvASuqlKm2f9nVpAeB\nLFN6TPdVlQWUHpRwWy/nA3A56eEVC5UeojAJOIQ0p6sL+Alp2CXA7cBDkuZ6/ieT5o49DizvceTk\nIGo/GGIicJKkZaRG7MG+vnyt5wD3AE95mRn4PC3P+xBgtZm90Uv5IdljZkka6T1NlXYizd9aAiwC\nXgP+xbftT+2HiUyU9Kxf44+AB6uUycZzFXCLpAWk3qw8effEr0nXYbHfE7dS0dvcy7kCXAzcQbrP\nX6oy32wo8HIvcYUitqovdtiKRZ6KiTwVE3kqJL63qJjIUzGRp/qpP36DuM932t3MrqxZOAAgaRYw\n1sx6azC0HEl3ApebWa3G4zbBe7WWAWeb2cqcMsakPg2rNcVk+2IiT8VEnorZ1vI0CbbE57B4KEEx\nkadiIk/5JGFmPUZV9dcG1v7Ab4C1Fd+FFcI2S9KBpKGYy4DzLeeXX1L/+0chhBCaYPjew1mzek3t\ngiGErVI0sEIIhUjKa3uFEEIIIQSX18DqizlYIYSwzYkx6cVEnoqJPBUTeSom8lRM5KmYyFP9ooEV\nQgghhBBCCJtJDBEMIXQTQwRDCCGEEGqLIYIhhBBCCCGEsIVFAyuEEBoQY9KLiTwVE3kqJvJUTOSp\nmMhTMZGn+kUDK4QQQgghhBA2k5iDFULoJr4HK4QQQn8zfHgba9a80uwwQouJ78EKIRSSGljx70II\nIYT+RMRn4lCveMhFCCFsVqVmB9AiSs0OoEWUmh1Aiyg1O4AWUWp2AC2i1OwAWkLMwarfFm1gSWqT\n9GzOtk5Jo7dk/Xkk7SNpsaQHM+tWNSOWPJJOlDS1QLm645Z0maQdcradL+kWX+6QNL7Gsc6X1FGj\nzAf1xlhL+Zh+j3UWKH+NpD9Jer/g8XvNv2+/r3jEIGmQpDl+742TdLyk5/z9QXm/K5n9a56rpB0l\n3S9phaRnJV2X2Xag1ze9nrhDCCGEEEJxfdGDtTX2t54JPGxmX86s2xrjLBJTI3FPBAY3sF+jMWyJ\n3FrOcp5ZwJGbUEcj2yuNBszMRpvZDOA84DozGw2sK3i8ImVuNLPPAocDx0s6lVTxi2b2OeAQSfvV\nGXvoob3ZAbSI9mYH0CLamx1Ai2hvdgAtor3ZAbSI9mYH0BLa29ubHULL6YsG1vaS7pa0XNK91XpO\nJJ0jaZm/rvd1AyRN9XVLJV3m6/f3XoAlkp5p8IPibsCbFeveysQz3uvskvRbXzdV0s2S5kl6WdJY\nX7+TpEc8lqWSzvD1bd6LMFXSSknTJJ3i+6+U9HkvN1jSHZLmS1ok6XQP4yPgvQLn8pYfZ7LHu1jS\naj/mYO/N6PI8jpN0CTAK6JQ01/f9tsc0Hzguc+y1pA/+vVnn5ZC0l6SZfm26JB1TTmkmt1dIWuBl\nOnzdFEkTMmU6JH0/r3yFT4C3ayXJzBaY2Ru1ymVsyL/3VpVzu0jSTl5miKQZfp3vysS/StIwXz5C\nqbd2T+Au4Eg/zkXA14Grs/v6PgMk3SDpaT/vC4ueq5mtM7PHfPljYDHwqYpib5B+B0IIIYQQwuZm\nZlvsBbQB64Fj/P0dwPd9uZP0F/2RwKvAMFKDby5whm97OHOsXfznfOAMXx4E7NBAXJOBiTnbDgZe\nAIb6+93851Rgui9/FnjJlwcCO/vy7pn1baQP6Qf7+2eAO3z5DGCmL18LnOvLuwIrgR0rYjoCuK3g\nue0KLCX1XowFfpXZNsR//jFzfiMy+d8OeAK4pcHr/a/Apb6sTH3v+89TyvH49vuA44HDgFLmOM8D\ne+eV9/cfVKl/JHB/jRjfb+C8ZgHH+vJgv09PBN7xOgU8CXwhk99hmWv3qC+fCMzKHHcqMDZzvyzz\n5QuBH2Xu8YVAWwPnuhvwH8C+FevnAp/vZT+Djsyr08Di1eMVeYk8RZ4iT1vrK/JUf56wUF1nZ2ez\nQ9hqdHZ2WkdHx4aX3zdUvrZjy/uTmc335buBS4CfZbYfCXSa2dsAkqYBJwDXAPtJuhn4A/CwpJ2B\nUWY2i3RGH9UbjCQBh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0iNyVGZbavMbLkvPw884svP\nAvv68hjgdEk/9PeDgH1IDS28vkXARQVP8W7gJjPrkrQW+AdJU4AHzOyJTMzlXqWj6Z7/6cABBeuq\ndDLwtx6zAZVzr8YAp0ha7PXvBBxgZlMl7SlpBLAX8LaZvS5pYrXypEZgD2b2Z+CrDcbem3nAz/3e\nnOmxASzwOpG0hHRNnyTTY9egMaRhfOP8/S6k8361XKDWuUoaCNwD/MLMXqnYvJr0O/BMfgiT6o86\nhBBCCGEb1t7e3m3I5OTJk6uW64sGltV4D1U+kJrZu5IOBU4FvgeMI80d6vXDqw81u9DrOc3M1mS2\nDSD1LnwIPFDHOZR9WCXm84A9gMPNbL3SQyd2qFJ+feb9ejbmXsBZZvZSA/F0I2kSqUF7J4CZvaT0\nIJHTgGskPWJm11TbdVPrdtWubWU9U8zs9irbZpCu8QhgeoHyterabMzsp5LuB74CzJM0xjdlr+8n\nbLymH7Nx+G3Vh4nUIOASM5vTSLzuNmClmf1jlW2/AmZLOsrMvrsJdfRz7c0OoEW0NzuAFtHe7ABa\nRHuzA2gR7c0OoEW0NzuAlhBzsOrXF3Ow2iQd7cvnAo9XbF8AnCBpmP/V/RzgMUm7AwPN7HfAj4HR\nZrYWeE3S12DDU9l2zB7MzH5pZodbepDAmopt681sX9Jf7r+RE++jwLjMHJqhOeXKjZJdgTe9cXUS\naahXZZnezAYu3bCDdFiBfXoGk+ZufRG4LLNuJLDOzO4BbiQNuwR4n9QrAvA0Kf9DJW1PauRUO/7F\n2XlSOeYCE7z8AElDyrv7z9nAd8pzmCSN8rlJAPcC3wTOIjW28srvUXHMenXbT9KZyjxpr+oO0qfN\n7Hkzu4E0XO+gGnWsIg0NhHQ+9ZoNTJC0ndd/QOV9XiPea0hDai/PKXIFcEE0rkIIIYQQNr++aGC9\nAFwsaTlpTsitvt4AvBF0JenLCLqAhWZ2H2kOTklSF+nhAFf6fuOBSyUtJQ3dGt5ATC+S5hz14EP6\nriU18rqAm7LxZov6z2mkBxcsBb4FrKhSptr+ZVeTHgSyTOkx3VdVFvAHJdzWy/kAXE56eMVCf4jC\nJOAQYIGfx09Iwy4BbgcekjTX8z+ZNLftcWB5jyMnBwF/qRHDROAkSctIjdiDfX35Ws8hDVt7ysvM\nAHb2bcuBIcBq84dR5JQfkj1mlqSR3tPUg6SfSnoN2FHpce0/8U37U/thIhOVHnm+lDSv7sEqZbLx\nXAXcImkBqTcrT9498WvSdVjs98StVPQ2552rpL2BHwEHZx7M8Z2KYkOBl3uJKxRSanYALaLU7ABa\nRKnZAbSIUrMDaBGlZgfQIkrNDqAlxPdg1U9pqkz/4vOddjezK2sWDgBImkV6IENvDYaWI+lO4HIz\nq9V43Cb4HMRlwNlmtjKnjPXhCMwWViKGlxRRIvJURInIUxElIk9FlIg8FVFiY55Ef/xMXESpVIph\ngjkkYWY9RlX11wbW/sBvgLXW/buwQthmSTqQNBRzGXC+5fzyRwMrhBBC/xMNrFC/aGCFEApJDawQ\nQgih/xg+vI01a15pdhihxeQ1sPriKYIhhBYTf3ipLYZMFBN5KibyVEzkqZjIUzGRp2IiT/Xri4dc\nhBBCCCGEEEK/EEMEQwjdSMqbnhVCCCGEEFzeEMHowQohhBBCCCGEzSQaWCGE0ID4XpBiIk/FRJ6K\niTwVE3kqJvJUTOSpftHACiGEEEIIIYTNJOZghRC6iTlYIYQQQgi1xWPaQwiFST3+rQghhH5l+N7D\nWbN6TbPDCCG0oOjBCiF0I8mY1OwoWsAqYL9mB9ECIk/FRJ6K6cs8TWrd7wSM7y0qJvJUTOQpXzxF\nMIQQQgghhBC2sC3awJLUJunZnG2dkkZvyfrzSNpH0mJJD2bWrWpGLHkknShpaoFydcct6TJJO+Rs\nO1/SLb7cIWl8jWOdL6mjRpkP6o2xlvIx/R7rLFD+Gkl/kvR+weP3mn/ffl/xiEHSIElz/N4bJ+l4\nSc/5+4Pyflcy+xc919GSlkl6UdIvMusP9Pqm1xN3yBG9DcVEnoqJPBUTeSokehuKiTwVE3mqX1/0\nYG2N/etnAg+b2Zcz67bGOIvE1EjcE4HBDezXaAxbIreWs5xnFnDkJtTRyPZKowEzs9FmNgM4D7jO\nzEYD6woer0iZfwYuMLMDgQMlnUqq+EUz+xxwiKT4mBJCCCGEsAX0RQNre0l3S1ou6d5qPSeSzvG/\nuC+TdL2vGyBpqq9bKukyX7+/9wIskfRMgx8UdwPerFj3Viae8V5nl6Tf+rqpkm6WNE/Sy5LG+vqd\nJD3isSyVdIavb5O0wvdbKWmapFN8/5WSPu/lBku6Q9J8SYskne5hfAS8V+Bc3vLjTPZ4F0ta7ccc\nLOl+X7/Me00uAUYBnZLm+r7f9pjmA8dljr2W9MG/N+u8HJL2kjTTr02XpGPKKc3k9gpJC7xMh6+b\nImlCpkyHpO/nla/wCfB2rSSZ2QIze6NWuYwN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0co9dbu\nCdwFHOnHuQj4OnB1dl/fZ4CkGyQ97ed9YdFzlTQCGGJmC33VnaQ/KGS9QfodCJtiq+rz3opFnoqJ\nPBUTeSokvreomMhTMZGn+vXFUwQ/A3zbzOZLugOYAPysvFHSSOB64HDgXWCON1JWA3ub2d94uV18\nl2mkv/rPkjSIxhqJA4H12RVmdrTXczDwI+BYM3tHUvaD6AgzO07SZ0k9IjOB/wLONLO1knYH5vs2\ngP2Bs8xsuaRngG/6/md4HWOBvwfmmtkFknYFFkh6xMyeAp7ymI4AvmtmF1WeSDluM+sAOvwY/w78\nE/Al4HUz+6ofZ4iZfSDpcqDdz28EMImU//eBErDYj3lTrUSa2b2Zt7cAJTMbK0nAzuViXv8pwAFm\ndpRvnyXpeGA68Avgl17+68CYvPJm9gTeaDOz1cDZfvyRwO3l890U2fwDPwAmmNlTkgaTrjnAYcDB\nwBpgnqQvmNmT9OxlMjN7S9LfAT8ws3Ij/FjgPjObKaktU/4C4F0zO9rv8XmSHjazVwuc696k352y\n1b4uaz3pdyBfdiDivsSwnBBCCCH0e6VSqVCDsy8aWH8ys/m+fDdwCZkGFmnYVqeZvQ0gaRpwAnAN\nsJ+km4E/AA9L2hkYZWazAMzso3qD8Q/qh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0gfXEdl\ntq0ys+W+/DzwiC8/S/rYCjAGOF3SD/39IGAfYGW5UjNbBPRoXOW4G7jJzLokrQX+QdIU4AFvmJRj\nLvcqHU33/E8HDihYV6WTgb/1mA2onHs1BjhF0mKvfydSA2qqpD29sbcX8LaZvS5pYrXywBNUYWZ/\nBja5cVXFPODnfm/O9NgAFnidSFpCuqZPkumxa9AY0jC+cf5+F9J5v1ousInnupr0O/BMbomTGjxy\nfxKNzmIiT8VEnoqJPBUSc2aKiTwVE3naqL29vVs+Jk+eXLVcXzSwevw1v0qZHh9IzexdSYcCpwLf\nA8aR5g71+uHVh5pd6PWcZmZrMtsGAH8EPgQeqOMcyj6sEvN5wB7A4Wa2XumhEztUKb8+8349G3Mv\nUi/XSw3E042kSaQG7Z0AZvaS0oNETgOu8Z6xa6rtuql1u1rzgwRMMbPbq2ybQbrGI0g9WrXK99mc\nOTP7qaT7ga+QepPG+Kbs9f2Ejdf0Yzb2rFZ9mEgNAi4xszkN7Ps68NeZ95/ydVm/AmZLOsrMvttA\nHSGEEEIIIUdfzMFqk3S0L58LPF6xfQFwgqRhkgYC5wCP+XC7gWb2O+DHwGgzWwu8JulrsOGpbDtm\nD2ZmvzSzw/1BAmsqtq03s31Jf7n/Rk68jwLjMnNohuaUKzdKdgXe9MbVSUBblTK9mQ1cumEH6bAC\n+/QMJs3d+iJwWWbdSGCdmd0D3Eh6yAKkoYDlIZdPk/I/VNL2pEZOteNfnJ0nlWMuaQhoeR7RkPLu\n/nM28J3yHCZJo3xuEsC9wDeBs0iNrbzye1Qcs17d9pN0pqTret1B+rSZPW9mNwALgYNq1LEKOMKX\nz2ogxtnABEnbef0HVN7nefyef09SeVjleODfKopdQXoIRjSuNkXMBSkm8lRM5KmYyFMhMWemmMhT\nMZGn+vVFA+sF4GJJy0kT62/19QYbPhBeSZr70wUsNLP7SPNGSpK6SA8HuNL3Gw9cKmkpaejW8AZi\nehEYVm2DD+m7ltTI6wLK85DyeuKmkR5csBT4FrCiSplq+5ddTXoQyDKlx3RfVVnAH5RwWy/nA3A5\n6eEVC/0hCpOAQ0hzurqAn5CGXQLcDjwkaa7nfzJp7tjjwPIeR04OAv5SI4aJwEmSlpEasQf7+vK1\nngPcAzzlZWbg87Q870OA1eWHUeSUH5I9Zpakkd7T1IOkn0p6DdhR6XHtP/FN+1P7YSITJT3r1/gj\n4MEqZbLxXAXcImkBqTcrT9498WvSdVjs98StVPQ293auwMXAHaT7/CUze6hi+1Dg5V7iCiGEEEII\nDVKrfkv5pvD5Trub2ZU1CwcAJM0CxppZbw2GliPpTuByM6vVeNwmeK/WMuBsM1uZU8aY1KdhhRDC\n1mcS9MfPSCGE4iRhZj1GVfXXBtb+wG+AtRXfhRXCNkvSgaShmMuA8y3nl19S//tHIYQQKgzfezhr\nVq+pXTCE0G9FAyuEUIikvLZXyCiVSvFkpQIiT8VEnoqJPBUTeSom8lRM5ClfXgOrL+ZghRBCCCGE\nEEK/ED1YIYRuogcrhBBCCKG26MEKIYQQQgghhC0sGlghhNCA+F6QYiJPxUSeiok8FRN5KibyVEzk\nqX7RwAohhBBCCCGEzSTmYIUQuok5WCGEEEIIteXNwdquGcGEELZu6fuIQwihfxk+vI01a15pdhgh\nhBYXQwRDCFVYvGq+OreCGFrhFXmKPLVOnt5441W2BTFnppjIUzGRp/pFAyuEEEIIIYQQNpPN0sCS\n1Cbp2ZxtnZJGb4566iVpH0mLJT2YWbeqGbHkkXSipKkFyq3yn7m5rig/RNJrkm7JHkPSsDpiq5kr\nv7779LL9fEn/WLTOgnGdXz4vSR2Sxtcof6SkLn8tlfSNAnVMlXRCje1j64z7eEnP+T35V5JulPSs\npJ/6eXy/xv5FzvWLkp7x81wo6aTMth9IeqHI+Yci2psdQItob3YALaK92QG0iPZmB9AS2tvbmx1C\nS4g8FRN5qt/mnINlm/FYm8uZwMNmdmVm3dYYZ5GYLGc5z9XAYw3Usynlt/RxGvUscISZrZc0AnhO\n0v80s0/6OI7zgOvM7B4ASRcCQ83MJHVspjreAr5qZmsk/TdgNvApADO7SdITwI3A9M1UXwghhBBC\nyNicQwS3l3S3pOWS7pW0Q2UBSedIWuav633dAO8NWOZ/db/M1+8vaY6kJf4X+f0aiGk34M2KdW9l\n4hnvdXZJ+q2vmyrpZknzJL1c7qWQtJOkRzK9A2f4+jZJK3y/lZKmSTrF918p6fNebrCkOyTNl7RI\n0ukexkfAewXO5a3KFZJuz/TMvCnp//H1RwB7AQ9X7gJc6vUvlXRg5tz+xa/BEkn/I6/OKv4CfOLH\n+ZIfe4mkOVXi3UPS/5T0tL+OVbJK0i6Zci9K2rNa+Sr1rwXW9Ragmf2Xma33tzsC7xVoXL1LujZI\nut57npZIuiFT5sQq98mJku7LnMs/+n12AfB14GpJd0n6N2BnYJGkcRV5+rSkB70H6rHydQI+KHCu\nS81sjS8/D+wgaftMkTXArjXOPRRSanYALaLU7ABaRKnZAbSIUrMDaAkxZ6aYyFMxkaf6bc4erM/w\n/7N372F2VHW+/98fMokRAgkhkEA0F+IZJA6JSUBEEVoBHWYGRBQU5aIPjyAXQS4eGUZJIojMIJwT\n9ScOwsQIkesAAyJDQLujB4hAroSEIBAFZBJACJIZBKG/vz9q7VC9u/betTsddnfyeT3Pfrp21aqq\n7/pWdbJXr7VqwxciYoGkK4GTgUsrGyXtDFwETCH78HpXaqQ8DYyOiEmpXOWD9lyyv/bfKmkQPWsM\nDgA68ysiYu90nonAucA+EfGipGG5YqMi4oOSdgduBW4C/gwcFhHrJe0ALEjbACYAn4yIFZIeBD6T\n9j80neNw4J+AX0TE8ZKGAvdLujsi7gPuSzFNA06MiBOqK1KJu2rdF9N+Y4A7gNmSBHyHrLfkoIKc\nPBsR0ySdBJwNnAB8A1iXuwZDa52zIIZPpX1GAJcD+0bEk1X5rJgFXBoR90p6J3BnREwmIQquAAAg\nAElEQVSUdAvwCWCOpPcBv4uI5yTNrS4PTKw6/yWVZUknZqvi8uoTp+P+GzAe+GyJep2R9htOdt3f\nnd5vlytWdJ9AQY9dRFwpaV/gtoi4KR3rTxExNS3ne7AuJ7sPHk9xXwYcEBH536eadc2V+RSwKCL+\nklvdSanf+xm55TY8LMfMzMy2dB0dHaUanL3ZwHoyIhak5auBL5NrYAF7Ae0R8QJA+vC8H3ABMF7S\nLODnwDxJQ4BdIuJWgIh4rdlgUkNjcoqlyEeAGyLixXSOdbltt6R1KyXtVDkk8G1l83I6gV1y21ZH\nxIq0/DBwd1p+CBiXlj8KHCLpq+n9IGAMsKpy0ohYSNbgaaaeg4EbgFMj4mlJpwC3R8QzWQqoft72\nzennQrJGDcCBwIZ5ORFRpket2vuB+RHxZDrGuoIyBwK7p2sDMETS1sD1wHnAHOAzvDl8rVb5QhHx\nr3W23Q/8jaTdgDsltUfEn0rU6yXgFUlXALcDP8ttK7pPNoqkbYAPADfk6j2wuly9uqbjvAf4Nt0b\n2c8DO0oaVuMaJTPKB73Famt1AP1EW6sD6CfaWh1AP9HW6gD6Bc+ZKcd5Ksd5elNbW1uXfMycObOw\n3Kacg1U076bbl+tExDpJk4GPAV8CjgC+UlS2y4Gkk4EvpvP8XWVYVNq2FfAE8CrZh+JmvVoQ8+eA\nEcCUNJdnNTC4oHxn7n2+t0BkvVy/7UE89VwG3BgR7en9PsC+KT/bkg3dfDkizq2K9Q16/3vQGn15\nkoC9q3pUAO5TNiR0BNm8uW/WK6+N+I6miFgl6XHgf5E1MhuVfyP1Ih1Adm+empah+D55na69rd2G\nyjawFfBipWerJyS9g6w37ZiI+F1+W0S8Iula4AlJn46IbkM5zczMzKznenMO1lhJlSFlnwV+XbX9\nfmA/ScMlDQCOAuan4XYDIuJm4OvA1IhYDzwl6eMAkgZJenv+YBHxg4iYEhFT842rtK0zIsYBD5Lr\nmanyS+CINAQMSdvXKFf54DyUbHhdp7Ins40tKFPPncBpG3aQ3ltin7pSb9WQiLi4si4ijo6IcRGx\nK9kQwJ/kGle13AWckjtut+F9yuaf7VznGAuAD0kam8oX5XMecHrumJNz224m6/FcketZqVe+NEnj\n0j1Hiu9dwG/T+zlK8+Rq7LsNMCwi/hM4E5hUq2j6+XtgoqSBKY8H1Cif32eDiHgZWJ2G91ViqHXO\noniHkvWyfS3Xo5zfPozsd2K0G1cbq6PVAfQTHa0OoJ/oaHUA/URHqwPoFzxnphznqRznqXm92cB6\nBDhF0gqyh0v8MK0PgNQIOofsX8fFwAMRcRswGuiQtBi4KpUBOJbsgQxLgXuAkT2I6VGg8LHkaUjf\nt8gaeYuBylyeWj1xc4G9UjxHAysLyhTtX3E+WW/SMmWPWf9mdQFJ0yTVnFNT4CxgD2UPuVgkqdHw\nwlqxXQAMV/bI8MVUjcFIQ9UmAC/UPHDE82TDG29Ox7i2oNjpwJ7KHrCxHDgxt+16sl7Ca0uW70bS\niTVysC+wVNKidJ4TcsMDJwHP1DnstsDP0nX/FXBGWl94n0TE0+kcy1NdFlWXqfO+4mjgeGUP1VgO\nHFpdoE5dTyW7Vufl7osRue1DgbURUfdhGWZmZmbWM4po9RO0N50032mHqse0W5PSfJ4vRMTZrY6l\nN0naFrgiIraY74VKwx1nRUTRExkrZaL1T9Y3M2sFsTl/LjKz3iWJiOg2Imlzb2BNAH4MrI+Ig1sc\njllLSTqLrJfw4oi4pk65zfcfBTOzOkaOHMuaNb9rdRhm1k9skQ0sM2uepPC/C411dHT4yUolOE/l\nOE/lOE/lOE/lOE/lOE+11Wpg9eYcLDMzMzMzsy2ae7DMrAv3YJmZmZk15h4sMzMzMzOzTcwNLDOz\nHvD3gpTjPJXjPJXjPJXjPJXjPJXjPDXPDSwzMzMzM7Ne4jlYZtaF52CZmZmZNVZrDtZftSIYM+vb\npG7/VphZHzNy9EjWPL2m1WGYmVkV92CZWReSghmtjqIfWA2Mb3UQ/YDzVE5P8jQDtrT/w/19POU4\nT+U4T+U4T7X5KYJmZmZmZmabWK80sCSNlfRQjW3tkqb2xnmaJWmMpEWS7sitW92KWGqRtL+k2SXK\nrU4/a+a6qvy2kp6S9N38MSQNbyK2hrlK13dMne3HSfpe2XOWjOu4Sr0kTZd0bIPye0lanF5LJX26\nxDlmS9qvwfbDm4x7X0nL0z35NkkXS3pI0j+nepzZYP+GdU3l/lHSbyWtlPTR3PqzJD1Spv5Wgntl\nynGeynGeSvFf0ctxnspxnspxnprXmz1YfXGcwmHAvIg4OLeuL8ZZJqaosVzL+cD8HpxnY8pv6uP0\n1EPAtIiYAnwM+P8kDWhBHJ8DLoyIqRHxKvBFYFJEfK23TiBpd+BIYHfgYOAHShOqIuIS4DjglN46\nn5mZmZl11ZsNrIGSrpa0QtL1kgZXF5B0lKRl6XVRWrdV6g1YlnoXTk/rJ0i6S9ISSQ9K6snf94YB\nz1atey4Xz7HpnIslzUnrZkuaJekeSY9VeikkbSPp7hTLUkmHpvVjU0/BbEmrJM2VdFDaf5WkPVO5\nrSVdKWmBpIWSDklhvAa8VKIuz1WvkPSjXM/Ms5K+kdZPA3YC5lXvApyWzr9U0l/n6vZv6RoskfSJ\nWucs8EfgjXScv03HXiLproJ4R0i6UdJv0msfZVZL2i5X7lFJOxaVLzj/euCVegFGxJ8jojO9fTvw\nUkS80aBe68iuDZIuSj1PSyT9S67M/gX3yf6SbsvV5XvpPjuerOFzvqSrJP0HMARYKOmIqjztKukO\nSQ9Iml+5TsDLjeoKfBy4NiJej4jfAb8F3pfbvgYY2uAYVkaf6gvvw5yncpynUvx9POU4T+U4T+U4\nT83rzacI7gZ8ISIWSLoSOBm4tLJR0s7ARcAUsg+vd6VGytPA6IiYlMpVPmjPJftr/62SBtGzxuAA\noDO/IiL2TueZCJwL7BMRL0oalis2KiI+mHoDbgVuAv4MHBYR6yXtACxI2wAmAJ+MiBWSHgQ+k/Y/\nNJ3jcOCfgF9ExPGShgL3S7o7Iu4D7ksxTQNOjIgTqitSibtq3RfTfmOAO4DZqbfiO2S9JQcV5OTZ\niJgm6STgbOAE4BvAutw1GFrrnAUxfCrtMwK4HNg3Ip6symfFLODSiLhX0juBOyNioqRbgE8AcyS9\nD/hdRDwnaW51eWBi1fkvqSxLOjFbFZdXnzgd99/IBuJ8tkS9zkj7DSe77u9O77fLFSu6T6Cgxy4i\nrpS0L3BbRNyUjvWniJialqfnil9Odh88nuK+DDggIvK/T7XqOpp0PyV/SOsqOinze9+eWx6Hhy+Z\nmZnZFq+jo6NUg7M3G1hPRsSCtHw18GVyDSxgL6A9Il4ASB+e9wMuAMZLmgX8HJgnaQiwS0TcChAR\nrzUbTGpoTE6xFPkIcENEvJjOsS637Za0bqWknSqHBL6tbF5OJ7BLbtvqiFiRlh8G7k7LD5F9PAX4\nKHCIpK+m94OAMcCqykkjYiFZg6eZeg4GbgBOjYinJZ0C3B4Rz2QpoPrJJjennwvJGjUABwIb5uVE\nRJketWrvB+ZHxJPpGOsKyhwI7J6uDcAQSVsD1wPnAXOAzwDXNShfKCL+tc62+4G/kbQbcKek9oj4\nU4l6vQS8IukK4HbgZ7ltRffJRpG0DfAB4IZcvQdWl6tX1waeB3aUNKzGNcp8uIdH35K40VmO81SO\n81SK54KU4zyV4zyV4zy9qa2trUs+Zs6cWViuNxtY1X+1L5p30+0xhhGxTtJksrkxXwKOAL5SVLbL\ngaSTyeawBPB3EbEmt20r4AngVbIPxc16tSDmzwEjgCkR0ansARCDC8p35t7newtE1sv12x7EU89l\nwI0RUelz2AfYN+VnW7Khmy9HxLlVsb5B738PWqMvTxKwd0T8pWr9fcqGhI4gmzf3zXrltRHf0RQR\nqyQ9DvwvskZmo/JvpF6kA8juzVPTMhTfJ6/Ttbe121DZBrYCXqz0bPXAH4B35t6/I60DICJekXQt\n8ISkT0dEt6GcZmZmZtZzvTkHa6ykypCyzwK/rtp+P7CfpOHKHjBwFDA/DbcbEBE3A18HpkbEeuAp\nSR8HkDRI0tvzB4uIH0TElPTAgDVV2zojYhzwILmemSq/BI5IQ8CQtH2NcpUPzkPJhtd1SvowMLag\nTD13Aqdt2EF6b4l96kq9VUMi4uLKuog4OiLGRcSuZEMAf5JrXNVyF7kHHxQN71M2/2znOsdYAHxI\n0thUviif84DTc8ecnNt2M1mP54pcz0q98qVJGpfuOVJ87yKbm4SkOUrz5Grsuw0wLCL+EzgTmFSr\naPr5e2CipIEpjwfUKJ/fZ4OIeBlYLelTuRhqnbPIrcBn0u/MeLK63p871jCy34nRblxtJM+ZKcd5\nKsd5KsVzQcpxnspxnspxnprXmw2sR4BTJK0ge7jED9P6AEiNoHOADmAx8EBE3EY2P6RD0mLgqlQG\n4FiyBzIsBe4BRvYgpkeBwseSpyF93yJr5C0GKnN5avXEzQX2SvEcDawsKFO0f8X5ZL1Jy5Q9Zv2b\n1QUkTZPUbf5QHWcBeyh7yMUiSY2GF9aK7QJguLJHhi8G2qriEtk8sxdqHjjiebLhjTenY1xbUOx0\nYE9lD9hYDpyY23Y9WS/htSXLdyPpxBo52BdYKmlROs8JueGBk4Bn6hx2W+Bn6br/CjgjrS+8TyLi\n6XSO5akui6rL1HlfcTRwvLKHaiwHDq0uUKuu6b6+HlhBNuT25Oj6TaRDgbUR0ehhGWZmZmbWA9qc\nvwU+zXfaISLOaVjYapL0HrIHmJzd6lh6k6RtgSsiYov5Xqg03HFWRBQ9kbFSJpjx1sVkZj00Azbn\n/8PNzPo6SUREtxFJm3sDawLwY2B91XdhmW1xJJ1F1kt4cURcU6fc5vuPgtlmZOTokax5ek3jgmZm\ntknUamD15hDBPiciHo+ID7lxZZY90j7NWazZuMqV9avBq729veUx9IeX87Tp8rQlNq48F6Qc56kc\n56kc56l5m3UDy8zMzMzM7K20WQ8RNLPmSQr/u2BmZmZW3xY5RNDMzMzMzOyt5AaWmVkPeEx6Oc5T\nOc5TOc5TOc5TOc5TOc5T89zAMjMzMzMz6yWeg2VmXXgOlpmZmVljteZg/VUrgjGzvk3q9m+FmVmh\nkSPHsmbN71odhplZn+EhgmZWIPxq+GrvAzH0h5fztLnnae3a3/NW8VyQcpyncpyncpyn5rmBZWZm\nZmZm1kt6pYElaaykh2psa5c0tTfO0yxJYyQtknRHbt3qVsRSi6T9Jc0uUW51+lkz11Xlt5X0lKTv\n5o8haXgTsTXMVbq+Y+psP07S98qes2Rcx1XqJWm6pGMblB8u6ZeSXs7no8E+syXt12D74U3Gva+k\n5emefJukiyU9JOmfUz3ObLB/mboeKOlBSUslPSDpw7ltZ0l6RNKnm4nbamlrdQD9RFurA+gn2lod\nQL/Q1tbW6hD6BeepHOepHOepeb3ZgxW9eKzechgwLyIOzq3ri3GWiSlqLNdyPjC/B+fZmPKb+jg9\n9Wfg68BZLY7jc8CFETE1Il4FvghMioiv9eI5ngP+ISImA58HrqpsiIhLgOOAU3rxfGZmZmaW05sN\nrIGSrpa0QtL1kgZXF5B0lKRl6XVRWrdV6g1Ylv7qfnpaP0HSXZKWpL/Ij+9BTMOAZ6vWPZeL59h0\nzsWS5qR1syXNknSPpMcqvRSStpF0d6534NC0fqyklWm/VZLmSjoo7b9K0p6p3NaSrpS0QNJCSYek\nMF4DXipRl+eqV0j6UYp9saRnJX0jrZ8G7ATMq94FOC2df6mkv87V7d/SNVgi6RO1zlngj8Ab6Th/\nm469RNJdBfGOkHSjpN+k1z7KrJa0Xa7co5J2LCpfcP71wCv1AoyI/4mIe4FXS9SnYh3ZtUHSRann\naYmkf8mV2b/gPtlf0m25unwv3WfHA0cC50u6StJ/AEOAhZKOqMrTrpLuSD1Q8yvXCXi5RF2XRsSa\ntPwwMFjSwFyRNcDQJvJgNXW0OoB+oqPVAfQTHa0OoF/wXJBynKdynKdynKfm9eZTBHcDvhARCyRd\nCZwMXFrZKGln4CJgCtmH17tSI+VpYHRETErlKh+055L9tf9WSYPoWWNwANCZXxERe6fzTATOBfaJ\niBclDcsVGxURH5S0O3ArcBNZL8hhEbFe0g7AgrQNYALwyYhYIelB4DNp/0PTOQ4H/gn4RUQcL2ko\ncL+kuyPiPuC+FNM04MSIOKG6IpW4q9Z9Me03BrgDmC1JwHfIeksOKsjJsxExTdJJwNnACcA3gHW5\nazC01jkLYvhU2mcEcDmwb0Q8WZXPilnApRFxr6R3AndGxERJtwCfAOZIeh/wu4h4TtLc6vLAxKrz\nX1JZlnRitioubxR3iXqdkY45nOy6vzu93y5XrOg+gYIeu4i4UtK+wG0RcVM61p8iYmpanp4rfjnZ\nffB4ysdlwAERkf99alhXSZ8CFkXEX3KrOyn1ez8jt9yGhy+ZmZnZlq6jo6NUg7M3G1hPRsSCtHw1\n8GVyDSxgL6A9Il4ASB+e9wMuAMZLmgX8HJgnaQiwS0TcChARrzUbTGpoTE6xFPkIcENEvJjOsS63\n7Za0bqWknSqHBL6tbF5OJ7BLbtvqiFiRlh8G7k7LDwHj0vJHgUMkfTW9HwSMAVZVThoRC8kaPM3U\nczBwA3BqRDwt6RTg9oh4JksB1c/bvjn9XEjWqAE4ENgwLyciyvSoVXs/MD8inkzHWFdQ5kBg93Rt\nAIZI2hq4HjgPmAN8BriuQflCEfGvPYi7kZeAVyRdAdwO/Cy3reg+2SiStgE+ANyQq/fA6nKN6irp\nPcC36d7Ifh7YUdKwGtcomVE+6C1WW6sD6CfaWh1AP9HW6gD6Bc8FKcd5Ksd5Ksd5elNbW1uXfMyc\nObOwXG82sKr/al8076bbl+tExDpJk4GPAV8CjgC+UlS2y4Gkk8nmsATwd5VhUWnbVsATZEPCbm+i\nDhX5oWSVOD4HjACmRESnsgdADC4o35l7n+8tEFkv1297EE89lwE3RkR7er8PsG/Kz7ZkQzdfjohz\nq2J9g97/HrRGX54kYO+qHhWA+5QNCR1BNm/um/XK6y38jqaIeCP1Ih1Adm+empah+D55na69rd2G\nyjawFfBipWerJyS9g6w37ZiI+F1+W0S8Iula4AlJn46IbkM5zczMzKznenMO1lhJlSFlnwV+XbX9\nfmA/ZU90GwAcBcxPw+0GRMTNZA8imBoR64GnJH0cQNIgSW/PHywifhARU9IDA9ZUbeuMiHHAg+R6\nZqr8EjgiDQFD0vY1ylU+OA8lG17XqezJbGMLytRzJ3Dahh2k95bYp67UWzUkIi6urIuIoyNiXETs\nSjYE8Ce5xlUtd5F78EHR8D5l8892rnOMBcCHJI1N5YvyOQ84PXfMybltN5P1eK7I9azUK99TXa6V\npDlK8+QKC2c9SsMi4j+BM4FJDY77e2CipIEpjwfUKN8tFoCIeBlYnYb3VWKodc6ieIeS9bJ9Ldej\nnN8+jOx3YrQbVxuro9UB9BMdrQ6gn+hodQD9gueClOM8leM8leM8Na83G1iPAKdIWkH2cIkfpvUB\nkBpB55D9L7IYeCAibgNGAx2SFpM98eyctN+xZA9kWArcA4zsQUyPAoWPJU9D+r5F1shbDFTm8tTq\niZsL7JXiORpYWVCmaP+K88l6k5Ype8z6N6sLSJomqZn5Q2cBeyh7yMUiSY2GF9aK7QJguLJHhi+m\naqxKGqo2AXih5oEjnicb3nhzOsa1BcVOB/ZU9oCN5cCJuW3Xk/USXluyfDeSTqyVg9TjeAlwnKQn\nJb07bZoEPFPnsNsCP0vX/VfAGWl94X0SEU+nuixPdVlUXabO+4qjgeOVPVRjOXBoQX1q1fVUsmt1\nXu6+GJHbPhRYGxF1H5ZhZmZmZj2jiFY/QXvTSfOddoiIcxoWtprSfJ4vRMTZrY6lN0naFrgiIraY\n74VKwx1nRUTRExkrZaL1T9Y3s/5DbM6fJczMapFERHQbkbS5N7AmAD8G1ld9F5bZFkfSWWS9hBdH\nxDV1yrmBZWZNcAPLzLZMtRpYvTlEsM+JiMcj4kNuXJllj7RPcxZrNq7eJL/88suvUq+RI/NTkjct\nzwUpx3kqx3kqx3lqXm8/Rc7MNgP+a3RjHR0dfnRtCc5TOc6TmdnmY7MeImhmzZMU/nfBzMzMrL4t\ncoigmZmZmZnZW8kNLDOzHvCY9HKcp3Kcp3Kcp3Kcp3Kcp3Kcp+a5gWVmZmZmZtZLPAfLzLrwHCwz\nMzOzxjwHy8zMzMzMbBNzA8vMupHkl1996jXqHaNa/WuxSXmOQznOUznOUznOUznOU/P8PVhm1t2M\nVgfQD6wGxrc6iH6gl/K0dsbajT+ImZnZW8BzsMysC0nhBpb1OTP8BdhmZta3SJtwDpaksZIeqrGt\nXdLU3jhPsySNkbRI0h25datbEUstkvaXNLtEudXpZ81cV5XfVtJTkr6bP4ak4U3E1jBX6fqOqbP9\nOEnfK3vOknEdV6mXpOmSjm1QfrikX0p6OZ+PBvvMlrRfg+2HNxn3vpKWp3vybZIulvSQpH9O9Tiz\nwf4N65rK/aOk30paKemjufVnSXpE0qebidvMzMzMyuvNOVh98U+LhwHzIuLg3Lq+GGeZmKLGci3n\nA/N7cJ6NKb+pj9NTfwa+DpzV4jg+B1wYEVMj4lXgi8CkiPhab51A0u7AkcDuwMHADyQJICIuAY4D\nTumt823R+tSfavow56kUz3Eox3kqx3kqx3kqx3lqXm82sAZKulrSCknXSxpcXUDSUZKWpddFad1W\nqTdgmaSlkk5P6ydIukvSEkkPSurJKP5hwLNV657LxXNsOudiSXPSutmSZkm6R9JjlV4KSdtIujvF\nslTSoWn92NRTMFvSKklzJR2U9l8lac9UbmtJV0paIGmhpENSGK8BL5Woy3PVKyT9KMW+WNKzkr6R\n1k8DdgLmVe8CnJbOv1TSX+fq9m/pGiyR9Ila5yzwR+CNdJy/TcdeIumugnhHSLpR0m/Sax9lVkva\nLlfuUUk7FpUvOP964JV6AUbE/0TEvcCrJepTsY7s2iDpotTztETSv+TK7F9wn+wv6bZcXb6X7rPj\nyRo+50u6StJ/AEOAhZKOqMrTrpLukPSApPmV6wS83KiuwMeBayPi9Yj4HfBb4H257WuAoU3kwczM\nzMya0JsPudgN+EJELJB0JXAycGllo6SdgYuAKWQfXu9KjZSngdERMSmVq3zQnkv21/5bJQ2iZ43B\nAUBnfkVE7J3OMxE4F9gnIl6UNCxXbFREfDD1BtwK3ETWC3JYRKyXtAOwIG0DmAB8MiJWSHoQ+Eza\n/9B0jsOBfwJ+ERHHSxoK3C/p7oi4D7gvxTQNODEiTqiuSCXuqnVfTPuNAe4AZqfeiu+Q9ZYcVJCT\nZyNimqSTgLOBE4BvAOty12BorXMWxPCptM8I4HJg34h4siqfFbOASyPiXknvBO6MiImSbgE+AcyR\n9D7gdxHxnKS51eWBiVXnv6SyLOnEbFVc3ijuEvU6Ix1zONl1f3d6v12uWNF9AgU9dhFxpaR9gdsi\n4qZ0rD9FxNS0PD1X/HKy++DxlI/LgAMiIv/7VKuuo0n3U/KHtK6ikzK/9+255XH4YQ5FnJNynKdS\n2traWh1Cv+A8leM8leM8leM8vamjo6NUj15vNrCejIgFaflq4MvkGljAXkB7RLwAkD487wdcAIyX\nNAv4OTBP0hBgl4i4FSAiXms2mNTQmJxiKfIR4IaIeDGdY11u2y1p3UpJO1UOCXxb2bycTmCX3LbV\nEbEiLT8M3J2WHyL7eArwUeAQSV9N7wcBY4BVlZNGxEKyBk8z9RwM3ACcGhFPSzoFuD0inslSQPXE\nu5vTz4VkjRqAA4EN83IiokyPWrX3A/Mj4sl0jHUFZQ4Edk/XBmCIpK2B64HzgDnAZ4DrGpQvFBH/\n2oO4G3kJeEXSFcDtwM9y24ruk40iaRvgA8ANuXoPrC63EXV9HthR0rAa1yjz4R4e3czMzGwz1dbW\n1qXBOXPmzMJym3IOVtG8m25P2Ugf8iYDHcCXgB/VKtvlQNLJaWjcIkmjqrZtRTbyf3eyD8XNyg8l\nq8TxOWAEMCUippANPRxcUL4z9z7fWyCyXq4p6TU+Ilax8S4DboyISp/DPsCpkp4g68k6RtKFBXV7\ng95/TH/da5a2753LwZg0fO8+YELqBTsM+Pd65Xs55roi4g2yIXY3Av8A/Gduc9F98jpdf6+6DZVt\nYCvgxTRPq1Lvv2li/z8A78y9f0daB0BEvAJcCzwhqaiH08ry3KJynKdSPMehHOepHOepHOepHOep\neb3ZwBorqTKk7LPAr6u23w/sp+yJbgOAo4D5abjdgIi4mexBBFMjYj3wlKSPA0gaJOnt+YNFxA/S\nh8+pEbGmaltnRIwDHiTXM1Pll8ARaQgYkravUa7ywXko2fC6TkkfBsYWlKnnTuC0DTtI7y2xT12p\nt2pIRFxcWRcRR0fEuIjYlWwI4E8i4twGh7qL3IMPiob3KZt/tnOdYywAPiRpbCpflM95wOm5Y07O\nbbuZrMdzRa5npV75nupyrSTNUZonV1g461EaFhH/CZwJTGpw3N8DEyUNTHk8oGwsABHxMrBa0qdy\nMdQ6Z5Fbgc+k35nxwLvIfvcqxxpG9jsxOiK6zZMzMzMzs43Tmw2sR4BTJK0ge7jED9P6AEiNoHPI\neqoWAw9ExG1k80M6JC0GrkplAI4leyDDUuAeYGQPYnoUKHwseRrS9y2yRt5ioDKXp1ZP3FxgrxTP\n0cDKgjJF+1ecT/YgkGXKHrP+zeoCkqZJamb+0FnAHrmevEbDC2vFdgEwXNkjwxcDbVVxiWye2Qs1\nDxzxPNnwxpvTMa4tKHY6sKeyB2wsB07MbbuerJfw2pLlu5F0Yq0cKHvk/CXAcSw5BAgAACAASURB\nVJKelPTutGkS8Eydw24L/Cxd918BZ6T1hfdJRDyd6rI81WVRdZk67yuOBo5X9lCN5cChBfUprGu6\nr68HVpANuT05un550FBgberJso3huUXlOE+leI5DOc5TOc5TOc5TOc5T8zbrLxpO8512iIhzGha2\nmiS9h+wBJme3OpbeJGlb4IqI2GK+Fyo9NGNWRBQ9kbFSxl80bH3PDH/RsJmZ9S2q8UXDm3sDawLw\nY2B91XdhmW1xJJ1F1kt4cURcU6fc5vuPgvVbI0ePZM3TaxoX7Kc6Ojr8V+ISnKdynKdynKdynKfa\najWwevshB31KRDwOfKjVcZj1BemR9pc0LIh7CsrwfzjlOE9mZral2ax7sMyseZLC/y6YmZmZ1Ver\nB6s3H3JhZmZmZma2RXMDy8ysB/y9IOU4T+U4T+U4T+U4T+U4T+U4T81zA8vMzMzMzKyXeA6WmXXh\nOVhmZmZmjXkOlpmZmZmZ2SbmBpaZdSPJL7+2+NeoUePest85z3Eox3kqx3kqx3kqx3lq3mb9PVhm\n1lMeIthYB9DW4hj6gw76a57Wru026sPMzKwhz8Eysy4khRtYZgDyl26bmVlNUh+agyVprKSHamxr\nlzT1rY4pnXuMpEWS7sitW92KWGqRtL+k2SXKrU4/a+a6qvy2kp6S9N3cunZJYxrsN1vSfg3iva3R\n+ZuRP6ak4yRNL7HPP0t6SNIySUeWKD9d0rENtp/ZZNy7SVosaaGk8ZJOk7RC0lWpHt9rsH/Dukqa\nLOneVNcl+bpKOkrSI5LOaCZuMzMzMyuvlXOw+uKfBQ8D5kXEwbl1fTHOMjFFjeVazgfm9yycpmLZ\nFMese3xJfwe8F5gEvB84W9KQTRBTI4cBN0TEtIhYDZwEHBgRx6TtzV7XIv8NHBMRewAHA/9X0nYA\nEXENsD/gBlav6Gh1AP1ER6sD6Bc8x6Ec56kc56kc56kc56l5rWxgDZR0dfoL/vWSBlcXSH9xX5Ze\nF6V1W6Vek2WSlko6Pa2fIOmu9Ff7ByWN70FMw4Bnq9Y9l4vn2HTOxZLmpHWzJc2SdI+kxyQdntZv\nI+nuFMtSSYem9WMlrUz7rZI0V9JBaf9VkvZM5baWdKWkBanH45AUxmvASyXq8lz1Ckk/SrEvlvSs\npG+k9dOAnYB5Vbv8EXijwXnWpZiQtFeqx5IU9zZV5y+sk6T7JO2eK9cuaWqdHOS9AqxvEONE4FeR\n+R9gGfC3DfZ5OR2b1NP0cKrXT3Nl3pNifUzSl1PZLj2Gks5KvV0HA18BTpL0C0mXAbsCd1Tu4dw+\nIyTdKOk36bVP2bpGxGMR8Xha/i+y+3nH3Pa1wNAGdTczMzOznoqIt/wFjAU6gfen91cCZ6bldmAq\nsDPwe2A4WUPwF8Chadu83LG2Sz8XAIem5UHA4B7ENRP4So1tE4FHgO3T+2Hp52zgurS8O/DbtDwA\nGJKWd8itH0vWIJmY3j8IXJmWDwVuSsvfAj6blocCq4C3V8U0Dbi8RK6XVa0bAzwMvANQyvkuwHHA\nd3t4TQcCjwNT0/sh6brtD9xar07A6cCMtH4UsLJB+Q3HrIrhkMpxqtYfBPw67TsixXlGE3X7AzCw\n6n6bDvw/sgfF7AA8n655l3wDZwHn5fY5M7ftidz9tCH3wFzgA2n5ncCKsnWtKvM+4OGC9S832C9g\neu7VHhB++bUFvggzM7OK9vb2mD59+oZX+n+C6lcrnyL4ZEQsSMtXA18GLs1t3wtoj4gXACTNBfYD\nLgDGS5oF/ByYl4Z77RIRtwJExGvNBiNJwOQUS5GPkA3vejGdY11u2y1p3UpJO1UOCXxb2fykTmCX\n3LbVEbEiLT8M3J2WHwLGpeWPAodI+mp6P4isYbSqctKIWAic0GQ9BwM3AKdGxNOSTgFuj4hnshTQ\n08dm7QY8ExGLUmzr0/nyZWrV6Qay3rMZwJHAjQ3KF4qI24Bu870i4i5JewH3kvXo3Evjnrm8pcBP\nJd1CutbJ7RHxOvBHSWuBkU0cE7JcF+X7QGB3vZm8IZK2jqz3Dahd1w0HlnYGfgIcU7D5BUkTIvV0\nFZvRMHgzMzOzLUlbWxttbW0b3s+cObOwXF+ag1X9Hgo+fKaGzWSygf1fAn5Uq2yXA0knp6FxiySN\nqtq2FbCarAfq9lLRd/VqQcyfI+stmRIRU8g+2A8uKN+Ze9/Jm4/OF/DJiJiSXuMjYhUb7zLgxoho\nT+/3AU6V9ATwHeAYSRf28NiNGmeFdYqIZ4DnJe0BfBq4LrdPr+QgIi5Mx/gY2X3/aBO7/z3wfbLe\n0wfS/QLdr+NfAa+T9WRVdBv6WoKAvXP1HpNvXDXcWdoW+BnwjxHxQEGRWcASSZ/vQWy2QUerA+gn\nOlodQL/gOQ7lOE/lOE/lOE/lOE/Na2UDa6ykvdPyZ8mGcOXdD+wnabikAcBRwHxJOwADIuJm4Otk\nQ9LWA09J+jiApEGS3p4/WET8IH1YnRoRa6q2dUbEOLLhep+uEe8vgSMkDU/n2L5GuUojYyjwbER0\nSvow2dCx6jL13AmctmEH6b0l9qkr9VYNiYiLK+si4uiIGBcRuwJnAz+JiHML9p1TmR9WwypgVJrP\nhaQh6brl1avTdcD/JhuCt7xE+dKUzdurXLdJwB6k+WaSLqzcNzX2FTAmIuYD5wDbkQ1/rGUtsKOk\n7SW9DfiHHoQ8j2zYZCWGyWV3lDSQrJdtTvodKXIu8K6I+HEPYjMzMzOzOlrZwHoEOEXSCrKHS/ww\nrQ+A1Ag6h+zPn4uBB9KwqNFAh6TFwFWpDMCxwGmSlgL30PxwLch6NYYXbUhD+r5F1shbDFySjzdf\nNP2cC+yV4jkaWFlQpmj/ivPJHgSyLD004ZvVBSRNk3R5nfpUOwvYI9eT18zwwknAM7U2RsRfyBqn\n35e0hKyR8LaqYvXq9O907726oE75biQdImlGwaaBwK8lLSe7z46OiM60bQ9gTcE+FQOAq9N1XAjM\niog/FZSr3LevpzgfIGsgriwo22WfAqcDeyp7OMpy4MTqAnXqeiSwL/D53HWeVFVmUGQPu7CN0tbq\nAPqJtlYH0C/kh5xYbc5TOc5TOc5TOc5T8/xFwzlprs8OEXFOw8JbkDTk7IqIqNW7129JuiO6PpZ/\ns5bmAS6NiJ3rlInabT+zLYm/aNjMzGpTX/qi4T7sJuCDyn3RsEFEvLw5Nq4AtrDG1VFkPYv/UqK0\nX35t8a+RI/Mjuzctz3Eox3kqx3kqx3kqx3lqXiufItjnpKeqfajVcZhtCpF90fA1Jctu4mj6v46O\nDg+bKMF5MjOzLY2HCJpZF5LC/y6YmZmZ1echgmZmZmZmZpuYG1hmZj3gMenlOE/lOE/lOE/lOE/l\nOE/lOE/NcwPLzMzMzMysl3gOlpl14TlYZmZmZo15DpaZmZmZmdkm5gaWmXUjyS+/Nslr1DtGtfr2\n7pM8x6Ec56kc56kc56kc56l5/h4sM+tuRqsD6AdWA+NbHUQ/UJWntTPWtiwUMzOzt4LnYJlZF5LC\nDSzbZGb4i6zNzGzzIPWhOViSxkp6qMa2dklT3+qY0rnHSFok6Y7cutWtiKUWSftLml2i3Or0s2au\nq8pvK+kpSd/NrWuXNKbBfrMl7dcg3tsanb8Z+WNKOk7S9BL7/LOkhyQtk3RkifLTJR3bYPuZTca9\nm6TFkhZKGi/pNEkrJF2V6vG9BvuXretxkh6VtCpfB0lHSXpE0hnNxG1mZmZm5bVyDlZf/BPmYcC8\niDg4t64vxlkmpqixXMv5wPyehdNULJvimHWPL+nvgPcCk4D3A2dLGrIJYmrkMOCGiJgWEauBk4AD\nI+KYtL3Z69qNpO2B84C9gL2B6ZKGAkTENcD+gBtYvaFP/emlD3OeSvEch3Kcp3Kcp3Kcp3Kcp+a1\nsoE1UNLV6S/410saXF0g/cV9WXpdlNZtlXpNlklaKun0tH6CpLskLZH0oKSezI4YBjxbte65XDzH\npnMuljQnrZstaZakeyQ9JunwtH4bSXenWJZKOjStHytpZdpvlaS5kg5K+6+StGcqt7WkKyUtSD0e\nh6QwXgNeKlGX56pXSPpRin2xpGclfSOtnwbsBMyr2uWPwBsNzrMuxYSkvVI9lqS4t6k6f2GdJN0n\nafdcuXZJU+vkIO8VYH2DGCcCv4rM/wDLgL9tsM/L6diknqaHU71+mivznhTrY5K+nMp26TGUdFbq\n7ToY+ApwkqRfSLoM2BW4o3IP5/YZIelGSb9Jr32aqOvHyP5I8FJErCO7phvqGhFrgaENjmFmZmZm\nPdTKh1zsBnwhIhZIuhI4Gbi0slHSzsBFwBSyD/F3pUbK08DoiJiUym2XdpkLXBgRt0oaRM8ajwOA\nzvyKiNg7nWcicC6wT0S8KGlYrtioiPhgaiTcCtwE/Bk4LCLWS9oBWJC2AUwAPhkRKyQ9CHwm7X9o\nOsfhwD8Bv4iI41MPxP2S7o6I+4D7UkzTgBMj4oTqilTirlr3xbTfGOAOYLYkAd8BPgccVFX+U40S\nFhFnpGMOBK4FjoiIRamH6JWq4oV1Svt9GpghaVTK5yJJ36pRPn/+6yvLqQE2LSJmVJ13KXCepEuB\nbYAPAw83qNelubdfA8ZFxF9y9xtk93AbWYNllaQfVHbvfri4Q9IPgZcrx5b0MaAt3U/H5crPAi6N\niHslvRO4E5hYsq6jgady7/+Q1uU1/t1ozy2Pww9zKOKclOM8ldLW1tbqEPoF56kc56kc56kc5+lN\nHR0dpXr0WtnAejIiFqTlq4Evk2tgkQ1xao+IFwAkzQX2Ay4AxkuaBfwcmJc+zO8SEbcCRMRrzQaT\nGhqTUyxFPkI2vOvFdI51uW23pHUrJe1UOSTwbWXzkzqBXXLbVkfEirT8MFBpNDxE9nEW4KPAIZK+\nmt4PAsYAqyonjYiFQLfGVYN6DgZuAE6NiKclnQLcHhHPZCmg20S9knYDnomIRSm29el8+TK16nQD\nWU/LDOBI4MYG5QtFxG1At/leEXGXpL2Ae8l6KO+lcc9c3lLgp5JuIV3r5PaIeB34o6S1wMgmjglZ\nrovyfSCwu95M3hBJW6feN6B2XUt6QdKEiHi8ZokP9/DIZmZmZpuptra2Lg3OmTNnFpbrS3OwiuaW\ndPvwmRo2k4EO4EvAj2qV7XIg6eQ0NG5R6iXJb9uKbKbA7sDtpaLv6tWCmD8HjACmRMQUsg/2gwvK\nd+bed/Jmo1dkvVxT0mt8RKxi410G3BgRlT6KfYBTJT1B1pN1jKQLe3jsRo2zwjpFxDPA85L2IOvJ\nui63T6/kICIuTMf4GNl9/2gTu/898H1gKvBAul+g+3X8K+B1sp7Qim5DX0sQsHeu3mPyjasG/kDX\nRug70rq8WcASSZ/vQWxW4blF5ThPpXiOQznOUznOUznOUznOU/Na2cAaK6kyjO2zwK+rtt8P7Cdp\nuKQBwFHA/DTcbkBE3Ax8HZiaekuekvRxAEmDJL09f7CI+EH6sDo1ItZUbeuMiHHAg2Qf8Iv8EjhC\n0vB0ju1rlKs0MoYCz0ZEp6QPA2MLytRzJ3Dahh2k95bYp67UWzUkIi6urIuIoyNiXETsCpwN/CQi\nzi3Yd47S/LAaVgGj0rBFJA1J1y2vXp2uA/43sF1ELC9RvjRl8/Yq120SsAdpvpmkCyv3TY19BYyJ\niPnAOcB2QL0HZKwFdpS0vaS3Af/Qg5DnARvmZUma3MS+dwIHSRqa7tGD0rq8c4F3RcSPexCbmZmZ\nmdXRygbWI8ApklaQPVzih2l9AKRG0DlkPVWLgQfSsKjRQIekxcBVqQzAscBpkpYC99D8cC3IejWG\nF21IQ/q+RdbIWwxcko83XzT9nAvsleI5GlhZUKZo/4rzyR4Esiw9NOGb1QUkTZN0eZ36VDsL2CPX\nk9fM8MJJwDO1NkbEX8gap9+XtISskfC2qmL16vTvdO+9uqBO+W4kHSJpRsGmgcCvJS0nu8+OjojK\nXLs9gDUF+1QMAK5O13EhMCsi/lRQrnLfvp7ifICsYbOyoGyXfQqcDuyp7OEoy4ETqwvUqmsawno+\n2R8LfgPMrBrOCjAoPezCNobnFpXjPJXiOQ7lOE/lOE/lOE/lOE/N8xcN56S5PjtExDkNC29BJG0L\nXBERtXr3+i1Jd1Q9ln+zluYBLo2IneuU8RcN26Yzw180bGZmmwfV+KJhN7ByJE0Afgys35I+dNuW\nQdJRZE9EnBMR/6dOOf+jYJvMyNEjWfN0vU7jLVNHR4f/SlyC81SO81SO81SO81RbrQZWK58i2Oek\np6p9qNVxmG0K6YuGrylZdhNH0//5P5xynCczM9vSuAfLzLqQFP53wczMzKy+Wj1YrXzIhZmZmZmZ\n2WbFDSwzsx7w94KU4zyV4zyV4zyV4zyV4zyV4zw1zw0sMzMzMzOzXuI5WGbWhedgmZmZmTXmOVhm\nZmZmZmabmBtYZtaNJL/88mszeo0aNa7V/6z0Cs8FKcd5Ksd5Ksd5ap6/B8vMCniIYGMdQFuLY+gP\nOnCeyuhgU+Zp7dpuI1jMzGwT8RwsM+tCUriBZba5kb9A3Mysl0l9aA6WpLGSHqqxrV3S1Lc6pnTu\nMZIWSbojt251K2KpRdL+kmaXKLc6/ayZ66ry20p6StJ3c+vaJY1psN9sSfs1iPe2RudvRv6Yko6T\nNL3EPm+ka7tY0i0lyk+XdGyD7Wc2Gfdu6fwLJY2XdJqkFZKuSvX4XoP9G9ZV0mRJ90p6SNISSUfm\nth0l6RFJZzQTt5mZmZmV18o5WH3xT2mHAfMi4uDcur4YZ5mYosZyLecD83sWTlOxbIpjljn+f0fE\n1IiYEhGHbYJ4yjgMuCEipkXEauAk4MCIOCZtb/a6Fvlv4JiI2AM4GPi/krYDiIhrgP0BN7B6RUer\nA+gnOlodQD/R0eoA+gXPBSnHeSrHeSrHeWpeKxtYAyVdnf6Cf72kwdUF0l/cl6XXRWndVqnXZJmk\npZJOT+snSLor/dX+QUnjexDTMODZqnXP5eI5Np1zsaQ5ad1sSbMk3SPpMUmHp/XbSLo7xbJU0qFp\n/VhJK9N+qyTNlXRQ2n+VpD1Tua0lXSlpQerxOCSF8RrwUom6PFe9QtKPUuyLJT0r6Rtp/TRgJ2Be\n1S5/BN5ocJ51KSYk7ZXqsSTFvU3V+QvrJOk+SbvnyrVLmlonB3mvAOsbxAjQ7ASEl9OxST1ND6d6\n/TRX5j0p1sckfTmV7dJjKOms1Nt1MPAV4CRJv5B0GbArcEflHs7tM0LSjZJ+k177lK1rRDwWEY+n\n5f8iu593zG1fCwxtMhdmZmZmVlZEvOUvYCzQCbw/vb8SODMttwNTgZ2B3wPDyRqCvwAOTdvm5Y61\nXfq5ADg0LQ8CBvcgrpnAV2psmwg8Amyf3g9LP2cD16Xl3YHfpuUBwJC0vENu/ViyBsnE9P5B4Mq0\nfChwU1r+FvDZtDwUWAW8vSqmacDlJXK9rGrdGOBh4B1kDY92YBfgOOC7PbymA4HHganp/ZB03fYH\nbq1XJ+B0YEZaPwpY2aD8hmNWxXBI5TgF215Lub4X+HiTdfsDMLDqfpsO/D+yB8XsADyfrnmXfANn\nAefl9jkzt+2J3P20IffAXOADafmdwIpm6por8z7g4YL1LzfYL2B67tUeEH755Ve/fhFmZrZx2tvb\nY/r06Rte6d9Wql+tfIrgkxGxIC1fDXwZuDS3fS+gPSJeAJA0F9gPuAAYL2kW8HNgnqQhwC4RcStA\nRLzWbDCSBExOsRT5CNnwrhfTOdbltt2S1q2UtFPlkMC3lc1P6gR2yW1bHREr0vLDwN1p+SFgXFr+\nKHCIpK+m94PIGkarKieNiIXACU3WczBwA3BqRDwt6RTg9oh4JktB0z09FbsBz0TEohTb+nS+fJla\ndbqBrPdsBnAkcGOD8oUi4jag1nyvsRHxX6ln85eSlkU2TK+MpcBPlc3dys/fuj0iXgf+KGktMLLk\n8SpEcb4PBHbXm8kbImnriPifSoEGdUXSzsBPgGMKNr8gaUKknq5iMxoGb2ZmZrYlaWtro62tbcP7\nmTNnFpbrS3Owqt9DwYfP1LCZTDZg/UvAj2qV7XIg6eQ0NG6RpFFV27YCVpP1QN1eKvquXi2I+XPA\nCGBKREwhG6o1uKB8Z+59J28+Ol/AJyObMzQlIsZHxCo23mXAjRHRnt7vA5wq6QngO8Axki7s4bEb\nNc4K6xQRzwDPS9oD+DRwXW6fXslBZMPlSI2qDmBKE7v/PfB9st7TB9L9At2v418Br5P1ZFV0G/pa\ngoC9c/Uek29cNdxZ2hb4GfCPEfFAQZFZwBJJn+9BbLZBR6sD6Cc6Wh1AP9HR6gD6Bc8FKcd5Ksd5\nKsd5al4rG1hjJe2dlj8L/Lpq+/3AfpKGSxoAHAXMl7QDMCAibga+TjYkbT3wlKSPA0gaJOnt+YNF\nxA/Sh9WpEbGmaltnRIwjG0L26Rrx/hI4QtLwdI7ta5SrNDKGAs9GRKekD5MNHasuU8+dwGkbdpDe\nW2KfulJv1ZCIuLiyLiKOjohxEbErcDbwk4g4t2DfOZX5YTWsAkal+VxIGpKuW169Ol0H/G+yIXjL\nS5QvTdIwSYPS8gjgg8CK9P7Cyn1TY18BYyJiPnAOsB3Z8Mda1gI7Stpe0tuAf+hByPPIhk1WYphc\ndkdJA8l62eak35Ei5wLviogf9yA2MzMzM6ujlQ2sR4BTJK0ge7jED9P6AEiNoHPI/qy3GHggDYsa\nDXRIWgxclcoAHAucJmkpcA/ND9cCeJRszlc3aUjft8gaeYuBS/Lx5oumn3OBvVI8RwMrC8oU7V9x\nPtmDQJalhyZ8s7qApGmSLq9Tn2pnAXvkevKaGV44CXim1saI+AtZ4/T7kpaQNRLeVlWsXp3+ne69\nVxfUKd+NpEMkzSjYtDvwYLpuvwAujIhH0rY9gDUF+1QMAK5O13EhMCsi/lRQrnLfvp7ifICsgbiy\noGyXfQqcDuyp7OEoy4ETqwvUqeuRwL7A53PXeVJVmUGRPezCNkpbqwPoJ9paHUA/0dbqAPqF/NAc\nq815Ksd5Ksd5ap6/aDgnzfXZISLOaVh4C5KGnF0REbV69/otSXdE18fyb9bSPMClEbFznTJRu+1n\nZv2Tv2jYzKy3qS990XAfdhPwQeW+aNggIl7eHBtXAFtY4+oosp7Ff2l1LJuHjlYH0E90tDqAfqKj\n1QH0C54LUo7zVI7zVI7z1LxWPkWwz0lPVftQq+Mw2xQi+6Lha8qV7unDJM2sLxo5cmzjQmZm1is8\nRNDMupAU/nfBzMzMrD4PETQzMzMzM9vE3MAyM+sBj0kvx3kqx3kqx3kqx3kqx3kqx3lqnhtYZmZm\nZmZmvcRzsMysC8/BMjMzM2vMc7DMzMzMzMw2MTewzMx6wGPSy3GeynGeynGeynGeynGeynGemufv\nwTKzbiR/D5b1npGjR7Lm6TWtDsPMzOwt4TlYZtaFpGBGq6OwzcoM8P81Zma2ufEcLDMzMzMzs02s\nJQ0sSWMlPVRjW7ukqW91TOncYyQtknRHbt3qVsRSi6T9Jc0uUW51+lkz11Xlt5X0lKTv5ta1SxrT\nYL/ZkvZrEO9tjc7fjPwxJR0naXqJfd5I13axpFtKlJ8u6dgG289sMu7d0vkXShov6TRJKyRdlerx\nvQb7l63rcZIelbQqXwdJR0l6RNIZzcRtNfSpfxn6MOepFM9xKMd5Ksd5Ksd5Ksd5al4r52D1xfEi\nhwHzIuKc3Lq+GGeZmKLGci3nA/N7Fk5TsWyKY5Y5/n9HREsa7jmHATdExIUAkk4CDoiIZyQdR/PX\ntRtJ2wPnAVMBAQsl/UdEvBQR10j6JfAA8H82piJmZmZmVqyVQwQHSro6/QX/ekmDqwukv7gvS6+L\n0rqtUq/JMklLJZ2e1k+QdJekJZIelDS+BzENA56tWvdcLp5j0zkXS5qT1s2WNEvSPZIek3R4Wr+N\npLtTLEslHZrWj5W08v9n7+7Dra7q/P8/X5CIiiJqgTpf0JguJyfvQEYdHTyaN6l5MyYVYdpMl9lo\natlopI0eU/Om9MrJ1Ex/yAiaWt6ghiLGIRIJhcNBBckKy5tBLaVB8/68f3+s94bP2WffrH06uDny\nflzXuc7ea63P5/Ne77057HXWWp/jxy2TNFXSgX78Mkm7e7uNJV0vaZ7PeBzuYbwF/CWjLy+VF0j6\nscfeLulFSf/l5aOBDwEzyg75M/Buneus9JiQNMb7scjj3qTs+hX7JOlhSR8ttJslaVSNHBS9Drxa\nJ0ZIg41GrPJz4zNNT3i/biq0+UeP9beSTvG2XWYMJX3dZ7sOAb4K/IekByVdDXwYmF56DxeO2UrS\nTyX92r/2aqCvB5N+SfAXM1tJek0/Uao0sxeAwQ3mIlTSk58w66PIU5aWlpZmh9AnRJ7yRJ7yRJ7y\nRJ4a18wZrB2AfzOzeZKuB04CLi9VStoauBjYjfQh/gEfpDwLbGtmO3u7zfyQqcB3zGyapAH0bPDY\nH+gsFpjZHn6dHYGzgL3M7BVJmxeaDTOzvX2QMA24HXgDOMrMXpW0JTDP6wBGAp8ysyWSHgU+68cf\n4dc4GjgbeNDMvihpMDBf0kwzexh42GMaDZxoZl8q70gp7rKyE/y44cB0YJIkAd8DJgAHlrU/pl7C\nzOxrfs4NgJ8A48xsoaRB+ACloGKf/LjPAK2Shnk+F0q6sEr74vVvLT32AdhoM2utEOqGnuu3gEvM\n7K46/bq88PQbwHZm9nbh/QbpPdxCGrAsk3RV6fDup7Ppkq4BVpXOLelgoMXfT8cX2l8BXG5mcyX9\nP+B+YMfMvm4LPFN4/pyXFdX/tzGr8Hg74kNyCCGEENZ7bW1tWUsmmzmD9Uczm+ePpwD7lNWPAWaZ\n2ctm1kkaQI0Ffg9s77NGBwOr/MP8NmY2DcDM3jKzNxoJxgcau5AGcJXsT1re9YpfY2Wh7k4vW0qa\nCYI0Y3KRpA5gJrCNpFLdcjNb4o+f8HqAx0gfZwEOAiZKagfagAFAl/1QzdMG8wAAIABJREFUZrag\n0uCqTj8HArcBXzGzZ0kD23vN7PlC3D2xA/C8mS302F71162oWp9uA0qDuU8DP63TviIzu7vK4Apg\nhJntThpIfr/BGc4O4CZJE+g6o3evmb1jZn8GXgCGNnBOSLmulO8DgCu939OAQZI2Ljao09d6XpY0\nsmaL/QpfMbiqLPYW5Yk8ZYk9DnkiT3kiT3kiT3kiT2u0tLTQ2tq6+quadWkPVqW9Jd0+fJrZSkm7\nkJZCfRkYR1p6VXNgIOkk4AS/zqFmtqJQ1480cHsTuLeBPpS8WSHmCcBWwG5m1ql004mBFdp3Fp53\nsuY1EWmW66kexFPL1cBPzaw0R7EXsI/nZ1PS0s1VZnZWD85db3BWtU+S/iRpJ9JM1omFqm7tfZar\nIWb2v/59uaQ20sxo7ke/w0iD+yOAsyV9zMvLX8cPAO+QZkJLui19zSBgDzN7uwfHPkeaVSv5O7rO\nR0GaIVsk6RQzu6EH1wghhBBCCFU0cwZrhKTSMrbPAXPK6ucDYyVtIak/MB6Y7cvt+pvZHcC3gFFm\n9irwjKQjASQNkLRR8WRmdpWZ7WZmo4qDK6/rNLPtgEdJH/Ar+QUwTtIWfo0hVdqVBhmDgRd9cLUf\nMKJCm1ruB05dfYC0a8YxNUk6GRhkZt8tlZnZsWa2nZl9GPhP4H8qDa4kTZbvD6tiGTDMly0iaZC/\nbkW1+nQLcCawmZk9ntE+m6TNfdkokrYC9gaW+PPvlN43VY4VMNzMZgMTgc2AQTUu9wLwQUlDJG0I\nfLIHIc8AVu/L8l8o5LofOFDSYH+PHuhlRWcBfx+Dq79RzOzliTxliT0OeSJPeSJPeSJPeSJPjWvm\nAOtJ4GRJS0g3l7jGyw3AB0ETSUvD2oFHzOxu0n6SNl8+daO3ATgOONWX5D1E48u1AH4DbFGpwpf0\nXUga5LUDlxXjLTb171OBMR7PscDSCm0qHV9yPmk2abHfNOHb5Q0kjZZ0bY3+lPs6sJPSTS4WSmpk\neeHOwPPVKn225TOkpW2LSIOEDcua1erTz/z4WwplF9Ro342kwyW1Vqj6KPCov24PkvbqPel1OwEr\nKhxT0h+Y4q/jAuAKM/u/Cu1K79t3PM5HSAObpRXadjmmgtOA3ZVujvI4XWf0gOp99SWs55N+WfBr\n4Lyy5awAA/xmFyGEEEIIoZfJbF28C3lzSDoD2LLsNu3rPUmbAteZWbXZvT5L0nQzO6TZcbxXfB9g\nh5ltXaON0frexdRnLSdmZ3IsByZD/F9TW1tbW/yWOEPkKU/kKU/kKU/kqTpJmFm3lWnN3IO1Lrod\nuGF9+9Bdj5mtovrSyT5tfXqdJY0n3RHx0rqNW9d2NGF9MnTbniwoCCGEEPqmmMEKIXQhyeLnQggh\nhBBCbdVmsJq5ByuEEEIIIYQQ3ldigBVCCD0QfxckT+QpT+QpT+QpT+QpT+QpT+SpcTHACiGEEEII\nIYReEnuwQghdxB6sEEIIIYT6Yg9WCCGEEEIIIaxlMcAKIYQeiDXpeSJPeSJPeSJPeSJPeSJPeSJP\njYu/gxVC6EbqNtsdQliHDB06ghUrnm52GCGEECqIPVghhC4kGcTPhRDWbSL+/w4hhOaKPVghhBBC\nCCGEsJY1ZYAlaYSkx6rUzZI06r2Oya89XNJCSdMLZcubEUs1kvaVNCmj3XL/XjXXZe03lfSMpP8u\nlM2SNLzOcZMkja0T7931rt+I4jklHS/p3Ixjpkt6RdK0zGucK+m4OvWn50cNknaQ1C5pgaTtJZ0q\naYmkG70fP6hzfN2+StpF0lxJj0laJOnThbrxkp6U9LVG4g7VtDU7gD6irdkB9BFtzQ6gT4i9IHki\nT3kiT3kiT41r5gzWuri24ShghpkdUihbF+PMicmqPK7mfGB2z8JpKJa1cc6c818KHLsW4mjEUcBt\nZjbazJYD/wEcYGaf9/pGX9dKXgM+b2Y7AYcA35e0GYCZ3QzsC8QAK4QQQghhLWnmAGsDSVP8N/i3\nShpY3sB/477Yvy72sn4+a7JYUoek07x8pKQH/Lf2j0ravgcxbQ68WFb2UiGe4/ya7ZIme9kkSVdI\nekjSbyUd7eWbSJrpsXRIOsLLR0ha6sctkzRV0oF+/DJJu3u7jSVdL2mez3gc7mG8Bfwloy8vlRdI\n+rHH3i7pRUn/5eWjgQ8BM8oO+TPwbp3rrPSYkDTG+7HI496k7PoV+yTpYUkfLbSbJWlUjRwUvQ68\nWidGzGxWTruCVX5ufKbpCe/XTYU2/+ix/lbSKd62y4yhpK/7bNchwFeB/5D0oKSrgQ8D00vv4cIx\nW0n6qaRf+9deuX01s9+a2e/88f+S3s8fLNS/AAxuIA+hqpZmB9BHtDQ7gD6ipdkB9AktLS3NDqFP\niDzliTzliTw1rpl3EdwB+DczmyfpeuAk4PJSpaStgYuB3Ugf4h/wQcqzwLZmtrO328wPmQp8x8ym\nSRpAzwaP/YHOYoGZ7eHX2RE4C9jLzF6RtHmh2TAz29sHCdOA24E3gKPM7FVJWwLzvA5gJPApM1si\n6VHgs378EX6No4GzgQfN7IuSBgPzJc00s4eBhz2m0cCJZval8o6U4i4rO8GPGw5MByZJEvA9YAJw\nYFn7Y+olzMy+5ufcAPgJMM7MFkoahA9QCir2yY/7DNAqaZjnc6GkC6u0L17/1tJjH4CNNrPWenFn\n9OvywtNvANuZ2duF9xuk93ALacCyTNJVpcO7n86mS7oGWFU6t6SDgRZ/Px1faH8FcLmZzZX0/4D7\ngR0b7aukfwI2KA24CjL+bRRP20J8+AshhBDC+q6trS1ryWQzZ7D+aGbz/PEUYJ+y+jHALDN72cw6\nSQOoscDvge191uhgYJV/mN/GzKYBmNlbZvZGI8H4QGMX0gCukv1Jy7te8WusLNTd6WVLSTNBAAIu\nktQBzAS2kVSqW25mS/zxE14P8BiwnT8+CJgoqZ20OH8A0GU/lJktqDS4qtPPgcBtwFfM7FnSwPZe\nM3u+EHdP7AA8b2YLPbZX/XUrqtan24DSYO7TwE/rtK/IzO7ujcFVBR3ATZIm0HVG714ze8fM/gy8\nAAxt8Lyicr4PAK70fk8DBknauNigXl/9FxT/A3yhQvXLkkbWDq218NVSu+l6q63ZAfQRbc0OoI9o\na3YAfULsBckTecoTecoTeVqjpaWF1tbW1V/VNHMGq9tv+Su06fbh08xWStoFOBj4MjCOtPSq5sBA\n0knACX6dQ81sRaGuH2ng9iZwbwN9KHmzQswTgK2A3cysU+mmEwMrtO8sPO9kzWsi0izXUz2Ip5ar\ngZ/6kjmAvYB9PD+bkpZurjKzs3pw7nqDs6p9kvQnSTuRZrJOLFR1a++zXO+lw0iD+yOAsyV9zMvL\nX8cPAO+QZkJLui19zSBgDzN7uwfHImlT4B7gm2b2SIUmVwCLJJ1iZjf05BohhBBCCKGyZs5gjZBU\nWsb2OWBOWf18YKykLST1B8YDs325XX8zuwP4FjDKzF4FnpF0JICkAZI2Kp7MzK4ys93MbFRxcOV1\nnWa2HfAo6QN+Jb8Axknawq8xpEq70iBjMPCiD672A0ZUaFPL/cCpqw+Qds04piZJJwODzOy7pTIz\nO9bMtjOzDwP/CfxPpcGVpMny/WFVLAOG+bJFJA3y162oVp9uAc4ENjOzxzPa90S3GSNJ3ym9byoe\nkGY2h5vZbGAisBkwqMY1XgA+KGmIpA2BT/YgzhnA6n1Z/guFLL5U805gsv8bqeQs4O9jcPW3aml2\nAH1ES7MD6CNamh1AnxB7QfJEnvJEnvJEnhrXzAHWk8DJkpaQbi5xjZcbgA+CJpLWTbQDj5jZ3cC2\nQJsvn7rR2wAcB5zqS/IeovHlWgC/AbaoVOFL+i4kDfLagcuK8Rab+vepwBiP51hgaYU2lY4vOZ80\nm7TYb5rw7fIGkkZLurZGf8p9HdhJ6SYXCyU1srxwZ+D5apU+2/IZ0tK2RaRBwoZlzWr16Wd+/C2F\nsgtqtO9G0uGSWqvU/dLPvb+kP0oq7TfbCVhR6RjXH5jir+MC4Aoz+78K7Urv23c8zkdIA8SlFdp2\nOaaC04DdlW6O8jhdZ/RK/anW10+Tltt+ofA671zWZoDf7CKEEEIIIfQyxV+CX0PSGcCWZjaxbuP1\niC85u87Mqs3u9VmSppfdlv99zfcBdpjZ1jXa2Lr51wnWNW3ErEOONiJPOdpoLE9iffz/u62tLX6b\nniHylCfylCfyVJ0kzKzbyrRm7sFaF90O3LC+feiux8xWUX3pZJ+2Pr3OksaT7oh4aUbrtR1OCOFv\nMHToiPqNQgghNEXMYIUQupBk8XMhhBBCCKG2ajNYzdyDFUIIIYQQQgjvKzHACiGEHoi/C5In8pQn\n8pQn8pQn8pQn8pQn8tS4GGCFEEIIIYQQQi+JPVghhC5iD1YIIYQQQn2xByuEEEIIIYQQ1rIYYIUQ\nQg/EmvQ8kac8kac8kac8kac8kac8kafGxd/BCiF0I8XfwQp/m6HbDmXFsyuaHUYIIYTwnos9WCGE\nLiQZrc2OIvR5rRD/v4QQQng/iz1YIYQQQgghhLCWNWWAJWmEpMeq1M2SNOq9jsmvPVzSQknTC2XL\nmxFLNZL2lTQpo91y/14112XtN5X0jKT/LpTNkjS8znGTJI2tE+/d9a7fiOI5JR0v6dyMY6ZLekXS\ntMxrnCvpuDr1p+dHDZJ2kNQuaYGk7SWdKmmJpBu9Hz+oc3xuX4+X9BtJy4p9kDRe0pOSvtZI3KGK\ndeonw7or1u7niTzliTzliTzliTzliTw1rpkzWOvi2pGjgBlmdkihbF2MMycmq/K4mvOB2T0Lp6FY\n1sY5c85/KXDsWoijEUcBt5nZaDNbDvwHcICZfd7rG31du5E0BDgHGAPsAZwraTCAmd0M7AvEACuE\nEEIIYS1p5gBrA0lT/Df4t0oaWN7Af+O+2L8u9rJ+PmuyWFKHpNO8fKSkByQtkvSopO17ENPmwItl\nZS8V4jnOr9kuabKXTZJ0haSHJP1W0tFevomkmR5Lh6QjvHyEpKV+3DJJUyUd6Mcvk7S7t9tY0vWS\n5vmMx+EexlvAXzL68lJ5gaQfe+ztkl6U9F9ePhr4EDCj7JA/A+/Wuc5KjwlJY7wfizzuTcquX7FP\nkh6W9NFCu1mSRtXIQdHrwKt1YsTMZuW0K1jl58Znmp7wft1UaPOPHutvJZ3ibbvMGEr6us92HQJ8\nFfgPSQ9Kuhr4MDC99B4uHLOVpJ9K+rV/7dVAXw8m/ZLgL2a2kvSafqKQhxeAwQ3kIVTTk58w66GW\nlpZmh9AnRJ7yRJ7yRJ7yRJ7yRJ4a18y7CO4A/JuZzZN0PXAScHmpUtLWwMXAbqQP8Q/4IOVZYFsz\n29nbbeaHTAW+Y2bTJA2gZ4PH/kBnscDM9vDr7AicBexlZq9I2rzQbJiZ7e2DhGnA7cAbwFFm9qqk\nLYF5XgcwEviUmS2R9CjwWT/+CL/G0cDZwINm9kWfgZgvaaaZPQw87DGNBk40sy+Vd6QUd1nZCX7c\ncGA6MEmSgO8BE4ADy9ofUy9hZvY1P+cGwE+AcWa2UNIgfIBSULFPftxngFZJwzyfCyVdWKV98fq3\nlh77AGy0mbXWizujX5cXnn4D2M7M3i683yC9h1tIA5Zlkq4qHd79dDZd0jXAqtK5JR0MtPj76fhC\n+yuAy81srqT/B9wP7JjZ122BZwrPn/Oyovr/NmYVHm9HDCZCCCGEsN5ra2vLWjLZzBmsP5rZPH88\nBdinrH4MMMvMXjazTtIAaizwe2B7nzU6GFjlH+a3MbNpAGb2lpm90UgwPtDYhTSAq2R/0vKuV/wa\nKwt1d3rZUtJMEICAiyR1ADOBbSSV6pab2RJ//ITXAzxG+jgLcBAwUVI70AYMALrshzKzBZUGV3X6\nORC4DfiKmT1LGtjea2bPF+LuiR2A581socf2qr9uRdX6dBtQGsx9GvhpnfYVmdndvTG4qqADuEnS\nBLrO6N1rZu+Y2Z+BF4ChDZ5XVM73AcCV3u9pwCBJGxcb/I19fVnSyJot9it8xeCqstiDlSXW7ueJ\nPOWJPOWJPOWJPOWJPK3R0tJCa2vr6q9qmjmD1e23/BXadPvwaWYrJe1CWgr1ZWAcaelVzYGBpJOA\nE/w6h5rZikJdP9LA7U3g3gb6UPJmhZgnAFsBu5lZp9JNJwZWaN9ZeN7JmtdEpFmup3oQTy1XAz/1\nJXMAewH7eH42JS3dXGVmZ/Xg3PUGZ1X7JOlPknYizWSdWKjq1t5nud5Lh5EG90cAZ0v6mJeXv44f\nAN4hzYSWdFv6mkHAHmb2dg+OfY40q1byd3Sdj4I0Q7ZI0ilmdkMPrhFCCCGEEKpo5gzWCEmlZWyf\nA+aU1c8HxkraQlJ/YDww25fb9TezO4BvAaPM7FXgGUlHAkgaIGmj4snM7Coz283MRhUHV17XaWbb\nAY+SPuBX8gtgnKQt/BpDqrQrDTIGAy/64Go/YESFNrXcD5y6+gBp14xjapJ0MjDIzL5bKjOzY81s\nOzP7MPCfwP9UGlxJmlzaH1bFMmCYL1tE0iB/3Ypq9ekW4ExgMzN7PKN9T3SbMZL0ndL7puIBaWZz\nuJnNBiYCmwGDalzjBeCDkoZI2hD4ZA/inAGs3pflv1DIdT9woKTB/h490MuKzgL+PgZXf6OY2csS\na/fzRJ7yRJ7yRJ7yRJ7yRJ4a18wB1pPAyZKWkG4ucY2XG4APgiaSloa1A4+Y2d2k/SRtvnzqRm8D\ncBxwqi/Je4jGl2sB/AbYolKFL+m7kDTIawcuK8ZbbOrfpwJjPJ5jgaUV2lQ6vuR80mzSYr9pwrfL\nG0gaLenaGv0p93VgJ6WbXCyU1Mjywp2B56tV+mzLZ0hL2xaRBgkbljWr1aef+fG3FMouqNG+G0mH\nS2qtUvdLP/f+kv4oqbTfbCdgRaVjXH9gir+OC4ArzOz/KrQrvW/f8TgfIQ1sllZo2+WYCk4Ddle6\nOcrjdJ3RK/WnYl99Cev5pF8W/Bo4r2w5K8AAv9lFCCGEEELoZTJbF+9C3hySzgC2NLOJdRuvRyRt\nClxnZtVm9/osSdPLbsv/vub7ADvMbOsabYzW9y6mPms5MYtVSyuYGW1tbfHbzwyRpzyRpzyRpzyR\npzyRp+okYWbdVqY1cw/Wuuh24Ib17UN3PWa2iupLJ/u09el1ljSedEfES+s2bl3b0YT3u6Hb9mQR\nQQghhND3xQxWCKELSRY/F0IIIYQQaqs2g9XMPVghhBBCCCGE8L4SA6wQQuiB+LsgeSJPeSJPeSJP\neSJPeSJPeSJPjYsBVgghhBBCCCH0ktiDFULoIvZghRBCCCHUF3uwQgghhBBCCGEtiwFWCCH0QKxJ\nzxN5yhN5yhN5yhN5yhN5yhN5alz8HawQQjdSt9nuEHrN0KEjWLHi6WaHEUIIIawVsQcrhNCFJIP4\nuRDWJhH/94QQQujrYg9WCCGEEEIIIaxlWQMsSSMkPValbpakUb0bVh5JwyUtlDS9ULa8GbFUI2lf\nSZMy2i0vtL+7WhtJW/RiXBWvU6ivGbe/L2bVadPr74/iOXNeb0k7S5orqUPSXZIGZRxT87ySVuVH\nvPqY70p6TNIlkraSNE/SAkn75Ly2mX29VNJSSYsk/UzSZoW6X0qaL+lDjcYeKmlrdgB9RFuzA+gT\nYo9DnshTnshTnshTnshT4xqZwVoX13McBcwws0MKZetinDkxWZXHjZ6nEfXO12jczZBz/euAM81s\nF+AO4MxeOG9P+n0CsLOZfQM4AFhsZqPN7FeZ58tpMwP4RzPbFXgK+Obqg83GAguAwxqOPIQQQggh\nZGlkgLWBpCmSlki6VdLA8gaSxkta7F8Xe1k/SZO8rEPSaV4+UtID/pv2RyVt34P4NwdeLCt7qRDP\ncX7NdkmTvWySpCskPSTpt5KO9vJNJM30WDokHeHlI3xGYJKkZZKmSjrQj18maXdvt7Gk6wuzEod7\nGG8Bf8noy0uFx4Ml3SPpSUlXFcpXr/GUdLrPhiwu5HRjP67dy8d5+RiPd5HHt0nxwpLu9ZnAdkkr\nJX0+M+53gZf9HP0KMzSLJJ1c3tjzNtdzfIvHe7CkWwttVs+sSTqovH2dvFXzER/EAMwEPpVxzEse\nwzBJsz0/iyXtvSZUXeB9nSvpg144qfSe8uer/PtdwCBggaQzgUuAo/y8A+n62k6Q9Guvu1pafceJ\nun01s5lm1ulP5wF/V9ZkBenfTfibtTQ7gD6ipdkB9AktLS3NDqFPiDzliTzliTzliTw1rpG7CO4A\n/JuZzZN0PXAScHmpUtLWwMXAbsBK4AEfpDwLbGtmO3u70pKlqcB3zGyapAH0bD9Yf6CzWGBme/h1\ndgTOAvYys1ckFT9UDjOzvSV9FJgG3A68ARxlZq9K2pL04XSatx8JfMrMlkh6FPisH3+EX+No4Gzg\nQTP7oqTBwHxJM83sYeBhj2k0cKKZfam8I6W43Rjgo8AfgfslHW1mt5cqlZbHHe/t+gO/ltTmcT5n\nZp/0dptK2gD4CTDOzBYqLY97vezahxXO+/8Bd5rZqlLc1ZjZs8Ax/vRLwAjSDI2V5RvP6beAj5vZ\n6z7IOB24CPiRpI3M7HXgM8BN3v7sCu0vqJY3SfcCXzSzFWWhPiHpCDObBnya7oOOSn0rnfdzwH1m\ndpEPdEqDvE2AuWb2LUmXkGanvlPpVH6+IyX9n5mVlja+AIw2s1P9eakP/+A5+Gcze1fSD4EJwJTM\nvhb9O+m1L+okvWfqaC08biE+JIcQQghhfdfW1pa1ZLKRQc0fzWyeP54C7FNWPwaYZWYv+2/QpwJj\ngd8D2/us0cHAKv+Qv41/4MXM3jKzNxqIBf+wuwtpAFfJ/sBtZvaKX2Nloe5OL1sKlPajCLhIUgdp\nlmMbrdmrstzMlvjjJ7we4DFgO398EDBRUjtp08EAYHgxIDNbUGlwVcF8M/uDpdts3Uz3XO8D3GFm\nb5jZa6QB4r94PAdKukjSPj5I2gF43swWegyvFmY4VpO0FXAjMN6Pa9QBwI885vJ8A+wJ7Ag85Dk6\nDhhuZu8C9wGHS+pPWr42rVr7WgGY2WFVBhz/Dpws6RHSwOitBvr1CPBvks4hDR5f8/I3zezn/ngB\na94H5XLvd15a/vdxYBTwiPd7f+DD3RpX72u6qHQ28LaZ3VRW9Rywc/1wWgtfLfWbr5famh1AH9HW\n7AD6hNjjkCfylCfylCfylCfytEZLSwutra2rv6ppZAarfP9Hpf0g3T5MmtlKSbsABwNfBsYBX63U\ntsuJpJNIswIGHFr8MCmpH2ng9iZwbwN9KHmzQswTgK2A3cysU+mGAgMrtO8sPO9kTQ5FmuV6qgfx\nlMvJdfeDzJ7yWahDgfMlPUgaTNbLdT/SQK7VB51rg0j75SZUqLsF+ArwCvCImb3mA+hq7RtiZr8h\nvf+Q9BEa2INkZnMkjfVjbpB0mZlNAd4uNHuXNe+Dd/BfXHgfNmgwXAGTzezsBo9bcwLpC6T3wP4V\nqm8HzpG0xMx27Ok1QgghhBBCZY3MYI2QVFw2Naesfj4wVtIWPhMxHpjtS736m9kdpCVio8zsVeAZ\nSUcCSBogaaPiyczsKjPbzcxGlf+m3sw6zWw74FHScqpKfgGMk9+ZTdKQKu1Kg4/BwIs+uNqPtNyt\nvE0t9wOnrj5A2jXjmGr2UNr71Y/Uv/JczyHt3xmotJ/qX4E5vkzzdZ+1+B5pJmQZMMyXJyJpkL8+\nRZcAHWZ2W6VglPZwTa4T8wPAiaVzV8j3PGBvSSO9fmMf7ADM9lhPYM2StlrtG1LYH9WP9B68xp9v\nI2lmnWOHk94X15NullG6I2K198TTwO7++Ei6DrBqvY9KdQ8CxxRiHuIxZJH0CeAM4Agze7NCk+OA\n6TG46g0tzQ6gj2hpdgB9QuxxyBN5yhN5yhN5yhN5alwjA6wnScuslpA2yV/j5aUlYSuAiaT1IO2k\nmYi7gW2BNl/udKO3gfRB71RfkvcQMLQH8f8GqHhra1/SdyFpkNcOXFaMt9jUv08Fxng8xwJLK7Sp\ndHzJ+aQbgSxWuqX9t8sbSBot6doa/SmZD1xJWo74OzO7s3htM2sHbiAtX3sYuNbMOoCdSHu/2oFz\ngAvM7G3SIO1KSYtId5nbsOx6XwcOUrrJxUJJnyyrHw78tU7M1wHPAIv9+uPLYv4T8AXgZs/xXNLy\nRXzJ4j3AJ/x7zfZUeQ2UbtYxrELVeEnLgCWkPWo3ePnWdJ2JqqQF6JC0kLR/6/u1YgB+DOzrOdgT\neK1QV2smspSnpaRB4Azv9wygW59q9PUHpJtpPOCv5VVl9UNIdxcMIYQQQghrgXzLTJ8k6QxgSzOb\nWLdx6DG/icONZvZ4s2PpTUp3OvyDmd3T7FjeK37TjMVm9qMabaz5d9/vC9qI2ZkcbXTPk+jL//es\nDW1tbfFb4gyRpzyRpzyRpzyRp+okYWbdVig1sgdrXXQ7aV/M9LK/hRV6kf/dpvcdM/ths2N4L0ma\nTdo3WOluhyGEEEIIoRf06RmsEELvSzNYIaw9Q4eOYMWKp5sdRgghhPA3eb/OYIUQ1oL4xUsIIYQQ\nQs/05I/7hhDCei/+LkieyFOeyFOeyFOeyFOeyFOeyFPjYoAVQgghhBBCCL0k9mCFELqQZPFzIYQQ\nQgihtmp7sGIGK4QQQgghhBB6SQywQgihB2JNep7IU57IU57IU57IU57IU57IU+NigBVCCCGEEEII\nvST2YIUQuoi/g9V3DN12KCueXdHsMEIIIYT1UrU9WDHACiF0IclobXYUIUtr/M2yEEIIoVniJhch\nhNCbljc7gL4h1u7niTzliTzliTzliTzliTw1LmuAJWmEpMeq1M2SNKp3w8ojabikhZKmF8rWqY89\nkvaVNCmj3fJC+7urtZG0RS/GVfE6hfqacfv7YladNr3+/iieM+f1lrSzpLmSOiTdJWlQxjE1zytp\nVX7Eq4/5rqTHJF0iaStJ8yQtkLRPzmub2dchkmZIWibpfkmDC3UTog2GAAAgAElEQVS/lDRf0oca\njT2EEEIIIeRpZAZrXVyHchQww8wOKZSti3HmxGRVHjd6nkbUO1+jcTdDzvWvA840s12AO4Aze+G8\nPen3CcDOZvYN4ABgsZmNNrNfZZ4vp81EYKaZ7QD8Avjm6oPNxgILgMMajjx0t32zA+gbWlpamh1C\nnxB5yhN5yhN5yhN5yhN5alwjA6wNJE2RtETSrZIGljeQNF7SYv+62Mv6SZrkZR2STvPykZIekLRI\n0qOSevJxZXPgxbKylwrxHOfXbJc02csmSbpC0kOSfivpaC/fRNJMj6VD0hFePkLSUj9umaSpkg70\n45dJ2t3bbSzp+sKsxOEexlvAXzL68lLh8WBJ90h6UtJVhfLVazwlne6zIYsLOd3Yj2v38nFePsbj\nXeTxbVK8sKR7fSawXdJKSZ/PjPtd4GU/R7/CDM0iSSeXN/a8zfUc3+LxHizp1kKb1TNrkg4qb18n\nb9V8xAcxADOBT2Uc85LHMEzSbM/PYkl7rwlVF3hf50r6oBdOKr2n/Pkq/34XMAhYIOlM4BLgKD/v\nQLq+thMk/drrrpZUqsvp65HAZH88mfRLiKIVpH83IYQQQghhLWhkgLUDcKWZ7QisAk4qVkraGrgY\naAF2Bcb4IGVXYFsz29lnEErLzqYCPzCzXYF/Bv63B/H3BzqLBWa2h8ezI3AW0GJmuwGnFZoNM7O9\ngcNJH3QB3gCOMrPdgf2BywrtRwLf9VmBHYDP+vFn+DUAzgYeNLM9/fjvSdrIzB42s695TKMlXVup\nI6W43RjgZOCjwN8XP7D7eUYBx3u7vYATJO0CfAJ4zsx2M7OdgfskbQD8BDjFc30A8HrZtQ8zs1HA\nF4GngTuLcVdjZs+a2TH+9EvACNIMza6k17cY85bAt4CPe44XAKeTBjz/JGkjb/oZ4CZvf3aF9lXz\n5gPFYRVCfaI0YAY+DfxdrX6VnfdzwH2en12ARV6+CTDX+zqHNDtV8VR+viOBv5rZKDO7FDgH+Ik/\nf6PQh3/wHPyzX7MTmNBAXz9kZi94+xVA+XLATtK/m9pmFb7WqUW365DIS5ZYu58n8pQn8pQn8pQn\n8pQn8rRGW1sbra2tq7+q+UAD5/yjmc3zx1OAU4DLC/VjgFlmVprRmAqMBS4Atpd0BfBzYIbSHpht\nzGwagJm91UAc+PlF+sA7pUqT/YHbzOwVv8bKQt2dXrZUa/ajCLhI0ljSh9BtCnXLzWyJP36CNCgA\neAzYzh8fBBwu6Qx/PgAYDiwrXdTMFpAGIvXMN7M/eD9vBvYBbi/U7wPcUfpgLul24F+A+0kDu4uA\ne83sV5I+BjxvZgs9hlf9mC4XlLQVcCNwjJk1vL+INHC72vyWZmX5BtgT2BF4yF+7DUgDlHcl3UfK\n3c9Iy9fOIA3Uu7WvFYCZVVv69u/ADyT9FzCNNDuX6xHgeh+o3mVmHV7+ppn93B8vIPW/km53lqmi\ntPzv48Ao4BHv90DghW6Nq/e12nlLniPltrb9Ms8eQgghhLCeaGlp6bJk8rzzzqvYrpEBVvkHtUr7\nQbp9mDSzlT67cjDwZWAc8NVKbbucSDqJNCtgwKH+2/hSXT/g98CbwL0N9KHkzQoxTwC2AnYzs06l\nGwoMrNC+s/C8kzU5FPApM3uqB/GUy8l194PMnvLZrUOB8yU9SBpM1st1P+BmoNXMlvYg3hwi7Zeb\nUKHuFuArwCvAI2b2mg8uqrVviJn9hvT+Q9JHaGAPkpnN8UH3YcANki4zsynA24Vm77LmffAOPjNc\nGBg2QsBkMzu7weNKXpA01Mxe8Bmu8iW0twPnSFris9Ghp2IPVpZYu58n8pQn8pQn8pQn8pQn8tS4\nRpYIjpBUXDY1p6x+PjBW0haS+gPjgdm+1Ku/md1BWiI2ymdRnpF0JICkAYUlYgCY2VW+1G1UcXDl\ndZ1mth3wKGk5VSW/AMbJ78wmaUiVdqXBx2DgRR9c7Uda7lbeppb7gVNXHyDtmnFMNXso7f3qR+pf\nea7nkPbvDPT9VP8KzPFlmq+b2U3A90gzIcuAYZJGe1yD/PUpugToMLPbKgWjtIdrcqW6ggeAE0vn\nrpDvecDekkZ6/cY+2AGY7bGeQFrOWK99Qwr7o/qR3oPX+PNtJM2sc+xw0vvietLNMkp3RKz2nnga\n2N0fH0nXAVat91Gp7kHgmELMQzyGXNOAL/jj44G7yuqPA6bH4CqEEEIIYe1oZID1JHCypCWkTfLX\neHlpSdgK0h3M2oB20kzE3cC2QJukdtIStIl+3HHAqZI6gIeAoT2I/zdAxVtb+5K+C0mDvHbW7Kmq\nNjs0lbRvrAM4FlhaoU2l40vOJ90IZLHSLe2/Xd6g1h6sMvOBK0nLEX9nZncWr21m7cANpOVrDwPX\n+tK1nYD53t9zgAvM7G3SIO1KSYuAGcCGZdf7OnCQ0k0uFkr6ZFn9cOCvdWK+DngGWOzXH18W859I\nH/xv9hzPJe1nw8w6gXtIe8juqdeeKq9BjX1J4yUtA5aQ9qjd4OVb03UmqpIWoEPSQtL+re/XigH4\nMbCv52BP4LVCXa2ZyFKelpIGgTO83zOAbn2q0ddLgAO9vx8n7YssGgL0xixriD1YWWLtfp7IU57I\nU57IU57IU57IU+PkW2b6JN/vtKWZTazbOPSYpEuAG83s8WbH0puU7nT4BzO7p9mxvFck/ZB0e/gf\n1WhjtL53MfVZy2n+MsFWWNd/hre1tcXykgyRpzyRpzyRpzyRpzyRp+okYWbdVij19QHWSNJMzqtl\nfwsrhFBG0mzSvsFjzey5Gu367g+F9czQbYey4tkV9RuGEEIIode9LwdYIYTeJ8ni50IIIYQQQm3V\nBliN7MEKIYTgYk16nshTnshTnshTnshTnshTnshT42KAFUIIIYQQQgi9JJYIhhC6iCWCIYQQQgj1\nxRLBEEIIIYQQQljLYoAVQgg9EGvS80Se8kSe8kSe8kSe8kSe8kSeGhcDrBBCCCGEEELoJbEHK4TQ\nRfwdrBDeX4YOHcGKFU83O4wQQnjfib+DFULIkgZY8XMhhPcPEf/XhxBC74ubXIQQQq9qa3YAfURb\nswPoI9qaHUCfEHtB8kSe8kSe8kSeGpc1wJI0QtJjVepmSRrVu2HlkTRc0kJJ0wtly5sRSzWS9pU0\nKaPd8kL7u6u1kbRFL8ZV8TqF+ppx+/tiVp02vf7+KJ4z5/WWdK6kZ/29slDSJzKOqXleSavyI159\nzHclPSbpEklbSZonaYGkfXJe28y+XippqaRFkn4mabNC3S8lzZf0oUZjDyGEEEIIeRqZwVoX1xcc\nBcwws0MKZetinDkxWZXHjZ6nEfXO12jczZB7/cvNbJR/3dcL5+1Jv08AdjazbwAHAIvNbLSZ/Srz\nfDltZgD/aGa7Ak8B31x9sNlYYAFwWMORhwpamh1AH9HS7AD6iJZmB9AntLS0NDuEPiHylCfylCfy\n1LhGBlgbSJoiaYmkWyUNLG8gabykxf51sZf1kzTJyzokneblIyU94L9pf1TS9j2If3PgxbKylwrx\nHOfXbJc02csmSbpC0kOSfivpaC/fRNJMj6VD0hFePsJnBCZJWiZpqqQD/fhlknb3dhtLur4wK3G4\nh/EW8JeMvrxUeDxY0j2SnpR0VaF89RpPSaf7bMjiQk439uPavXycl4/xeBd5fJsULyzpXp/ZaZe0\nUtLnM+N+F3jZz9GvMEOzSNLJ5Y09b3M9x7d4vAdLurXQZvXMmqSDytvXyVst3dbH1vGSxzBM0mzP\nz2JJe68JVRd4X+dK+qAXTiq9p/z5Kv9+FzAIWCDpTOAS4Cg/70C6vrYTJP3a666WVKqr21czm2lm\nnf50HvB3ZU1WkP7dhBBCCCGEtcHM6n4BI4BOYE9/fj1wuj+eBYwCtgb+AGxBGrg9CBzhdTMK59rM\nv88DjvDHA4CBObGUxXUe8NUqdTsCTwJD/Pnm/n0ScIs//ijwlD/uDwzyx1sWykeQBhs7+vNHgev9\n8RHA7f74QuBz/ngwsAzYqCym0cC1dfq0L/BXv65IMxJHe91yz+8ooAMYCGwCPA7sAhwN/Khwrk2B\nDYDfAaO8bJC/PvsC08quPQpYBGzag9fiy8CtrLlxSinfpffHlsDsUk6AM4Fved6fLpRfBYyv1r54\nzgox3AsMq1B+ruduEXAdMLiBfp0OfNMfC9jEH3cCh/rjS4CzCu+vowvH/1+Vx8cD/114Xnpt/wGY\nBvT38h8Cx+b2tazNtNJ7slD2X8B/1jnO4NzC1ywDi69uX5GXyFNfyRP2fjFr1qxmh9AnRJ7yRJ7y\nRJ7WmDVrlp177rmrv/znK+VfHyDfH81snj+eApwCXF6oHwPMMrPSjMZUYCxwAbC9pCuAnwMzJA0C\ntjGzaaTI3mogDvz8Ig0qplRpsj9wm5m94tdYWai708uWFvajCLhI0ljSh+dtCnXLzWyJP34CmOmP\nHwO288cHAYdLOsOfDwCGkwZa+PUWAF/K6N58M/uD9/NmYB/g9kL9PsAdZvaGt7kd+BfgfuB7ki4C\n7jWzX0n6GPC8mS30GF71Y7pcUNJWwI3AMWbW8P4i0pK3q83M/Dory+r3JA16H/LXbgNgrpm9K+k+\nUu5+Rlq+dgZpvUy39rUCMLNqS9+uAr5tZibpAtL79ouZ/XoEuF7SBsBdZtbh5W+a2c/98QJS/yvJ\nnTkz//5x0oD0Ee/3QOCFbo2r9zVdVDobeNvMbiqreo6stUit9ZuEEEIIIaxHWlpauiyZPO+88yq2\na2SAZXWeQ4UPk2a2UtIuwMGkWY5xwFcrte1yIukk0p4VI80UrCjU9QN+D7xJ+k1+o96sEPMEYCtg\nNzPrVLqhwMAK7TsLzztZk0MBnzKzp3oQT7mcXHc/yOwppZs/HAqcL+lB0mCyXq77ATcDrWa2tAfx\n5hBpJnNChbpbgK8ArwCPmNlrPrio1r4hZlZcWvdjoOrNPSocO8cH3YcBN0i6zMymAG8Xmr3LmvfB\nO/jS28LAsBECJpvZ2Q0et+YE0hdI74H9K1TfDpwjaYmZ7djTawSIPTO5WpodQB/R0uwA+oTYC5In\n8pQn8pQn8tS4RvZgjZC0hz/+HDCnrH4+MFbSFpL6k5Z5zZa0JWm50x2kJWGjfBblGUlHAkgaIGmj\n4snM7Coz283STQlWlNV1mtl2pOV6n6kS7y+AcfI7s0kaUqVdafAxGHjRB1f7kZbolbep5X7g1NUH\nSLtmHFPNHkp7v/qR+lee6zmk/TsDfT/VvwJzJG0NvO6zFt8jzYQsA4ZJGu1xDfLXp+gSoMPMbqsU\njO/hmlwn5geAE0vnrpDvecDekkZ6/caSPuJ1sz3WE4CfZLRviKRhhadHk5ZUImkbSTMrH7X62OGk\n98X1pOWFpTsiVntPPA3s7o+PpOsAq9b7qFT3IHBMYU/XEI8hi9IdEs8gLb99s0KT44DpMbgKIYQQ\nQlg7GhlgPQmcLGkJaZP8NV5eWhK2AphI+mMe7aSZiLuBbYE2Se2kJWgT/bjjgFMldQAPAUN7EP9v\nSPtWuvElfReSBnntwGXFeItN/ftUYIzHcyywtEKbSseXnE+6EchipVvaf7u8gaTRkq6t0Z+S+cCV\npOWIvzOzO4vXNrN24AbS8rWHSfu6OoCdgPne33OAC8zsbdIg7UpJi0h7ujYsu97XgYOUbnKxUNIn\ny+qHk/aF1XId8Ayw2K8/vizmPwFfAG72HM8FdvC6TuAe4BP+vWZ7qrwGSjfrGFah6lJ/XRaR9p59\nzcu3putMVCUtQIekhcCnge/XioE0Q7av52BP4LVCXa2ZyFKelpJ+ETHD+z0D6NanGn39AWmf3QP+\nWl5VVj+EdHfB8Ddra3YAfURbswPoI9qaHUCfEH+PJ0/kKU/kKU/kqXGlGxL0Sb7faUszm1i3cegx\nSZcAN5rZ482OpTcp3enwD2Z2T7Njea9I+iHp9vA/qtHGMlelrufaiGVdOdqIPOVoY+3lSfTl/+uL\n2traYrlShshTnshTnshTdZIws24rlPr6AGskaSbnVev6t7BCCGUkzSbtGzzWzJ6r0a7v/lAIIXQz\ndOgIVqx4utlhhBDC+877coAVQuh9kix+LoQQQggh1FZtgNXIHqwQQggu1qTniTzliTzliTzliTzl\niTzliTw1LgZYIYQQQgghhNBLYolgCKGLWCIYQgghhFBfLBEMIYQQQgghhLUsBlghhNADsSY9T+Qp\nT+QpT+QpT+QpT+QpT+SpcTHACiGEEEIIIYReEnuwQghdxN/BCs0wdNuhrHh2RbPDCCGEELLF38EK\nIWSRZLQ2O4qw3mmF+P8ohBBCXxI3uQghhN60vNkB9BGRpyyxxyFP5ClP5ClP5ClP5KlxWQMsSSMk\nPValbpakUb0bVh5JwyUtlDS9ULZO/XcuaV9JkzLaLS+0v7taG0lb9GJcFa9TqK8Zt78vZtVp0+vv\nj+I5c15vSedKetbfKwslfSLjmJrnlbQqP+LVx3xX0mOSLpG0laR5khZI2ifntc3s6xBJMyQtk3S/\npMGFul9Kmi/pQ43GHkIIIYQQ8jQyg7Uurt04CphhZocUytbFOHNisiqPGz1PI+qdr9G4myH3+peb\n2Sj/uq8XztuTfp8A7Gxm3wAOABab2Wgz+1Xm+XLaTARmmtkOwC+Ab64+2GwssAA4rOHIQ3fbNzuA\nPiLylKWlpaXZIfQJkac8kac8kac8kafGNTLA2kDSFElLJN0qaWB5A0njJS32r4u9rJ+kSV7WIek0\nLx8p6QFJiyQ9Kqkn/w1vDrxYVvZSIZ7j/JrtkiZ72SRJV0h6SNJvJR3t5ZtImumxdEg6wstHSFrq\nxy2TNFXSgX78Mkm7e7uNJV1fmJU43MN4C/hLRl9eKjweLOkeSU9KuqpQvnqNp6TTfTZkcSGnG/tx\n7V4+zsvHeLyLPL5NiheWdK/P7LRLWinp85lxvwu87OfoV5ihWSTp5PLGnre5nuNbPN6DJd1aaLN6\nZk3SQeXt6+Stlm7rY+t4yWMYJmm252expL3XhKoLvK9zJX3QCyeV3lP+fJV/vwsYBCyQdCZwCXCU\nn3cgXV/bCZJ+7XVXSyrV5fT1SGCyP55M+iVE0QrSv5sQQgghhLAWNDLA2gG40sx2BFYBJxUrJW0N\nXAy0ALsCY3yQsiuwrZntbGa7AKVlZ1OBH5jZrsA/A//bg/j7A53FAjPbw+PZETgLaDGz3YDTCs2G\nmdnewOGkD7oAbwBHmdnuwP7AZYX2I4Hv+qzADsBn/fgz/BoAZwMPmtmefvz3JG1kZg+b2dc8ptGS\nrq3UkVLcbgxwMvBR4O+LH9j9PKOA473dXsAJknYBPgE8Z2a7mdnOwH2SNgB+ApziuT4AeL3s2oeZ\n2Sjgi8DTwJ3FuKsxs2fN7Bh/+iVgBGmGZlfS61uMeUvgW8DHPccLgNOBmcA/SdrIm34GuMnbn12h\nfdW8+UBxWJVwv+KDoetUWDZXo2+l834OuM/zswuwyMs3AeZ6X+eQZqcqnsrPdyTwV59BuxQ4B/iJ\nP3+j0Id/8Bz8s1+zE5jQQF8/ZGYvePsVQPlywE7Sv5vaZhW+1qlFt+uQyEueyFOW2OOQJ/KUJ/KU\nJ/KUJ/K0RltbG62trau/qvlAA+f8o5nN88dTgFOAywv1Y4BZZlaa0ZgKjAUuALaXdAXwc2CGpEHA\nNmY2DcDM3mogDvz8In3gnVKlyf7AbWb2il9jZaHuTi9bqjX7UQRcJGks6UPoNoW65Wa2xB8/QRoU\nADwGbOePDwIOl3SGPx8ADAeWlS5qZgtIA5F65pvZH7yfNwP7ALcX6vcB7ih9MJd0O/AvwP2kgd1F\nwL1m9itJHwOeN7OFHsOrfkyXC0raCrgROMbMGt5fRBq4XW1+G7CyfAPsCewIPOSv3QakAcq7ku4j\n5e5npOVrZ5AG6t3a1wrAzKotfbsK+LaZmaQLSO/bL2b26xHgeh+o3mVmHV7+ppn93B8vIPW/ktyZ\ns9Lyv48Do4BHvN8DgRe6Na7e12rnLXmOlNva9ss8ewghhBDCeqKlpaXLksnzzjuvYrtGBljlH9Qq\n7Qfp9mHSzFb67MrBwJeBccBXK7XtciLpJNKsgAGH+m/jS3X9gN8DbwL3NtCHkjcrxDwB2ArYzcw6\nlW4oMLBC+87C807W5FDAp8zsqR7EUy4n190PMnvKZ7cOBc6X9CBpMFkv1/2Am4FWM1vag3hziLRf\nbkKFuluArwCvAI+Y2Ws+uKjWviFmVlxa92Og6s09Khw7xwfdhwE3SLrMzKYAbxeavcua98E7+Mxw\nYWDYCAGTzezsBo8reUHSUDN7wWe4ypfQ3g6cI2mJz0aHnoq9RXkiT1lij0OeyFOeyFOeyFOeyFPj\nGlkiOEJScdnUnLL6+cBYSVtI6g+MB2b7Uq/+ZnYHaYnYKJ9FeUbSkQCSBhSWiAFgZlf5UrdRxcGV\n13Wa2XbAo6TlVJX8AhgnvzObpCFV2pUGH4OBF31wtR9puVt5m1ruB05dfYC0a8Yx1eyhtPerH6l/\n5bmeQ9q/M9D3U/0rMMeXab5uZjcB3yPNhCwDhkka7XEN8ten6BKgw8xuqxSM0h6uyZXqCh4ATiyd\nu0K+5wF7Sxrp9RtL+ojXzfZYTyAtZ6zXviFlS+mOBh738m0kzax81Opjh5PeF9cD13mcUP098TSw\nuz8+kq4DrFrvo1Ldg8AxhT1dQzyGXNOAL/jj44G7yuqPA6bH4CqEEEIIYe1oZID1JHCypCWkTfLX\neHlpSdgK0h3M2oB20kzE3cC2QJukdtIStIl+3HHAqZI6gIeAoT2I/zdAxVtb+5K+C0mDvHbW7Kmq\nNjs0lbRvrAM4FlhaoU2l40vOJ90IZLHSLe2/Xd6g1h6sMvOBK0nLEX9nZncWr21m7cANpOVrDwPX\n+tK1nYD53t9zgAvM7G3SIO1KSYuAGcCGZdf7OnCQ0k0uFkr6ZFn9cOCvdWK+DngGWOzXH18W859I\nH/xv9hzPJe1nw8w6gXtIe8juqdeeKq9BjX1Jl/rrsgjYFyjtLduarjNRlbQAHZIWAp8Gvl8rBtIM\n2b6egz2B1wp1tWYiS3laSvpFxAzv9wygW59q9PUS4EBJy0jLDS8uqx8C9MYsa4i9RXkiT1lij0Oe\nyFOeyFOeyFOeyFPj5Ftm+iTf77SlmU2s2zj0mKRLgBvN7PFmx9KblO50+Aczu6fZsbxXJP2QdHv4\nH9VoY7S+dzH1WcuJ5W85cvPUCn35/6O/VVtbWyzDyRB5yhN5yhN5yhN5qk4SZtZthVJfH2CNJM3k\nvFr2t7BCCGUkzSbtGzzWzJ6r0a7v/lAIfdbQbYey4tkV9RuGEEII64j35QArhND7JFn8XAghhBBC\nqK3aAKuRPVghhBBcrEnPE3nKE3nKE3nKE3nKE3nKE3lqXAywQgghhBBCCKGXxBLBEEIXsUQwhBBC\nCKG+WCIYQgghhBBCCGtZDLBCCKEHYk16nshTnshTnshTnshTnshTnshT42KAFUIIIYQQQgi9JPZg\nhRC6iL+DFULzDB06ghUrnm52GCGEEDLE38EKIWRJA6z4uRBCc4j4fzmEEPqGuMlFCCH0qrZmB9BH\ntDU7gD6irdkB9AmxFyRP5ClP5ClP5KlxWQMsSSMkPValbpakUb0bVh5JwyUtlPT/s3fncXJVdd7H\nP98EkG0IEDUBhMBklEdUCAEUBZMGd9kxLAEGhgEdXzqigDgoahIWBQXGCKLymAloMgiMKLuEpTsT\ngjEhZAGyyBIWcQI8QjQ4ipj+PX/cU8nt6lpONR26m3zfr1e/+tY55977u7+qdOrUOefW7aWyFX0R\nSz2SxkqamtFuRan9zfXaSNq2F+OqeZ5SfcO40+uivUmbXn99lI+Z83xLGifpIUlrcmNpdlxJq/Oi\n7bLPtyU9KOkiSW+UNEfSfEn75zy3mdf6LUlLJS2U9DNJW5Xq/lvSXElvbjV2MzMzM8vTyghWf5yz\ncDgwIyI+Virrj3HmxBR1tls9TiuaHa/VuPtCzvkfBI4AZvbicXty3Z8Edo+IfwM+CCyOiL0i4t7M\n4+W0mQG8IyJGAY8AX167c8QYYD5wUMuRWw1tfR3AANHW1wEMEG19HcCA0NbW1tchDAjOUx7nKY/z\n1LpWOlgbS5omaYmk6yRtWt1A0nhJi9PPhalskKSpqWyRpM+n8pGS7kyftN8vaZcexL818FxV2fOl\neE5M51wg6epUNlXSZEmzJT0q6chUvoWku1IsiyQdmspHpBGBqZKWS5ou6UNp/+WS9k7tNpc0pTQq\ncUgK46/AHzKu5fnS9hBJt0haJumKUvnaOZ6SzkijIYtLOd087bcglR+VyvdJ8S5M8W1RPrGkW9NI\n4AJJqyT9Y2bca4AX0jEGlUZoFkr6bHXjlLf7Uo6vTfF+RNJ1pTZrR9Ykfbi6fZO81RQRyyPikXL+\nMjyfYhguaWbKz2JJ+60LVeena71P0ptS4dTKayo9Xp1+3whsCcyX9CXgIuDwdNxN6frcHi/p16nu\n+5IqdTnXeldEdKaHc4C3VDVZSfHvxszMzMzWh4ho+gOMADqBfdPjKcAZabsdGA1sBzwJbEvRcbsb\nODTVzSgda6v0ew5waNreBNg0J5aquCYBX6hTtxuwDNgmPd46/Z4KXJu23w48krYHA1um7aGl8hEU\nnY3d0uP7gSlp+1DghrR9AXBc2h4CLAc2q4ppL+DKJtc0FvjfdF5RjEgcmepWpPyOBhYBmwJbAA8B\newBHAj8sHevvgI2Bx4DRqWzL9PyMBW6qOvdoYCHwdz14Lj4NXMe6G6dU8l15fQylGEHaLJV/Cfhq\nyvsTpfIrgPH12pePWSOGW4HhDWKsuV+T6zoD+HLaFrBF2u4EPp62LwK+Unp9HVna/491tk8Cvlt6\nXHlu/w9wEzA4lX8POKHVa01tbqq8JktlXwO+2GS/gAmln/aA8E+3H+fFeVofeSKsvvb29r4OYUBw\nnvI4T3mcp3Xa29tjwoQJa3/S32yqfzYi31MRMSdtTwM+BwsvocoAACAASURBVFxaqt8HaI+IyojG\ndGAMcD6wi6TJwG3ADElbAttHxE0Ukf21hThIxxdFp2JanSYHAtdHxIvpHKtKdb9IZUtL61EEfFPS\nGIo3z9uX6lZExJK0/TBwV9p+ENg5bX8YOETSWenxJsBOFB0t0vnmA5/KuLy5EfFkus5rgP2BG0r1\n+wM/j4i/pDY3AO8H7gAulvRN4NaIuFfSO4HfRcQDKYaX0j5dTijpjcBPgHER0fL6Ioopb9+PiEjn\nWVVVvy9Fp3d2eu42Bu6LiDWSfkmRu59RTF87i2K+TLf2jQKIiPUx9W0eMEXSxsCNEbEolb8cEbel\n7fkU119L7qhZpN8foOiQzkvXvSnwbLfGTa5V0jnAKxHxn1VVz5A1F2li8yZmZmZmG5C2trYuUyYn\nTZpUs10rHaxo8hhqvJmMiFWS9gA+QjHKcRTwhVptuxxI+gzFmpWgGClYWaobBDwOvEzxSX6rXq4R\n8/HAG4E9I6JTxQ0FNq3RvrP0uJN1ORTwiSimor1aObnuvlPEIypu4vBx4DxJd1N0JpvlehBwDTAx\nIpb2IN4cohjJPL5G3bXAvwIvAvMi4k+pc1Gv/WsmImalTvdBwFWSLomIacArpWZrWPc6+Btp6m2p\nY9gKAVdHxDk9jVnSP1G8Bg6sUX0D8HVJSyJit56ew8BrZnK19XUAA0RbXwcwIHgtSB7nKY/zlMd5\nal0ra7BGSHpP2j4OmFVVPxcYI2lbSYMppnnNlDSUYrrTzymmhI1OoyhPSzoMQNImkjYrHywiroiI\nPSNidLlzleo6I2Jniul6x9SJ9x7gKKU7s0napk67SudjCPBc6lwdQDFFr7pNI3cAp63dQRqVsU89\n71Gx9msQxfVV53oWxfqdTdN6qiOAWZK2A/6cRi0uphgJWQ4Ml7RXimvL9PyUXQQsiojrawWT1nBd\n3STmO4F/qRy7Rr7nAPtJGpnqN5f01lQ3M8X6SeCnGe1fjfJap+0l3dWwsbQTxetiCvCjFGeX41R5\nAtg7bR9G1w5Wo9dRpe5uYFxpTdc2KYYskj5KMQJ4aES8XKPJicDt7lyZmZmZrR+tdLCWAZ+VtIRi\nkfwPUnllSthK4GyKL/NYQDEScTOwA9AhaQHFFLSz034nAqdJWgTMBob1IP7fUKxb6SZN6buAopO3\nALikHG+5afo9HdgnxXMCsLRGm1r7V5xHcSOQxSpuaX9udQNJe0m6ssH1VMwFLqeYjvhYRPyifO6I\nWABcRTF97VcU67oWAe8C5qbr/TpwfkS8QtFJu1zSQoo1XW+oOt+ZwIdV3OTiAUkHV9XvRLEurJEf\nAU8Di9P5x1fF/P+AfwKuSTm+D9g11XUCtwAfTb8btqfOc6DiZh3Da5QfLulpimmKt2jdbf23o+tI\nVC1twCJJDwBHA99pFAPwf4GxKQf7An8q1TUaiazkaSnFBxEz0nXPAGpdU81rBS6jWGd3Z3our6iq\n34bi7oL2qnX0dQADREdfBzBAdPR1AAOCv48nj/OUx3nK4zy1rnJDggEprXcaGhFnN21sPSbpIuAn\nEfFQX8fSm1Tc6fDJiLilr2N5rUj6HsXt4X/YoE1kzkrdwHXgaV05OnCecnRQ5EkM5P+X17eOjg5P\nV8rgPOVxnvI4T/VJIiK6zVAa6B2skRQjOS9F1+/CMrMqkmZSrBs8ISKeadDOHSyzPuMOlpnZQPG6\n7GCZWe8rOlhm1heGDRvBypVP9HUYZmaWoV4Hq5U1WGa2gaj1nQ7+6frT3t7e5zEMhB/nqbU8uXPV\nmNeC5HGe8jhPeZyn1rmDZWZmZmZm1ks8RdDMupAU/rtgZmZm1pinCJqZmZmZma1n7mCZmfWA56Tn\ncZ7yOE95nKc8zlMe5ymP89Q6d7DMzMzMzMx6iddgmVkXXoNlZmZm1ly9NVgb9UUwZta/Sd3+Vmyw\nhu0wjJW/XdnXYZiZmdkA4REsM+tCUjCxr6PoRyYW3wtWraOjg7a2ttc8nIHGecrjPOVxnvI4T3mc\npzzOU32+i6CZmZmZmdl6ltXBkjRC0oN16tolje7dsPJI2knSA5JuL5Wt6ItY6pE0VtLUjHYrSu1v\nrtdG0ra9GFfN85TqG8adXhftTdr0+uujfMyc51vSOEkPSVqTG0uz40panRdtl32+LelBSRdJeqOk\nOZLmS9o/57nNvNZtJM2QtFzSHZKGlOr+W9JcSW9uNXbrzp/m5XGe8jhPeZynPM5THucpj/PUulZG\nsPrjXMLDgRkR8bFSWX+MMyemqLPd6nFa0ex4rcbdF3LO/yBwBDCzF4/bk+v+JLB7RPwb8EFgcUTs\nFRH3Zh4vp83ZwF0RsStwD/DltTtHjAHmAwe1HLmZmZmZZWmlg7WxpGmSlki6TtKm1Q0kjZe0OP1c\nmMoGSZqayhZJ+nwqHynpTkkLJd0vaZcexL818FxV2fOleE5M51wg6epUNlXSZEmzJT0q6chUvoWk\nu1IsiyQdmspHSFqa9lsuabqkD6X9l0vaO7XbXNKU0qjEISmMvwJ/yLiW50vbQyTdImmZpCtK5Wvn\neEo6I42GLC7ldPO034JUflQq3yfFuzDFt0X5xJJuTSOBCyStkvSPmXGvAV5IxxhUGqFZKOmz1Y1T\n3u5LOb42xfsRSdeV2qwdWZP04er2TfJWU0Qsj4hHyvnL8HyKYbikmSk/iyXtty5UnZ+u9T5Jb0qF\nUyuvqfR4dfp9I7AlMF/Sl4CLgMPTcTel63N7vKRfp7rvS2vvONH0WoHDgKvT9tUUH0KUraT4d2Ov\nkr8XJI/zlMd5yuM85XGe8jhPeZyn1rVyF8FdgZMjYo6kKcBngEsrlZK2Ay4E9gRWAXemTspvgR0i\nYvfUbqu0y3TgGxFxk6RN6Nl6sMFAZ7kgIt6TzrMb8BXgvRHxoqTym8rhEbGfpLcDNwE3AH8BDo+I\nlyQNBeakOoCRwCciYomk+4Fj0/6HpnMcCZwD3B0Rp6RpWXMl3RURvwJ+lWLaC/iXiPhU9YVU4k72\nAd4OPAXcIenIiLihUqlimttJqd1g4NeSOlKcz0TEwand30naGPgpcFREPCBpS+DPVec+qHTc/wB+\nERGrK3HXExG/Bcalh58CRlCM0ERVvkk5/SrwgYj4c+pknAF8E/ihpM0i4s/AMcB/pvbn1Gh/fr28\nSboVOCUiXvUt30rHPQ74ZUR8M3V0Kp28LYD7IuKrki6iGJ36Rq1DpeMdJumPEVGZ2vgssFdEnJYe\nV67h/6QcvC8i1kj6HnA8MC3zWt8cEc+mc65U9+mAnRSvmcbKEz93Bnry8YeZmZnZ60hHR0dWh7OV\nDtZTETEnbU8DPkepg0XxZr89IiojGtOBMRRviHeRNBm4DZiR3uRvHxE3AUTEX1uIg3R8AXukWGo5\nELg+Il5M51hVqvtFKltaegMq4JuSxlC8Cd2+VLciIpak7YeBu9L2gxRvPwE+DBwi6az0eBNgJ2B5\n5aQRMZ+iI9LM3Ih4Ml3nNcD+FJ3Aiv2Bn0fEX1KbG4D3A3cAF0v6JnBrRNwr6Z3A7yLigRTDS2mf\nLieU9EbgJ8C41Llq1QeB71e+QKkq3wD7ArsBs9NztzFFB2WNpF9S5O5nFNPXzgLaarVvFEClo9jL\n5gFTUkf1xohYlMpfjojb0vZ8iuuvJXfUrDL97wPAaGBeuu5NgWe7Nc6/1upphc9Q5LaxAzKPvgHz\nnPQ8zlMe5ymP85THecrjPOVxntZpa2vrko9JkybVbNdKB6v6jVqt9SDd3kxGxCpJewAfAT4NHAV8\noVbbLgeSPkMxKhDAx8uf1EsaBDwOvAzc2sI1VLxcI+bjgTcCe0ZEp4obCmxao31n6XEn63IoilGu\nR3oQT7WcXHffKeKRNAr1ceA8SXdTdCab5XoQcA0wMSKW9iDeHKJYL3d8jbprgX8FXgTmRcSfUuei\nXvvXTETMSp3ug4CrJF0SEdOAV0rN1rDudfA30mhsqWPYCgFXR8Q5PQz5WUnDIuJZScPpPoX2BuDr\nkpZExG49PIeZmZmZ1dHKtLwRksrTpmZV1c8FxkjaVtJgYDwwM031GhwRP6eYIjY6jaI8LekwAEmb\nSNqsfLCIuCIi9oyI0dXToCKiMyJ2Bu6nmE5Vyz3AUUp3ZpO0TZ12lc7HEOC51Lk6gGK6W3WbRu4A\nTlu7gzQqY5963qNi7dcgiuurzvUsivU7m6pYT3UEMCtN0/xzRPwncDHFSMhyYHianoikLdPzU3YR\nsCgirq8VjIo1XFfXqiu5E/iXyrFr5HsOsJ+kkal+c0lvTXUzU6yfpJjO2Kz9q1Fe67S9pLsaNpZ2\nonhdTAF+lOLscpwqTwB7p+3D6NrBavQ6qtTdDYwrrenaJsWQ6ybgn9L2ScCNVfUnAre7c/XqeU56\nHucpj/OUx3nK4zzlcZ7yOE+ta6WDtQz4rKQlFIvkf5DKK1PCVlLcwawDWEAxEnEzsAPQIWkBxRS0\ns9N+JwKnSVoEzAaG9SD+3wA1b22dpvRdQNHJWwBcUo633DT9ng7sk+I5AVhao02t/SvOo7gRyGIV\nt7Q/t7qBpL0kXdngeirmApdTTEd8LCJ+UT53RCwArqKYvvYr4Mo0de1dFGu/FgBfB86PiFcoOmmX\nS1oIzADeUHW+M4EPq7jJxQOSDq6q3wn43yYx/wh4Gliczj++Kub/R/HG/5qU4/so1vUREZ3ALcBH\n0++G7anzHKi4WcfwGuWHS3qaYpriLVp3W//t6DoSVUsbsEjSA8DRwHcaxQD8X2BsysG+wJ9KdY1G\nIit5WkrxQcSMdN0zgFrXVPNaKTrLH5K0nGK64YVV9dsAvTHKamZmZmY1KC2ZGZDSeqehEXF208bW\nY+kmDj+JiIf6OpbepOJOh09GxC19HctrJd00Y3FE/LBBm2DiaxdTvzcRBvLfSTMzM1s/JBER3WYo\nDfQO1kiKkZyXqr4Ly8yqSJpJsW7whIh4pkG7gftHYT0YtsMwVv72Vd+Y0szMzF5n6nWwenJr9H4j\nIh6LiPe7c2XWXESMjYgDGnWuSm39k37qda48Jz2P85THecrjPOVxnvI4T3mcp9YN6A6WmZmZmZlZ\nfzKgpwiaWe+TFP67YGZmZtbY63KKoJmZmZmZWX/iDpaZWQ94Tnoe5ymP85THecrjPOVxnvI4T61z\nB8vMzMzMzKyXeA2WmXXhNVhmZmZmzdVbg7VRXwRjZv2b1O1vhfUzw4aNYOXKJ/o6DDMzM6viKYJm\nVkP4p+lPe5+e/9lnn6z/9PUjnrufx3nK4zzlcZ7yOE95nKfWNexgSRoh6cE6de2SRq+fsBqTtJOk\nByTdXipb0Rex1CNprKSpGe36VdxlObE1ayNpgqQzei+qrseUNFXSmCbtt5Z0g6RFkuZI2i3jHO2S\ndmpS39LrX9I4SUsk3Z0eXyNpoaTPp+s4ssn+Odd6XLrORZLulbR7qe4SSQ9LGttK3GZmZmaWL2cE\nqz8uxjgcmBERHyuV9cc4c2Lqj3FXDPT4K74CLIiIPYCTgO/2URynAKdGxAckDQf2johRETG5F8/x\nODAmXev5wJWViog4EzgX+OdePN8GrK2vAxgQ2tra+jqEAcF5yuM85XGe8jhPeZyn1uV0sDaWNC19\n8n6dpE2rG0gaL2lx+rkwlQ1Kn7gvTp+mfz6Vj5R0Z/rk/n5Ju/Qg7q2B56rKni/Fc2I65wJJV6ey\nqZImS5ot6dHKaIGkLSTdlWJZJOnQVD5C0tK033JJ0yV9KO2/XNLeqd3mkqakkZH5kg5JYfwV+EPG\ntTyfjjNc0sw0MrdY0n6pfLWk81O+7pP0plR+cOmcM0rlEyT9OLVdLunUVD42Hf8WScskXaHCyZL+\nvZS7UyVdUp3TZvHXy3uZpL+XdLukeSmWt0naStITpTabS3pK0uBa7WucfxVFrhvZDbgHICKWAztX\n8tXA74E19V7HydGSfp3yWXm+TpJ0Wel6bpY0RtLXgP2BKZK+BdwB7JCe7/2r8jRaUke67tslDcu9\n1oiYExGV190cYIeqJisp/v2YmZmZ2foQEXV/gBFAJ7BvejwFOCNttwOjge2AJ4FtKTpsdwOHproZ\npWNtlX7PAQ5N25sAmzaKoU5ck4Av1KnbDVgGbJMeb51+TwWuTdtvBx5J24OBLdP20FL5CIo3s7ul\nx/cDU9L2ocANafsC4Li0PQRYDmxWFdNewJVNrukM4MtpW8AWabsT+Hjavgj4SuVcpX1PAb6dticA\nC1JuhwJPAcOBscD/pusSMAM4EtgCeBQYnPafDbyjB89JvbxPKL1m7gJGpu13A3en7Z8DY9P20ZVc\nNWi/9pg1XhcH1yi/ALikdJy/AntmXle913F7KecfA+5M2ycB3y21v5liRKmyz56l19fiUrup6fnY\nKD0HQ0v5mJJ7rVVtvlj9ugPeD9zSZL+A8E/Tn/Y+Pj8xELS3t/d1CAOC85THecrjPOVxnvI4T/Wl\n/4u7vZfKuYvgUxExJ21PAz4HXFqq3wdoj4gXACRNB8ZQTE/aRdJk4DZghqQtge0j4iaKiJqNPHQj\nScAeKZZaDgSuj4gX0zlWlep+kcqWSnpz5ZDAN1WsbekEti/VrYiIJWn7YYo3/AAPAjun7Q8Dh0g6\nKz3eBNiJoqNFOt984FNNLm0exejGxsCNEbEolb8cEbel7fnAB9P2jpKuo+jgbgysKB3rxpTb30u6\nh6JT8QdgbkQ8CcX6H2D/iLhBxZqggyUtAzaKiIebxFpLo7wjaQvgfcD16TkkxQ1wHXAMMBM4Fvhe\nk/Y1RcSEOlUXApMlPUDx3C0A1mRe1+NUvY5LdTek3/MpOkw5mt2eb1fgncCd6boHAb+rbtTgWouT\nSAcAJ1OMmpU9A7xN0hsi4uX6R5hY2m7D0+HMzMxsQ9fR0ZF104+cDlY0eQw13jRGxCpJewAfAT4N\nHAV8oVbbLgeSPgN8Mp3n4xGxslQ3iOIN78vArRmxVyu/oazEcTzwRoqRhU4VN23YtEb7ztLjTtbl\nTsAnIuKRHsSzVkTMSp28g4CrJF0SEdOAV0rN1pTOexlwcUTcquKmBeU33OXnSNR+zsrtplCsU1pG\nMZKyPgwCXoyIWjeGuAm4QNI2FCNG9wBbNmjfkohYTWndUXqOH8/ct9br+NRUXXk9lJ+Xv9F16m23\nKbVNCHgoIvZrcb91ByhubHEl8NFKh7ciIh6XtBR4UtIH6nemJ/b09BuQtr4OYEDw3P08zlMe5ymP\n85THecrjPK3T1tbWJR+TJk2q2S5nDdYISe9J28cBs6rq5wJjJG0raTAwHpgpaSjFtLOfA18FRkfE\nS8DTkg4DkLSJpM3KB4uIKyJiz4gYXe5cpbrOiNiZYrreMXXivQc4StK26Rzb1GlX6WANAZ5LnasD\n6DoSkfNlQHcAp63dQRqVsU/3YIo71j0XEVOAH1F0NBrFsBXrRjZOqqo7LOV2KMXUwHmpfB8Va8sG\nUeTvXoCImAvsSPHcXVMnvqVNLqFh3lMnZ4WkcaVj7p7q/kTxnE6mmL4Wjdq3StKQNDKIpE8CM9Nr\nERXr77ZrsG+313G9pun3E8CotL5tR4rRw7qHr1G2HHiTpH3T+TdSxl0PS/HuBPwM+MeIeKxG/e7A\nLhQjyT0ZqTQzMzOzBnI6WMuAz0paQrE4/gepvFisUXSCzgY6KKZezYuImykW13dIWgD8JLUBOBE4\nTdIiirUmlQX8rfgNxZqvbtKUvgsoOnkLgMoNG+qNxE2n6HgsAk4AltZoU2v/ivMobgSyWMUt7c+t\nbiBpL0lXdt+1izZgUZrGdjTwnSbnnQT8l6R5dL8ZxWKK5+M+4NxSR/V+4HKK6Y6PpU5DxXXA7Fh3\ng4Ry/EObxN4o72UnAKeouGHHQxRr2SqupRhN/Gmp7PgG7buRNEnSwTWq3g48lDqJHwEqN1wRMBJ4\nocFh672Oa76eImI2RSfrYYrncH51mzqPK/u/AowDLpK0kOLf1HtbuNavUfzbuELFzUbmVtVvAzwR\nEZ019rWWdPR1AAOCvz8lj/OUx3nK4zzlcZ7yOE+tU7E+a2BJ652GRsTZTRtvYCRNAFZHxKVV5WOB\nMyOiZidF0s3ApRHRXqPuIGCXiLh8fcTcVyS9Azg5Ir7Y17G8ViQdDRwREeMbtIn6/Xpbp4O+nSYo\nBsLf746ODk8vyeA85XGe8jhPeZynPM5TfZKIiG4zkgZqB2skcBXwUnT9LqwNXqsdLElDKKZ5LoiI\nY1+7SO21puL2+++nuFvl3Q3auYM1IAyMDpaZmdnr1euqg2Vm60/RwbL+btiwEaxc+URfh2FmZrbB\nqtfBylmDZWYbmFrf6eCfrj/t7e19ev6B0rny3P08zlMe5ymP85THecrjPLXOHSwzMzMzM7Ne4imC\nZtaFpPDfBTMzM7PGPEXQzMzMzMxsPXMHy8ysBzwnPY/zlMd5yuM85XGe8jhPeZyn1rmDZWZmZmZm\n1ku8BsvMuvAaLDMzM7Pm6q3B2qgvgjGz/k3q9rfC+sCwHYax8rcr+zoMMzMza4FHsMysC0nBxL6O\nYgBYAeyyns8xsfhOsoGso6ODtra2vg6j33Oe8jhPeZynPM5THuepvh7dRVDSCEkP1qlrlzS6twJs\nhaSdJD0g6fZS2Yq+iKUeSWMlTc1o16/iLsuJrVkbSRMkndF7UXU9pqSpksY0ab+1pBskLZI0R9Ju\nGedol7RTk/qWXv+SxklaIunu9PgaSQslfT5dx5FN9m96randdyU9ko49qlR+iaSHJY1tJW4zMzMz\ny5dzk4v++PHp4cCMiPhYqaw/xpkTU3+Mu2Kgx1/xFWBBROwBnAR8t4/iOAU4NSI+IGk4sHdEjIqI\nyb11AkkfA0ZGxFuBfwF+UKmLiDOBc4F/7q3zbdDW9+jV64Q/9czjPOVxnvI4T3mcpzzOU+tyOlgb\nS5qWPnm/TtKm1Q0kjZe0OP1cmMoGpU/cF6eRg8+n8pGS7kyfrt8vqSdvU7YGnqsqe74Uz4npnAsk\nXZ3KpkqaLGm2pEcrowWStpB0V4plkaRDU/kISUvTfsslTZf0obT/ckl7p3abS5qSRkbmSzokhfFX\n4A8Z1/J8Os5wSTPTyNxiSful8tWSzk/5uk/Sm1L5waVzziiVT5D049R2uaRTU/nYdPxbJC2TdIUK\nJ0v691LuTpV0SXVOm8VfL+9lkv5e0u2S5qVY3iZpK0lPlNpsLukpSYNrta9x/lUUuW5kN+AegIhY\nDuxcyVcDvwfW1HsdJ0dL+nXKZ+X5OknSZaXruVnSGElfA/YHpkj6FnAHsEN6vvevytNoSR3pum+X\nNKyFaz0M+HG61l8DQ0r7A6yk+PdjZmZmZutBTgdrV+DyiNgNWA18plwpaTvgQqANGAXskzopo4Ad\nImL3NHJQmS43HbgsIkYB7wP+pwdxDwY6ywUR8Z4Uz24UIxZtEbEnUH5DPDwi9gMOAS5KZX8BDo+I\nvYEDgUtK7UcC346IXVMejk37n5XOAXAOcHdE7Jv2v1jSZhHxq4g4PcW0l6Qra11IJW7gOOCXETEa\n2ANYmMq3AO5L+ZoFfDKVz4qIfSNiL+Ba4Eulw76L4vl4H/B1FaMlAPsAnwXeDvwDcARwHXCIpMGp\nzcnAf1TFVldm3iuuBP41IvahyOH3I+KPwAKtm7Z2cMrDmlrta5z/9IiYk2KYJOngGuddBFQ61O8G\ndgLe0uS6xkXEM9R/HQMMTtd/OnRZtdRtVC8izgPuB46LiC8BhwKPRsToiLi30k7SRsBlwCfSdU8F\nvtHCte4APF16/Ewqq+ik+Pdjr1a/ndzbv/j7U/I4T3mcpzzOUx7nKY/z1Lqcuwg+VXlTB0wDPgdc\nWqrfB2iPiBcAJE0HxgDnA7tImgzcBsyQtCWwfUTcBBARzT6N70aSKDog0+o0ORC4PiJeTOdYVar7\nRSpbKunNlUMC31SxtqUT2L5UtyIilqTth4G70vaDwM5p+8MUHZSz0uNNKN7AL6+cNCLmA59qcmnz\nKEY3NgZujIhFqfzliLgtbc8HPpi2d5R0HbAdsDFd3+7dmHL7e0n3AO+mGE2bGxFPQrH+B9g/Im5Q\nsSboYEnLgI0i4uEmsdbSKO9I2oKiw3d9eg5JcUPRyTsGmAkcC3yvSfuaImJCnaoLgcmSHqB47hYA\nazKv63GqXseluhvS7/nAiMzjNbs9367AO4E703UPAn5X3ajBtTbzDPA2SW+IiJfrtmovbe+Mp8OZ\nmZnZBq+joyOrw5nTwar+NL7WmptubxojYpWkPYCPAJ8GjgK+UKttlwNJn6EYpQng4xGxslQ3iOIN\n78vArRmxVyu/oazEcTzwRmDPiOhUcdOGTWu07yw97mRd7kQx2vBID+JZKyJmpU7eQcBVki6JiGnA\nK6Vma0rnvQy4OCJuTaM/5Tfc5edI1F8nVSmfQjH6tIyuIzS9aRDwYhqhq3YTcIGkbYDRFNP5tmzQ\nviURsZrSuqP0HD+euW+t1/Gpqbryeig/L3+j68hwtym1TQh4KI2U9sQzwI6lx29JZQBExOOSlgJP\nSvpA3c70AT08+4bEnc4snrufx3nK4zzlcZ7yOE95nKd12trauuRj0qRJNdvlTBEcIak8jW1WVf1c\nYIykbdM0s/HATElDKaZQ/Rz4KjA6Il4CnpZ0GICkTSRtVj5YRFwREXumqVMrq+o6I2JniqlWx9SJ\n9x7gKEnbpnNsU6ddpYM1BHguda4OoOtIRM6XAd0BnLZ2h9Jd21qh4o51z0XEFOBHFB2NRjFsxbqR\njZOq6g5LuR0KjKUYHYNi+uaI1FE9BrgXICLmUrwpHw9cUye+pU0uoWHeUydnhaRxpWPunur+RPGc\nTgZuiULd9q2SNCSNDCLpk8DM9FpExfq77Rrs2+11XK9p+v0EMEqFHSlGD+sevkbZcuBNkvZN599I\nGXc9LLkJODHtuy+wKiKeLV3P7hRdg+17OFJpZmZmZg3kdLCWAZ+VtIRicXzlrmQBkDpBZwMdFFOv\n5kXEzRTrPjokLQB+ktpA8ebvNEmLgNlAeQF+rt8A9uLOlAAAIABJREFU29aqSFP6LqDo5C1g3Zqq\neiNx0yk6HouAE4ClNdrU2r/iPIobgSxWcUv7c6sbNFqDVdIGLErT2I4GvtPkvJOA/5I0j+43o1hM\n8XzcB5xb6qjeD1xOMd3xsdRpqLgOmB0R3W7MkToZDTXIe9kJwCkqbtjxEMU6pIprKUYTf1oqO75B\n+24arEt6O/BQ6iR+hLQ+LE3BGwm80OCw9V7HNV9PETGbopP1MMVzOL+6TZ3Hlf1fAcYBF0laSPFv\n6r2515qmk66Q9CjwQ6rWTALbAE9ERGf1vtYir8HK4rn7eZynPM5THucpj/OUx3lq3YD8ouG03mlo\nRJzdtPEGRtIEYHVEXFpVPhY4MyJqdlIk3QxcGhHtNeoOAnaJiMvXR8x9RdI7gJMj4ot9HctrRdLR\nwBERMb5BG3/RcA5/0XAWf0FlHucpj/OUx3nK4zzlcZ7qU50vGh6oHayRwFXAS1XfhbXBa7WDJWkI\nxTTPBRFx7GsXqb3WVNx+//3AlyPi7gbt3MHqLyYO/A6WmZnZ69XrqoNlZuuPJP9R6CeG7TCMlb9d\n2byhmZmZvebqdbBy1mCZ2QYmIvzT5Ke9vX29n+P10Lny3P08zlMe5ymP85THecrjPLXOHSwzMzMz\nM7Ne4imCZtaFpPDfBTMzM7PGPEXQzMzMzMxsPXMHy8ysBzwnPY/zlMd5yuM85XGe8jhPeZyn1rmD\nZWZmZmZm1ku8BsvMuvAaLDMzM7PmvAbLzLJJqvkzfPjOfR2amZmZWb/mDpaZ1RA1f5599sk+jao/\n8Zz0PM5THucpj/OUx3nK4zzlcZ5a17CDJWmEpAfr1LVLGr1+wmpM0k6SHpB0e6lsRV/EUo+ksZKm\nZrTrV3GX5cTWrI2kCZLO6L2ouh5T0lRJYzL2+a6kRyQtlDQqo327pJ2a1Lf0+pc0TtISSXenx9ek\neD6fruPIJvs3vVZJx0lalH7ulbR7qe4SSQ9LGttK3GZmZmaWb6OMNv1xMcbhwIyIOLtU1h/jzImp\nP8ZdMdDjB0DSx4CREfFWSe8BfgDs2wehnAKcGhH3SRoO7B0Rb00xNu2MZ3ocGBMRf5D0UeBK0rVG\nxJmS5gL/DMzspfNtsNra2vo6hAHBecrjPOVxnvI4T3mcpzzOU+typghuLGla+uT9OkmbVjeQNF7S\n4vRzYSoblD5xX5w+Tf98Kh8p6c70yf39knbpQdxbA89VlT1fiufEdM4Fkq5OZVMlTZY0W9KjldEC\nSVtIuivFskjSoal8hKSlab/lkqZL+lDaf7mkvVO7zSVNkTRH0nxJh6Qw/gr8IeNank/HGS5pZhqZ\nWyxpv1S+WtL5KV/3SXpTKj+4dM4ZpfIJkn6c2i6XdGoqH5uOf4ukZZKuUOFkSf9eyt2pki6pzmmz\n+OvlvUzS30u6XdK8FMvbJG0l6YlSm80lPSVpcK32Nc6/iiLXjRwG/BggIn4NDJE0rMk+vwfW1Hsd\nJ0dL+nXKZ+X5OknSZaXruVnSGElfA/YHpkj6FnAHsEN6vvevytNoSR3pum8vxdr0WiNiTkRUXndz\ngB2qmqyk+PdjZmZmZutBTgdrV+DyiNgNWA18plwpaTvgQqANGAXskzopo4AdImL3iNgDqHxCPx24\nLCJGAe8D/qcHcQ8GOssFEfGeFM9uwFeAtojYEyi/IR4eEfsBhwAXpbK/AIdHxN7AgcAlpfYjgW9H\nxK4pD8em/c9K5wA4B7g7IvZN+18sabOI+FVEnJ5i2kvSlbUupBI3cBzwy4gYDewBLEzlWwD3pXzN\nAj6ZymdFxL4RsRdwLfCl0mHfRfF8vA/4uorREoB9gM8Cbwf+ATgCuA44RNLg1OZk4D+qYqsrM+8V\nVwL/GhH7UOTw+xHxR2CB1k1bOzjlYU2t9jXOf3pEzEkxTJJ0cI3z7gA8XXr8DN07HtXHHRcRz1D/\ndQwwOF3/6cDE8u41jncecD9wXER8CTgUeDQiRkfEvZV2kjYCLgM+ka57KvCNFq617FTg9qqyTop/\nP/YqeU56Hucpj/OUx3nK4zzlcZ7yOE+ty5ki+FTlTR0wDfgccGmpfh+gPSJeAJA0HRgDnA/sImky\ncBswQ9KWwPYRcRNARDQbeehGkig6INPqNDkQuD4iXkznWFWq+0UqWyrpzZVDAt9UsbalE9i+VLci\nIpak7YeBu9L2g8DOafvDFB2Us9LjTYCdgOWVk0bEfOBTTS5tHsXoxsbAjRGxKJW/HBG3pe35wAfT\n9o6SrgO2AzYGymuhbky5/b2ke4B3U4ymzY2IJ6FY/wPsHxE3qFgTdLCkZcBGEfFwk1hraZR3JG1B\n0eG7Pj2HpLih6OQdQzFt7Vjge03a1xQRE3oQdzOPU/U6LtXdkH7PB0ZkHq/brTyr7Aq8E7gzXfcg\n4HfVjZpdq6QDKDrL+1dVPQO8TdIbIuLl+keYWNpuSz9mZmZmG66Ojo6sDmdP1mDVWnPT7U1jRKyS\ntAfwEeDTwFHAF2q17XIg6TMUozQBfDwiVpbqBlG84X0ZuDUj9mrlN5SVOI4H3gjsGRGdKm7asGmN\n9p2lx52sy50oRhse6UE8a0XErNTJOwi4StIlETENeKXUbE3pvJcBF0fErWn0p/yGu/wcifrrpCrl\nUyhGn5bRdYSmNw0CXkwjdNVuAi6QtA0wGrgH2LJB+1Y9A+xYevyWVNZUndfxqam68nooPy9/o+vI\ncLcptU0IeCiNlPaIihtbXAl8tNLhrYiIxyUtBZ6U9IH6nemJPT39BsNz0vM4T3mcpzzOUx7nKY/z\nlMd5Wqetra1LPiZNmlSzXc4UwREqbgwAxTS2WVX1c4ExkrZN08zGAzMlDaWYQvVz4KvA6Ih4CXha\n0mEAkjaRtFn5YBFxRUTsmaZOrayq64yInSmmWh1TJ957gKMkbZvOsU2ddpUO1hDgudS5OoCuIxHN\nRhugWEtz2todMu5QVzOY4o51z0XEFOBHFB2NRjFsxbqRjZOq6g5LuR0KjKUYHYNi+uaI1FE9BrgX\nICLmUnRAxgPX1IlvaZNLaJj3iFgNrJA0rnTM3VPdnyie08nALVGo274HbgJOTMfYF1gVEc+mx3el\naa411Xod12uafj8BjFJhR4rRw7qHr1G2HHhTihNJG6Xpl1nS6+hnwD9GxGM16ncHdqEYSe7JSKWZ\nmZmZNZDTwVoGfFbSEorF8T9I5QGQOkFnAx3AAmBeRNxMscalQ9IC4CepDRRvdE+TtAiYDTS72UAt\nvwG2rVWRpvRdQNHJW8C6NVX1RuKmU3Q8FgEnAEtrtKm1f8V5FDcCWazilvbnVjdotAarpA1YJOkB\n4GjgO03OOwn4L0nz6H4zisUUz8d9wLmljur9wOUU0x0fS52GiuuA2aUbJJTjH9ok9kZ5LzsBOEXF\nDTseoliHVHEtxWjiT0tlxzdo3029dUlpiuUKSY8CPyStI0xT8EYCLzQ4bL3Xcc3XU0TMpuhkPUzx\nHM6vblPncWX/V4BxwEWSFlL8m3pv7rUCX6P4t3GFipuNzK2q3wZ4IiI6u+9qrfCc9DzOUx7nKY/z\nlMd5yuM85XGeWqeIfn+X7W7SeqehVbdpN4q7CAKrI+LSqvKxwJkRUbOTIulm4NKIaK9RdxCwS0Rc\nvj5i7iuS3gGcHBFf7OtYXiuSjgaOiIjxDdpE/X69GIh/M9aHjo4OT5vI4DzlcZ7yOE95nKc8zlMe\n56k+SUREtxlJA7WDNRK4CngpIj7Wx+H0K612sCQNoZjmuSAijn3tIrXXmorb778f+HJE3N2gnTtY\nZmZmZk28rjpYZrb+uINlZmZm1ly9DlbOGiwz2+Co5s+wYbl3o3/985z0PM5THucpj/OUx3nK4zzl\ncZ5al3ObdjPbwHiUyszMzKxnPEXQzLqQFP67YGZmZtaYpwiamZmZmZmtZ+5gmZn1gOek53Ge8jhP\neZynPM5THucpj/PUOnewzMzMzMzMeonXYJlZF16DZWZmZtac12CZmZmZmZmtZ+5gmVk3kmr+DH/L\n8L4Ord/wnPQ8zlMe5ymP85THecrjPOVxnlrn78Eys+4m1i5+duKzr2kYZmZmZgNNwxEsSSMkPVin\nrl3S6PUTVmOSdpL0gKTbS2Ur+iKWeiSNlTQ1o12/irssJ7ZmbSRNkHRG70XV9ZiSpkoak7HPdyU9\nImmhpFEZ7dsl7dSkvqXXv6RxkpZIujs9vibF8/l0HUc22f9VXaukSyQ9LGlsK3FbbW1tbX0dwoDg\nPOVxnvI4T3mcpzzOUx7nqXU5UwT742r3w4EZEfGxUll/jDMnpv4Yd8VAjx8ASR8DRkbEW4F/AX7Q\nR6GcApwaER+QNBzYOyJGRcTk3jpBo2uNiDOBc4F/7q3zmZmZmVlXOR2sjSVNS5+8Xydp0+oGksZL\nWpx+Lkxlg9In7oslLZL0+VQ+UtKd6dP1+yXt0oO4twaeqyp7vhTPiemcCyRdncqmSposabakRyuj\nBZK2kHRXimWRpENT+QhJS9N+yyVNl/ShtP9ySXundptLmiJpjqT5kg5JYfwV+EPGtTyfjjNc0sw0\nMrdY0n6pfLWk81O+7pP0plR+cOmcM0rlEyT9OLVdLunUVD42Hf8WScskXaHCyZL+vZS7UyVdUp3T\nZvHXy3uZpL+XdLukeSmWt0naStITpTabS3pK0uBa7WucfxVFrhs5DPgxQET8GhgiaViTfX4PrKn3\nOk6OlvTrlM/K83WSpMtK13OzpDGSvgbsD0yR9C3gDmCH9HzvX5Wn0ZI60nXfXoq1N651JcW/H3uV\nPCc9j/OUx3nK4zzlcZ7yOE95nKfW5XSwdgUuj4jdgNXAZ8qVkrYDLgTagFHAPqmTMgrYISJ2j4g9\ngMp0uenAZRExCngf8D89iHsw0FkuiIj3pHh2A74CtEXEnkD5DfHwiNgPOAS4KJX9BTg8IvYGDgQu\nKbUfCXw7InZNeTg27X9WOgfAOcDdEbFv2v9iSZtFxK8i4vQU016Srqx1IZW4geOAX0bEaGAPYGEq\n3wK4L+VrFvDJVD4rIvaNiL2Aa4EvlQ77Lorn433A11WMlgDsA3wWeDvwD8ARwHXAIZIGpzYnA/9R\nFVtdmXmvuBL414jYhyKH34+IPwILtG7a2sEpD2tqta9x/tMjYk6KYZKkg2ucdwfg6dLjZ1JZo+sa\nFxHPUP91DDA4Xf/pdF211G1ULyLOA+4HjouILwGHAo9GxOiIuLfSTtJGwGXAJ9J1TwW+0YvX2knx\n78fMzMzM1oOcm1w8VXlTB0wDPgdcWqrfB2iPiBcAJE0HxgDnA7tImgzcBsyQtCWwfUTcBBARzT6N\n70aSKDog0+o0ORC4PiJeTOdYVar7RSpbKunNlUMC31SxtqUT2L5UtyIilqTth4G70vaDwM5p+8MU\nHZSz0uNNgJ2A5ZWTRsR84FNNLm0exejGxsCNEbEolb8cEbel7fnAB9P2jpKuA7YDNgbKa6FuTLn9\nvaR7gHdTjKbNjYgnoVj/A+wfETeoWBN0sKRlwEYR8XCTWGtplHckbUHR4bs+PYekuKHo5B0DzASO\nBb7XpH1NETGhB3E38zhVr+NS3Q3p93xgRObxun1XQpVdgXcCd6brHgT8rrrRq7jWZ4C3SXpDRLxc\nt1V7aXtnoCfjzK9znpOex3nK4zzlcZ7yOE95nKc8ztM6HR0dWSN6OR2s6k/ja6256famMSJWSdoD\n+AjwaeAo4Au12nY5kPQZilGaAD4eEStLdYMo3vC+DNyaEXu18hvKShzHA28E9oyIThU3bdi0RvvO\n0uNO1uVOFKMNj/QgnrUiYlbq5B0EXCXpkoiYBrxSaramdN7LgIsj4tY0+lN+w11+jkT9dVKV8ikU\no0/L6DpC05sGAS+mEbpqNwEXSNoGGA3cA2zZoH2rngF2LD1+Syprqs7r+NRUXXk9lJ+Xv9F1ZLjb\nlNomBDyURkp7ouG1RsTjkpYCT0r6QN3O9AE9PLuZmZnZ61RbW1uXDuekSZNqtsuZIjhCUnka26yq\n+rnAGEnbpmlm44GZkoZSTKH6OfBVYHREvAQ8LekwAEmbSNqsfLCIuCIi9kxTp1ZW1XVGxM4UU62O\nqRPvPcBRkrZN59imTrtKB2sI8FzqXB1A15GIZqMNUKylOW3tDhl3qKsZTHHHuuciYgrwI4qORqMY\ntmLdyMZJVXWHpdwOBcZSjI5BMX1zROqoHgPcCxARcynelI8HrqkT39Iml9Aw7xGxGlghaVzpmLun\nuj9RPKeTgVuiULd9D9wEnJiOsS+wKiKeTY/vStNca6r1Oq7XNP1+Ahilwo4Uo4d1D1+jbDnwphQn\nkjZK0y9z1b3WVLY7xXjU9j0cqbTEc9LzOE95nKc8zlMe5ymP85THeWpdTgdrGfBZSUsoFsdX7koW\nAKkTdDbQASwA5kXEzRTrPjokLQB+ktpA8ebvNEmLgNlAs5sN1PIbYNtaFWlK3wUUnbwFrFtTVW8k\nbjpFx2MRcAKwtEabWvtXnEdxI5DFKm5pf251g0ZrsEragEWSHgCOBr7T5LyTgP+SNI/uN6NYTPF8\n3AecW+qo3g9cTjHd8bHUaai4DpgdEd1uzJE6GQ01yHvZCcApKm7Y8RDFOqSKaylGE39aKju+Qftu\n6q1LSlMsV0h6FPghaR1hmoI3EnihwWHrvY5rvp4iYjZFJ+thiudwfnWbOo8r+78CjAMukrSQ4t/U\ne1/ttZZsAzwREZ3V+5qZmZnZq6eIfn+X7W7SeqehEXF208YbGEkTgNURcWlV+VjgzIio2UmRdDNw\naUS016g7CNglIi5fHzH3FUnvAE6OiC/2dSyvFUlHA0dExPgGbaLeFw0zEQbi3wwzMzOz3iaJiOg2\nIylnBKs/ugHYT6UvGraekTRE0nLgT7U6VwARcevrrXMFEBEPb2Cdq0uAL1JMQTUzMzOz9WBAjmCZ\n2fojqe4fhWE7DGPlb1fWq96gdHR0+M5KGZynPM5THucpj/OUx3nK4zzVV28EK+cugma2gfEHL2Zm\nZmY94xEsM+tCUvjvgpmZmVljr7c1WGZmZmZmZv2OO1hmZj3g7wXJ4zzlcZ7yOE95nKc8zlMe56l1\n7mCZmZmZmZn1Eq/BMrMuvAbLzMzMrDmvwTIzMzMzM1vPfJt2M+tG6vZhjJmZbQA2pO879Pc75XGe\nWucOlpl1N7GvAxgAVgC79HUQA4DzlMd5yuM85XkVeXp24rO9GorZhshTBK3PSVrd1zFUSBot6UFJ\nU0plK/oolj0kfaz0+CRJEzL2u13Si5JuqiofL2mZpNPXR7wbHL/Jy+M85XGe8jhPeZynLB6VyeM8\ntc4dLOsP+tMdFU4AvhcRp5TKWopPUm/9uxoFfLyqLCeWb1FcR9cdI64BxgLuYJmZmZmtJ+5gWb8h\naZKkBZIekPRbSVMkjZC0VNJUScslTZf0IUmz0+O90777SLpP0nxJ90p6aw/D2Bp4rqrs+XSOsZJm\nSroljQRdUYp9taSLJS0A9k0jYR2S5qURpWGp3WmSHpa0UNJ/prLN07XOSfEfImlj4Fzg6JSPo4D/\nBV5qdgER0V6vXUQ8CwxpOSvWXZ+Maw5AzlMe5ymP85THecri73fK4zy1zmuwrN+IiAnABElDgP8G\nLktVI4FPRMQSSfcDx0bEfpIOBc4BjgCWAvtHRKekDwDfBMb1IIzBQGdVXO8pPdwHeDvwFHCHpCMj\n4gZgC+BXEfFFSRsBM4FDI+L3ko4GvgGcAvwbsHNEvCJpq3TMc4C7I+KUdO1zgbuArwN7RcRp1UFK\nOiTVTezBNfqDFTMzM7P1xB0s64+mAZdExEJJI4AVEbEk1T1M0fkAeBAYkba3Bn6cRq6CHry2U8fo\nHazr2NUyNyKeTO2vAfYHbgDWpN8AuwLvBO5UcTu+QcDvUt0i4D8l/QL4RSr7MHCIpLPS402AnRrF\nGhE3AzfnX10XL0gaGRGP1W3RXtreGc/nr8U5yeM85XGe8jhPeZynLF5blMd5WqejoyNrRM8dLOtX\nJE0EnoqIH5eKXy5td5Yed7LuNXwecE9EHJk6ZeUuQuXY5wMHARERo6vq3kIxcvRoRNzfIMTqNVCV\nx38ufTuvgIciYr8a+x8EjAEOBc6R9K7U/hMR8UhVTPs2iOPVmAwslPS5iLiqZosD1tOZzczMzAao\ntra2Lh3OSZMm1WznqULWHwjWTnv7IPD5WvVNDAGeSdsn12oQEV+NiD2rO1ep7rfADkUYamtwnnen\ndWGDgGOAWTViXA68qdJBkrSRpN1S3U4RMRM4G9iKYmrhHcDaaYCSRqXN1alNT4j6efsK8A91O1eW\nx2sc8jhPeZynPM5THucpi9cW5XGeWucOlvUHlZGf04HtgXnpxg4Tq+qrt8u+BVwoaT49fF2nEahH\ngW0bNLsfuJxiquJjEVGZ5rc2roh4hWL910WSFgILgPemKYjTJC0C5gOTI+KPFKNvG0taLOlBiptb\nQDEKt1vpJhdrpRthTKwVoKT/Bq4FDpT0lKQPVTXZJN3swszMzMx6mdbNajIzSd8DHoyIH9SoGwuc\nGRGHvvaR9Q5JbwYWRcR2DdqEv2jYzGwDNRH83tAsjyQiotuMIY9gmXX1Y+Dk8hcNv15IGg/MoBjt\nMzMzM7P1wCNYZtaFJP9RMDPbQA3bYRgrf7uyr8N4TXR0dPgOeRmcp/rqjWD5LoJm1o0/eGnO/+Hk\ncZ7yOE95nKc8zpNZ3/IIlpl1ISn8d8HMzMysMa/BMjMzMzMzW8/cwTIz6wF/L0ge5ymP85THecrj\nPOVxnvI4T61zB8vMzMzMzKyXeA2WmXXhNVhmZmZmzXkNlpmZmZmZ2Xrm27SbWTdStw9jzMzMzAac\nvvhuN08RNLMuJAUT+zqKAWAFsEtfBzEAOE95nKc8zlMe5ymP85RnoOdp4vr7fk9PETTrZySNkPRg\nC+2/JWmppIWSfiZpqxbPt6uk+yT9RdIZrUdsXQzk/2xeS85THucpj/OUx3nK4zzlcZ5a5g6WWd9q\n5SOVGcA7ImIU8Ajw5RbP9Xvgc8C3W9zPzMzMzDK5g2XWtzaWNE3SEv3/9u49yJK6POP490FQEQUV\nw1Jh5RZBRQV2uYUC4mAUbwHRRAyKAhpjFZRsAhgTk8IlmggYJQQvkYgrohhBUYHyhsAoKyLI7nJH\no1wES1YtAVeNROTNH6cHembOzPYZZvfMst9P1anp/p3uc9556uzsvNO/7k7OTfL4JLslWZ5kWZLr\nkvweoKq+XlUPNvtdCcwf5I2q6udVdQ3wwCx/D+un24ZdwDrCnLoxp27MqRtz6sacujGngdlgScP1\nTOADVbUTsAo4qqquqaoFVbUQ+Ar9jzi9EfjyWqxTkiRJHXgVQWm4flRVVzbLn6Q3he/9AEleAywA\nDmjvkOQfgd9V1TlrrKrLWsvb4vzrfsykG3Pqxpy6MaduzKkbc+rGnB4yOjrK6OjoarezwZKGa+I5\nWAWQ5LnACcB+7bv+JjkCeBnwgn4vluTdwMuBao6Azcz+M95TkiTpUWlkZISRkZGH1k888cS+2zlF\nUBqubZLs1Sy/FliaZDPgHOANVfWLsQ2TvAR4G3BQVd3f78Wq6p9a0wun442uHinnpHdjTt2YUzfm\n1I05dWNO3ZjTwDyCJQ3XLcDRSZYANwAfBg4Btgb+K707/o4djTodeCxwcXMj4Cur6qiub5RkHvBd\n4EnAg0kWATtV1a9m8xuSJElan3mjYUnjeKNhSZL0qLHYGw1LkiRJ0jrLI1iSxkniDwVJkvSoMG+r\nedx9191r5LWnOoLlOViSJvEPL6s3Ojo67kpC6s+cujGnbsypG3Pqxpy6MafBeQRL0jhJyp8LkiRJ\n0/McLEmSJElaw2ywJGkGutzJXebUlTl1Y07dmFM35tSNOQ3OBkuSJEmSZonnYEkax3OwJEmSVs9z\nsCRJkiRpDbPBkjRJEh8+fMzBx5bztxz2j4eh8lyQbsypG3PqxpwG532wJE22eNgFrANuA7YbdhHr\nAHPqpmNOKxevXOOlSJIeGc/BkuaYJLcBu1XVL5Israp9kzwfOL6qDnwEr3sm8GfAyqraeZrtygZL\nmqMWeyNwSZorEs/BktYVD/32VFX79hufoSXAix/ha0iSJGkaNljSkCR5S5LlSZYluTXJJWNPtbZZ\n1dplsyQXJbklyYcGfb+qWgrc8wjL1pjbhl3AOsKcujGnTjwXpBtz6sacujGnwdlgSUNSVR+pqgXA\nnsCdwPv6bdZa3gM4Gng28Iwkr1rzVUqSJGkQXuRCGr7/AC6tqi+tZrurquoOgCSfBvYFzl8jFV3W\nWt4WL1LQj5l0Y07dmFMnIyMjwy5hnWBO3ZhTN+b0sNHR0U5H9GywpCFKcgTw9Ko6qsPmE8/BGree\nZE/gI834CVV10YwL23/Ge0qSJD0qjYyMjGs4TzzxxL7bOUVQGpIkuwHHAYdNt1lrea8k2yTZAHgN\nsLS9YVVdVVULqmrhNM1VJrymZspzZroxp27MqRPPBenGnLoxp27MaXA2WNLwHA08BbisudDFGc14\n+8hUe/kq4APAjcAPq+rzg7xZknOAK4Adk/woyZEzL12SJEn9eB8sSeN4HyxpDlvsfbAkaa7wPliS\nJEmStIbZYEnSTHjOTDfm1I05deK5IN2YUzfm1I05Dc6rCEqabPGwC5DUz7yt5g27BEnSangOlqRx\nkpQ/FyRJkqbnOViSJEmStIbZYEnSDDgnvRtz6sacujGnbsypG3PqxpwGZ4MlSZIkSbPEc7AkjeM5\nWJIkSavnOViSJEmStIbZYEnSDDgnvRtz6sacujGnbsypG3PqxpwGZ4MlaZIkfR9bzt9y2KVJkiTN\naZ6DJWmcJDXljYYXgz8zJEmSPAdL6iTJg0ne21o/LskJQ6plm6aeo1tjpyd5wzDqkSRJ0urZYEnj\n3Q+8KslTh11I46fAoiQbDrsQjeec9G7MqRtz6sacujGnbsypG3ManA2WNN4DwBnAsROfaI4oXZJk\nRZKLk8xvxpckOS3Jt5L8IMmrWvscn+SqZp8OkRXqAAAKwklEQVR3zqCenwGXAEf0qWfXJN9uXvtz\nSTZrxi9LclKS7yS5Jck+zfgGSU5pxlckefMM6pEkSdI0bLCk8Qr4IPC6JE+a8NzpwJKq2hU4p1kf\ns2VV7QMcCJwMkORFwA5VtSewANg9yb4zqOdk4PgkE+f4ngW8rannBqDdwD2mqvYC/hYeOqPqTcC9\nzfiewF8n2WbAetQYGRkZdgnrBHPqxpy6MaduzKkbc+rGnAbntCNpgqr6VZKzgEXA/7ae2ht4ZbN8\nNk0j1fhCs+/NSbZoxg4AXpRkGRBgE2AHYOmA9dye5ErgdWNjSTYFNquqsdc6Czi3tdv5zddrgLEm\n6gDgeUle3axv2tRzx6Q3vay1vC2w3SAVS5IkPfqMjo52mjLpESypv9PoHfHZpDU23eXz7m8tp/X1\nPVW1sKoWVNWOVbWkvVOSg5MsT7IsycJpXv89wNsnjE26ak2fen7Pw39ICfDWppYFVfVHVfX1vnvv\n33rYXPXlnPRuzKkbc+rGnLoxp27MqRtzetjIyAiLFy9+6DEVGyxpvABU1T30jgi9qfXcFcChzfJh\nwOXTvQbwVeCNSTYBSPKHSf6gvWFVfaFpdhZW1bJp6vkecBNwULP+S+AXY+dXAa8HvtGhnqPGLpiR\nZIckG0+xjyRJkmbA+2BJLUl+WVWbNstbALcCJ1fVu5JsDSwBNqd38Ykjq+quJB8DLqqq8/u8xluB\nsYtJrAIOq6rbOtayDXBhVe3crO8MLAPeWFWfSLIL8J/Axk2dR1bVfUkuBY6vqmVJNgeurqrtm3O4\n3k3vPLHQu0LhwVW1asL7eh8sSZKk1ZjqPlg2WJLGscGSJElaPW80LEmzyDnp3ZhTN+bUjTl1Y07d\nmFM35jQ4GyxJky3u/5i31bwhFTT3rFixYtglrBPMqRtz6sacujGnbsypG3ManJdplzSJ0wBX7957\n7x12CesEc+rGnLoxp27MqRtz6sacBucRLEmSJEmaJTZYkjQDt99++7BLWCeYUzfm1I05dWNO3ZhT\nN+Y0OK8iKGmcJP5QkCRJ6sDLtEuSJEnSGuQUQUmSJEmaJTZYkiRJkjRLbLAkSZIkaZbYYEkCIMlL\nktyS5PtJ3j7seuaSJGcmWZnkutbYU5J8Lcn3knw1yWbDrHHYksxPcmmSG5Ncn+SYZtycWpI8Lsl3\nkixvsvrXZtyc+kiyQZJlSS5o1s2pjyS3J7m2+Vxd1YyZ1QRJNktyXpKbm39/e5nTeEl2bD5Hy5qv\n9yU5xpwGY4MliSQbAB8AXgw8Bzg0ybOGW9WcsoReNm1/D3y9qp4JXAr8w1qvam55ADi2qp4D7A0c\n3XyGzKmlqu4H9q+qBcDOwAuS7IM5TWURcFNr3Zz6exAYqaoFVbVnM2ZWk50GfKmqng3sAtyCOY1T\nVd9vPkcLgd2AXwOfx5wGYoMlCWBP4H+q6o6q+h3w38ArhlzTnFFVS4F7Jgy/AjirWT4LOHitFjXH\nVNXdVbWiWf4VcDMwH3OapKp+0yw+jt7/w/dgTpMkmQ+8DPhoa9ic+guTf6czq5YkmwL7VdUSgKp6\noKruw5ym80Lgh1V1J+Y0EBssSQBbAXe21u9qxjS1LapqJfSaC2CLIdczZyTZFtgVuBKYZ07jNdPe\nlgN3A6NVdRPm1M+pwNuA9v1kzKm/Ai5OcnWSv2rGzGq87YCfJ1nSTH87I8kTMKfpvAY4p1k2pwHY\nYEnS7PCmgkCSJwKfBRY1R7Im5rLe51RVDzZTBOcD+yUZwZzGSfJyYGVzVHTSTTxb1uucWvZppnS9\njN703P3wMzXRhsBC4INNVr+mN+3NnPpIshFwEHBeM2ROA7DBkgTwY2Dr1vr8ZkxTW5lkHkCSLYGf\nDrmeoUuyIb3m6uyq+mIzbE5TqKpfAl8CdsecJtoHOCjJrcCn6Z2rdjZwtzlNVlU/ab7+DPgCvWnf\nfqbGuwu4s6q+26x/jl7DZU79vRS4pqp+3qyb0wBssCQBXA08I8k2SR4L/CVwwZBrmmvC+L+kXwAc\n0SwfDnxx4g7roY8BN1XVaa0xc2pJ8rSxq28l2Rh4EbAccxqnqt5RVVtX1fb0fh5dWlWvBy7EnMZJ\n8oTmyDFJNgEOAK7Hz9Q4zfS2O5Ps2Az9KXAj5jSVQ+n9cWOMOQ0gVR7hk9S7TDu9KyxtAJxZVScN\nuaQ5I8k5wAiwObASeCe9vxKfBzwduAM4pKruHVaNw9ZcCe+b9H6xq+bxDuAq4FzMCYAkz6N3gvjY\nRQnOrqp/S/JUzKmvJM8Hjquqg8xpsiTb0bvKW9GbBvepqjrJrCZLsgu9i6ZsBNwKHAk8BnMapzk3\n7Q5g+6pa1Yz5eRqADZYkSZIkzRKnCEqSJEnSLLHBkiRJkqRZYoMlSZIkSbPEBkuSJEmSZokNliRJ\nkiTNEhssSZIkSZolNliSJGmdkuSiJJuuxffbJclLW+sHJvm7tfX+ktYt3gdLkiStFUkeU1W/H3Yd\n/UxXW5LDgd2r6q1ruSxJ6yCPYEmStJ5J8oYk1yZZnuSsZmybJJckWZHk4iTzm/ElST6U5NtJfpBk\nJMnHk9yU5GOt11yV5P1Jbmj237wZvyzJqUmuBo5J8rQkn03yneaxd7Pd85t6liW5JskmSbZM8o1m\n7Lok+zTb3pbkqc3ysUmub55f1PpebkpyRlPPV5I8rk8OS5J8OMmVwMlJ9khyRfP+S5PskGQj4J+B\nQ5o6Xp3k8CSnT5ebpPWXDZYkSeuRJDsB7wBGqmoBsKh56nRgSVXtCpzTrI95clXtDRwLXACcUlU7\nATsn2bnZZhPgqqp6LvBN4J2t/Teqqj2q6lTgNOD9VbUX8BfAmc02xwFHVdVCYD/gt8Brga80Y7sA\nK5ptq/leFgKHA3sAewNvTrJLs80zgNObeu4D/nyKSLaqqj+uquOBm4F9q2q3pv73VNXvgBOAz1TV\nwqo6r13DanKTtB7acNgFSJKkteoFwHlVdQ9AVd3bjO8NvLJZPhs4ubXPhc3X64GfVNVNzfqNwLbA\ndcCDwLnN+CeBz7X2/0xr+YXAs5OkWX9ikicA3wJOTfIp4Pyq+nFz1OvM5ijSF6vq2gnfy77A56vq\ntwBJzqfXnF0I3FZV1zfbXdPU2c95reUnA59IsgO9BqrL70kTczulwz6SHsU8giVJkuDhIzL93N98\nfbC1PLY+VRPSfr1ft5YD7FVVC5rH1lX1m6o6GXgTsDHwrSQ7VtXlwJ8APwY+nuSwAb6fdp2/n6bO\ndm3vAi6tqucBBwKP7/A+E3Pz5HZpPWeDJUnS+uVS4NWtc5ie0oxfARzaLB8GXD7F/plifAN6U/4A\nXgcsnWK7r/HwtETGpvQl2b6qbqyqU4CrgWcl2Rr4aVWdCXwUWDihhsuBg5M8Pskm9I4kXT5hm0Fs\nSq+ZAziyNb6qea6frrlJWk/YYEmStB5ppvf9C/CNJMuB9zVPHQMcmWQFvQZprAma7gjNxKNUeya5\nHhihd2GIfvsvAnZvLrJxA/CWZvxvmotVXAv8H/Dl5nWuTbIMOAT49/ZrVtVy4OP0GrJvA2e0phF2\nOZI0cZv3AicluYbxvyNdBuw0dpGLCftMlZuk9ZSXaZckSY9YklVV9aRh1yFJw+YRLEmSNBv8i60k\n4REsSZIkSZo1HsGSJEmSpFligyVJkiRJs8QGS5IkSZJmiQ2WJEmSJM0SGyxJkiRJmiX/DxZWyxUh\naxJjAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "ratios = compression_ratios() \n", - "labels = ['%s - %s' % (c, o)\n", - " for c, o in compression_configs]\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression ratio', fontsize=14, y=1.01)\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "ax.set_xlim(0, max(ratios)+3)\n", - "ax.set_ylim(0, len(ratios))\n", - "ax.set_xlabel('compression ratio')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();\n" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "@functools.lru_cache(maxsize=None)\n", - "def compression_decompression_times(repeat=3, number=1):\n", - " c = list()\n", - " d = list()\n", - " for compression, compression_opts in compression_configs:\n", - " \n", - " def compress():\n", - " zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", - " t = timeit.Timer(stmt=compress, globals=locals())\n", - " compress_times = t.repeat(repeat=repeat, number=number)\n", - " c.append(compress_times)\n", - " \n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", - " def decompress():\n", - " z[:]\n", - " \n", - " t = timeit.Timer(stmt=decompress, globals=locals())\n", - " decompress_times = t.repeat(repeat=repeat, number=number)\n", - " d.append(decompress_times)\n", - " \n", - " log(compression, compression_opts, compress_times, decompress_times)\n", - " \n", - " return c, d\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcVcWZ//HPlwaCoAhowqYsEjEuAWk3FAIoanSiSAwi\niMsoxjjmF3FLhnGjiRE1I05MoiZmEDUaI86QDOCggtI4GgnKqoCorIriwqKiEZV+fn+cunD69l2b\nPrdbeN6v13n1vafqVNV57hFP3ao6V2aGc84555xzzrmd16i+G+Ccc84555xzuwrvYDnnnHPOOedc\nHfEOlnPOOeecc87VEe9gOeecc84551wd8Q6Wc84555xzztUR72A555xzzjnnXB3xDpZzzrmvFEmz\nJP26vttRXyRdLWlVfbfDOedcZt7Bcs65BkzSNyTdKekNSZ9JelPS45JOre+21aPvA/9W342oZ/4j\nls4510A1ru8GOOecy0xSZ+BvwIfAvwKLib4YOxG4B+hSb43LQlITM/siyTrMbHOS5TvnnHM7w0ew\nnHOu4boHqAKOMLP/NrPXzWy5md0F9EhlkrS/pL9I+ihs/y2pYyx9jKSXJZ0vaZWkLZImSGoi6Sdh\nVOwDSf8erzzkHSPpj5I+lvSOpKvT8lRJuizUuQW4Oew/RNK00J53Jf1JUtvYcYdJminpw1D2Akn9\nQ1pjSb+WtC6M2q2RNC52bLUpgpJaSXpA0kZJn0qaIemQWPoFoY4TQhy2SHomdGCzkvQjScsl/UPS\n+5KmS2oU0iZKmirpOknrQ/n3SfpaWhk/C6OPn0paJGlEWnoHSX8Obd8YYvbNDGW8E2J5P7BnrnY7\n55yrX97Bcs65BkhSa+C7wG/N7B/p6Wb2UcgnYArwdaA/MADoAPwl7ZAuwCDge0RT7IYCjwOHE42I\njQRGSToj7bgrgSVAL+BGYJykwWl5bgxlHQbcJakdMJtoxO1IYCDQAvif2DF/At4O6T2BCuCzkDYK\nOCO08ZvA2cDymlHa7gHgKOD08PdT4Im0zs7XgNHAPwO9gVbA77IVKOkI4LfAGKA7cALwRFq2/kQd\n3ROAM4GTgdtiZdwMXAj8C3AwcAvwu9T0Tkl7ALOAT4DvhHa9DcyU1CzkGQrcBNwAlAOvAVfliIVz\nzrn6Zma++eabb741sI2oo1AFnJEn30nAF8D+sX1dgW3ACeH9GKKb+D1jeR4D3gUax/bNAn4de78K\neDKtvj8Az8beVwG/SsszFpiRtq91yHtkeP8hcF6Wc7oz/fi09O3tBA4M5faJpbcENgMXhfcXhHh8\nM5bnHOAfOer4PrAJaJElfSKwEdgjtm8E8A9gD6A5UUevT9px/wFMC68vApanpZcBHwBDwvvngd+l\n5ZkBrKzva9Q333zzzbfMm49gOedcw6QC830LeNvM3kztMLNVRCMhh8TyrTWzLbH37wKvmdmXafu+\nkVb+CxneH5K2b17a+yOA/mHa3MeSPgbWEj2YoVvIcwcwQdLTkq6VdFDs+PuBXpJek/RbSf8URuoy\n+RZR52lOaodFo3svp7Vzq5m9EXv/NtBUUqss5c4A1gCrJT0UplemT81bbNVHF18AmoZzPARoRjSS\nFo/DpcABIX85cEBa+mai0bVUnA6On1usHueccw2UP+TCOecapteJOiQHU31qXTHiT5pLf/CEZdlX\nVot6Pkl73wiYBlxNzY7iuwBmNlbSQ8CpwCnAGEk/MrP7zWxBWB/1XaLphQ8AC4lG64oRP/8vs6Rl\n/KLRzLZIKgf6hXpHE02PPNLM1ueoM3W+qXJPA95My/NFLM8CoimQ6XHamKMO55xzDZiPYDnnXANk\nZpuAJ4H/J6l5erqkvcPLZUAHSZ1iaQcQrcNaUgdN6Z32/thQZy7zgUOJRs1Wpm3bO2NmtsLMfmtm\npwETgItjaZ+Y2WQz+zHRurGB6Q9/CJYR/b/s2NQOSS2Bb7OT529mVWZWaWbXEa0Ta0HUYUr5dlhH\nlXIssBVYASwNr7tkiEGqwzWfaI3Zhgx5Uk9KXEbmz8A551wD5R0s55xruH5MNLLxkqQhkrpLOkjS\nvwCLAMxsJtF0uIclHSHpSOAh4CUzq6yDNvSW9K+Svinph8C5RNP7crkL2BuYJOloSV0lnSjp95Ja\nSGoWpv71l9RZ0jFAX0KHSNKVkoZJ+lboVI0gWrP1VnpFYdrfFOD3kvpK+nY4/w+BR/K0M+s0TEnf\nk3S5pMND53UE0dP7lsayNQbuU/TExJOIHmJxr5n9I0zHvB24XdKFkrpJ6hmeTJjqSD5MNKL3P5L6\nSeoS/t4uKTVF8E7gAkkXh8/g34Cj85yXc865euRTBJ1zroEys1Vhmtq1wK1AR2AD8ApwRSzrIODX\nwDPh/Qzg8jpqxh1ET8q7HtgC3GBm8ScU1vjBWzN7R1Ifog7HdKK1SGuBp4hGdUT00IuJQPtwTlOB\nn4YiPg6vvxnKXwCcamappwym1/nPwK+IplI2A54DTjGzrXnOLdeP9W4GBhM9va850ajUSDP7WyzP\nbKJO4SyiB1v8F9HvlaXicIOk9URTJe8GPiKa6vjLkP4PSf2IPttJRJ3St0N5m0KeSZK6Ar8I7ZgC\njA/n7JxzrgGSmf8YvHPOuZokrQJ+Y2b5Rqx2O5ImAvuY2aD6botzzrmGxacIOuecc84551wd8Q6W\nc865bHyKg3POOVcknyLonHPOOeecc3XER7Ccc84555xzro54B8s555xzzjnn6oh3sJxzzjnnnHOu\njvjvYDnnqpHkCzOdc8455wpgZjV+tN5HsJxzNZiZb3W8XXDBBfXehl1189h6XL9Km8fVY/tV2zyu\n2bdsvIPlnHMl0KVLl/puwi7LY5sMj2syPK7J8dgmw+NaPO9gOeecc84551wd8Q6Wc64GSXWytWvX\npb5PpcFo1apVfTdhl+WxTYbHNRke1+R4bJPhcS2eP+TCOZdB3Tzn4t13a6z73G0dfvjh9d2EXZbH\nNhke12R4XJPjsU2Gx7V4yrVAyzm3+4meIlhX/y4o5yJQ55xzzkXrnNasWVPfzXBZdO7cmdWrV9fY\nLwmrz6cISuos6eUsabMklZeqLWl1d5I0X9L02L5V9dGWbCT1lzSxgHw52y3p4/C3vaRJ4fUFkn5T\nm/IKrHOMpKvylVOMeJmSJkrqlyd/f0mbw+c8X9L1BdQxS1KnPOlFXbOShkhaKunp8P4RSQsljQrn\ncWae4ws513MkLQrbc5J6xNLGS1oiqX8x7XbOOedcstasWVPvT8TzLftWbOe31GuwGuJX2YOBp8zs\n1Ni+htjOQtqUL48BmNk7Zja0gOPqos6G4lkzKw/bL+qpDSOBi81soKR2wJFmdriZ3VmHdawE+plZ\nT+AXwL2pBDO7Gvg5cFEd1ucKVFlZWd9N2GV5bJPhcU2GxzU5HlvXUJS6g9VE0kPhW/xJkpqlZ5A0\nXNLisN0a9jUK394vDt/Mjwr7u0maEUYBXpLUtRZtagW8l7bv/Vh7zg91LpD0QNg3UdKdkp6X9EZq\n5EFSC0kzQ1sWSRoU9neWtCwct1zSw5JOCscvl3RkyNdc0gRJcyTNk3R6aMbnwIcFnMv7oZyxob3z\nJb0laULqdGLtiY8mdgojMssl3ZgpDvnqzBarOEkHSJou6UVJsyV1l9RS0upYnuaS1koqy5Q/Q/2b\nieKTT7GLgTYA27Jde8FQSX+X9KqkPqH91UYEJU2V1E/SDUBfYIKkXwJPAh3DZ9S3WkOlckmV4byn\nS2pb6Lma2RwzS10rc4COaVnWE13zzjnnnHMuAaV+yMVBwIVmNifc9F8G3JFKlNQeuBXoRXQzOSN0\nUt4COppZj5CvZTjkYWCcmU2R1JTadRjLgKr4DjM7JtRzCHAtcKyZbZIUvzFtZ2Z9JB0MTAEmA58B\ng81si6R9iG5wp4T83YAfmNlSSS8Bw8Lxg0IdZwLXAU+b2UhJewNzJc00sxeAF0KbjgB+ZGaXpJ9I\nqt1mNgYYE8p4Fkjd8MdHm+KvjwIODe1/UdI0M5ufKi+XAmOVcm9o+wpJRwP3hNGcBZL6m9ls4DTg\nCTPbJqlGfmBgWv1Xpl5LGgu8aGbTMtR9rKSFwDrgp2a2NM95DQlllpP52gMoM7NjJJ0KVAAnpQ7P\nUN5Nkk4ArjKzBZLuAqaaWXkod2T425jo8xpkZhskDQXGASOLONeUi4HpafuqiK75PCpirweEze2M\nAQMG1HcTdlke22R4XJPhcU2Ox9YlrbKysqCR0lJ3sNaa2Zzw+iHgJ8Q6WEQ3+rPMbCOApIeBfkRT\nnbpKuhP4X+ApSXsCHcxsCoCZFTKKUY0kAT1DWzI5AXjMzDaFOjbH0v4a9i2T9I1UkcAtitbJVAEd\nYmmrYjf1S4CZ4fXLQJfw+mTgdEk/De+bAp2A5alKzWweUKNzlcVDwB1mtjBPvhmpc5M0mWikZX6B\ndaTkihWSWgDHAY+FuAM0CX8nAWcDs4FhwF158mcUOpaZzAM6mdmnoTP0VyDTaFgmK0m79mJpk2Pl\ndy6wvHwjaQcBhxF9uSCiLw3eTs+U41yjSqTjgQuJPsu4dUB3SV8zs63ZS6jI00znnHPOud3LgAED\nqnXkx44dmzFffa/ByrR+p8YNaLhZ7wlUApcCf8iWt1pB0mWxqXLt0tIaAauAg4HHC2p9dfGb01Q7\nRgD7Ar3MrBfR1MNmGfJXxd5XsaOjK6JRrl5h62pmy6kFSRVEHdoaU/UyKORz2VmNgE1hDVTq/A4L\naVOAUyS1BsqBZ/LkL4qZbTGzT8Pr6URTVdsUeGy2aw92fIbb2PEZfkn1/65qTIPNQ8ArsfPumbY+\nMH8B0YMt7iUaBdsUTzOzlcAyYI2kQ4tsm9sJvjYgOR7bZHhck+FxTc6uFtt27brU2e9SZtoK/a3K\nrl278swzz2RMe+655zj44IPr5Hxz1ZPPZ599xumnn06rVq04++yzAbj++uv5+te/TocOHVizZg2N\nGjWiqqoqT0l1o9QdrM6SUtPOzgH+Ly19LtBPUhtJZcBwYHaYbldmZn8BrgfKzWwL8KakMwAkNZW0\nR7wwM7s73KSWm9n6tLQqM+sCvEQ0epLJM8BZqZvx0AHIJNXB2ht4z8yqwghC5wx5cnkSuHz7AVKt\nfnhA0dqtE4FR6UlZDjlJUqsQv8HA8xnKXJan2pyxMrOPgVWShsTK7BHSPiH6HO4Eplkka/5ixdYw\nEaYaKjZKOjNMTc12bI1rL1vW8Hc1cLgi+wNH52pahn3Lga9L6h3qbxymXxZE0ZMP/xs4z8xWZEjv\nAXQlGv1dUmi5zjnnnCutd99dQ/SddzJbVP7O6du3L8uW7bhF3JlO0s74r//6L95//302bdrEo48+\nyptvvskdd9zBq6++yttvRxOBdkyISl6pO1ivAj+WtJRoof3vwv7U0+3WA6OJRgsWEK0xmUq0UL9S\n0gLgjyEPwPnA5ZIWEXUKtt9IF+E1IONoRpjSdzNRJ28BMD7e3njW8Pdh4KjQnnOJRgrS82Q6PuUm\notGVxYoeQvHz9AySjghrk3K5EuhAtJ5qfhjNylXvXKLpbguJpvlVmx4YOhk55YhV3LnASEUPJXkF\nGBRLe5RoBPDPsX0jcuSvQdHDPU7LkDRE0iuhXb8imoaYmiLaDdiYo9hs117Ga8DMnifqZC0Jdc1L\nz5Plfer4L4AhwG2K1owtAI4t4lxvILqe7w6jt3PT0lsDq82sNF/huO18bUByPLbJ8Lgmw+OaHI/t\n7mvNmjV07959eydqzZo17LvvvuyzT95b2GTkeub77rABPwVure92NOQN+B7w/+q7HQmc16HA7fXd\njhKf81DgkTx5DKyONsw555xzuWX6/2Xd/v+49v+P7tKli91yyy12yCGHWJs2beyiiy6yrVu3mplZ\nZWWl7bfffmZmdt5551mjRo2sefPmttdee9m///u/1yjrgw8+sNNOO81atWplbdq0sX79+lWr5/bb\nb7cePXpYq1atbNiwYdvruf/++61v377VypJkK1assDFjxljTpk2tSZMmttdee9nvf/9722OPPays\nrMz22msvu/DCC2316tXWqFEj27Ztm5mZffjhhzZy5Ehr37697bfffnb99ddbVVVVUZ9PbH+Ne6lS\nj2A1RJOBPor90LCrzsweN7Pf1nc76pqZLTGza+q7HaUiaTxwDfCfBeSuk61t20Kf/bHr29XWBjQk\nHttkeFyT4XFNjsc2OX/605+YMWMGK1asYPny5fziFzt+UjQ1avTggw/SqVMnpk2bxkcffcQ119S8\nxRo/fjz7778/GzZs4L333mPcuHHV0h977DGeeuopVq1axaJFi7j//vtr1JP+vqKigmuvvZZhw4bx\n0UcfcckllzB9+nQ6dOjARx99xH333VejHRdccAFNmzZl5cqVLFiwgBkzZvCf/1nA7VGBdvsOlpmt\nMLPvWJEPEnDuq8bMrjazo83s6QLy1sm2fv3qEpyZc84555L0k5/8hA4dOtCqVSuuu+46Hnnkkax5\no4GdzJo0acI777zDqlWrKCsro0+fPtXSR40aRdu2bWnVqhWnn346CxdmfxB2rnpyeffdd5k+fTr/\n8R//QbNmzdh333254oorcp5TsXb7DpZzzpWCrw1Ijsc2GR7XZHhck+OxTc5+++23/XXnzp23Pzii\nWD/72c/o1q0bJ598Mt/85je57bbbqqW3bbvjcQrNmzdny5YttWtwDmvXruWLL76gffv2tGnThtat\nW3PppZfywQcf1Fkdpf4dLOecc84559xXyJtvvrn99Zo1a+jQoUPGfPme1NeiRQtuv/12br/9dpYu\nXcrxxx/P0UcfzfHHH5/3uE8//XT7+/Xr19f6qYD7778/zZo1Y8OGDYk9WdBHsJxzrgR8bUByPLbJ\n8Lgmw+OaHI9tcu666y7WrVvHxo0bGTduHMOGDcuYr127dqxcuTJrOY8//jgrVkS/IrPXXnvRuHFj\nysrK8tbfs2dPlixZwuLFi9m6dWvWH/jNJTWlsF27dpx88slceeWVfPzxx5gZK1eu5Nlnny26zGy8\ng+Wcc84551wDEz0oqm4eOrUzD6KSxDnnnLN9Wt+BBx7IddddlzHv6NGjuemmm2jTpg133HFHjfTX\nX3+dE088kb322os+ffrw4x//mH79+m2vJ5sDDzyQG2+8kYEDB9K9e3e+853vFNT29PNIefDBB/n8\n88855JBDaNOmDWeddRbr16/PcXSRddV2gZhzbtckyfzfBeecc650JNX6oQ0uedk+n7C/Rs/QR7Cc\nc84555xzro54B8s5V4Okndra7deuvk+hwfG1Acnx2CbD45oMj2tyPLauofCnCDrnaqrYucPfrXi3\nTprhnHPOOfdV42uwnHPVSLKd7WBRUfsfAHTOOed2N74Gq2FrsGuwJHWW9HKWtFmSykvVlrS6O0ma\nL2l6bN+q+mhLNpL6S5pYQL6c7Zb0cfjbXtKk8PoCSb+pTXkF1jlG0lX5yilGvExJEyX1y5O/v6TN\n4XOeL+n6AuqYJalTnvSirllJQyQtlfR0eP+IpIWSRoXzODPP8XnPNeT7taTXQ9mHx/aPl7REUv9i\n2u2cc8455wpX6jVYDbFrPhh4ysxOje1riO0spE358hiAmb1jZkMLOK4u6mwonjWz8rD9op7aMBK4\n2MwGSmoHHGlmh5vZnXVVgaRTgW5mdiDwI+B3qTQzuxr4OXBRXdXnCudrA5LjsU2GxzUZHtfkeGxd\nQ1HqDlYTSQ+Fb/EnSWqWnkHScEmLw3Zr2NcofHu/WNIiSaPC/m6SZoRv6l+S1LUWbWoFvJe27/1Y\ne84PdS6Q9EDYN1HSnZKel/RGauRBUgtJM0NbFkkaFPZ3lrQsHLdc0sOSTgrHL5d0ZMjXXNIESXMk\nzZN0emjG58CHBZzL+6GcsaG98yW9JWlC6nRi7YmPJnYKIzLLJd2YKQ756swWqzhJB0iaLulFSbMl\ndZfUUtLqWJ7mktZKKsuUP0P9m4nik0+xP9W9AdiW7doLhkr6u6RXJfUJ7a82IihpqqR+km4A+gIT\nJP0SeBLoGD6jvtUaKpVLqgznPV1S2yLO9QzgQQAz+zuwd+x4gPVE17xzzjnnnEtAqR9ycRBwoZnN\nCTf9lwHbf4VMUnvgVqAX0c3kjNBJeQvoaGY9Qr6W4ZCHgXFmNkVSU2rXYSwDquI7zOyYUM8hwLXA\nsWa2SVL8xrSdmfWRdDAwBZgMfAYMNrMtkvYB5oQ0gG7AD8xsqaSXgGHh+EGhjjOB64CnzWykpL2B\nuZJmmtkLwAuhTUcAPzKzS9JPJNVuMxsDjAllPAukbvjjo03x10cBh4b2vyhpmpnNT5WXS4GxSrk3\ntH2FpKOBe8JozgJJ/c1sNnAa8ISZbZNUIz8wMK3+K1OvJY0FXjSzaRnqPlbSQmAd8FMzW5rnvIaE\nMsvJfO0BlJnZMWHUqAI4KXV4hvJuknQCcJWZLZB0FzDVzMpDuSPD38ZEn9cgM9sgaSgwDhhZ4Ll2\nBN6MvV8X9qWeOlFFdM3nNiv2ugtQm68uXDUDBgyo7ybssjy2yfC4JsPjmhyPrUtaZWVlQSOlpR7B\nWmtmc8Lrh4i+0Y87CphlZhvNrIqoA9UPWAl0DaNG3wU+lrQn0MHMpgCY2edm9lkxjZEkoCdRBy6T\nE4DHzGxTqGNzLO2vYd8y4BupIoFbJC0CZgIdJKXSVsVu6peEdICXiW5hAU4GRktaAFQCTYFq64DM\nbF6mzlUWDwF3mNnCPPlmmNnmEL/J1PxcCpErVkhqARwHPBbO7/dAamRlEnB2eD0MeDRP/ozMbEyW\nztU8oJOZHQ78lvDZFajGtRdLmxwrv7CfQ88/knYQcBjRlwsLiDrdHdIz5TjXfNYB3SV9LWeu42Ob\nd66cc865kmu3X7ud/tmUXFtD+kmV448/nvvuu6/Wx1944YW0adOG3r17A3DPPffQrl07WrZsycaN\nG2nUqBErV67c6XYOGDCAioqK7Vs2pR7BSv9mP9P6nRo3oGa2WVJP4LvApcBZwBWZ8lYrSLoM+GGo\n55/MbH0srRHRzfNW4PEiziFla4Y2jwD2BXqZWZWiB0A0y5C/Kva+ih2fg4hGuV6vRXuqkVRB1KGt\nMVUvg0I+l53VCNiUGrFJMwW4WVJroBx4BtgzR/6imNmW2Ovpku6W1MbMNhZwbKZr7+KQnPoMt7Hj\nM/yS6l9c1JgGm4eAV8ysT5HHpawD9o+93y/sA8DMVkpaBqyRNNDMltSyHlekyspK/3Y1IR7bZHhc\nk+FxTc6uFtt317270z+bkrP8XeQnVZ577jmefvpp3n77bZo1a8aXX37J1Vdfzdy5cznssMOA6Gl/\npVTqEazOklLTzs4B/i8tfS7QT1IbSWXAcGB2mG5XZmZ/Aa4HysNN85uSzgCQ1FTSHvHCzOxuM+sV\nHmywPi2tysy6AC+xY/Qk3TPAWZLahDpaZ8mX+tT2Bt4LnavjqT6qUcgn+yRw+fYDYk+AK4aitVsn\nAqPSk7IccpKkViF+g4HnM5S5LE+1OWNlZh8DqyQNiZXZI6R9QvQ53AlMs0jW/MWKr0EKUw2V6lwp\nWjPXPsexNa69bFnD39XA4YrsDxydq2kZ9i0Hvi6pd6i/cZh+WagpwPnh2N7AZjPb/i9oiGFXotFf\n71w555xz7itt9erVdOnShWbNou+0169fz9atWzn44IO35yn1I/BL3cF6FfixpKVEC+1TTzhLPd1u\nPTCaaHrcAqI1JlOJ1pBUhilTfwx5ILqRvDxMyXuePFPIsngNaJMpIUzpu5mok7cAGB9vbzxr+Psw\ncFRoz7nAsgx5Mh2fchPRg0AWK3oIxc/TM0g6IqxNyuVKomllLyp6iEJFnnrnEk13W0g0zW9+Wp37\n5KkvV6zizgVGKnooySvAoFjao0QjgH+O7RuRI38Nih7ucVqGpCGSXgnt+hXRNMTUFNFuQK6RrGzX\nXsZrwMyeJ+pkLQl1zUvPk+V96vgvgCHAbYrWjC0Aji30XM3sf4k6pm8QTau8LC1La2B1mILrSmhX\n+la1ofHYJsPjmgyPa3I8tsno2rUr48ePp2fPnrRu3Zrhw4fz+ec7nrn1hz/8gQMPPJB9992XwYMH\n884772QsZ+vWrZx33nnsu+++tG7dmmOOOYb339/xPLXVq1fTt29fWrZsySmnnMLGjdHt2ezZs9l/\n//2rldW1a1eeeeYZ7rvvPn74wx/ywgsv0LJlS0aMGMG3vvUtAFq3bs2JJ55Yox2ff/4511xzDZ07\nd6Z9+/ZcdtllbN26tUa+nbHb/9CwpJ8C+5jZ6LyZd1OSvgd0NbPf1ndb6pKkQ4keunJNfbelVBQ9\nNOP7ZjY8Rx7/oWHnnHOuhJThh2wlJTpFsND/V3ft2pW2bdvyP//zP3zta1/juOOO44orruCSSy7h\nmWee4eyzz2bmzJkccsghXH311SxatIjZs2fXKOfee+/l8ccfZ9KkSTRt2pSFCxdy4IEHsueee3L8\n8cfz1ltv8cQTT7DffvtxyimncOyxxzJu3Dhmz57Neeedx9q1a6u1acKECZxwwgk88MADTJgwgWef\nfRaANWvWcMABB/Dll19unxrYqFEj3njjDQ444ACuvPJKVq1axQMPPEDjxo0555xzOOyww7j55puz\nxiDT5xPbX2NGUqnXYDVEk4H7JU1P+y0sF5hZbdaoNXhhitzu1LkaD3wH+Le8mSt2rq62HWszmLxr\n29XWBjQkHttkeFyT4XFNjsc2OaNGjaJt2+j/7aeffjoLF0bPT/vTn/7EyJEj6dmzJwC33HILrVu3\nZu3atXTqVO05bTRp0oQNGzbw2muv8e1vf5tevXpVS7/wwgvp1q0bAEOHDmXq1Kk71WYzy7j26g9/\n+AMvv/wye++9NwCjR49mxIgROTtYxdrtO1hmtoLoptO5XVr4oeFC8ybZFOecc859haQ6VwDNmzff\nPg3w7bff5ogjjtie1qJFC/bZZx/WrVtXo4N1/vnn89ZbbzFs2DA+/PBDRowYwbhx4ygri349pl27\ndtXq2LJlC3Xt/fff59NPP63W5qqqqjq/7yn1GiznnNst+beqyfHYJsPjmgyPa3I8tqXXoUMH1qxZ\ns/39J5/ThHwuAAAgAElEQVR8woYNG+jYsWONvGVlZdxwww0sWbKEv/3tb0ybNo0HH3wwbx0tWrTg\n008/3f5+27Zt1dZuFWPfffelefPmLFmyhI0bN7Jx40Y2b97Mhx9+WKvysvEOlnPOOeecc65ow4cP\nZ+LEiSxevJitW7dy7bXX0rt37xqjVxBN4XzllVeoqqpizz33pEmTJttHr3Lp3r07n332GdOnT+fL\nL7/kF7/4RbWHbGSSbURKEj/84Q+54oortnfS1q1bx1NPPVXA2RZut58i6JxzpeBrA5LjsU2GxzUZ\nHtfk7GqxbduxbaK/VVXoeulcvyE1cOBAbrrpJs4880w2b97Mcccdx5///OeMedevX8+ll17KunXr\n2HPPPRk2bBjnnntu3jpatmzJ3XffzciRI6mqquJnP/sZ++23X1Ftjr+/7bbbGDt2LL17994+2vYv\n//IvnHzyyTnLLMZu/xRB51x1ksz/Xah7u9r/+BsSj20yPK7J8Lgm56sc22xPqXMNQ7FPEfQOlnOu\nGu9gOeecc6XlHayGrdgOlq/Bcs4555xzzrk64h0s51wNknaJrV27LvUdyu0qKyvruwm7LI9tMjyu\nyfC4Jsdj6xoKf8iFcy6DXWOawrvvZl8065xzzjmXBF+D5ZyrRpLtKh0s8DntzjnnGj5fg9WwNdg1\nWJI6S3o5S9osSeWlakta3Z0kzZc0PbZvVX20JRtJ/SVNLCBfznZL+jj8bS9pUnh9gaTf1Ka8Ausc\nI+mqfOUUI16mpImS+uXJP0jSIkkLJL0k6YQC6pglqeaPOFRPL+qalTRE0lJJT4f3j0haKGlUOI8z\n8xxfyLmeE851kaTnJPWIpY2XtERS/2La7Zxzzrlkde7cud6n1fuWfevcuXNRn2ep12A1xK75YOAp\nMzs1tq8htrOQNuXLYwBm9o6ZDS3guLqosyGYaWY9zawXcCFwbz21YyRwsZkNlNQOONLMDjezO+uw\njpVAPzPrCfyC2Lma2dXAz4GL6rA+VyBfG5Acj20yPK7J8Lgm56sc29WrV2NmDXKbNWtWvbehvrfV\nq1cX9XmWuoPVRNJD4Vv8SZKapWeQNFzS4rDdGvY1Ct/eLw7fzI8K+7tJmqFoFOAlSV1r0aZWwHtp\n+96Pted87Rj9eCDsmyjpTknPS3pDYeRBUgtJM0NbFkkaFPZ3lrQsHLdc0sOSTgrHL5d0ZMjXXNIE\nSXMkzZN0emjG58CHBZzL+6GcsaG98yW9JWlC6nRi7YmPJnZSNCKzXNKNmeKQr85ssYqTdICk6ZJe\nlDRbUndJLSWtjuVpLmmtpLJM+TPUv5koPlmZ2aext3sCHxRwXhuAbdmuvWCopL9LelVSn9D+aiOC\nkqZK6ifpBqAvMEHSL4EngY7hM+qbFqdySZXhvKdLSv0SYCHnOsfMUtfKHKBjWpb1RNe8c84555xL\nQKkfcnEQcKGZzQk3/ZcBd6QSJbUHbgV6Ed1MzgidlLeAjmbWI+RrGQ55GBhnZlMkNaV2HcYyoCq+\nw8yOCfUcAlwLHGtmmyTFb0zbmVkfSQcDU4DJwGfAYDPbImkfohvcKSF/N+AHZrZU0kvAsHD8oFDH\nmcB1wNNmNlLS3sBcSTPN7AXghdCmI4Afmdkl6SeSareZjQHGhDKeBVI3/PHRpvjro4BDQ/tflDTN\nzOanysulwFil3BvavkLS0cA9Fo3mLJDU38xmA6cBT5jZNkk18gMD0+q/MvVa0ljgRTObll6xpMHA\nLUA74LsFnNeQcFw5ma89gDIzO0bSqUAFcFLq8Azl3aRoauJVZrZA0l3AVDMrD+WODH8bE31eg8xs\ng6ShwDhgZKHnGnMxMD1tXxXRNZ9HRez1gLC5nfFV/fHLrwKPbTI8rsnwuCbHY5sMj+sOlZWVBY2U\nlrqDtdbM5oTXDwE/IdbBIrrRn2VmGwEkPQz0I5rq1FXSncD/Ak9J2hPoYGZTAMws5zf7mUgS0DO0\nJZMTgMfMbFOoY3Ms7a9h3zJJ30gVCdyiaJ1MFdAhlrbKzJaG10uAmeH1y0CX8Ppk4HRJPw3vmwKd\ngOWpSs1sHlCjc5XFQ8AdZrYwT74ZqXOTNJlopGV+gXWk5IoVkloAxwGPhbgDNAl/JwFnA7OBYcBd\nefJnFDqW2dL+Cvw1jBb9kaizX4iVpF17sbTJ4e88oNDJufkea3cQcBjRlwsi+tLg7fRMuc4VQNLx\nRNMh+6YlrQO6S/qamW3NXkJFnmY655xzzu1eBgwYUK3DOXbs2Iz56nsNVqb1OzVuQMPNek+gErgU\n+EO2vNUKki6LTZVrl5bWCFgFHAw8XlDrq4vfnKbaMQLYF+hl0Xqf94BmGfJXxd5XsaOjK6JRrl5h\n62pmy6kFSRVEHdoaU/UyKORz2VmNgE1mVh47v8NC2hTgFEmtgXLgmTz5a83MngMahxHGQvJnu/Zg\nx2e4jR2f4ZdU/++qxjTYPAS8EjvvnlZ9fWD+AqIHW9xLNAq2KZ5mZiuBZcAaSYcW2Ta3E77KawMa\nOo9tMjyuyfC4JsdjmwyPa/FK3cHqLCk17ewc4P/S0ucC/SS1kVQGDAdmh5vhMjP7C3A9UG5mW4A3\nJZ0BIKmppD3ihZnZ3eEmtdzM1qelVZlZF+AlotGTTJ4BzpLUJtTROku+VAdrb+A9M6sKIwidM+TJ\n5Ung8u0HSIcXcEzNxkRrt04ERqUnZTnkJEmtQvwGA89nKHNZnmpzxsrMPgZWSRoSK7NHSPuE6HO4\nE5hmkaz5iyWpW+x1eahzQ3g/M0xNzXZsjWsvW9bwdzVwuCL7A0fnalqGfcuBr0vqHepvHKZfFkTR\nkw//GzjPzFZkSO8BdCUa/V1SaLnOOeecc64wpe5gvQr8WNJSooX2vwv7U0+3Ww+MJhotWEC0xmQq\n0UL9SkkLiKZ3jQ7HnQ9cLmkRUacg9TCAYrwGtMmUEKb03UzUyVsAjI+3N541/H0YOCq051yikYL0\nPJmOT7mJ6EEgixU9hOLn6RkkHRHWJuVyJdCBaD3V/DCalaveuUTT3RYSTfOrNj2wkNGeHLGKOxcY\nqeihJK8Ag2JpjxKNAP45tm9Ejvw1KHq4x2kZkn4g6RVJ84k6ccNCfhGtjduYo9hs117Ga8DMnifq\nZC0BfkU0fZBcx6Qd/wUwBLhN0kKi/w6OLeJcbyC6nu8Oo7dz09JbA6vNrKrmoS5JPoc9OR7bZHhc\nk+FxTY7HNhke1+Lt9j80HNY77WNmo/Nm3k1J+h7Q1cx+W99tqUthityFZnZNfbelVMJDM75vZsNz\n5PEfGnbOOeecy0P1/UPDDdhkoI9iPzTsqjOzx3e1zhWAmS3ZzTpX44FrgP+s77bsjnwOe3I8tsnw\nuCbD45ocj20yPK7FK/VTBBucsE7lO/XdDueSZtEPDReokCWDDV/btsX98rpzzjnn3M7a7acIOueq\nk2T+74JzzjnnXG4+RdA555xzzjnnEuYdLOecKwGfw54cj20yPK7J8Lgmx2ObDI9r8byD5Zxzzjnn\nnHN1xNdgOeeq8TVYzjnnnHP5+Ros55xzzjnnnEuYd7Ccc64EfA57cjy2yfC4JsPjmhyPbTI8rsXb\n7X8HyzlXk/TV/B2sth3bsv6t9fXdDOecc87txnwNlnOuGklGRX23opYqwP9Nc84551wp+Bos55xz\nzjnnnEtYyTpYkjpLejlL2ixJ5aVqS1rdnSTNlzQ9tm9VfbQlG0n9JU0sIF/Odkv6OPxtL2lSeH2B\npN/UprwC6xwj6ap85RQjXqakiZL65ck/SNIiSQskvSTphALqmCWpU570oq5ZSUMkLZX0dHj/iKSF\nkkaF8zgzz/F5zzXk+7Wk10PZh8f2j5e0RFL/Ytrt6obPYU+OxzYZHtdkeFyT47FNhse1eKVeg9UQ\n5+4MBp4ys9GxfQ2xnYW0KV8eAzCzd4ChBRxXF3U2BDPNbAqApG8DfwG+WQ/tGAlcbGZ/k9QOONLM\nDgztytuBLoSkU4FuZnagpGOA3wG9AczsaklzgYuA2XVRn3POOeecq67UUwSbSHoofIs/SVKz9AyS\nhktaHLZbw75G4dv7xWEkYlTY303SjPBN/UuSutaiTa2A99L2vR9rz/mx0Y8Hwr6Jku6U9LykN1Ij\nD5JaSJoZ2rJI0qCwv7OkZeG45ZIelnRSOH65pCNDvuaSJkiaI2mepNNDMz4HPizgXN4P5YwN7Z0v\n6S1JE1KnE2tPfDSxUxiRWS7pxkxxyFdntljFSTpA0nRJL0qaLam7pJaSVsfyNJe0VlJZpvwZ6t9M\nFJ+szOzT2Ns9gQ8KOK8NwLZs114wVNLfJb0qqU9of7URQUlTJfWTdAPQF5gg6ZfAk0DH8Bn1TYtT\nuaTKcN7TJbUt9FyBM4AHw3n/Hdg7djzAeqJr3pXYgAED6rsJuyyPbTI8rsnwuCbHY5sMj2vxSj2C\ndRBwoZnNCTf9lwF3pBIltQduBXoR3UzOCJ2Ut4COZtYj5GsZDnkYGGdmUyQ1pXYdxjKgKr7DzI4J\n9RwCXAsca2abJMVvTNuZWR9JBwNTgMnAZ8BgM9siaR9gTkgD6Ab8wMyWSnoJGBaOHxTqOBO4Dnja\nzEZK2huYK2mmmb0AvBDadATwIzO7JP1EUu02szHAmFDGs0Dqhj8+2hR/fRRwaGj/i5Kmmdn8VHm5\nFBirlHtD21dIOhq4x8wGhg5ZfzObDZwGPGFm2yTVyA8MTKv/ytRrSWOBF81sWnrFkgYDtwDtgO8W\ncF5DwnHlZL72AMrM7BhFo0YVwEmpwzOUd5OiqYlXmdkCSXcBU82sPJQ7MvxtTPR5DTKzDZKGAuOA\nkQWea0fgzdj7dWHfu+F9FdE1n9us2OsuQG2+unDOOeec24VUVlYWNGWy1CNYa81sTnj9ENE3+nFH\nAbPMbKOZVRF1oPoBK4GuYdTou8DHkvYEOqSmfpnZ52b2WTGNkSSgJ1EHLpMTgMfMbFOoY3Ms7a9h\n3zLgG6kigVskLQJmAh0kpdJWmdnS8HpJSAd4megWFuBkYLSkBUAl0BSotg7IzOZl6lxl8RBwh5kt\nzJNvhpltDvGbTM3PpRC5YoWkFsBxwGPh/H4PpEZWJgFnh9fDgEfz5M/IzMZk6lyFtL+a2cHA6cAf\nizivGtdeLG1y+DsP6Fxgefmef34QcBjRlwsLiDrdHdIz5TrXPNYB3SV9LWeu42Obd67qhM9hT47H\nNhke12R4XJPjsU2Gx3WHAQMGUFFRsX3Lpr7XYGVav1PjBtTMNkvqSTTycClwFnBFprzVCpIuA34Y\n6vknM1sfS2tEdPO8FXi8iHNI2ZqhzSOAfYFeZlal6AEQzTLkr4q9r2LH5yCiUa7Xa9GeaiRVEHVo\na0zVy6CQz2VnNQI2pUZs0kwBbpbUGigHniGaypctf62Z2XOSGkvax8w2FJA/07V3cUhOfYbb2PEZ\nfkn1Ly5qTIPNQ8ArZtanyONS1gH7x97vF/YBYGYrJS0D1kgaaGZLalmPc84555zLoNQjWJ0VLbwH\nOAf4v7T0uUA/SW0klQHDgdlhul2Zmf0FuB4oN7MtwJuSzgCQ1FTSHvHCzOxuM+tlZuXxzlVIqzKz\nLsBL7Bg9SfcMcJakNqGO1lnypTpYewPvhc7V8VQf1Sjkl1ufBC7ffkDsCXDFULR260RgVHpSlkNO\nktQqxG8w8HyGMpflqTZnrMzsY2CVpCGxMnuEtE+IPoc7gWkWyZq/WJK6xV6Xhzo3hPczw9TUbMfW\nuPayZQ1/VwOHK7I/cHSupmXYtxz4uqTeof7GYfploaYA54djewObzSw1PTAVw65Eo7/euSohn8Oe\nHI9tMjyuyfC4JsdjmwyPa/FK3cF6FfixpKVEC+1/F/annm63HhhNND1uAdEak6lEa0gqw5SpP4Y8\nEN1IXh6m5D1PnilkWbwGtMmUEKb03UzUyVsAjI+3N541/H0YOCq051xgWYY8mY5PuYnoQSCLFT2E\n4ufpGSQdEdYm5XIl0bSyF8NDFCry1DuXaLrbQqJpfvPT6twnT325YhV3LjBS0UNJXgEGxdIeJRoB\n/HNs34gc+WtQ9HCP0zIk/UDSK5LmE3XihoX8IlobtzFHsdmuvYzXgJk9T9TJWgL8imj6ILmOSTv+\nC2AIcJukhUT/HRxb6Lma2f8SdUzfIJpWeVlaltbA6jAF1znnnHPO1TGZfRWesp0cST8F9kl7TLuL\nkfQ9oKuZ/ba+21KXJB1K9NCVa+q7LaUSHprxfTMbniOPUVG6NtWpCmio/6ZVVlb6t4AJ8dgmw+Oa\nDI9rcjy2yfC4ZicJM6sxI6nUa7AaosnA/ZKmm9mp9d2YhsjMarNGrcELU+R2p87VeOA7wL/lzVyR\ndGuS0bZjbQaxnXPOOefqzm4/guWcq06S+b8LzjnnnHO5ZRvBKvUaLOecc84555zbZXkHyznnSsB/\nRyQ5HttkeFyT4XFNjsc2GR7X4nkHyznnnHPOOefqiK/Bcs5V42uwnHPOOefy8zVYzjnnnHPOOZcw\n72A551wJ+Bz25Hhsk+FxTYbHNTke22R4XIvnv4PlnKtBqjHavUtq27Yz69evru9mOOecc24X4muw\nnHPVSDLYXf5dEP5voHPOOedqw9dgOeecc84551zCStbBktRZ0stZ0mZJKi9VW9Lq7iRpvqTpsX2r\n6qMt2UjqL2liAflytlvSx+Fve0mTwusLJP2mNuUVWOcYSVflK6cY8TIlTZTUL0/+gyT9TdJnhbYl\nXJOd8qQXdc1KGiJpqaSnw/tHJC2UNCqcx5l5ji/kXM+RtChsz0nqEUsbL2mJpP7FtNvVDZ/DnhyP\nbTI8rsnwuCbHY5sMj2vxSr0GqyHOxRkMPGVmo2P7GmI7C2lTvjwGYGbvAEMLOK4u6mwINgA/Ifqs\n69NI4GIz+5ukdsCRZnYgRJ2nOqpjJdDPzD6UdApwL9AbwMyuljQXuAiYXUf1Oeecc865mFJPEWwi\n6aHwLf4kSc3SM0gaLmlx2G4N+xqFb+8Xh2/mR4X93STNCKMAL0nqWos2tQLeS9v3fqw954c6F0h6\nIOybKOlOSc9LeiM18iCphaSZoS2LJA0K+ztLWhaOWy7pYUknheOXSzoy5GsuaYKkOZLmSTo9NONz\n4MMCzuX9UM7Y0N75kt6SNCF1OrH2xEcTO4URmeWSbswUh3x1ZotVnKQDJE2X9KKk2ZK6S2opaXUs\nT3NJayWVZcqfof7NRPHJysw+MLN5wJcFnE/KBmBbtmsvGCrp75JeldQntL/aiKCkqZL6SboB6AtM\nkPRL4EmgY/iM+qbFqVxSZTjv6ZLaFnGuc8wsda3MATqmZVlPdM27EhswYEB9N2GX5bFNhsc1GR7X\n5Hhsk+FxLV6pR7AOAi40sznhpv8y4I5UoqT2wK1AL6KbyRmhk/IW0NHMeoR8LcMhDwPjzGyKpKbU\nrsNYBlTFd5jZMaGeQ4BrgWPNbJOk+I1pOzPrI+lgYAowGfgMGGxmWyTtQ3SDOyXk7wb8wMyWSnoJ\nGBaOHxTqOBO4DnjazEZK2huYK2mmmb0AvBDadATwIzO7JP1EUu02szHAmFDGs0Dqhj8+2hR/fRRw\naGj/i5Kmmdn8VHm5FBirlHtD21dIOhq4x8wGhg5ZfzObDZwGPGFm2yTVyA8MTKv/ytRrSWOBF81s\nWr52F3BeQ0KZ5WS+9gDKzOwYSacCFcBJqcMzlHeTpBOAq8xsgaS7gKlmVh7KHRn+Nib6vAaZ2QZJ\nQ4FxwMhanOvFwPS0fVVE13weFbHXA8LmnHPOObf7qqysLGjKZKlHsNaa2Zzw+iGib/TjjgJmmdlG\nM6si6kD1I5r21DWMGn0X+FjSnkAHM5sCYGafm9lnxTRGkoCeRB24TE4AHjOzTaGOzbG0v4Z9y4Bv\npIoEbpG0CJgJdJCUSltlZkvD6yUhHeBloEt4fTIwWtICoBJoClRbB2Rm8zJ1rrJ4CLjDzBbmyTfD\nzDaH+E2m5udSiFyxQlIL4DjgsXB+vwdSIzOTgLPD62HAo3nyZ2RmY+qic5WmxrUXS5sc/s4DOhdY\nXr7nnx8EHEb05cICok53h/RM+c5V0vHAhcC/piWtA7pL+lruZlTEtgF5muwK4XPYk+OxTYbHNRke\n1+R4bJPhcd1hwIABVFRUbN+yqe81WJnW79S4ATWzzZJ6At8FLgXOAq7IlLdaQdJlwA9DPf9kZutj\naY2Ibp63Ao8XcQ4pWzO0eQSwL9DLzKoUPQCiWYb8VbH3Vez4HEQ0yvV6LdpTjaQKog5tjal6GRTy\nueysRsCm1IhNminAzZJaA+XAM8CeOfKXTJZr7+KQnPoMt7HjM/yS6l9c1JgGm4eAV8ysT+1aDIoe\nbHEvcEqqw5tiZislLQPWSBpoZktqW49zzjnnnKup1CNYnSWlpp2dA/xfWvpcoJ+kNpLKgOHA7DDd\nrszM/gJcD5Sb2RbgTUlnAEhqKmmPeGFmdreZ9TKz8njnKqRVmVkX4CV2jJ6kewY4S1KbUEfrLPlS\nHay9gfdC5+p4qo9qFPLLrU8Cl28/QDq8gGNqNiZau3UiMCo9KcshJ0lqFeI3GHg+Q5nL8lSbM1Zm\n9jGwStKQWJk9QtonRJ/DncA0i2TNv5OqxUDRmrn2WTNnuPbylLsaOFyR/YGjC21LsBz4uqTeof7G\nYfplQRQ9+fC/gfPMbEWG9B5AV6LRX+9clZDPYU+OxzYZHtdkeFyT47FNhse1eKXuYL0K/FjSUqKF\n9r8L+1NPt1sPjCaaHreAaI3JVKKF+pVhytQfQx6A84HLw5S858kzhSyL14A2mRLClL6biTp5C4Dx\n8fbGs4a/DwNHhfacCyzLkCfT8Sk3ET0IZLGih1D8PD2DpCPC2qRcriSaVvZieIhCRZ565xJNd1tI\nNM1vflqd++SpL1es4s4FRip6KMkrwKBY2qNEI4B/ju0bkSN/DYoe7nFahv1tJb1JFJfrFD1EY88w\nRbQbsDFHsdmuvYzXgJk9T9TJWgL8imj6ILmOSTv+C2AIcJukhUT/HRxb6LkCNxBdz3eHtW1z09Jb\nA6vDFFznnHPOOVfHZPZVeMp2ciT9FNgn7THtLkbS94CuZvbb+m5LXZJ0KNFDV66p77aUSnhoxvfN\nbHiOPPbVePp+XRCl+jewsrLSvwVMiMc2GR7XZHhck+OxTYbHNTtJmFmNGUmlXoPVEE0G7pc03cxO\nre/GNERmVps1ag1emCK3O3WuxgPfAf6tgNxJN6dBaNu20GeTOOecc84VZrcfwXLOVSfJ/N8F55xz\nzrncso1glXoNlnPOOeecc87tsryD5ZxzJeC/I5Icj20yPK7J8Lgmx2ObDI9r8byD5ZxzzjnnnHN1\nxNdgOeeq8TVYzjnnnHP5+Ros55xzzjnnnEuYd7Ccc64EfA57cjy2yfC4JsPjmhyPbTI8rsXz38Fy\nztUg7R6/gxXXtmNb1r+1vr6b4ZxzzrmvOF+D5ZyrRpJRUd+tqAcV4P8eOuecc65QvgbLOeecc845\n5xJWsg6WpM6SXs6SNktSeanaklZ3J0nzJU2P7VtVH23JRlJ/SRMLyJez3ZI+Dn/bS5oUXl8g6Te1\nKa/AOsdIuipfOcWIlylpoqR+efIfJOlvkj4rtC3hmuyUJ72oa1bSEElLJT0d3j8iaaGkUeE8zsxz\nfN5zDfl+Len1UPbhsf3jJS2R1L+Ydru64XPYk+OxTYbHNRke1+R4bJPhcS1eqddgNcT5N4OBp8xs\ndGxfQ2xnIW3Kl8cAzOwdYGgBx9VFnQ3BBuAnRJ91fRoJXGxmf5PUDjjSzA6EqPNUFxVIOhXoZmYH\nSjoG+B3QG8DMrpY0F7gImF0X9TnnnHPOuepKPUWwiaSHwrf4kyQ1S88gabikxWG7NexrFL69Xyxp\nkaRRYX83STPCN/UvSepaiza1At5L2/d+rD3nhzoXSHog7Jso6U5Jz0t6IzXyIKmFpJmhLYskDQr7\nO0taFo5bLulhSSeF45dLOjLkay5pgqQ5kuZJOj0043PgwwLO5f1QztjQ3vmS3pI0IXU6sfbERxM7\nhRGZ5ZJuzBSHfHVmi1WcpAMkTZf0oqTZkrpLailpdSxPc0lrJZVlyp+h/s1E8cnKzD4ws3nAlwWc\nT8oGYFu2ay8YKunvkl6V1Ce0v9qIoKSpkvpJugHoC0yQ9EvgSaBj+Iz6psWpXFJlOO/pktoWeq7A\nGcCD4bz/DuwdOx5gPdE170pswIAB9d2EXZbHNhke12R4XJPjsU2Gx7V4pR7BOgi40MzmhJv+y4A7\nUomS2gO3Ar2IbiZnhE7KW0BHM+sR8rUMhzwMjDOzKZKaUrsOYxlQFd9hZseEeg4BrgWONbNNkuI3\npu3MrI+kg4EpwGTgM2CwmW2RtA8wJ6QBdAN+YGZLJb0EDAvHDwp1nAlcBzxtZiMl7Q3MlTTTzF4A\nXghtOgL4kZldkn4iqXab2RhgTCjjWSB1wx8fbYq/Pgo4NLT/RUnTzGx+qrxcCoxVyr2h7SskHQ3c\nY2YDQ4esv5nNBk4DnjCzbZJq5AcGptV/Zeq1pLHAi2Y2LV+7CzivIaHMcjJfewBlZnaMolGjCuCk\n1OEZyrtJ0gnAVWa2QNJdwFQzKw/ljgx/GxN9XoPMbIOkocA4YGSB59oReDP2fl3Y9254X0V0zec2\nK/a6C1Cbry6cc84553YhlZWVBU2ZLPUI1tr/z97dB1lV3fn+f39AGUVRwQcezIiG0hhujIpRY3S0\n9RcxMdF4URIfMliJo0lhqZiYKn6Z3HSjRIzGRJ0px2gcEgVrkHtJBjWKQLodRRGE5kFB1AkasYJ6\no6hMER3le/9Y3wO7T5/d54He3QjfV1VX77P32nt993fvA2edtdZuM1voy9NI3+hnHQu0mtlbZraZ\n1IA6GfgjcIj3Gp0BvCdpT2CYmc0GMLMPzOyv9QQjScCRpAZcJacBM83sba9jQ2bb73zdauCA0iGB\nKZKWA/OAYZJK29aa2Spffs63A6wkfYQFGA1MlNQOtAH9gA7zgMxsSaXGVY5pwM/NbFmVcnPNbIPn\nbxUygQ8AACAASURBVBadr0stusoVkvYAvgDM9PP7JVDqWbkf+IYvnw/MqFK+IjNr7o7GVZlO915m\n2yz/vQQYXuPxqj3//FPAZ0hfLrSTGt3Dygttw7m+Bhwm6W+6LHVq5icaV90ixrAXJ3JbjMhrMSKv\nxYncFiPyulVTUxMtLS1bfvL09hysSvN3On0ANbMNko4EzgC+C4wFJlQq2+FA0njgUq/nTDNbn9nW\nh/Th+X3goTrOoeT9CjFfBOwHHG1mm5UeALFbhfKbM683s/U6iNTL9WID8XQgqYXUoO00VK+CWq7L\ntuoDvF3qsSkzG/iJpIHAKOAPwJ5dlO8xOffeP/jm0jX8iK3X8EM6fnHRaRhsFQKeNbMTG4uY14C/\nzbz+hK8DwMz+KGk18Iqk/8/MnmuwnhBCCCGEUEFP92ANV5p4D3Ah8HjZ9kXAyZIGSeoLXAA85sPt\n+prZb4EfAaPMbCPwqqSvAUjqJ2n37MHM7HYzO9rMRmUbV75ts5kdDDzD1t6Tcn8Axkoa5HUMzClX\namDtDbzhjatT6dirUctfbp0DXLllh8wT4OqhNHfri8BV5Ztydjld0j6ev3OABRWOubpKtV3mysze\nA9ZKOi9zzM/6tv8iXYdbgQctyS2/jTrkQGnO3NDcwhXuvSrHfRk4SsnfAsfVGotbA+wv6fNe/y4+\n/LJWs4Fxvu/ngQ1mVhoeWMrhIaTe32hc9aAYw16cyG0xIq/FiLwWJ3JbjMhr/Xq6gfU8cLmkVaSJ\n9nf4+tLT7dYDE0nD49pJc0weIM0hafMhU/d6GUgfJK/0IXkLqDKELMcLwKBKG3xI309Ijbx24OZs\nvNmi/ns6cKzH801gdYUylfYvuY70IJAVSg+huLa8gKRjfG5SV64mDStb7A9RaKlS7yLScLdlpGF+\nS8vq3LdKfV3lKuubwCVKDyV5Fjg7s20GqQfw3zLrLuqifCdKD/f4aoX1gyW9SsrLPyo9RGNPHyI6\nAniri8Pm3XsV7wEzW0BqZD0H3EIaPkhX+5Tt/9/AecBPJS0jvQ9OqPVczez3pIbpS6RhlePLigwE\nXvYhuCGEEEIIoZvJ7OPwlO3iSPoBsG/ZY9pDhqSvAIeY2T/3dizdSdL/ID105ZrejqWn+EMz/qeZ\nXdBFGaOl52LabrRAkf8etrW1xbeABYncFiPyWozIa3Eit8WIvOaThJl1GpHU03OwtkezgF9LetjM\nvtzbwWyPzKyROWrbPR8itzM1rm4G/g74/6sWbik6mu3P4AMb6QAPIYQQQuhop+/BCiF0JMni34UQ\nQgghhK7l9WD19BysEEIIIYQQQthhRQMrhBB6QPwdkeJEbosReS1G5LU4kdtiRF7rFw2sEEIIIYQQ\nQugmMQcrhNBBzMEKIYQQQqgu5mCFEEIIIYQQQsGigRVCCD0gxrAXJ3JbjMhrMSKvxYncFiPyWr9o\nYIUQOpFU2M+QIQf39umFEEIIIRQm5mCFEDqQZFDkvwsi/t0JIYQQwsddzMEKIYQQQgghhIIV2sCS\nNFzSypxtrZJGFVl/HkkHSVoq6eHMurW9EUseSadImlpDubrjlnSVpN1ytl0s6TZfbpY0rsqxLpbU\nXKXMe/XGWE3pmH6PtdZQvlXS85La/drvV6V8l/n37Q/UGXM/SXO9/rGSTpL0rL8+PO+9ktm/6rlK\n2l3Sg5JWS1op6frMtsO8vhn1xB26R4xhL07kthiR12JEXosTuS1G5LV+PdGDtT2OBToHeNTMvpxZ\ntz3GWUtMjcQ9AejfwH6NxlBEbi1nuSsXmNnRZjbKzP5vnXU0sr3cKMC8/pnARcD1ZjYK2FTj8Wop\nc5OZfRo4GjhJ0hmkil8ws88AR0g6pM7YQwghhBBCDXqigbWrpGmSVkm6v1LPiaQLJK3wnxt8XR9J\nU33dcklX+foR3guwTNIzDX5Q3Ad4o2zdm5l4xnmd7ZJ+4+umSrpV0gJJL0ka4+v3kDTPY1ku6Wxf\nP9x7EaZKWiNpuqTTff81kj7n5fpLulvSQklLJJ3lYXwAvFPDubzpx5mU6Z1Z58fs770Z7Z7HsZKu\nAIYBrZLm+77f8pgWAidmjr2R9MG/K5u8HJIOkDTLr027pM+XUprJ7TWSFnmZZl83RdL4TJlmSd/L\nK1/mI+CtGvIE9d3vW/LvvVWl3C6RtIeXGSBppl/nezPxr5U0yJeP8d6z/YF7gWP9OJcBXweuy+7r\n+/SRdKOkp/28L631XM1sk5k95ssfAkuBT5QVe530Hgg9qKmpqbdD2GFFbosReS1G5LU4kdtiRF7r\nt0sP1PEp4FtmtlDS3cB44OeljZKGAjeQvm3fAMz1Rso64EAz+6yX28t3mU761n+2pH401kjsC2zO\nrjCz472ekcAPgRPM7G1J2Q+iQ8zsREmfBmYDs4C/AueY2UZJ+wILfRvACOBcM1sl6RngfN//bK9j\nDPCPwHwzu0TS3sAiSfPM7CngKY/pGOA7ZnZZ+YmU4jazZqDZj/EfwD8DXwJeM7Ov+nEGmNl7kq4G\nmvz8hgAtpPy/C7SRPpRjZjdXS6SZ3Z95eRvQZmZjJAnYs1TM6z8dONTMjvPtsyWdBMwAbgFu9/Jf\nB0bnlTezJ/BGm5mtA87z4w8F7iqdbwW/lvTfwCwzm1zlvLbkH/g+MN7MnpLUn3TNAY4CRgLrgQWS\nvmBmT9K5l8nM7E1J/wB838xKjfATgAfMbJak4ZnylwAbzOx4v8cXSHrUzF6p41zxe/csUm6zNpPe\nA11oySw3+U8IIYQQws6rra2tpiGTPdGD9SczW+jL04CTyrYfC7Sa2VtmtpnUgDoZ+CNwiPcanQG8\nJ2lPYJiZzQYwsw/M7K/UwT+oH0lqwFVyGjDTzN72OjZktv3O160GDigdEpgiaTkwDxgmqbRtrZmt\n8uXnfDvASuBgXx4NTJTUTmrc9AMOygZkZksqNa5yTANuNrN2r+d07yE6ycxKc6HE1l6l49ma/w9J\njZ1GnQb8i8dsmfpKRns8S0mNuE+RGlDLgP0lDZH0WeAtM3str3xe5Wb25y4aHBea2RHA3wF/J+mb\ndZzXAuAX3vs30O9TgEVepwHL2HpNOz1Npk6jgXF+TzwNDKLsvKucK5L6AvcBt5jZy2Wb15HeA11o\nyfw01R55yBVj2IsTuS1G5LUYkdfiRG6LEXndqqmpiZaWli0/eXqiB6vTt/kVynT6QGpmGyQdCZwB\nfBcYS5o71OWHVx9qdqnXc6aZrc9s60NquL0PPFTHOZS8XyHmi4D9gKPNbLPSQyd2q1B+c+b1Zrbm\nXqRerhcbiKcDSS2kBu09AGb2otKDRM4EJnvPWKWem21tEJRUmx8kYIqZ3VVh20zSNR7C1kZeV+Xr\nmv9kZn/23/8l6T7gOFJjtJZ9fyrpQeArpN6k0b4pe30/Yus1/ZCtX15UfJhIFQKuMLO5Dexbciew\nxsz+qcK2XwJzJB1nZt/ZhjpCCCGEEEKZnujBGi7peF++EHi8bPsi4GRJg/xb9wuAx3y4XV8z+y3w\nI2CUmW0EXpX0NdjyVLbdswczs9szDzJYX7Zts5kdDDwDfCMn3j8AYzNzaAbmlCs1SvYG3vDG1anA\n8AplujIHuHLLDtJRNezTOZg0d+uLwFWZdUOBTWZ2H3AT6SELkIYCloZcPk3K/0BJu5IaOZWOf3l2\nnlSO+aQhoKV5RANKu/vvOcC3S3OYJA3zuUkA9wPnA+eSGlt55fcrO2ZVkvr6/YSf41eBZ/31Oco8\naS9n/0+a2XNmdiOwGDi8SpVrgWN8+dxa48yYA4yXtIvXf2j5fV4l3snAXmZ2dU6Ra4BLonHVs2IM\ne3Eit8WIvBYj8lqcyG0xIq/164kG1vPA5ZJWkSbW3+HrDcAbQRNJw+PagcVm9gBwINDmw6Tu9TIA\n44ArfUjeAmBwAzG9QBp21YkP6fsJqZHXDpTmIeX1xE0nPbhgOfBNYHWFMpX2L7mO9CCQFUqP6b62\nvIA/KOHOLs4H4GrSwysW+0MUWoAjSHO62oEfA6Xeq7uARyTN9/xPIs0dexxY1enIyeHAX6rEMAE4\nVdIKUiN2pK8vXeu5pGFrT3mZmfg8Lc/7AGCdmb3eRfkB2WNmSRrqPU3l/obUY7OMNNRwnecA0jy5\nag8TmaD0yPPlpIdfPFyhTDaea4HbJC0i9WblybsnfkW6Dkv9nriDst7mvHOVdCBpft9IbX0wx7fL\nig0EXuoirhBCCCGE0CCl6SM7F0k/APY1s4lVCwcAJM0Gxvg8rR2GpHuAq82sWuNxh+BzEFcA55nZ\nmpwyVuxfLRA74787bW1t8S1gQSK3xYi8FiPyWpzIbTEir/kkYWadRlX1RA/W9mgWcKIyf2g4dM3M\nzt7RGlcAZjZuJ2pcHUbqJW4n9eKGEEIIIYRutlP2YIUQ8qUerOIMHjyc9etfLrKKEEIIIYTC5fVg\n9cRTBEMIHzPxxUsIIYQQQmN21iGCIYTQo+LviBQncluMyGsxIq/FidwWI/Jav2hghRBCCCGEEEI3\niTlYIYQOJFn8uxBCCCGE0LV4imAIIYQQQgghFCwaWCGE0ANiDHtxIrfFiLwWI/JanMhtMSKv9YsG\nVgghhBBCCCF0k5iDFULooLv+DtbgAwezft367jhUCCGEEMJ2J28OVjSwQggdSDJauuFALfH3tEII\nIYSw44qHXIQQQi+KMezFidwWI/JajMhrcSK3xYi81q/QBpak4ZJW5mxrlTSqyPrzSDpI0lJJD2fW\nre2NWPJIOkXS1BrK1R23pKsk7Zaz7WJJt/lys6RxVY51saTmKmXeqzfGakrH9HustYbyrZKel9Tu\n136/KuW7zL9vf6DOmPtJmuv1j5V0kqRn/fXhee+VzP61nusoSSskvSDplsz6w7y+GfXEHUIIIYQQ\natcTPVjb4xihc4BHzezLmXXbY5y1xNRI3BOA/g3s12gMReTWcpa7coGZHW1mo8zs/9ZZRyPby40C\nzOufCVwEXG9mo4BNNR6vljL/AlxiZocBh0k6g1TxC2b2GeAISYfUGXvYRk1NTb0dwg4rcluMyGsx\nIq/FidwWI/Jav55oYO0qaZqkVZLur9RzIukC/8Z9haQbfF0fSVN93XJJV/n6Ed4LsEzSMw1+UNwH\neKNs3ZuZeMZ5ne2SfuPrpkq6VdICSS9JGuPr95A0z2NZLulsXz9c0mrfb42k6ZJO9/3XSPqcl+sv\n6W5JCyUtkXSWh/EB8E4N5/KmH2dSpndmnR+zv6QHff0K7zW5AhgGtEqa7/t+y2NaCJyYOfZG0gf/\nrmzyckg6QNIsvzbtkj5fSmkmt9dIWuRlmn3dFEnjM2WaJX0vr3yZj4C3asgT1He/b8m/91aVcrtE\n0h5eZoCkmX6d783Ev1bSIF8+xnvP9gfuBY7141wGfB24Lruv79NH0o2SnvbzvrTWc5U0BBhgZot9\n1T2kLxSyXie9B0IIIYQQQjfbpQfq+BTwLTNbKOluYDzw89JGSUOBG4CjgQ3AXG+krAMONLPPerm9\nfJfppG/9Z0vqR2ONxL7A5uwKMzve6xkJ/BA4wczelpT9IDrEzE6U9GlgNjAL+CtwjpltlLQvsNC3\nAYwAzjWzVZKeAc73/c/2OsYA/wjMN7NLJO0NLJI0z8yeAp7ymI4BvmNml5WfSCluM2sGmv0Y/wH8\nM/Al4DUz+6ofZ4CZvSfpaqDJz28I0ELK/7tAG7DUj3lztUSa2f2Zl7cBbWY2RpKAPUvFvP7TgUPN\n7DjfPlvSScAM4Bbgdi//dWB0XnkzewJvtJnZOuA8P/5Q4K7S+Vbwa0n/Dcwys8lVzmtL/oHvA+PN\n7ClJ/UnXHOAoYCSwHlgg6Qtm9iSde5nMzN6U9A/A982s1Ag/AXjAzGZJGp4pfwmwwcyO93t8gaRH\nzeyVGs71QNJ7p2Sdr8vaTHoP5MsORDwYiP6ubdbW1hbfAhYkcluMyGsxIq/FidwWI/K6VVtbW01z\n0nqigfUnM1voy9OAK8g0sIBjgVYzewtA0nTgZGAycIikW4HfA49K2hMYZmazAczsg3qD8Q/qR3os\nlZwGzDSzt72ODZltv/N1qyUdUDokMEXSyaQPrsMy29aa2Spffg6Y58srSR9bAUYDZ0n6gb/uBxwE\nrClVamZLgE6NqxzTgJvNrF3SRuBnkqYAD3nDpBRzqVfpeDrmfwZwaI11lTsN+HuP2YDyuVejgdMl\nLfX69yA1oKZK2t8bewcAb5nZa5ImVCoPPEEFZvZnIK9xdaGZ/dl7n2ZJ+qaZ5d0D5RYAv/B7c5bH\nBrDI60TSMtI1fZJMj12DRpOG8Y3113uRzvuVUoEq51rNOtJ74JncEqc2eOQQQgghhB1UU1NTh8bm\npEmTKpbriQZWp2/zK5Tp9IHUzDZIOhI4A/guMJY0d6jLD68+1OxSr+dMM1uf2dYH+CPwPvBQHedQ\n8n6FmC8C9gOONrPNSg+d2K1C+c2Z15vZmnuRerlebCCeDiS1kBq09wCY2YtKDxI5E5jsPWOVem62\ntUFQUm1+kIApZnZXhW0zSdd4CKlHq1r5uuY/lRpCZvZfku4DjiO/kV2+708lPQh8hdSbNNo3Za/v\nR2y9ph+ytWe14sNEqhBwhZnNbWDf14C/zbz+hK/L+iUwR9JxZvadBuoIDYhv/4oTuS1G5LUYkdfi\nRG6LEXmtX0/MwRou6XhfvhB4vGz7IuBkSYMk9QUuAB7z4XZ9zey3wI+AUWa2EXhV0tdgy1PZds8e\nzMxuzzzIYH3Zts1mdjDpm/tv5MT7B2BsZg7NwJxypUbJ3sAb3rg6FRheoUxX5gBXbtlBOqqGfToH\nk+ZufRG4KrNuKLDJzO4DbiI9ZAHSUMDSkMunSfkfKGlXUiOn0vEvz86TyjGfNAS0NI9oQGl3/z0H\n+HZpDpOkYT43CeB+4HzgXFJjK6/8fmXHrEpSX7+f8HP8KvCsvz5H0vVV9v+kmT1nZjcCi4HDq1S5\nFjjGl8+tNc6MOcB4Sbt4/YeW3+d5/J5/R1JpWOU44N/Lil1DeghGNK5CCCGEELpZTzSwngcul7SK\nNLH+Dl9vsOUD4UTS3J92YLGZPUCaN9ImqZ30cICJvt844EpJy0lDtwY3ENMLwKBKG3xI309Ijbx2\noDQPKa8nbjrpwQXLgW8CqyuUqbR/yXWkB4GsUHpM97XlBZQelHBnF+cDcDXp4RWLlR6i0AIcQZrT\n1Q78mDTsEuAu4BFJ8z3/k0hzxx4HVnU6cnI48JcqMUwATpW0gtSIHenrS9d6LnAf8JSXmYnP0/K8\nDwDWmdnrXZQfkD1mlqSh3tNU7m9IPTbLSPPL1nkOIM2Tq/YwkQmSVvo1/gB4uEKZbDzXArdJWkTq\nzcqTd0/8inQdlvo9cQdlvc1dnCvA5cDdpPv8RTN7pGz7QOClLuIKBYi/I1KcyG0xIq/FiLwWJ3Jb\njMhr/ZSmyuxcfL7TvmY2sWrhAICk2cAYM+uqwfCxI+ke4Gozq9Z43CF4r9YK4DwzW5NTxmjphspa\nYGf89yVPTBIuTuS2GJHXYkReixO5LUbkNZ8kzKzTqKqdtYE1Avg1sLHsb2GFsMOSdBhpKOYK4GLL\nefNL6pZ/FAYfOJj169ZXLxhCCCGE8DEUDawQQk0k5bW9QgghhBCCy2tg9cQcrBBC2OnFGPbiRG6L\nEXktRuS1OJHbYkRe6xcNrBBCCCGEEELoJjFEMITQQQwRDCGEEEKoLoYIhhBCCCGEEELBooEVQgg9\nIMawFydyW4zIazEir8WJ3BYj8lq/aGCFEEIIIYQQQjeJOVghhA666+9g7QgGDx7O+vUv93YYIYQQ\nQtgOxd/BCiHUJDWw4t+FRMS/kSGEEEKoJB5yEUIIvSjGsBcncluMyGsxIq/FidwWI/Jav0IbWJKG\nS1qZs61V0qgi688j6SBJSyU9nFm3tjdiySPpFElTayhXd9ySrpK0W862iyXd5svNksZVOdbFkpqr\nlHmv3hirKR3T77HWGso/LKld0rOSfiVplyrlu8y/b3+gzpj7SZrr995YSSd5PEslHZ73XsnsX/Vc\nJe0u6UFJqyWtlHR9ZtthXt+MeuIOIYQQQgi164kerO1xfM05wKNm9uXMuu0xzlpiaiTuCUD/BvZr\nNIYicms5y3nGmtnRZvYZYB/gG3XW0cj2cqMAM7NRZjYTuAi43sxGAZtqPF4tZW4ys08DRwMnSTqD\nVPELfv5HSDqkztjDNmpqaurtEHZYkdtiRF6LEXktTuS2GJHX+vVEA2tXSdMkrZJ0f6WeE0kXSFrh\nPzf4uj6Spvq65ZKu8vUjvBdgmaRnGvyguA/wRtm6NzPxjPM62yX9xtdNlXSrpAWSXpI0xtfvIWme\nx7Jc0tm+frj3IkyVtEbSdEmn+/5rJH3Oy/WXdLekhZKWSDrLw/gAeKeGc3nTjzPJ410qaZ0fs7/3\nZrR7HsdKugIYBrRKmu/7fstjWgicmDn2RtIH/65s8nJIOkDSLL827ZI+X0ppJrfXSFrkZZp93RRJ\n4zNlmiV9L698mY+At6olycxKMe4K9AP+UmWXLfn33qpSbpdI2sPLDJA006/zvZn410oa5MvHKPXW\n7g/cCxzrx7kM+DpwXXZf36ePpBslPe3nfWmt52pmm8zsMV/+EFgKfKKs2Ouk90AIIYQQQuhuZlbY\nDzAc2Ax83l/fDXzPl1tJ3+gPBV4BBpEafPOBs33bo5lj7eW/FwJn+3I/YLcG4poETMjZNhJ4Hhjo\nr/fx31OBGb78aeBFX+4L7OnL+2bWDyd9SB/pr58B7vbls4FZvvwT4EJf3htYA+xeFtMxwJ01ntve\nwHJS78UY4JeZbQP89x8z5zckk/9dgCeA2xq83v8GXOnLytT3rv8+vRSPb38AOAk4CmjLHOc54MC8\n8v76vQr1DwUe7CK+R0gNqxl1ntds4ARf7u/36SnA216ngCeBL2TyOyhz7f7gy6cAszPHnQqMydwv\nK3z5UuCHmXt8MTC8nnMt3bvAfwIHl62fD3yui/0MmjM/rQa2k/5g3aW1tbXbjhU6itwWI/JajMhr\ncSK3xYi8btXa2mrNzc1bfvxzQqfPUl3OQ+kmfzKzhb48DbgC+Hlm+7FAq5m9BSBpOnAyMBk4RNKt\nwO+BRyXtCQwzs9mkM/qg3mAkCTjSY6nkNGCmmb3tdWzIbPudr1st6YDSIYEpkk4mNSaHZbatNbNV\nvvwcMM+XVwIH+/Jo4CxJP/DX/YCDSA0tvL4lwGU1nuI04GYza5e0EfiZpCnAQ2b2RCbmUq/S8XTM\n/wzg0BrrKnca8PceswHlc69GA6dLWur17wEcamZTJe0vaQhwAPCWmb0maUKl8qRGYCdm9mfgq3nB\nmdmXJPUD7pc0zszuqfG8FgC/8HtzlscGsMjrRNIy0jV9kkyPXYNGk4bxjfXXe5HO+5XMuXR5rpL6\nAvcBt5jZy2Wb15HeA8/kh9BSf9QhhBBCCDuwpqamDkMmJ02aVLFcTzSwyueMVJpD0ukDqZltkHQk\ncAbwXWAsae5Qlx9efajZpV7PmWa2PrOtD6l34X3goTrOoeT9CjFfBOwHHG1mm5UeOrFbhfKbM683\nszX3As41sxcbiKcDSS2kBu09AGb2otKDRM4EJkuaZ2aTK+26rXW7Ste2vJ4pZnZXhW0zSdd4CDCj\nhvLV6qocoNkHkv4PcBxQUwPLzH4q6UHgK8ACSaN9U/b6fsTWa/ohW4ffVnyYSBUCrjCzuQ3sW3In\nsMbM/qnCtl8CcyQdZ2bf2YY6Qh1iDHtxIrfFiLwWI/JanMhtMSKv9euJOVjDJR3vyxcCj5dtXwSc\nLGmQf+t+AfCYpH2Bvmb2W+BHwChL82helfQ12PJUtt2zBzOz2y09zGBUtnHl2zab2cGkb+7zHnLw\nB2BsZg7NwJxypUbJ3sAb3rg6lTTUq7xMV+YAV27ZQTqqhn06B5Pmbn0RuCqzbiiwyczuA24iDbsE\neJfUKwLwNCn/A31+0lgqkHR5dp5UjvnAeC/fR9KA0u7+ew7w7dIcJknDfG4SwP3A+cC5pMZWXvn9\nyo5ZldI8uSG+vAupobTMX5+jzJP2cvb/pJk9Z2Y3kobrHV6lyrWkoYH4+dRrDjDeY0XSoeX3eZV4\nJ5OG1F6dU+Qa4JJoXIUQQgghdL+eaGA9D1wuaRVpTsgdvj5N9kiNoIlAG9AOLDazB0hzcNoktZMe\nDjDR9xsHXClpOWno1uAGYnqBNOeoEx/S9xNSI68duDkbb7ao/55OenDBcuCbwOoKZSrtX3Id6UEg\nK5Qe031teQF/UMKdXZwPwNWkh1cs9ocotABHAIv8PH5MGnYJcBfwiKT5nv9JpLltjwOrOh05OZzq\nD4aYAJwqaQWpETvS15eu9VzSsLWnvMxMYE/ftgoYAKwzs9e7KD8ge8wsSUO9p6ncHsBsH8a3BHgV\n+FffNoLqDxOZoPTI8+WkeXUPVyiTjeda4DZJi0i9WXny7olfka7DUr8n7qCstznvXCUdCPwQGJl5\nMMe3y4oNBF7qIq5QgPg7IsWJ3BYj8lqMyGtxIrfFiLzWT2mqzM7F5zvta2YTqxYOAEiaTXogQ1cN\nho8dSfcAV5tZtcbjDsHnIK4AzjOzNTllbPv8qwW9QXTXv5FtbW0xzKIgkdtiRF6LEXktTuS2GJHX\nfJIws06jqnbWBtYI4NfARuv4t7BC2GFJOow0FHMFcLHlvPlTAysADB48nPXrX+7tMEIIIYSwHYoG\nVgihJpLy2l4hhBBCCMHlNbB6Yg5WCCHs9GIMe3Eit8WIvBYj8lqcyG0xIq/1iwZWCCGEEEIIIXST\nGCIYQugghgiGEEIIIVQXQwRDCCGEEEIIoWDRwAohhB4QY9iLE7ktRuS1GJHX4kRuixF5rV80sEII\nIYQQQgihm8QcrBBCBzvj38EafOBg1q9b39thhBBCCOFjJP4OVgihJpKMlt6Oooe1QPxbGEIIIYR6\nxEMuQgihF8UY9uJEbosReS1G5LU4kdtiRF7rV2gDS9JwSStztrVKGlVk/XkkHSRpqaSHM+vW0G7j\nMgAAIABJREFU9kYseSSdImlqDeXqjlvSVZJ2y9l2saTbfLlZ0rgqx7pYUnOVMu/VG2M1pWP6PdZa\nQ/mHJbVLelbSryTtUqV8l/n37Q/UGXM/SXP93hsr6SSPZ6mkw/PeK5n9az3XUZJWSHpB0i2Z9Yd5\nfTPqiTuEEEIIIdSuJ3qwtsdxN+cAj5rZlzPrtsc4a4mpkbgnAP0b2K/RGIrIreUs5xlrZkeb2WeA\nfYBv1FlHI9vLjQLMzEaZ2UzgIuB6MxsFbKrxeLWU+RfgEjM7DDhM0hmkil/w8z9C0iF1xh62UVNT\nU2+HsMOK3BYj8lqMyGtxIrfFiLzWrycaWLtKmiZplaT7K/WcSLrAv3FfIekGX9dH0lRft1zSVb5+\nhPcCLJP0TIMfFPcB3ihb92YmnnFeZ7uk3/i6qZJulbRA0kuSxvj6PSTN81iWSzrb1w+XtNr3WyNp\nuqTTff81kj7n5fpLulvSQklLJJ3lYXwAvFPDubzpx5nk8S6VtM6P2V/Sg75+hfeaXAEMA1olzfd9\nv+UxLQROzBx7I+mDf1c2eTkkHSBpll+bdkmfL6U0k9trJC3yMs2+boqk8ZkyzZK+l1e+zEfAW9WS\nZGalGHcF+gF/qbLLlvx7b1Upt0sk7eFlBkia6df53kz8ayUN8uVjlHpr9wfuBY7141wGfB24Lruv\n79NH0o2SnvbzvrTWc5U0BBhgZot91T2kLxSyXie9B0IIIYQQQjfrcphUN/kU8C0zWyjpbmA88PPS\nRklDgRuAo4ENwFxvpKwDDjSzz3q5vXyX6aRv/WdL6kdjjcS+wObsCjM73usZCfwQOMHM3paU/SA6\nxMxOlPRpYDYwC/grcI6ZbZS0L7DQtwGMAM41s1WSngHO9/3P9jrGAP8IzDezSyTtDSySNM/MngKe\n8piOAb5jZpeVn0gpbjNrBpr9GP8B/DPwJeA1M/uqH2eAmb0n6Wqgyc9vCNBCyv+7QBuw1I95c7VE\nmtn9mZe3AW1mNkaSgD1Lxbz+04FDzew43z5b0knADOAW4HYv/3VgdF55M3sCb7SZ2TrgPD/+UOCu\n0vmWk/QIcCwwz8weqXJeW/IPfB8Yb2ZPSepPuuYARwEjgfXAAklfMLMn6dzLZGb2pqR/AL5vZqVG\n+AnAA2Y2S9LwTPlLgA1mdrzf4wskPWpmr9RwrgeS3jsl63xd1mbSeyBfdiDiwUD0d22ztra2+Baw\nIJHbYkReixF5LU7kthiR163a2tpqmpPWEw2sP5nZQl+eBlxBpoFF+sDbamZvAUiaDpwMTAYOkXQr\n8HvgUUl7AsPMbDaAmX1QbzD+Qf1Ij6WS04CZZva217Ehs+13vm61pANKhwSmSDqZ9MF1WGbbWjNb\n5cvPAfN8eSXpYyvAaOAsST/w1/2Ag4A1pUrNbAnQqXGVYxpws5m1S9oI/EzSFOAhb5iUYi71Kh1P\nx/zPAA6tsa5ypwF/7zEbUD73ajRwuqSlXv8epAbUVEn7e2PvAOAtM3tN0oRK5YEnqMDM/gxUbFz5\n9i95g+V+SePM7J4az2sB8Au/N2d5bACLvE4kLSNd0yfJ9Ng1aDRpGN9Yf70X6bxfyZxLl+daxTrS\ne+CZ3BKnNnjkEEIIIYQdVFNTU4fG5qRJkyqW64kGVqdv8yuU6fSB1Mw2SDoSOAP4LjCWNHeoyw+v\nPtTsUq/nTDNbn9nWB/gj8D7wUB3nUPJ+hZgvAvYDjjazzUoPnditQvnNmdeb2Zp7kXq5Xmwgng4k\ntZAatPcAmNmLSg8SOROY7D1jkyvtuq11u2rzgwRMMbO7KmybSbrGQ0g9WtXKNzSvy8w+kPR/gONI\nw+dq2eenkh4EvkLqTRrtm7LX9yO2XtMP2dqzWvFhIlUIuMLM5jaw72vA32Zef8LXZf0SmCPpODP7\nTgN1hAbEt3/FidwWI/JajMhrcSK3xYi81q8n5mANl3S8L18IPF62fRFwsqRBkvoCFwCP+XC7vmb2\nW+BHwCifR/OqpK/Blqey7Z49mJnd7g8zGJVtXPm2zWZ2MOmb+7yHHPwBGJuZQzMwp1ypUbI38IY3\nrk4Fhlco05U5wJVbdpCOqmGfzsGkuVtfBK7KrBsKbDKz+4CbSA9ZgDQUsDTk8mlS/gf6/KSxVCDp\n8uw8qRzzSUNAS/OIBpR2999zgG+X5jBJGuZzkwDuB84HziU1tvLK71d2zKqU5skN8eVdSA2lZf76\nHEnXV9n/k2b2nJndCCwGDq9S5VrgGF8+t9Y4M+YA4z1WJB1afp/n8Xv+HUmlYZXjgH8vK3YN6SEY\n0bgKIYQQQuhmPdHAeh64XNIq0sT6O3y9wZYPhBNJc3/agcVm9gBp3kibpHbSwwEm+n7jgCslLScN\n3RrcQEwvAIMqbfAhfT8hNfLagdI8pLyeuOmkBxcsB74JrK5QptL+JdeRHgSyQukx3deWF1B6UMKd\nXZwPwNWkh1csVnqIQgtwBGlOVzvwY9KwS4C7gEckzff8TyLNHXscWNXpyMnhVH8wxATgVEkrSI3Y\nkb6+dK3nAvcBT3mZmfg8Lc/7AGCdmb3eRfkB2WNmSRrqPU3l9iDN31oGLAFeBf7Vt42g+sNEJkha\n6df4A+DhCmWy8VwL3CZpEak3K0/ePfEr0nVY6vfEHZT1NndxrgCXA3eT7vMXK8w3Gwi81EVcoQDx\nd0SKE7ktRuS1GJHX4kRuixF5rZ/SVJmdi8932tfMJlYtHACQNBsYY2ZdNRg+diTdA1xtZtUajzsE\n79VaAZxnZmtyyhgtPRpW72uBov8tjEnCxYncFiPyWozIa3Eit8WIvOaThJl1GlW1szawRgC/BjaW\n/S2sEHZYkg4jDcVcAVxsOW9+STvdPwqDDxzM+nXrqxcMIYQQQnDRwAoh1ERSXtsrhBBCCCG4vAZW\nT8zBCiGEnV6MYS9O5LYYkddiRF6LE7ktRuS1ftHACiGEEEIIIYRuEkMEQwgdxBDBEEIIIYTqYohg\nCCGEEEIIIRQsGlghhNADYgx7cSK3xYi8FiPyWpzIbTEir/WLBlYIIYQQQgghdJOYgxVC6GBn/DtY\nYccxePBw1q9/ubfDCCGEsBOIv4MVQqhJamDFvwvh40rE/2shhBB6QjzkIoQQelVbbwewA2vr7QB2\nSDHvohiR1+JEbosRea1foQ0sScMlrczZ1ippVJH155F0kKSlkh7OrFvbG7HkkXSKpKk1lKs7bklX\nSdotZ9vFkm7z5WZJ46oc62JJzVXKvFdvjNWUjun3WGsN5SdL+pOkd2s8fpf59+0P1B4xSOonaa7f\ne2MlnSTpWX99eN57JbN/1XOVtLukByWtlrRS0vWZbYd5fTPqiTuEEEIIIdSuJ3qwtsexGucAj5rZ\nlzPrtsc4a4mpkbgnAP0b2K/RGIrIreUs55kNHLsNdTSyvdwowMxslJnNBC4CrjezUcCmGo9XS5mb\nzOzTwNHASZLOIFX8gpl9BjhC0iF1xh62WVNvB7ADa+rtAHZITU1NvR3CDinyWpzIbTEir/XriQbW\nrpKmSVol6f5KPSeSLpC0wn9u8HV9JE31dcslXeXrR3gvwDJJzzT4QXEf4I2ydW9m4hnndbZL+o2v\nmyrpVkkLJL0kaYyv30PSPI9luaSzff1w70WYKmmNpOmSTvf910j6nJfrL+luSQslLZF0lofxAfBO\nDefyph9nkse7VNI6P2Z/781o9zyOlXQFMAxolTTf9/2Wx7QQODFz7I2kD/5d2eTlkHSApFl+bdol\nfb6U0kxur5G0yMs0+7opksZnyjRL+l5e+TIfAW9VS5KZLTKz16uVy9iSf++tKuV2iaQ9vMwASTP9\nOt+biX+tpEG+fIxSb+3+wL3AsX6cy4CvA9dl9/V9+ki6UdLTft6X1nquZrbJzB7z5Q+BpcAnyoq9\nTnoPhBBCCCGE7mZmhf0Aw4HNwOf99d3A93y5lfSN/lDgFWAQqcE3Hzjbtz2aOdZe/nshcLYv9wN2\nayCuScCEnG0jgeeBgf56H/89FZjhy58GXvTlvsCevrxvZv1w0of0kf76GeBuXz4bmOXLPwEu9OW9\ngTXA7mUxHQPcWeO57Q0sJ/VejAF+mdk2wH//MXN+QzL53wV4Aritwev9b8CVvqxMfe/679NL8fj2\nB4CTgKOAtsxxngMOzCvvr9+rUP9Q4MEqMb7bwHnNBk7w5f5+n54CvO11CngS+EImv4My1+4PvnwK\nMDtz3KnAmMz9ssKXLwV+mLnHFwPDGzjXfYD/BA4uWz8f+FwX+xk0Z35aDSx+tvkn8tgzucVC92ht\nbe3tEHZIkdfiRG6LEXndqrW11Zqbm7f8+P85lP/sQvH+ZGYLfXkacAXw88z2Y4FWM3sLQNJ04GRg\nMnCIpFuB3wOPStoTGGZms0ln9EG9wUgScKTHUslpwEwze9vr2JDZ9jtft1rSAaVDAlMknUxqTA7L\nbFtrZqt8+Tlgni+vBA725dHAWZJ+4K/7AQeRGlp4fUuAy2o8xWnAzWbWLmkj8DNJU4CHzOyJTMyl\nXqXj6Zj/GcChNdZV7jTg7z1mA8rnXo0GTpe01OvfAzjUzKZK2l/SEOAA4C0ze03ShErlSY3ATszs\nz8BXG4y9KwuAX/i9OctjA1jkdSJpGemaPkmmx65Bo0nD+Mb6671I5/1KqUC1c5XUF7gPuMXMXi7b\nvI70HngmP4SW+qMOIYQQQtiBNTU1dRgyOWnSpIrleqKBZVVeQ4UPpGa2QdKRwBnAd4GxpLlDXX54\n9aFml3o9Z5rZ+sy2PqTehfeBh+o4h5L3K8R8EbAfcLSZbVZ66MRuFcpvzrzezNbcCzjXzF5sIJ4O\nJLWQGrT3AJjZi0oPEjkTmCxpnplNrrTrttbtKl3b8nqmmNldFbbNJF3jIcCMGspXq6vbmNlPJT0I\nfAVYIGm0b8pe34/Yek0/ZOvw24oPE6lCwBVmNreReN2dwBoz+6cK234JzJF0nJl9ZxvqCHVp6u0A\ndmBNvR3ADinmXRQj8lqcyG0xIq/164k5WMMlHe/LFwKPl21fBJwsaZB/634B8JikfYG+ZvZb4EfA\nKDPbCLwq6Wuw5alsu2cPZma3m9nRlh4ksL5s22YzO5j0zf03cuL9AzA2M4dmYE65UqNkb+ANb1yd\nShrqVV6mK3OAK7fsIB1Vwz6dg0lzt74IXJVZNxTYZGb3ATeRhl0CvEvqFQF4mpT/gZJ2JTVyKh3/\n8uw8qRzzgfFevo+kAaXd/fcc4NulOUyShvncJID7gfOBc0mNrbzy+5Uds14d9pN0jjJP2qu4g/RJ\nM3vOzG4kDdc7vEoda0lDAyGdT73mAOMl7eL1H1p+n1eJdzJpSO3VOUWuAS6JxlUIIYQQQvfriQbW\n88DlklaR5oTc4esNwBtBE0l/yKQdWGxmD5Dm4LRJaic9HGCi7zcOuFLSctLQrcENxPQCac5RJz6k\n7yekRl47cHM23mxR/z2d9OCC5cA3gdUVylTav+Q60oNAVig9pvva8gL+oIQ7uzgfgKtJD69Y7A9R\naAGOABb5efyYNOwS4C7gEUnzPf+TSHPbHgdWdTpycjjwlyoxTABOlbSC1Igd6etL13ouadjaU15m\nJrCnb1sFDADWmT+MIqf8gOwxsyQN9Z6mTiT9VNKrwO5Kj2v/sW8aQfWHiUxQeuT5ctK8uocrlMnG\ncy1wm6RFpN6sPHn3xK9I12Gp3xN3UNbbnHeukg4EfgiMzDyY49tlxQYCL3URVyhEW28HsANr6+0A\ndkjxt2+KEXktTuS2GJHX+ilNldm5+Hynfc1sYtXCAQBJs0kPZOiqwfCxI+ke4Gozq9Z43CH4HMQV\nwHlmtianjPXgCMydSBsxlK0obWzNrdgZ/18rQltbWwwNKkDktTiR22JEXvNJwsw6jaraWRtYI4Bf\nAxut49/CCmGHJekw0lDMFcDFlvPmjwZW+HiLBlYIIYSeEQ2sEEJNUgMrhI+nwYOHs379y70dRggh\nhJ1AXgOrJ+ZghRA+Zir9TYf42baf1tbWXo9hR/3J5jYaV90n5l0UI/JanMhtMSKv9YsGVgghhBBC\nCCF0kxgiGELoQJLFvwshhBBCCF2LIYIhhBBCCCGEULBoYIUQQg+IMezFidwWI/JajMhrcSK3xYi8\n1i8aWCGEEEIIIYTQTWIOVgihg5iDFUIIIYRQXd4crF16I5gQwvZN6vRvRQg7ncEHDmb9uvW9HUYI\nIYSPmejBCiF0IMlo6e0odkBrgUN6O4gdVFG5bUl/E25n1dbWRlNTU2+HscOJvBYncluMyGu+eIpg\nCCGEEEIIIRSs0AaWpOGSVuZsa5U0qsj680g6SNJSSQ9n1q3tjVjySDpF0tQaytUdt6SrJO2Ws+1i\nSbf5crOkcVWOdbGk5ipl3qs3xmpKx/R7rLWG8pMl/UnSuzUev8v8+/YHao8YJPWTNNfvvbGSTpL0\nrL8+PO+9ktm/1nMdJWmFpBck3ZJZf5jXN6OeuEM3id6r4kRuCxHfWBcj8lqcyG0xIq/164kerO1x\nfMU5wKNm9uXMuu0xzlpiaiTuCUD/BvZrNIYicms5y3lmA8duQx2NbC83CjAzG2VmM4GLgOvNbBSw\nqcbj1VLmX4BLzOww4DBJZ5AqfsHMPgMcISk+koYQQgghFKAnGli7SpomaZWk+yv1nEi6wL9xXyHp\nBl/XR9JUX7dc0lW+foT3AiyT9EyDHxT3Ad4oW/dmJp5xXme7pN/4uqmSbpW0QNJLksb4+j0kzfNY\nlks629cPl7Ta91sjabqk033/NZI+5+X6S7pb0kJJSySd5WF8ALxTw7m86ceZ5PEulbTOj9lf0oO+\nfoX3mlwBDANaJc33fb/lMS0ETswceyPpg39XNnk5JB0gaZZfm3ZJny+lNJPbayQt8jLNvm6KpPGZ\nMs2SvpdXvsxHwFvVkmRmi8zs9WrlMrbk33urSrldImkPLzNA0ky/zvdm4l8raZAvH6PUW7s/cC9w\nrB/nMuDrwHXZfX2fPpJulPS0n/eltZ6rpCHAADNb7KvuIX2hkPU66T0QetJ21Ue+g4ncFiL+9k0x\nIq/FidwWI/Jav554iuCngG+Z2UJJdwPjgZ+XNkoaCtwAHA1sAOZ6I2UdcKCZfdbL7eW7TCd96z9b\nUj8aayT2BTZnV5jZ8V7PSOCHwAlm9rak7AfRIWZ2oqRPk3pEZgF/Bc4xs42S9gUW+jaAEcC5ZrZK\n0jPA+b7/2V7HGOAfgflmdomkvYFFkuaZ2VPAUx7TMcB3zOyy8hMpxW1mzUCzH+M/gH8GvgS8ZmZf\n9eMMMLP3JF0NNPn5DQFaSPl/F2gDlvoxb66WSDO7P/PyNqDNzMZIErBnqZjXfzpwqJkd59tnSzoJ\nmAHcAtzu5b8OjM4rb2ZP4I02M1sHnOfHHwrcVTrfbZHNP/B9YLyZPSWpP+maAxwFjATWAwskfcHM\nnqRzL5OZ2ZuS/gH4vpmVGuEnAA+Y2SxJwzPlLwE2mNnxfo8vkPSomb1Sw7keSHrvlKzzdVmbSe+B\nfNmBiAcTQ7BCCCGEsNNra2urqcHZEw2sP5nZQl+eBlxBpoFFGrbVamZvAUiaDpwMTAYOkXQr8Hvg\nUUl7AsPMbDaAmX1QbzD+Qf1Ij6WS04CZZva217Ehs+13vm61pANKhwSmSDqZ9MF1WGbbWjNb5cvP\nAfN8eSXpYyvAaOAsST/w1/2Ag4A1pUrNbAnQqXGVYxpws5m1S9oI/EzSFOAhb5iUYi71Kh1Px/zP\nAA6tsa5ypwF/7zEbUD73ajRwuqSlXv8epAbUVEn7e2PvAOAtM3tN0oRK5YEnqMDM/gxsc+OqggXA\nL/zenOWxASzyOpG0jHRNnyTTY9eg0aRhfGP99V6k836lVGAbz3Ud6T3wTG6JUxs8csgXjdTiRG4L\nEfMuihF5LU7kthiR162ampo65GPSpEkVy/VEA6vTt/kVynT6QGpmGyQdCZwBfBcYS5o71OWHVx9q\ndqnXc6aZrc9s6wP8EXgfeKiOcyh5v0LMFwH7AUeb2Walh07sVqH85szrzWzNvUi9XC82EE8HklpI\nDdp7AMzsRaUHiZwJTPaescmVdt3Wul21+UECppjZXRW2zSRd4yGkHq1q5XtszpyZ/VTSg8BXSL1J\no31T9vp+xNZr+iFbe1YrPkykCgFXmNncBvZ9DfjbzOtP+LqsXwJzJB1nZt9poI4QQgghhJCjJ+Zg\nDZd0vC9fCDxetn0RcLKkQZL6AhcAj/lwu75m9lvgR8AoM9sIvCrpa7DlqWy7Zw9mZreb2dH+IIH1\nZds2m9nBpG/uv5ET7x+AsZk5NANzypUaJXsDb3jj6lRgeIUyXZkDXLllB+moGvbpHEyau/VF4KrM\nuqHAJjO7D7iJ9JAFSEMBS0Munyblf6CkXUmNnErHvzw7TyrHfNIQ0NI8ogGl3f33HODbpTlMkob5\n3CSA+4HzgXNJja288vuVHbNeHfaTdI6k67vcQfqkmT1nZjcCi4HDq9SxFjjGl89tIMY5wHhJu3j9\nh5bf53n8nn9HUmlY5Tjg38uKXUN6CEY0rnpSzBMqTuS2EDHvohiR1+JEbosRea1fTzSwngcul7SK\nNLH+Dl9vsOUD4UTS3J92YLGZPUCaN9ImqZ30cICJvt844EpJy0lDtwY3ENMLwKBKG3xI309Ijbx2\noDQPKa8nbjrpwQXLgW8CqyuUqbR/yXWkB4GsUHpM97XlBfxBCXd2cT4AV5MeXrHYH6LQAhxBmtPV\nDvyYNOwS4C7gEUnzPf+TSHPHHgdWdTpycjjwlyoxTABOlbSC1Igd6etL13oucB/wlJeZic/T8rwP\nANaVHkaRU35A9phZkoZ6T1Mnkn4q6VVgd6XHtf/YN42g+sNEJkha6df4A+DhCmWy8VwL3CZpEak3\nK0/ePfEr0nVY6vfEHZT1Nnd1rsDlwN2k+/xFM3ukbPtA4KUu4gohhBBCCA3SzvhX6n2+075mNrFq\n4QCApNnAGDPrqsHwsSPpHuBqM6vWeNwheK/WCuA8M1uTU8Zo6dGwQtg+tcDO+H9kCCGE2kjCzDqN\nqtpZG1gjgF8DG8v+FlYIOyxJh5GGYq4ALracN7+kne8fhRAqGHzgYNavW1+9YAghhJ1SXgOrJ4YI\nbnfM7D/N7O+icRV2Jv6Hho8ys3F5jatM2fjp5p/W1tZej2FH/Skqtzt74yrmXRQj8lqcyG0xIq/1\n2ykbWCGEEEIIIYRQhJ1yiGAIIZ8ki38XQgghhBC6FkMEQwghhBBCCKFg0cAKIYQeEGPYixO5LUbk\ntRiR1+JEbosRea1fNLBCCCGEEEIIoZvEHKwQQgcxByuEEEIIobq8OVi79EYwIYTtW/p7xD1v8ODh\nrF//cq/UHUIIIYTQHWKIYAihAuuVn9dff6VHzq43xBj24kRuixF5LUbktTiR22JEXusXDawQQggh\nhBBC6Cbd0sCSNFzSypxtrZJGdUc99ZJ0kKSlkh7OrFvbG7HkkXSKpKk1lFvrv3NzXVZ+gKRXJd2W\nPYakQXXEVjVXfn0P6mL7xZL+qdY6a4zr4tJ5SWqWNK5K+WMltfvPcknfqKGOqZJOrrJ9TJ1xnyTp\nWb8n/0bSTZJWSvqpn8f3quxfy7l+UdIzfp6LJZ2a2fZ9Sc/Xcv6h+zU1NfV2CDusyG0xIq/FiLwW\nJ3JbjMhr/bpzDtb2OCv+HOBRM5uYWbc9xllLTJaznOc64LEG6tmW8kUfp1ErgWPMbLOkIcCzkv63\nmX3Uw3FcBFxvZvcBSLoUGGhmJqm5m+p4E/iqma2X9D+AOcAnAMzsZklPADcBM7qpvhBCCCGEkNGd\nQwR3lTRN0ipJ90varbyApAskrfCfG3xdH+8NWOHful/l60dImitpmX8jf0gDMe0DvFG27s1MPOO8\nznZJv/F1UyXdKmmBpJdKvRSS9pA0L9M7cLavHy5pte+3RtJ0Saf7/mskfc7L9Zd0t6SFkpZIOsvD\n+AB4p4ZzebN8haS7Mj0zb0j6X77+GOAA4NHyXYArvf7lkg7LnNu/+jVYJul/5tVZwV+Aj/w4X/Jj\nL5M0t0K8+0n635Ke9p8TlKyVtFem3AuS9q9UvkL9G4FNXQVoZn81s83+cnfgnRoaVxtI1wZJN3jP\n0zJJN2bKnFLhPjlF0gOZc/knv88u+X/s3X28XeOd///XWyZpSkhECNLmRvodlY6kSaiaKkfRjpmh\nqqXUXfvwQItSN50avRGlqlW+37T90dGaNCWtikGjmhE0J+0DKXIrEtFqOqhJUKIyo5Tz/v2xrs06\n+6x9d5yVHSef5+NxHvZe67PW+lyfvU/s61zXtTZwJHCRpGsl/QwYAiySdERVnXaRNDeNQC2ovE7A\nC020dZnttenxQ8BgSQNzIWuBoQ3aHkoQc9jLE7UtR9S1HFHX8kRtyxF1bV1fjmDtCnzK9kJJ1wCn\nAldUdkraCbgUmEz24fWO1El5Ahhle2KKq3zQnkX21/45kgbRu87gAKArv8H2Xuk6E4Dzgb1tPydp\nWC5sR9vvk7QbMAe4CfgLcJjtDZK2AxamfQDjgY/aXinpAeCodPyh6RqHA18E7rJ9oqShwH2S7rR9\nL3BvymkqcIrtk6sbUsm7attJ6bjRwFxghiQB3yIbLTmooCZP2Z4q6TPAucDJwJeB9bnXYGitaxbk\n8LF0zAjgamAf249V1bNiOnCF7XskvR243fYESbcAHwFmSnoP8AfbT0uaVR0PTKi6/uWVx5JOyTb5\n6uoLp/P+OzAO+EQT7TorHTec7HV/Z3q+TS6s6H0CBSN2tq+RtA9wq+2b0rn+bHtKepwfwbqa7H3w\naMr7KuAA2/nfp5ptzcV8DFhs+6+5zV009Xs/Lfe4I/2EEEIIIWy+Ojs7m+pw9mUH6zHbC9Pj64DP\nkutgAXsC820/C5A+PO8LXAyMkzQd+AUwT9IQYGfbcwBsv9xqMqmjMSnlUuQDwGzbz6VrrM/tuyVt\nWyVph8opga8rW5fTBeyc27fG9sr0+CHgzvT4QWBsevxB4BBJn0/PBwGjgdWVi9peRNYDx74QAAAg\nAElEQVThaaWdg4HZwOm2n5B0GnCb7SezElB9v+2b038XkXVqAA4EXluXY7uZEbVq7wUW2H4snWN9\nQcyBwG7ptQEYImlL4AbgK8BM4Chen75WK76Q7X+rs+8+4O8k7QrcLmm+7T830a7ngRcl/QC4Dfh5\nbl/R++QNkbQV8PfA7Fy7B1bH1WtrOs+7gK/Ts5P9DLC9pGE1XqNkWvNJh6bEHPbyRG3LEXUtR9S1\nPFHbckRdX9fR0dGtHhdeeGFhXJlrsIrW3fT4ch3b6yVNAj4EfBo4AvhcUWy3E0mnAiel6/xjZVpU\n2rcF8HvgJbIPxa16qSDnY4ARwOS0lmcNMLggviv3PD9aILJRrt/2Ip96rgJutD0/Pd8b2CfVZ2uy\nqZsv2D6/KtdX6fvvQWv05UkC9qoaUQG4V9mU0BFk6+a+Wi9eb+A7mmyvlvQo8H/IOpmN4l9No0gH\nkL03T0+Pofh98grdR1t7TJVtYAvgucrIVm9IehvZaNpxtv+Q32f7RUnXA7+X9HHbPaZyhhBCCCGE\n3uvLNVhjJFWmlH0C+HXV/vuAfSUNlzQAOBpYkKbbDbB9M/AlYIrtDcDjkj4MIGmQpLfmT2b7StuT\nbU/Jd67Svi7bY4EHyI3MVPklcESaAoakbWvEVT44DyWbXtel7M5sYwpi6rkdOOO1A6R3N3FMXWm0\naojtyyrbbB9re6ztXcimAP4o17mq5Q7gtNx5e0zvU7b+bKc651gIvF/SmBRfVM95wJm5c07K7buZ\nbMRzZW5kpV580ySNTe85Un7vAH6bns9UWidX49itgGG2/xM4G5hYKzT997+ACZIGpjoeUCM+f8xr\nbL8ArEnT+yo51LpmUb5DyUbZvpAbUc7vH0b2OzEqOlcbV8xhL0/UthxR13JEXcsTtS1H1LV1fdnB\nehg4TdJKsptLfC9tN0DqBJ0HdAJLgPtt3wqMAjolLQGuTTEAx5PdkGEZcDcwshc5PQIU3pY8Ten7\nGlknbwlQWctTayRuFrBnyudYYFVBTNHxFReRjSYtV3ab9a9WB0iaKqnmmpoC5wC7K7vJxWJJjaYX\n1srtYmC4sluGL6FqwU2aqjYeeLbmie1nyKY33pzOcX1B2JnAHspusLECOCW37wayUcLrm4zvQdIp\nNWqwD7BM0uJ0nZNz0wMnAk/WOe3WwM/T6/4r4Ky0vfB9YvuJdI0VqS2Lq2PqPK84FjhR2U01VgCH\nVgfUaevpZK/VV3LvixG5/UOBdbbr3iwjhBBCCCH0jux230G7PGm903ZVt2kPLUrreT5l+9x259KX\nJG0N/MD2ZvO9UGm643TbRXdkrMS4fXfWF/3536QQQggh9B+SsN1jRlJ/72CNB34IbLB9cJvTCaGt\nJJ1DNkp4me2f1Ilr2z8KI0eOYe3aP7Tr8iGEEEIITavVwerLKYKbHNuP2n5/dK5CyG5pn9Ys1uxc\n5WLb8tOfO1cxh708UdtyRF3LEXUtT9S2HFHX1vXrDlYIIYQQQgghbEz9eopgCKF1khz/LoQQQggh\n1LdZThEMIYQQQgghhI0pOlghhLARxBz28kRtyxF1LUfUtTxR23JEXVsXHawQQgghhBBC6COxBiuE\n0E2swQohhBBCaCzWYIUQmiapz392fNuO7W5WCCGEEELpYgQrhNCNJDOthBNPy75fa3PV2dlJR0dH\nu9Pol6K25Yi6liPqWp6obTmirrXFCFYIIYQQQgghlKxPOliSxkh6sMa++ZKm9MV1WiVptKTFkubm\ntq1pRy61SNpP0owm4tak/9asdVX81pIel/Tt/DkkDW8ht4a1Sq/v6Dr7T5D0nWav2WReJ1TaJekC\nScc3iN9T0pL0s0zSx5u4xgxJ+zbYf3iLee8jaUV6T75F0mWSHpT0jdSOsxsc37CtKe5fJf1W0ipJ\nH8xtP0fSw820P/S9+OtfeaK25Yi6liPqWp6obTmirq3ryxGsTXHuz2HAPNsH57Ztink2k5NrPK7l\nImBBL67zRuLLPk9vPQhMtT0Z+BDw/0ka0IY8jgEusT3F9kvAScBE21/oqwtI2g04EtgNOBi4UpIA\nbF8OnACc1lfXCyGEEEII3fVlB2ugpOskrZR0g6TB1QGSjpa0PP1cmrZtkUYDlqfRhTPT9vGS7pC0\nVNIDksb1IqdhwFNV257O5XN8uuYSSTPTthmSpku6W9LvKqMUkraSdGfKZZmkQ9P2MWmkYIak1ZJm\nSTooHb9a0h4pbktJ10haKGmRpENSGi8DzzfRlqerN0j6fm5k5ilJX07bpwI7APOqDwHOSNdfJulv\nc2379/QaLJX0kVrXLPAn4NV0nn9I514q6Y6CfEdIulHSb9LP3sqskbRNLu4RSdsXxRdcfwPwYr0E\nbf/Fdld6+lbgeduvNmjXerLXBkmXppGnpZK+mYvZr+B9sp+kW3Nt+U56n51I1vG5SNK1kn4GDAEW\nSTqiqk67SJor6X5JCyqvE/BCo7YCHwaut/2K7T8AvwXek9u/Fhja4ByhBPE9IuWJ2pYj6lqOqGt5\norbliLq27m/68Fy7Ap+yvVDSNcCpwBWVnZJ2Ai4FJpN9eL0jdVKeAEbZnpjiKh+0Z5H9tX+OpEH0\nrjM4AOjKb7C9V7rOBOB8YG/bz0kalgvb0fb70mjAHOAm4C/AYbY3SNoOWJj2AYwHPmp7paQHgKPS\n8YemaxwOfBG4y/aJkoYC90m60/a9wL0pp6nAKbZPrm5IJe+qbSel40YDc4EZabTiW2SjJQcV1OQp\n21MlfQY4FzgZ+DKwPvcaDK11zYIcPpaOGQFcDexj+7GqelZMB66wfY+ktwO3254g6RbgI8BMSe8B\n/mD7aUmzquOBCVXXv7zyWNIp2SZfXX3hdN5/B8YBn2iiXWel44aTve7vTM+3yYUVvU+gYMTO9jWS\n9gFutX1TOtefbU9Jjy/IhV9N9j54NOV9FXCA7fzvU622jiK9n5I/pm0VXTTzez8/93gsWdVCCCGE\nEDZjnZ2dTXU4+7KD9ZjthenxdcBnyXWwgD2B+bafBUgfnvcFLgbGSZoO/AKYJ2kIsLPtOQC2X241\nmdTRmJRyKfIBYLbt59I11uf23ZK2rZK0Q+WUwNeVrcvpAnbO7Vtje2V6/BBwZ3r8INnHU4APAodI\n+nx6PggYDayuXNT2IrIOTyvtHAzMBk63/YSk04DbbD+ZlYDqO5vcnP67iKxTA3Ag8Nq6HNvNjKhV\ney+wwPZj6RzrC2IOBHZLrw3AEElbAjcAXwFmAkcBP20QX8j2v9XZdx/wd5J2BW6XNN/2n5to1/PA\ni5J+ANwG/Dy3r+h98oZI2gr4e2B2rt0Dq+PqtbWBZ4DtJQ2r8Rpl9u/l2UNNMYe9PFHbckRdyxF1\nLU/UthxR19d1dHR0q8eFF15YGNeXHazqv9oXrbvpcRtD2+slTSJbG/Np4Ajgc0Wx3U4knUq2hsXA\nP9pem9u3BfB74CWyD8Wteqkg52OAEcBk213KbgAxuCC+K/c8P1ogslGu3/Yin3quAm60XRlz2BvY\nJ9Vna7Kpmy/YPr8q11fp29cfGrxmaf9etv9atf1eZVNCR5Ctm/tqvfjX+x2ts71a0qPA/yHrZDaK\nfzWNIh1A9t48PT2G4vfJK3Qfbe0xVbaBLYDnKiNbvfBH4O25529L2wCw/aKk64HfS/q47R5TOUMI\nIYQQQu/15RqsMZIqU8o+Afy6av99wL6Shiu7wcDRwII03W6A7ZuBLwFTbG8AHpf0YQBJgyS9NX8y\n21fanpxuGLC2al+X7bHAA+RGZqr8EjgiTQFD0rY14iofnIeSTa/rkrQ/MKYgpp7bgTNeO0B6dxPH\n1JVGq4bYvqyyzfaxtsfa3oVsCuCPcp2rWu4gd+ODoul9ytaf7VTnHAuB90sak+KL6jkPODN3zkm5\nfTeTjXiuzI2s1ItvmqSx6T1Hyu8dZGuTkDRTaZ1cjWO3AobZ/k/gbGBirdD03/8CJkgamOp4QI34\n/DGvsf0CsEbSx3I51LpmkTnAUel3ZhxZW+/LnWsY2e/EqOhcbVwxh708UdtyRF3LEXUtT9S2HFHX\n1vVlB+th4DRJK8luLvG9tN0AqRN0HtAJLAHut30r2fqQTklLgGtTDMDxZDdkWAbcDYzsRU6PAIW3\nJU9T+r5G1slbAlTW8tQaiZsF7JnyORZYVRBTdHzFRWSjScuV3Wb9q9UBkqZK6rF+qI5zgN2V3eRi\nsaRG0wtr5XYxMFzZLcOXAB1VeYlsndmzNU9sP0M2vfHmdI7rC8LOBPZQdoONFcApuX03kI0SXt9k\nfA+STqlRg32AZZIWp+ucnJseOBF4ss5ptwZ+nl73XwFnpe2F7xPbT6RrrEhtWVwdU+d5xbHAicpu\nqrECOLQ6oFZb0/v6BmAl2ZTbU939232HAutsN7pZRgghhBBC6AV1/+zVv6T1TtvZPq9hcKhJ0rvI\nbmBybrtz6UuStgZ+YHuz+V6oNN1xuu2iOzJWYsy0Ei4+DfrzvzchhBBC2LxIwnaPGUn9vYM1Hvgh\nsKHqu7BC2OxIOodslPAy2z+pE1fKPwojR41k7RNrGweGEEIIIbwJ1Opg9eUUwU2O7Udtvz86VyFk\nt7RPaxZrdq5ysX3+s7l3rmIOe3mituWIupYj6lqeqG05oq6t69cdrBBCCCGEEELYmPr1FMEQQusk\nOf5dCCGEEEKob7OcIhhCCCGEEEIIG1N0sEIIYSOIOezlidqWI+pajqhreaK25Yi6ti46WCGEEEII\nIYTQR2INVgihm1iDFUIIIYTQWK01WH/TjmRCCJs2qce/FWEjGTlyDGvX/qHdaYQQQgihl2KKYAih\ngOOnz3/mNxW3bt1/NfMChZxYH1COqGs5oq7lidqWI+rauuhghRBCCCGEEEIf6ZMOlqQxkh6ssW++\npCl9cZ1WSRotabGkublta9qRSy2S9pM0o4m4Nem/NWtdFb+1pMclfTt/DknDW8itYa3S6zu6zv4T\nJH2n2Ws2mdcJlXZJukDS8Q3ih0v6paQX8vVocMwMSfs22H94i3nvI2lFek++RdJlkh6U9I3UjrMb\nHN9MWw+U9ICkZZLul7R/bt85kh6W9PFW8g59paPdCfRbHR0d7U6hX4q6liPqWp6obTmirq3ryxGs\nTXFV/GHAPNsH57Ztink2k5NrPK7lImBBL67zRuLLPk9v/QX4EnBOm/M4BrjE9hTbLwEnARNtf6EP\nr/E08M+2JwGfBK6t7LB9OXACcFofXi+EEEIIIeT0ZQdroKTrJK2UdIOkwdUBko6WtDz9XJq2bZFG\nA5anv7qfmbaPl3SHpKXpL/LjepHTMOCpqm1P5/I5Pl1ziaSZadsMSdMl3S3pd5VRCklbSbozNzpw\naNo+RtKqdNxqSbMkHZSOXy1pjxS3paRrJC2UtEjSISmNl4Hnm2jL09UbJH0/5b5E0lOSvpy2TwV2\nAOZVHwKcka6/TNLf5tr27+k1WCrpI7WuWeBPwKvpPP+Qzr1U0h0F+Y6QdKOk36SfvZVZI2mbXNwj\nkrYvii+4/gbgxXoJ2v5f2/cALzXRnor1ZK8Nki5NI09LJX0zF7NfwftkP0m35trynfQ+OxE4ErhI\n0rWSfgYMARZJOqKqTrtImptGoBZUXifghSbausz22vT4IWCwpIG5kLXA0BbqEPpMZ7sT6LdifUA5\noq7liLqWJ2pbjqhr6/ryLoK7Ap+yvVDSNcCpwBWVnZJ2Ai4FJpN9eL0jdVKeAEbZnpjiKh+0Z5H9\ntX+OpEH0rjM4AOjKb7C9V7rOBOB8YG/bz0kalgvb0fb7JO0GzAFuIhsFOcz2BknbAQvTPoDxwEdt\nr5T0AHBUOv7QdI3DgS8Cd9k+UdJQ4D5Jd9q+F7g35TQVOMX2ydUNqeRdte2kdNxoYC4wQ5KAb5GN\nlhxUUJOnbE+V9BngXOBk4MvA+txrMLTWNQty+Fg6ZgRwNbCP7ceq6lkxHbjC9j2S3g7cbnuCpFuA\njwAzJb0H+IPtpyXNqo4HJlRd//LKY0mnZJt8daO8m2jXWemcw8le93em59vkworeJ1AwYmf7Gkn7\nALfavimd68+2p6THF+TCryZ7Hzya6nEVcIDt/O9Tw7ZK+hiw2PZfc5u7aOr3flrucQcxvS2EEEII\nm7vOzs6mOpx92cF6zPbC9Pg64LPkOljAnsB8288CpA/P+wIXA+MkTQd+AcyTNATY2fYcANsvt5pM\n6mhMSrkU+QAw2/Zz6Rrrc/tuSdtWSdqhckrg68rW5XQBO+f2rbG9Mj1+CLgzPX4QGJsefxA4RNLn\n0/NBwGhgdeWitheRdXhaaedgYDZwuu0nJJ0G3Gb7yawEVN9v++b030VknRqAA4HX1uXYbmZErdp7\ngQW2H0vnWF8QcyCwW3ptAIZI2hK4AfgKMBM4Cvhpg/hCtv+tF3k38jzwoqQfALcBP8/tK3qfvCGS\ntgL+Hpida/fA6rhGbZX0LuDr9OxkPwNsL2lYjdcomdZ80qFJHe1OoN+K9QHliLqWI+panqhtOaKu\nr+vo6OhWjwsvvLAwri87WNV/tS9ad9Pjy3Vsr5c0CfgQ8GngCOBzRbHdTiSdSraGxcA/VqZFpX1b\nAL8nmxJ2WwttqMhPJavkcQwwAphsu0vZDSAGF8R35Z7nRwtENsr1217kU89VwI2256fnewP7pPps\nTTZ18wXb51fl+ip9/z1ojb48ScBeVSMqAPcqmxI6gmzd3FfrxWsjfkeT7VfTKNIBZO/N09NjKH6f\nvEL30dYeU2Ub2AJ4rjKy1RuS3kY2mnac7T/k99l+UdL1wO8lfdx2j6mcIYQQQgih9/pyDdYYSZUp\nZZ8Afl21/z5gX2V3dBsAHA0sSNPtBti+mexGBFNsbwAel/RhAEmDJL01fzLbV9qenG4YsLZqX5ft\nscAD5EZmqvwSOCJNAUPStjXiKh+ch5JNr+tSdme2MQUx9dwOnPHaAdK7mzimrjRaNcT2ZZVtto+1\nPdb2LmRTAH+U61zVcge5Gx8UTe9Ttv5spzrnWAi8X9KYFF9Uz3nAmblzTsrtu5lsxHNlbmSlXnxv\ndXutJM1UWidXGJyNKA2z/Z/A2cDEBuf9L2CCpIGpjgfUiO+RC4DtF4A1aXpfJYda1yzKdyjZKNsX\nciPK+f3DyH4nRkXnamPrbHcC/VasDyhH1LUcUdfyRG3LEXVtXV92sB4GTpO0kuzmEt9L2w2QOkHn\nkX3KWALcb/tWYBTQKWkJ2R3PzkvHHU92Q4ZlwN3AyF7k9AhQeFvyNKXva2SdvCVAZS1PrZG4WcCe\nKZ9jgVUFMUXHV1xENpq0XNlt1r9aHSBpqqRW1g+dA+yu7CYXiyU1ml5YK7eLgeHKbhm+hKq5TGmq\n2njg2Zontp8hm954czrH9QVhZwJ7KLvBxgrglNy+G8hGCa9vMr4HSafUqkEacbwcOEHSY5LemXZN\nBJ6sc9qtgZ+n1/1XwFlpe+H7xPYTqS0rUlsWV8fUeV5xLHCisptqrAAOLWhPrbaeTvZafSX3vhiR\n2z8UWGe77s0yQgghhBBC78hu9x20y5PWO21n+7yGwaGmtJ7nU7bPbXcufUnS1sAPbG823wuVpjtO\nt110R8ZKjNt/Z/3NmejP/y6HEEII/YUkbPeYkdTfO1jjgR8CG6q+CyuEzY6kc8hGCS+z/ZM6cdHB\naqvoYIUQQghvBrU6WH05RXCTY/tR2++PzlUI2S3t05rFmp2r1yl+2vQzcmR+eWdoRqwPKEfUtRxR\n1/JEbcsRdW1dX99FLoTQD8QISt/r7OyMW92GEEIIm4F+PUUwhNA6SY5/F0IIIYQQ6tsspwiGEEII\nIYQQwsYUHawQQtgIYg57eaK25Yi6liPqWp6obTmirq2LDlYIIYQQQggh9JFYgxVC6CbWYIUQQggh\nNBZrsEIIIYQQQgihZNHBCiH0ICl+evGz49t2rFnTmMNenqhtOaKu5Yi6lidqW46oa+vie7BCCD1N\na3cCb07rpq1rdwohhBBCaLNYgxVC6EaSo4PVS9PiS5pDCCGEzUWpa7AkjZH0YI198yVN6YvrtErS\naEmLJc3NbVvTjlxqkbSfpBlNxK1J/61Z66r4rSU9Lunb+XNIGt5Cbg1rlV7f0XX2nyDpO81es8m8\nTqi0S9IFko5vED9c0i8lvZCvR4NjZkjat8H+w1vMex9JK9J78i2SLpP0oKRvpHac3eD4hm1Ncf8q\n6beSVkn6YG77OZIelvTxVvIOIYQQQgjN68s1WJvin20PA+bZPji3bVPMs5mcXONxLRcBC3pxnTcS\nX/Z5eusvwJeAc9qcxzHAJban2H4JOAmYaPsLfXUBSbsBRwK7AQcDV0oSgO3LgROA0/rqeqF5MYe9\nPFHbckRdyxF1LU/UthxR19b1ZQdroKTrJK2UdIOkwdUBko6WtDz9XJq2bZFGA5ZLWibpzLR9vKQ7\nJC2V9ICkcb3IaRjwVNW2p3P5HJ+uuUTSzLRthqTpku6W9LvKKIWkrSTdmXJZJunQtH1MGimYIWm1\npFmSDkrHr5a0R4rbUtI1khZKWiTpkJTGy8DzTbTl6eoNkr6fcl8i6SlJX07bpwI7APOqDwHOSNdf\nJulvc2379/QaLJX0kVrXLPAn4NV0nn9I514q6Y6CfEdIulHSb9LP3sqskbRNLu4RSdsXxRdcfwPw\nYr0Ebf+v7XuAl5poT8V6stcGSZemkaelkr6Zi9mv4H2yn6Rbc235TnqfnUjW8blI0rWSfgYMARZJ\nOqKqTrtImivpfkkLKq8T8EKjtgIfBq63/YrtPwC/Bd6T278WGNpCHUIIIYQQQgv68iYXuwKfsr1Q\n0jXAqcAVlZ2SdgIuBSaTfXi9I3VSngBG2Z6Y4ioftGeR/bV/jqRB9K4zOADoym+wvVe6zgTgfGBv\n289JGpYL29H2+9JowBzgJrJRkMNsb5C0HbAw7QMYD3zU9kpJDwBHpeMPTdc4HPgicJftEyUNBe6T\ndKfte4F7U05TgVNsn1zdkEreVdtOSseNBuYCM9JoxbfIRksOKqjJU7anSvoMcC5wMvBlYH3uNRha\n65oFOXwsHTMCuBrYx/ZjVfWsmA5cYfseSW8Hbrc9QdItwEeAmZLeA/zB9tOSZlXHAxOqrn955bGk\nU7JNvrpR3k2066x0zuFkr/s70/NtcmFF7xMoGLGzfY2kfYBbbd+UzvVn21PS4wty4VeTvQ8eTfW4\nCjjAdv73qVZbR5HeT8kf07aKLpr5vZ+fezwW6M2fN0I3HR0d7U6h34raliPqWo6oa3mituWIur6u\ns7OzqRG9vuxgPWZ7YXp8HfBZch0sYE9gvu1nAdKH532Bi4FxkqYDvwDmSRoC7Gx7DoDtl1tNJnU0\nJqVcinwAmG37uXSN9bl9t6RtqyTtUDkl8HVl63K6gJ1z+9bYXpkePwTcmR4/SPbxFOCDwCGSPp+e\nDwJGA6srF7W9iKzD00o7BwOzgdNtPyHpNOA2209mJaB64d3N6b+LyDo1AAcCr63Lsd3MiFq19wIL\nbD+WzrG+IOZAYLf02gAMkbQlcAPwFWAmcBTw0wbxhWz/Wy/ybuR54EVJPwBuA36e21f0PnlDJG0F\n/D0wO9fugdVxb6CtzwDbSxpW4zXK7N/Ls4cQQggh9FMdHR3dOpwXXnhhYVyZa7CK1t30uMtG+pA3\nCegEPg18v1ZstxNJp6apcYsl7Vi1bwtgDdk6lNuayr67/FSySh7HACOAybYnk009HFwQ35V7nh8t\nENko1+T0M872at64q4AbbVfGHPYGTpf0e7KRrOMkXVLQtlfp+9v0133N0v69cjUYnabv3QuMT6Ng\nhwH/US++j3Ouy/arZFPsbgT+GfjP3O6i98krdP+96jFVtoEtgOfSOq1Ku/+uheP/CLw99/xtaRsA\ntl8Ergd+L6lohDOUJOawlydqW46oazmiruWJ2pYj6tq6vuxgjZFUmVL2CeDXVfvvA/ZVdke3AcDR\nwII03W6A7ZvJbkQwxfYG4HFJHwaQNEjSW/Mns31l+vA5xfbaqn1dtscCD5AbmanyS+CINAUMSdvW\niKt8cB5KNr2uS9L+wJiCmHpuB8547QDp3U0cU1carRpi+7LKNtvH2h5rexeyKYA/sn1+g1PdQe7G\nB0XT+5StP9upzjkWAu+XNCbFF9VzHnBm7pyTcvtuJhvxXJkbWakX31vdXitJM5XWyRUGZyNKw2z/\nJ3A2MLHBef8LmCBpYKrjAc3mAmD7BWCNpI/lcqh1zSJzgKPS78w44B1kv3uVcw0j+50YZbvHOrkQ\nQgghhPDG9GUH62HgNEkryW4u8b203QCpE3Qe2UjVEuB+27eSrQ/plLQEuDbFABxPdkOGZcDdwMhe\n5PQIUHhb8jSl72tknbwlQGUtT62RuFnAnimfY4FVBTFFx1dcRHYjkOXKbrP+1eoASVMltbJ+6Bxg\n99xIXqPphbVyuxgYruyW4UuAjqq8RLbO7NmaJ7afIZveeHM6x/UFYWcCeyi7wcYK4JTcvhvIRgmv\nbzK+B0mn1KqBslvOXw6cIOkxSe9MuyYCT9Y57dbAz9Pr/ivgrLS98H1i+4nUlhWpLYurY+o8rzgW\nOFHZTTVWAIcWtKewrel9fQOwkmzK7anu/sVMQ4F1aSQrbEQxh708UdtyRF3LEXUtT9S2HFHX1vXr\nLxpO6522s31ew+BQk6R3kd3A5Nx259KXJG0N/MD2ZvO9UOmmGdNtF92RsRITXzTcW9Pii4ZDCCGE\nzYVqfNFwf+9gjQd+CGyo+i6sEDY7ks4hGyW8zPZP6sT1338USjZy1EjWPrG2cF9nZ2f8FbAkUdty\nRF3LEXUtT9S2HFHX2mp1sPr6JgebFNuPAu9vdx4hbArSLe0vbxhIjMKEEEIIIfRWvx7BCiG0TpLj\n34UQQgghhPpqjWD15U0uQgghhBBCCGGzFh2sEELYCOJ7RMoTtS1H1LUcUdfyRG3LEXVtXXSwQggh\nhBBCCKGPxBqsEEI3sQYrhBBCCKGxWIMVQgghhBBCCCWLDlYIoQdJ8dPPfnbcccPzrOIAACAASURB\nVGy731alifUB5Yi6liPqWp6obTmirq3r19+DFULorZgi2Pc6gY62XX3duh4zGEIIIYRQgliDFULo\nRpKjg9UfKb5AOoQQQuhDm9QaLEljJD1YY998SVM2dk7p2qMlLZY0N7dtTTtyqUXSfpJmNBG3Jv23\nZq2r4reW9Likb+e2zZc0usFxMyTt2yDfWxtdvxX5c0o6QdIFTRzzDUkPSlou6cgm4i+QdHyD/We3\nmPeukpZIWiRpnKQzJK2UdG1qx3caHN+wrZImSbontXVpvq2Sjpb0sKSzWsk7hBBCCCE0r51rsDbF\nP6UeBsyzfXBu26aYZzM5ucbjWi4CFvQunZZyKeOcdc8v6R+BdwMTgfcC50oaUkJOjRwGzLY91fYa\n4DPAgbaPS/tbfV2L/A9wnO3dgYOB/ydpGwDbPwH2A6KD1Rad7U6g34r1AeWIupYj6lqeqG05oq6t\na2cHa6Ck69Jf8G+QNLg6IP3FfXn6uTRt2yKNmiyXtEzSmWn7eEl3pL/aPyBpXC9yGgY8VbXt6Vw+\nx6drLpE0M22bIWm6pLsl/U7S4Wn7VpLuTLksk3Ro2j5G0qp03GpJsyQdlI5fLWmPFLelpGskLUwj\nHoekNF4Gnm+iLU9Xb5D0/ZT7EklPSfpy2j4V2AGYV3XIn4BXG1xnfcoJSXumdixNeW9Vdf3CNkm6\nV9Juubj5kqbUqUHei8CGBjlOAH7lzP8Cy4F/aHDMC+ncpJGmh1K7fpyLeVfK9XeSPptiu40YSjon\njXYdDHwO+IykuyRdBewCzK28h3PHjJB0o6TfpJ+9m22r7d/ZfjQ9/m+y9/P2uf3rgKEN2h5CCCGE\nEHqpnTe52BX4lO2Fkq4BTgWuqOyUtBNwKTCZ7EP8HamT8gQwyvbEFLdNOmQWcIntOZIG0bvO4wCg\nK7/B9l7pOhOA84G9bT8naVgubEfb70udhDnATcBfgMNsb5C0HbAw7QMYD3zU9kpJDwBHpeMPTdc4\nHPgicJftEyUNBe6TdKfte4F7U05TgVNsn1zdkEreVdtOSseNBuYCMyQJ+BZwDHBQVfzHGhXM9lnp\nnAOB64EjbC9OI0QvVoUXtikd93FgmqQdUz0XS/pajfj89W+oPE4dsKm2p1VddxnwFUlXAFsB+wMP\nNWjXFbmnXwDG2v5r7v0G2Xu4g6zDslrSlZXDe57OcyV9D3ihcm5JHwI60vvphFz8dOAK2/dIejtw\nOzChybaSi3kPMLDS4cpp4ncjf9oO2nlzhv6jo90J9FsdHR3tTqFfirqWI+panqhtOaKur+vs7Gxq\nRK+dHazHbC9Mj68DPkuugwXsCcy3/SyApFnAvsDFwDhJ04FfAPPSh/mdbc8BsP1yq8mkjsaklEuR\nD5BN73ouXWN9bt8tadsqSTtUTgl8Xdn6pC5g59y+NbZXpscPAZVOw4PA2PT4g8Ahkj6fng8CRgOr\nKxe1vQjo0blq0M7BwGzgdNtPSDoNuM32k1kJ6O2txnYFnrS9OOW2IV0vH1OrTbPJRs+mAUcCNzaI\nL2T7VqDHei/bd0jaE7iHbETnHhqPzOUtA34s6RbSa53cZvsV4E+S1gEjWzgnZLUuqveBwG56vXhD\nJG2ZRt+A2m197cTZHyh+BBxXsPtZSeMLOl450xomH0IIIYSwOeno6OjW4bzwwgsL4zalNVhFa0t6\nfPhMHZtJZAsaPg18v1ZstxNJp6apcYvTKEl+3xbAGmA34Lamsu/upYKcjwFGAJNtTyb7YD+4IL4r\n97yL1zu9Ihvlmpx+xtlezRt3FXCj7fnp+d7A6ZJ+TzaSdZykS3p57kads8I22X4SeEbS7mQjWT/N\nHdMnNbB9STrHh8je94+0cPg/Ad8FpgD3p/cL9Hwd/wZ4hWwktKLH1NcmCNgr1+7R+c5Vw4OlrYGf\nA/9q+/6CkOnAUkmf7EVuodc6251AvxXrA8oRdS1H1LU8UdtyRF1b184O1hhJlWlsnwB+XbX/PmBf\nScMlDQCOBhak6XYDbN8MfAmYkkZLHpf0YQBJgyS9NX8y21emD6tTbK+t2tdleyzwANkH/CK/BI6Q\nNDxdY9sacZVOxlDgKdtdkvYHxhTE1HM7cMZrB0jvbuKYutJo1RDbl1W22T7W9ljbuwDnAj+yfX7B\nsTOV1ofVsBrYMU1bRNKQ9Lrl1WvTT4F/AbaxvaKJ+KYpW7dXed0mAruT1ptJuqTyvqlxrIDRthcA\n5wHbAPVukLEO2F7StpLeAvxzL1KeB7y2LkvSpGYPTFM1bwFmpt+RIucD77D9w17kFkIIIYQQ6mhn\nB+th4DRJK8luLvG9tN0AqRN0HtmffZcA96dpUaOATklLgGtTDMDxwBmSlgF30/p0LchGNYYX7UhT\n+r5G1slbAlyezzcfmv47C9gz5XMssKogpuj4iovIbgSyPN004avVAZKmSrq6TnuqnQPsnhvJa2V6\n4UTgyVo7bf+VrHP6XUlLyToJb6kKq9em/6Dn6NXFdeJ7kHSIpGkFuwYCv5a0gux9dqztylq73YG1\nBcdUDACuS6/jImC67T8XxFXet6+kPO8n6yCuKojtdkyBM4E9lN0cZQVwSnVAnbYeCewDfDL3Ok+s\nihmUbnYRNqqOdifQb8X6gHJEXcsRdS1P1LYcUdfWxRcN56S1PtvZPq9h8GYkTTn7ge1ao3tvWpLm\nVt2Wv19L6wCX2d6pTkx80XC/FF80HEIIIfQlbUpfNLwJuwl4n3JfNBzA9gv9sXMFsJl1ro4mG1n8\nZhPR8dPPfkaOzM9S7l9ifUA5oq7liLqWJ2pbjqhr69p5F8FNTrqr2vvbnUcIZUhfNPyTJmNLzmbz\n09nZGdMsQgghhM1ATBEMIXQjyfHvQgghhBBCfTFFMIQQQgghhBBKFh2sEELYCGIOe3mituWIupYj\n6lqeqG05oq6tiw5WCCGEEEIIIfSRWIMVQugm1mCFEEIIITQWa7BCCCGEEEIIoWTRwQoh9CApfuKn\n9J8d37Zjn7xfY31AOaKu5Yi6lidqW46oa+vie7BCCD1Na3cC/dAaYFy7k9i0rJu2rt0phBBCCH0u\n1mCFELqR5OhghY1iWnypdQghhDcvaRNagyVpjKQHa+ybL2nKxs4pXXu0pMWS5ua2rWlHLrVI2k/S\njCbi1qT/1qx1VfzWkh6X9O3ctvmSRjc4boakfRvke2uj67cif05JJ0i6oIljviHpQUnLJR3ZRPwF\nko5vsP/sFvPeVdISSYskjZN0hqSVkq5N7fhOg+ObbesJkh6RtDrfBklHS3pY0lmt5B1CCCGEEJrX\nzjVYm+KfLQ8D5tk+OLdtU8yzmZxc43EtFwELepdOS7mUcc6655f0j8C7gYnAe4FzJQ0pIadGDgNm\n255qew3wGeBA28el/a2+rj1I2hb4CrAnsBdwgaShALZ/AuwHRAerHTapP9X0L7E+oBxR13JEXcsT\ntS1H1LV17exgDZR0XfoL/g2SBlcHpL+4L08/l6ZtW6RRk+WSlkk6M20fL+kOSUslPSCpN6sdhgFP\nVW17OpfP8emaSyTNTNtmSJou6W5Jv5N0eNq+laQ7Uy7LJB2ato+RtCodt1rSLEkHpeNXS9ojxW0p\n6RpJC9OIxyEpjZeB55toy9PVGyR9P+W+RNJTkr6ctk8FdgDmVR3yJ+DVBtdZn3JC0p6pHUtT3ltV\nXb+wTZLulbRbLm6+pCl1apD3IrChQY4TgF8587/AcuAfGhzzQjo3aaTpodSuH+di3pVy/Z2kz6bY\nbiOGks5Jo10HA58DPiPpLklXAbsAcyvv4dwxIyTdKOk36WfvFtr6IbI/Ejxvez3Za/paW22vA4Y2\nOEcIIYQQQuildt7kYlfgU7YXSroGOBW4orJT0k7ApcBksg/xd6ROyhPAKNsTU9w26ZBZwCW250ga\nRO86jwOArvwG23ul60wAzgf2tv2cpGG5sB1tvy91EuYANwF/AQ6zvUHSdsDCtA9gPPBR2yslPQAc\nlY4/NF3jcOCLwF22T0wjEPdJutP2vcC9KaepwCm2T65uSCXvqm0npeNGA3OBGZIEfAs4BjioKv5j\njQpm+6x0zoHA9cARthenEaIXq8IL25SO+zgwTdKOqZ6LJX2tRnz++jdUHqcO2FTb06quuwz4iqQr\ngK2A/YGHGrTritzTLwBjbf81936D7D3cQdZhWS3pysrhPU/nuZK+B7xQObekDwEd6f10Qi5+OnCF\n7XskvR24HZjQZFtHAY/nnv8xbctr/LsxP/d4LHFzhr4QNSxNR0dHu1Pol6Ku5Yi6lidqW46o6+s6\nOzubGtFrZwfrMdsL0+PrgM+S62CRTXGab/tZAEmzgH2Bi4FxkqYDvwDmpQ/zO9ueA2D75VaTSR2N\nSSmXIh8gm971XLrG+ty+W9K2VZJ2qJwS+Lqy9UldwM65fWtsr0yPHwIqnYYHyT7OAnwQOETS59Pz\nQcBoYHXlorYXAT06Vw3aORiYDZxu+wlJpwG32X4yKwE9Fuo1aVfgSduLU24b0vXyMbXaNJtspGUa\ncCRwY4P4QrZvBXqs97J9h6Q9gXvIRijvofHIXN4y4MeSbiG91slttl8B/iRpHTCyhXNCVuuieh8I\n7KbXizdE0pZp9A2o3dYmPStpvO1Ha0bs38szhxBCCCH0Ux0dHd06nBdeeGFh3Ka0BqtobUmPD5+p\nYzMJ6AQ+DXy/Vmy3E0mnpqlxi9MoSX7fFmQrJHYDbmsq++5eKsj5GGAEMNn2ZLIP9oML4rtyz7t4\nvdMrslGuyelnnO3VvHFXATfaroxR7A2cLun3ZCNZx0m6pJfnbtQ5K2yT7SeBZyTtTjaS9dPcMX1S\nA9uXpHN8iOx9/0gLh/8T8F1gCnB/er9Az9fxb4BXyEZCK3pMfW2CgL1y7R6d71w18Ee6d0Lflrbl\nTQeWSvpkL3ILvRVrsEoT6wPKEXUtR9S1PFHbckRdW9fODtYYSZVpbJ8Afl21/z5gX0nDJQ0AjgYW\npOl2A2zfDHwJmJJGSx6X9GEASYMkvTV/MttXpg+rU2yvrdrXZXss8ADZB/wivwSOkDQ8XWPbGnGV\nTsZQ4CnbXZL2B8YUxNRzO3DGawdI727imLrSaNUQ25dVttk+1vZY27sA5wI/sn1+wbEzldaH1bAa\n2DFNW0TSkPS65dVr00+BfwG2sb2iifimKVu3V3ndJgK7k9abSbqk8r6pcayA0bYXAOcB2wD1bpCx\nDthe0raS3gL8cy9Snge8ti5L0qQWjr0dOEjS0PQePShtyzsfeIftH/YitxBCCCGEUEc7O1gPA6dJ\nWkl2c4nvpe0GSJ2g88hGqpYA96dpUaOATklLgGtTDMDxwBmSlgF30/p0LchGNYYX7UhT+r5G1slb\nAlyezzcfmv47C9gz5XMssKogpuj4iovIbgSyPN004avVAZKmSrq6TnuqnQPsnhvJa2V64UTgyVo7\nbf+VrHP6XUlLyToJb6kKq9em/6Dn6NXFdeJ7kHSIpGkFuwYCv5a0gux9dqztylq73YG1BcdUDACu\nS6/jImC67T8XxFXet6+kPO8n69isKojtdkyBM4E9lN0cZQVwSnVArbamKawXkf2x4DfAhVXTWQEG\npZtdhI0p1mCVJtYHlCPqWo6oa3mituWIurYuvmg4J6312c72eQ2DNyOStgZ+YLvW6N6blqS5Vbfl\n79fSOsBltneqExNfNBw2jmnxRcMhhBDevFTji4ajg5UjaTzwQ2DD5vShO2weJB1NdkfEmbb/b524\n+EchbBQjR41k7RP1BpCb09nZGX9hLUHUtRxR1/JEbcsRda2tVgernXcR3OSku6q9v915hFCG9EXD\nP2kytuRsNj/xP6gQQghh8xAjWCGEbiQ5/l0IIYQQQqiv1ghWO29yEUIIIYQQQgj9SnSwQghhI4jv\nESlP1LYcUddyRF3LE7UtR9S1ddHBCiGEEEIIIYQ+EmuwQgjdxBqsEEIIIYTGYg1WCCGEEEIIIZQs\nbtMeQuhB6vHHmBBCCG8CI0eOYe3aP7Q7jbaIr8MoR9S1ddHBCiEUiCmCfa8T6GhzDv1VJ1HbMnQS\ndS1DJ2XWdd26+ANZCO0Wa7BCCN1IcnSwQgjhzUrxZfEhbCSb1BosSWMkPVhj33xJUzZ2TunaoyUt\nljQ3t21NO3KpRdJ+kmY0Ebcm/bdmravit5b0uKRv57bNlzS6wXEzJO3bIN9bG12/FflzSjpB0gVN\nHPNqem2XSLqlifgLJB3fYP/ZLea9a7r+IknjJJ0haaWka1M7vtPg+IZtlTRJ0j2SHpS0VNKRuX1H\nS3pY0lmt5B1CCCGEEJrXzptcbIp/XjkMmGf74Ny2TTHPZnJyjce1XAQs6F06LeVSxjmbOf//2J5i\ne7Ltw0rIpxmHAbNtT7W9BvgMcKDt49L+Vl/XIv8DHGd7d+Bg4P9J2gbA9k+A/YDoYLVFZ7sT6Mc6\n251AP9XZ7gT6qc52J9Bvxfc1lSPq2rp2drAGSrou/QX/BkmDqwPSX9yXp59L07Yt0qjJcknLJJ2Z\nto+XdEf6q/0Dksb1IqdhwFNV257O5XN8uuYSSTPTthmSpku6W9LvJB2etm8l6c6UyzJJh6btYySt\nSsetljRL0kHp+NWS9khxW0q6RtLCNOJxSErjZeD5JtrydPUGSd9PuS+R9JSkL6ftU4EdgHlVh/wJ\neLXBddannJC0Z2rH0pT3VlXXL2yTpHsl7ZaLmy9pSp0a5L0IbGiQI0Crk9JfSOcmjTQ9lNr141zM\nu1Kuv5P02RTbbcRQ0jlptOtg4HPAZyTdJekqYBdgbuU9nDtmhKQbJf0m/ezdbFtt/872o+nxf5O9\nn7fP7V8HDG2xFiGEEEIIoUntvMnFrsCnbC+UdA1wKnBFZaeknYBLgclkH+LvSJ2UJ4BRtiemuG3S\nIbOAS2zPkTSI3nUeBwBd+Q2290rXmQCcD+xt+zlJw3JhO9p+X+okzAFuAv4CHGZ7g6TtgIVpH8B4\n4KO2V0p6ADgqHX9ousbhwBeBu2yfKGkocJ+kO23fC9ybcpoKnGL75OqGVPKu2nZSOm40MBeYIUnA\nt4BjgIOq4j/WqGC2z0rnHAhcDxxhe7GkIaQOSk5hm9JxHwemSdox1XOxpK/ViM9f/4bK49QBm2p7\nWkGqb0m1fhn4hu2fNWjXFbmnXwDG2v5r7v0G2Xu4g6zDslrSlZXDe57OcyV9D3ihcm5JHwI60vvp\nhFz8dOAK2/dIejtwOzChhbZWYt4DDKx0uHKa+N3In7aDWOjeFzranUA/1tHuBPqpjnYn0E91tDuB\nfivudFeOqOvrOjs7mxrRa2cH6zHbC9Pj64DPkutgAXsC820/CyBpFrAvcDEwTtJ04BfAvPRhfmfb\ncwBsv9xqMqmjMSnlUuQDZNO7nkvXWJ/bd0vatkrSDpVTAl9Xtj6pC9g5t2+N7ZXp8UNApdPwIDA2\nPf4gcIikz6fng4DRwOrKRW0vAnp0rhq0czAwGzjd9hOSTgNus/1kVoKWR3oqdgWetL045bYhXS8f\nU6tNs8lGz6YBRwI3NogvZPtWoNZ6rzG2/zuNbP5S0vI0Ta8Zy4AfK1u7lV+/dZvtV4A/SVoHjGzy\nfBWiuN4HArvp9eINkbSl7f+tBDRoa+UPFD8CjivY/ayk8QUdr5xpDZMPIYQQQticdHR0dOtwXnjh\nhYVxm9IarKK1JT0+fKaOzSSyScyfBr5fK7bbiaRT09S4xWmUJL9vC2ANsBtwW1PZd/dSQc7HACOA\nybYnk03VGlwQ35V73sXrnV6RjXJNTj/jbK/mjbsKuNH2/PR8b+B0Sb8nG8k6TtIlvTx3o85ZYZts\nPwk8I2l3spGsn+aO6ZMapOlypE5VJ9nIaLP+CfguMAW4P71foOfr+DfAK2QjoRU9pr42QcBeuXaP\nzneuGh4sbQ38HPhX2/cXhEwHlkr6ZC9yC73W2e4E+rHOdifQT3W2O4F+qrPdCfRbsVaoHFHX1rWz\ngzVGUmUa2yeAX1ftvw/YV9JwSQOAo4EFabrdANs3A18CpqTRksclfRhA0iBJb82fzPaV6cPqFNtr\nq/Z12R4LPED2Ab/IL4EjJA1P19i2RlylkzEUeMp2l6T9gTEFMfXcDpzx2gHSu5s4pq40WjXE9mWV\nbbaPtT3W9i7AucCPbJ9fcOxMpfVhNawGdkzTFpE0JL1uefXa9FPgX4BtbK9oIr5pkoalaaNIGgG8\nD1iZnl9Sed/UOFbAaNsLgPOAbYAhdS63Dthe0raS3gL8cy9Snge8ti5L0qRmD0xTNW8BZqbfkSLn\nA++w/cNe5BZCCCGEEOpoZwfrYeA0SSvJbi7xvbTdAKkTdB7Zn3qWAPenaVGjgE5JS4BrUwzA8cAZ\nkpYBd9P6dC2AR4DhRTvSlL6vkXXylgCX5/PNh6b/zgL2TPkcC6wqiCk6vuIishuBLE83TfhqdYCk\nqZKurtOeaucAu+dG8lqZXjgReLLWTtt/JeucflfSUrJOwluqwuq16T/oOXp1cZ34HiQdImlawa7d\ngAfS63YX2Vq9h9O+3YG1BcdUDACuS6/jImC67T8XxFXet6+kPO8n6yCuKojtdkyBM4E9lN0cZQVw\nSnVAnbYeCewDfDL3Ok+sihmUbnYRNqqOdifQj3W0O4F+qqPdCfRTHe1OoN+KtULliLq2Lr5oOCet\n9dnO9nkNgzcjacrZD2zXGt1705I0t+q2/P1aWge4zPZOdWLii4ZDCOFNK75oOISNRZvSFw1vwm4C\n3qfcFw0HsP1Cf+xcAWxmnaujyUYWv9nuXDZPne1OoB/rbHcC/VRnuxPopzrbnUC/FWuFyhF1bV07\n7yK4yUl3VXt/u/MIoQzpi4Z/0lx0b28mGUIIoZ1GjhzTOCiEUKqYIhhC6EaS49+FEEIIIYT6Yopg\nCCGEEEIIIZQsOlghhLARxBz28kRtyxF1LUfUtTxR23JEXVsXHawQQgghhBBC6COxBiuE0E2swQoh\nhBBCaCzWYIUQQgghhBBCyaKDFUIIG0HMYS9P1LYcUddyRF3LE7UtR9S1dfE9WCGEHqT4HqwQQghh\nYxo5aiRrn1jb7jRCH4g1WCGEbiSZae3OIoQQQtjMTIP4XP7mEmuwQgghhBBCCKFkbelgSRoj6cEa\n++ZLmrKxc0rXHi1psaS5uW1r2pFLLZL2kzSjibg16b81a10Vv7WkxyV9O7dtvqTRDY6bIWnfBvne\n2uj6rcifU9IJki5o4phX02u7RNItTcRfIOn4BvvPbjHvXdP1F0kaJ+kMSSslXZva8Z0Gxzfb1hMk\nPSJpdb4Nko6W9LCks1rJO/SRTepfkn4maluOqGs5oq7lidqWItZgta6da7A2xTHQw4B5ts/LbdsU\n82wmJ9d4XMtFwILepdNSLmWcs5nz/4/ttnTccw4DZtu+BEDSZ4ADbD8p6QRaf117kLQt8BVgCiBg\nkaSf2f8/e2cfblVV7f/PFxJRUURNULqgcXu8Wr6B/MyrF4/mS2q+XJOMMK3bY5amll7NtKuH1NRK\nf3kzNZMfcgVNLVSUSEQ5ZCKCcDioIGlhvl3UUgrN9zN+f8yxYZ199uvpLDYcxud59nPWmnPMOccc\na7FZY40x57a/mtmtkh4E5gP/9x+ZSBAEQRAEQVCaRqYIbiRpkr/Bv11S32IBf+O+2D+Xe1kvj5os\nltQm6UwvHybpfkmLJD0maccu6LQl8EpR2asZfU70MVslTfSyCZKulvSwpGckHevlm0ma6bq0STrK\ny4dKWurtlkmaLOlgb79M0l4ut6mk8ZLmesTjSFfjXeCvNczl1eICST933VslvSLpv7x8BLAtMKOo\nyV+AD6qMs9J1QtJIn8ci13uzovFLzknSI5J2zsjNkjS8gg2yvAW8UUVHSM5GPazyvvFI05M+r1sy\nMh93XZ+RdLrLdogYSjrbo12HAd8Evi7pAUnXAR8Fphfu4UybbST9UtKj/tmnjrkeSnpJ8FczW0m6\npp8uVJrZy0D/Om0RdAdd+UYKaiNsmw9h13wIu+ZH2DYXmpqaGq3CekcjI1g7AV82s7mSxgOnAlcV\nKiVtB1wO7El6iL/fnZQXgMFmtpvLbeFNJgPfN7OpkvrQNeexN9CeLTCzvX2cXYDzgX3M7HVJW2bE\nBpnZvu4kTAWmAG8Dx5jZG5K2BuZ6HcAw4LNmtkTSY8Dnvf1RPsaxwAXAA2b2FUn9gXmSZprZI8Aj\nrtMI4BQz+2rxRAp6F5Wd7O2GANOBCZIE/AgYCxxcJH9cNYOZ2be8z42AXwCjzWyhpH64g5Kh5Jy8\n3fFAs6RBbs+Fki4tI58d//bCsTtgI8ysuYSqG7ut3wWuMLO7q8zrqszpt4EdzOy9zP0G6R5uIjks\nyyRdW2jeuTubLul6YFWhb0mHAk1+P52Ukb8auMrM5kj6J+A+YJca5zoYeD5z/qKXZan+b2NW5ngH\n4j+tIAiCIAg2eFpaWmpKmWykg/Wcmc3140nA6WQcLGAkMMvMXgOQNBkYBVwC7CjpauDXwAx/mN/e\nzKYCmNm79SrjjsburkspDiSld73uY6zM1N3lZUslbVvoErhMaX1SO7B9pm65mS3x4yeBgtPwOOlx\nFuAQ4EhJ5/h5H2AIsKwwqJktADo5V1Xm2Re4A/iGmb0g6TRgmqepFfTuCjsBL5nZQtftDR8vK1Nu\nTneQIi3NwOeAX1aRL4mZ3QOUW+811Mz+1yObD0pabGa1Zmu3Abcord3Krt+aZmbvA3+R9DIwsMb+\nCojS9j4I2FlrjNdP0qZm9veCQJW5VuM1ScPM7A9lJQ7oYs9BeZYTjmpehG3zIeyaD2HX/Ajb5kJL\nS0tEsZympqYOthg3blxJuXVpDVaptSWdHj7NbKWk3UmpUF8DRpNSryo6BpJOBU72cQ43sxWZul7A\nH4F3gGl1zKHAOyV0HgtsA+xpZu1Km070LSHfnjlvZ801ESnK9XQX9KnEdcAvzawQo9gH2M/tszkp\ndXOVmZ3fhb6rOWdl5yTpz5J2JUWyTslUdZL3KFddmNn/+t/lklpIkdFaif3A3AAAIABJREFUHawj\nSM79UcAFkj7h5cXX8UPA+6RIaIFOqa81IGBvM3uvC21fJEXVCnyEjvEoSBGyRZJON7ObujBGEARB\nEARBUIZGrsEaKqmQxvYF4KGi+nnAKElbSeoNjAFme7pdbzO7E/guMNyjJc9LOhpAUh9Jm2Q7M7Nr\nzWxPMxueda68rt3MdgAeIz3gl+JBYLSkrXyMAWXkCk5Gf+AVd64OAIaWkKnEfcAZqxtIe9TQpiIe\nrepnZj8slJnZCWa2g5l9FPhP4H9KOVeSJsrXh5VhGTDI0xaR1M+vW5ZKc7oNOBfYwsyeqEG+ZiRt\n6WmjSNoG2BdY4uffL9w3ZdoKGGJms4HzgC2AfhWGexn4sKQBkjYGPtMFlWcAq9dl+QuFWrkPOFhS\nf79HD/ayLOcD/xzO1Vom3qrmR9g2H8Ku+RB2zY+wbS5E9Kp+GulgPQWcJmkJaXOJ673cANwJOg9o\nAVqB+Z4WNRhokdQK3OwyACcCZ0hqAx6m/nQtgN8DW5Wq8JS+S0lOXitwZVbfrKj/nQyMdH1OAJaW\nkCnVvsDFpGjSYt804XvFApJGSLqhwnyKORvYVWmTi4WS6kkv3A14qVylR1uOB66RtIjkJGxcJFZp\nTr/y9rdlyi6pIN8JSUdKai5RtTPwmF+3B0hr9Z7yul2BSj+b3huY5NdxAXC1mf2thFzhvn3f9ZxP\ncmyWlpDt0KYEZwJ7KW2O8gQdI3pA+bl6CuvFpJcFjwLjitJZAfr4ZhdBEARBEARBN6P4xeg1+Fqf\nrYu2ad/gkbQ5cKOZlYvurbdImm5mhzVaj7WFrwNsM7PtKsgYzWtPpw2GWBuQH2HbfAi75kPYNT/W\nd9s2w7r4XB5rsMojCTPrlJnWyDVY6yJTgJs2tIfuapjZKsqnTq7XbEjXWdIY0o6IP6gq3Jy3NkEQ\nBEEQZBk4uCvJV8G6SESwgiDogCSL74UgCIIgCILKlItgNXINVhAEQRAEQRAEQY8iHKwgCIK1QC0/\nTBh0jbBtPoRd8yHsmh9h23wIu9ZPOFhBEARBEARBEATdRKzBCoKgA7EGKwiCIAiCoDqxBisIgiAI\ngiAIgiBnwsEKgiBYC0QOe36EbfMh7JoPYdf8CNvmQ9i1fuJ3sIIg6ITUKdodBEEQ9FAGDhzKihXP\nNlqNIOgxxBqsIAg6IMkgvheCIAg2HEQ8DwZB/cQarCAIgiAIgiAIgpxpiIMlaaikx8vUzZI0fG3r\n5GMPkbRQ0vRM2fJG6FIOSftLmlCD3HL/W9bWRfKbS3pe0n9nymZJGlKl3QRJo6roe0+18esh26ek\nkyRdVEOb6ZJelzS1xjEuknRilfqzatcaJO0kqVXSAkk7SjpD0hJJN/s8flKlfdW5Stpd0hxJj0ta\nJOlzmboxkp6S9K169A66i5ZGK9CDaWm0Aj2UlkYr0ENpabQCPZZYK5QPYdf6aWQEa12MRR8DzDCz\nwzJl66KetehkZY7LcTEwu2vq1KVLHn3W0v8PgBNy0KMejgHuMLMRZrYc+DpwkJl90evrva6leBP4\nopntChwG/FjSFgBmdiuwPxAOVhAEQRAEQU400sHaSNIkf4N/u6S+xQL+xn2xfy73sl4eNVksqU3S\nmV4+TNL9/tb+MUk7dkGnLYFXispezehzoo/ZKmmil02QdLWkhyU9I+lYL99M0kzXpU3SUV4+VNJS\nb7dM0mRJB3v7ZZL2crlNJY2XNNcjHke6Gu8Cf61hLq8WF0j6ueveKukVSf/l5SOAbYEZRU3+AnxQ\nZZyVrhOSRvo8FrnemxWNX3JOkh6RtHNGbpak4RVskOUt4I0qOmJms2qRy7DK+8YjTU/6vG7JyHzc\ndX1G0uku2yFiKOlsj3YdBnwT+LqkByRdB3wUmF64hzNttpH0S0mP+mefWudqZs+Y2R/8+H9J9/OH\nM/UvA/3rsEPQbTQ1WoEeTFOjFeihNDVagR5KU6MV6LE0NTU1WoUeSdi1fhq5i+BOwJfNbK6k8cCp\nwFWFSknbAZcDe5Ie4u93J+UFYLCZ7eZyW3iTycD3zWyqpD50zXnsDbRnC8xsbx9nF+B8YB8ze13S\nlhmxQWa2rzsJU4EpwNvAMWb2hqStgbleBzAM+KyZLZH0GPB5b3+Uj3EscAHwgJl9RVJ/YJ6kmWb2\nCPCI6zQCOMXMvlo8kYLeRWUne7shwHRggiQBPwLGAgcXyR9XzWBm9i3vcyPgF8BoM1soqR/uoGQo\nOSdvdzzQLGmQ23OhpEvLyGfHv71w7A7YCDNrrqZ3DfO6KnP6bWAHM3svc79BuoebSA7LMknXFpp3\n7s6mS7oeWFXoW9KhQJPfTydl5K8GrjKzOZL+CbgP2KXeuUr6P8BGBYcrQw3/NrLdNhEPBEEQBEEQ\nbOi0tLTUlDLZSAfrOTOb68eTgNPJOFjASGCWmb0GIGkyMAq4BNhR0tXAr4EZ/jC/vZlNBTCzd+tV\nxh2N3V2XUhxISu963cdYmam7y8uWStq20CVwmdL6pHZg+0zdcjNb4sdPAgWn4XFgBz8+BDhS0jl+\n3gcYAiwrDGpmC4BOzlWVefYF7gC+YWYvSDoNmGZmLyUT0NX9uXcCXjKzha7bGz5eVqbcnO4gRc+a\ngc8Bv6wiXxIzuwfo1vVeThtwi6S78GvtTDOz94G/SHoZGFhnv6K0vQ8CdtYa4/WTtKmZ/b0gUG2u\n/oLif4Avlqh+TdKwEo5Xhuaqygf10kI4qnnRQtg2D1oIu+ZBC2HXfGhpaYloSw6EXdfQ1NTUwRbj\nxo0rKddIB6vTW/4SMp0ePs1spaTdgUOBrwGjSalXFR0DSacCJ/s4h5vZikxdL+CPwDvAtDrmUOCd\nEjqPBbYB9jSzdqVNJ/qWkG/PnLez5pqIFOV6ugv6VOI64JeeMgewD7Cf22dzUurmKjM7vwt9V3PO\nys5J0p8l7UqKZJ2Sqeok71GutckRJOf+KOACSZ/w8uLr+CHgfVIktECn1NcaELC3mb3XhbZI2hy4\nF/iOmc0vIXI1sEjS6WZ2U1fGCIIgCIIgCErTyDVYQyUV0ti+ADxUVD8PGCVpK0m9gTHAbE+3621m\ndwLfBYZ7tOR5SUcDSOojaZNsZ2Z2rZntaWbDs86V17Wb2Q7AY6QH/FI8CIyWtJWPMaCMXMHJ6A+8\n4s7VAcDQEjKVuA84Y3UDaY8a2lTEo1X9zOyHhTIzO8HMdjCzjwL/CfxPKedK0kT5+rAyLAMGedoi\nkvr5dctSaU63AecCW5jZEzXId4VOESNJ3y/cNyUbpCjSEDObDZwHbAH0qzDGy8CHJQ2QtDHwmS7o\nOQNYvS7LXyjUhKdq3gVM9H8jpTgf+OdwrtY2TY1WoAfT1GgFeihNjVagh9LUaAV6LBFlyYewa/00\n0sF6CjhN0hLS5hLXe7kBuBN0HimW3grM97SowUCLpFbgZpcBOBE4Q1Ib8DD1p2sB/B7YqlSFp/Rd\nSnLyWoErs/pmRf3vZGCk63MCsLSETKn2BS4mRZMW+6YJ3ysWkDRC0g0V5lPM2cCuSptcLJRUT3rh\nbsBL5So92nI8cI2kRSQnYeMisUpz+pW3vy1TdkkF+U5IOlJSc5m633rfB0p6TlJhvdmuwIpSbZze\nwCS/jguAq83sbyXkCvft+67nfJKDuLSEbIc2JTgT2Etpc5Qn6BjRK8yn3Fw/B+wHfClznXcrkunj\nm10EQRAEQRAE3Yzil7vX4Gt9tjaz86oKb0B4ytmNZlYuurfeIml60bb8PRpfB9hmZttVkLF189cJ\n1ndaiDfXedFC2DYPWgi75kEL655dRU94Hoy1QvkQdi2PJMysU2ZaI9dgrYtMAW7a0B66q2Fmqyif\nOrlesyFdZ0ljSDsi/qAG6bzVCYIgCNYRBg4cWl0oCIKaiQhWEAQdkGTxvRAEQRAEQVCZchGsRq7B\nCoIgCIIgCIIg6FGEgxUEQbAWqOWHCYOuEbbNh7BrPoRd8yNsmw9h1/oJBysIgiAIgiAIgqCbiDVY\nQRB0INZgBUEQBEEQVCfWYAVBEARBEARBEORMOFhBEARrgchhz4+wbT6EXfMh7JofYdt8CLvWT/wO\nVhAEnZDid7CCIAiCoLsZOHggK15Y0Wg1gpyJNVhBEHRAktHcaC2CIAiCoAfSDPHs3XOINVhBEARB\nEARBEAQ50xAHS9JQSY+XqZslafja1snHHiJpoaTpmbLljdClHJL2lzShBrnl/resrYvkN5f0vKT/\nzpTNkjSkSrsJkkZV0feeauPXQ7ZPSSdJuqiGNtMlvS5pao1jXCTpxCr1Z9WuNUjaSVKrpAWSdpR0\nhqQlkm72efykSvta53qSpN9LWpadg6Qxkp6S9K169A66iXXqm6SHEbbNh7BrPoRd8yNsmwuxBqt+\nGhnBWhfjo8cAM8zssEzZuqhnLTpZmeNyXAzM7po6demSR5+19P8D4IQc9KiHY4A7zGyEmS0Hvg4c\nZGZf9Pp6r2snJA0ALgRGAnsDF0nqD2BmtwL7A+FgBUEQBEEQ5EQjHayNJE3yN/i3S+pbLOBv3Bf7\n53Iv6+VRk8WS2iSd6eXDJN0vaZGkxyTt2AWdtgReKSp7NaPPiT5mq6SJXjZB0tWSHpb0jKRjvXwz\nSTNdlzZJR3n5UElLvd0ySZMlHeztl0nay+U2lTRe0lyPeBzparwL/LWGubxaXCDp5657q6RXJP2X\nl48AtgVmFDX5C/BBlXFWuk5IGunzWOR6b1Y0fsk5SXpE0s4ZuVmShlewQZa3gDeq6IiZzapFLsMq\n7xuPND3p87olI/Nx1/UZSae7bIeIoaSzPdp1GPBN4OuSHpB0HfBRYHrhHs602UbSLyU96p996pjr\noaSXBH81s5Wka/rpjB1eBvrXYYegu+jKN1JQG2HbfAi75kPYNT/CtrnQ1NTUaBXWOxq5i+BOwJfN\nbK6k8cCpwFWFSknbAZcDe5Ie4u93J+UFYLCZ7eZyW3iTycD3zWyqpD50zXnsDbRnC8xsbx9nF+B8\nYB8ze13SlhmxQWa2rzsJU4EpwNvAMWb2hqStgbleBzAM+KyZLZH0GPB5b3+Uj3EscAHwgJl9xSMQ\n8yTNNLNHgEdcpxHAKWb21eKJFPQuKjvZ2w0BpgMTJAn4ETAWOLhI/rhqBjOzb3mfGwG/AEab2UJJ\n/XAHJUPJOXm744FmSYPcngslXVpGPjv+7YVjd8BGmFlzNb1rmNdVmdNvAzuY2XuZ+w3SPdxEcliW\nSbq20LxzdzZd0vXAqkLfkg4Fmvx+OikjfzVwlZnNkfRPwH3ALjXOdTDwfOb8RS/LUv3fxqzM8Q7E\nf1pBEARBEGzwtLS01JQy2UgH6zkzm+vHk4DTyThYpBSnWWb2GoCkycAo4BJgR0lXA78GZvjD/PZm\nNhXAzN6tVxl3NHZ3XUpxICm963UfY2Wm7i4vWypp20KXwGVK65Page0zdcvNbIkfPwkUnIbHSY+z\nAIcAR0o6x8/7AEOAZYVBzWwB0Mm5qjLPvsAdwDfM7AVJpwHTzOylZAK6uj/3TsBLZrbQdXvDx8vK\nlJvTHaRISzPwOeCXVeRLYmb3AN263stpA26RdBd+rZ1pZvY+8BdJLwMD6+xXlLb3QcDOWmO8fpI2\nNbO/FwT+wbm+JmmYmf2hrMQBXew5KM9ywlHNi7BtPoRd8yHsmh9h21xoaWmJKJbT1NTUwRbjxo0r\nKddIB6vTW/4SMp0ePs1spaTdSalQXwNGk1KvKjoGkk4FTvZxDjezFZm6XsAfgXeAaXXMocA7JXQe\nC2wD7Glm7UqbTvQtId+eOW9nzTURKcr1dBf0qcR1wC89ZQ5gH2A/t8/mpNTNVWZ2fhf6ruaclZ2T\npD9L2pUUyTolU9VJ3qNca5MjSM79UcAFkj7h5cXX8UPA+6RIaIFOqa81IGBvM3uvC21fJEXVCnyE\njvEoSBGyRZJON7ObujBGEARBEARBUIZGrsEaKqmQxvYF4KGi+nnAKElbSeoNjAFme7pdbzO7E/gu\nMNyjJc9LOhpAUh9Jm2Q7M7NrzWxPMxueda68rt3MdgAeIz3gl+JBYLSkrXyMAWXkCk5Gf+AVd64O\nAIaWkKnEfcAZqxtIe9TQpiIerepnZj8slJnZCWa2g5l9FPhP4H9KOVeSJsrXh5VhGTDI0xaR1M+v\nW5ZKc7oNOBfYwsyeqEG+K3SKGEn6fuG+KdkgRZGGmNls4DxgC6BfhTFeBj4saYCkjYHPdEHPGcDq\ndVn+QqFW7gMOltTf79GDvSzL+cA/h3O1lom3qvkRts2HsGs+hF3zI2ybCxG9qp9GOlhPAadJWkLa\nXOJ6LzcAd4LOA1qAVmC+p0UNBloktQI3uwzAicAZktqAh6k/XQvg98BWpSo8pe9SkpPXClyZ1Tcr\n6n8nAyNdnxOApSVkSrUvcDEpmrTYN034XrGApBGSbqgwn2LOBnZV2uRioaR60gt3A14qV+nRluOB\nayQtIjkJGxeJVZrTr7z9bZmySyrId0LSkZKay9T91vs+UNJzkgrrzXYFKv2kem9gkl/HBcDVZva3\nEnKF+/Z913M+ybFZWkK2Q5sSnAnspbQ5yhN0jOgV5lNyrp7CejHpZcGjwLiidFaAPr7ZRRAEQRAE\nQdDNKH5Neg2+1mdrMzuvqvAGhKTNgRvNrFx0b71F0vSibfl7NL4OsM3MtqsgYzSvPZ02GGJtQH6E\nbfMh7JoPYdf8WB9s2wzr27N3rMEqjyTMrFNmWiPXYK2LTAFu2tAeuqthZqsonzq5XrMhXWdJY0g7\nIv6gqnBz3toEQRAEwYbHwMFdSbAK1jcighUEQQckWXwvBEEQBEEQVKZcBKuRa7CCIAiCIAiCIAh6\nFOFgBUEQrAVq+WHCoGuEbfMh7JoPYdf8CNvmQ9i1fsLBCoIgCIIgCIIg6CZiDVYQBB2INVhBEARB\nEATViTVYQRAEQRAEQRAEORMOVhAEwVogctjzI2ybD2HXfAi75kfYNh/CrvUTv4MVBEEnpE7R7qCb\nGDhwKCtWPNtoNYIgCIIgyIlYgxUEQQckGcT3Qn6I+N4NgiAIgvWfWIMVBEEQBEEQBEGQMzU5WJKG\nSnq8TN0sScO7V63akDRE0kJJ0zNlyxuhSzkk7S9pQg1yyzPy95STkbRVN+pVcpxMfUW9/b6YVUWm\n2++PbJ+1XG9Ju0maI6lN0t2S+tXQpmK/klbVrvHqNj+U9LikKyRtI2mupAWS9qvl2tY41x9IWipp\nkaRfSdoiU/dbSfMkbVuv7kF30NJoBXossT4gH8Ku+RB2zY+wbT6EXeunngjWupjTcgwww8wOy5St\ni3rWopOVOa63n3qo1l+9ejeCWsa/ETjXzHYH7gTO7YZ+uzLvk4HdzOzbwEHAYjMbYWa/q7G/WmRm\nAB83sz2Ap4HvrG5sNgpYABxRt+ZBEARBEARBTdTjYG0kaZKkJZJul9S3WEDSGEmL/XO5l/WSNMHL\n2iSd6eXDJN3vb9ofk7RjF/TfEnilqOzVjD4n+pitkiZ62QRJV0t6WNIzko718s0kzXRd2iQd5eVD\nPSIwQdIySZMlHeztl0nay+U2lTQ+E5U40tV4F/hrDXN5NXPcX9K9kp6SdG2mfHWOp6SzPBqyOGPT\nTb1dq5eP9vKRru8i12+z7MCSpnkksFXSSklfrFHvD4DXvI9emQjNIkmnFQu73ea4jW9zfQ+VdHtG\nZnVkTdIhxfJV7FaOj7kTAzAT+GwNbV51HQZJmu32WSxp3zWq6hKf6xxJH/bCCYV7ys9X+d+7gX7A\nAknnAlcAx3i/fel4bcdKetTrrpNW7zhRda5mNtPM2v10LvCRIpEVpH83wVqnqdEK9FiampoarUKP\nJOyaD2HX/Ajb5kPYtX7q2UVwJ+DLZjZX0njgVOCqQqWk7YDLgT2BlcD97qS8AAw2s91crpCyNBn4\nvplNldSHrq0H6w20ZwvMbG8fZxfgfGAfM3tdUvahcpCZ7StpZ2AqMAV4GzjGzN6QtDXp4XSqyw8D\nPmtmSyQ9Bnze2x/lYxwLXAA8YGZfkdQfmCdpppk9AjziOo0ATjGzrxZPpKC3MxLYGXgOuE/SsWY2\npVCplB53ksv1Bh6V1OJ6vmhmn3G5zSVtBPwCGG1mC5XS494qGvuITL//D7jLzFYV9C6Hmb0AHOen\nXwWGkiI0VmRv3KbfBT5lZm+5k3EWcBnwM0mbmNlbwPHALS5/QQn5S8rZTdI04CtmtqJI1SclHWVm\nU4HP0dnpKDW3Qr9fAH5jZpe5o1Nw8jYD5pjZdyVdQYpOfb9UV97f0ZL+ZmaF1MaXgRFmdoafF+bw\nL26DfzWzDyT9FBgLTKpxrln+g3Tts7ST7pkqNGeOmwjnIAiCIAiCDZ2WlpaaUibrcWqeM7O5fjwJ\n2K+ofiQwy8xe8zfok4FRwB+BHT1qdCiwyh/yt/cHXszsXTN7uw5d8Ifd3UkOXCkOBO4ws9d9jJWZ\nuru8bClQWI8i4DJJbaQox/Zas1ZluZkt8eMnvR7gcWAHPz4EOE9SK2mxRR9gSFYhM1tQyrkqwTwz\n+5OlrcZupbOt9wPuNLO3zexNkoP4b67PwZIuk7SfO0k7AS+Z2ULX4Y1MhGM1krYBbgbGeLt6OQj4\nmetcbG+ATwK7AA+7jU4EhpjZB8BvgCMl9Salr00tJ19JATM7oozD8R/AaZLmkxyjd+uY13zgy5Iu\nJDmPb3r5O2b2az9ewJr7oJha9zsvpP99ChgOzPd5Hwh8tJNw+bmmQaULgPfM7JaiqheB3aqr05z5\nNFUXD2qgpdEK9FhifUA+hF3zIeyaH2HbfAi7rqGpqYnm5ubVn3LUE8EqXv9Raj1Ip4dJM1spaXfg\nUOBrwGjgm6VkO3QknUqKChhwePZhUlIvkuP2DjCtjjkUeKeEzmOBbYA9zaxdaUOBviXk2zPn7ayx\noUhRrqe7oE8xtdi6cyOzpz0KdThwsaQHSM5kNVv3Ijlyze505oFI6+XGlqi7DfgG8Dow38zedAe6\nnHxdmNnvSfcfkj5GHWuQzOwhSaO8zU2SrjSzScB7GbEPWHMfvI+/uPA5bFSnugImmtkFdbZb04H0\nJdI9cGCJ6inAhZKWmNkuXR0jCIIgCIIgKE09EayhkrJpUw8V1c8DRknayiMRY4DZnurV28zuJKWI\nDTezN4DnJR0NIKmPpE2ynZnZtWa2p5kNL35Tb2btZrYD8BgpnaoUDwKj5TuzSRpQRq7gfPQHXnHn\n6gBSuluxTCXuA85Y3UDao4Y25dhbae1XL9L8im39EGn9Tl+l9VT/DjzkaZpvedTiR6RIyDJgkKcn\nIqmfX58sVwBtZnZHKWWU1nBNrKLz/cAphb5L2HsusK+kYV6/qTs7ALNd15NZk9JWSb4uMuujepHu\nwev9fHtJM6u0HUK6L8aTNsso7IhY7p54FtjLj4+mo4NV6T4q1D0AHJfReYDrUBOSPg2cAxxlZu+U\nEDkRmB7OVSNoarQCPZZYH5APYdd8CLvmR9g2H8Ku9VOPg/UUKc1qCWmR/PVeXkgJWwGcR8qDaSVF\nIu4BBgMtnu50s8tAetA7w1PyHgYGdkH/3wMlt7b2lL5LSU5eK3BlVt+sqP+dDIx0fU4AlpaQKdW+\nwMWkjUAWK21p/71iAUkjJN1QYT4F5gHXkNIR/2Bmd2XHNrNW4CZS+tojwA1m1gbsSlr71QpcCFxi\nZu+RnLRrJC0i7TK3cdF4ZwOHKG1ysVDSZ4rqhwB/r6LzjcDzwGIff0yRzn8GvgTc6jaeQ0pfxFMW\n7wU+7X8rylPmGiht1jGoRNUYScuAJaQ1ajd5+XZ0jESVoglok7SQtH7rx5V0AH4O7O82+CTwZqau\nUiSyYKelJCdwhs97BtBpThXm+hPSZhr3+7W8tqh+AGl3wSAIgiAIgiAH5Etm1ksknQNsbWbnVRUO\nuoxv4nCzmT3RaF26E6WdDv9kZvc2Wpe1hW+asdjMflZBxhq/+35PpIXkr4v1+Xt3XaSlpSXesOZA\n2DUfwq75EbbNh7BreSRhZp0ylOpZg7UuMoW0LmZ60W9hBd2I/25Tj8PMftpoHdYmkmaT1g2W2u0w\nCIIgCIIg6AbW6whWEATdT4pgBXkxcOBQVqx4ttFqBEEQBEHwD9JTI1hBEORAvHgJgiAIgiDoGl35\ncd8gCIKgTuJ3RPIjbJsPYdd8CLvmR9g2H8Ku9RMOVhAEQRAEQRAEQTcRa7CCIOiAJIvvhSAIgiAI\ngsqUW4MVEawgCIIgCIIgCIJuIhysIAiCtUDksOdH2DYfwq75EHbNj7BtPoRd6yccrCAIgiAIgiAI\ngm4i1mAFQdCB+B2s9ZOBgwey4oUVjVYjCIIgCDYYyq3BCgcrCIIOSDKaG61FUDfN8ftlQRAEQbA2\niU0ugiAIGsnyRivQc4n1AfkQds2HsGt+hG3zIexaPzU5WJKGSnq8TN0sScO7V63akDRE0kJJ0zNl\n69RjjKT9JU2oQW55Rv6ecjKStupGvUqOk6mvqLffF7OqyHT7/ZHts5brLWk3SXMktUm6W1K/GtpU\n7FfSqto1Xt3mh5Iel3SFpG0kzZW0QNJ+tVzbGuc6QNIMScsk3Sepf6but5LmSdq2Xt2DIAiCIAiC\n2qgngrUu5p4cA8wws8MyZeuinrXoZGWO6+2nHqr1V6/ejaCW8W8EzjWz3YE7gXO7od+uzPtkYDcz\n+zZwELDYzEaY2e9q7K8WmfOAmWa2E/Ag8J3Vjc1GAQuAI+rWPPjH2bHRCvRcmpqaGq1CjyTsmg9h\n1/wI2+ZD2LV+6nGwNpI0SdISSbdL6lssIGmMpMX+udzLekma4GVtks708mGS7pe0SNJjkrry+LEl\n8EpR2asZfU70MVslTfSyCZKulvSwpGckHevlm0ma6bq0STrKy4dKWurtlkmaLOlgb79M0l4ut6mk\n8ZmoxJGuxrvAX2uYy6uZ4/6S7pX0lKRrM+WrczwlneXRkMUZm27q7Vq9fLSXj3R9F7l+m2UHljTN\nI4GtklZK+mKNen8AvOZ99MpEaBZJOq1Y2O02x218m+t7qKTbMzIqbMp2AAAgAElEQVSrI2uSDimW\nr2K3cnzMnRiAmcBna2jzquswSNJst89iSfuuUVWX+FznSPqwF04o3FN+vsr/3g30AxZIOhe4AjjG\n++1Lx2s7VtKjXnedpEJdLXM9GpjoxxNJLyGyrCD9uwmCIAiCIAhyoB4HayfgGjPbBVgFnJqtlLQd\ncDnQBOwBjHQnZQ9gsJnt5hGEQtrZZOAnZrYH8K/A/3ZB/95Ae7bAzPZ2fXYBzgeazGxP4MyM2CAz\n2xc4kvSgC/A2cIyZ7QUcCFyZkR8G/NCjAjsBn/f25/gYABcAD5jZJ739jyRtYmaPmNm3XKcRkm4o\nNZGC3s5I4DRgZ+Cfsw/s3s9w4CSX2wc4WdLuwKeBF81sTzPbDfiNpI2AXwCnu60PAt4qGvsIMxsO\nfAV4Frgrq3c5zOwFMzvOT78KDCVFaPYgXd+szlsD3wU+5TZeAJxFcnj+j6RNXPR44BaXv6CEfFm7\nuaM4qISqTxYcZuBzwEcqzauo3y8Av3H77A4s8vLNgDk+14dI0amSXXl/RwN/N7PhZvYD4ELgF37+\ndmYO/+I2+Fcfsx0YW8dctzWzl11+BVCcDthO+ndTmVmZzzqVdLseE3bMjVgfkA9h13wIu+ZH2DYf\nwq5raGlpobm5efWnHB+qo8/nzGyuH08CTgeuytSPBGaZWSGiMRkYBVwC7CjpauDXwAylNTDbm9lU\nADN7tw498P5FeuCdVEbkQOAOM3vdx1iZqbvLy5ZqzXoUAZdJGkV6CN0+U7fczJb48ZMkpwDgcWAH\nPz4EOFLSOX7eBxgCLCsMamYLSI5INeaZ2Z98nrcC+wFTMvX7AXcWHswlTQH+DbiP5NhdBkwzs99J\n+gTwkpktdB3e8DYdBpS0DXAzcJyZ1b2+iOS4XWe+jVmRvQE+CewCPOzXbiOSg/KBpN+QbPcrUvra\nOSRHvZN8JQXMrFzq238AP5H0X8BUUnSuVuYD491RvdvM2rz8HTP7tR8vIM2/FJ12lilDIf3vU8Bw\nYL7Puy/wcifh8nMt12+BF0m2rcwBNfYeBEEQBEGwgdDU1NQhZXLcuHEl5epxsIof1EqtB+n0MGlm\nKz26cijwNWA08M1Ssh06kk4lRQUMONzfxhfqegF/BN4BptUxhwLvlNB5LLANsKeZtSttKNC3hHx7\n5rydNTYU8Fkze7oL+hRTi607NzJ72qNbhwMXS3qA5ExWs3Uv4Fag2cyWdkHfWhBpvdzYEnW3Ad8A\nXgfmm9mb7lyUk68LM/s96f5D0seoYw2SmT3kTvcRwE2SrjSzScB7GbEPWHMfvI9HhjOOYT0ImGhm\nF9TZrsDLkgaa2cse4SpOoZ0CXChpiUejg7VFrMHKjVgfkA9h13wIu+ZH2DYfwq71U0+K4FBJ2bSp\nh4rq5wGjJG0lqTcwBpjtqV69zexOUorYcI+iPC/paABJfTIpYgCY2bWe6jY861x5XbuZ7QA8Rkqn\nKsWDwGj5zmySBpSRKzgf/YFX3Lk6gJTuVixTifuAM1Y3kPaooU059lZa+9WLNL9iWz9EWr/T19dT\n/TvwkKdpvmVmtwA/IkVClgGDJI1wvfr59clyBdBmZneUUkZpDdfEUnUZ7gdOKfRdwt5zgX0lDfP6\nTd3ZAZjtup5MSmesJl8XmfVRvUj34PV+vr2kmVXaDiHdF+NJm2UUdkQsd088C+zlx0fT0cGqdB8V\n6h4AjsvoPMB1qJWpwJf8+CTg7qL6E4Hp4VwFQRAEQRDkQz0O1lPAaZKWkBbJX+/lhZSwFaQdzFqA\nVlIk4h5gMNAiqZWUgnaetzsROENSG/AwMLAL+v8eKLm1taf0XUpy8lpZs6aqXHRoMmndWBtwArC0\nhEyp9gUuJm0EslhpS/vvFQtUWoNVxDzgGlI64h/M7K7s2GbWCtxESl97BLjBU9d2Beb5fC8ELjGz\n90hO2jWSFgEzgI2LxjsbOERpk4uFkj5TVD8E+HsVnW8EngcW+/hjinT+M+nB/1a38RzSejbMrB24\nl7SG7N5q8pS5BhXWJY2RtAxYQlqjdpOXb0fHSFQpmoA2SQtJ67d+XEkH4OfA/m6DTwJvZuoqRSIL\ndlpKcgJn+LxnAJ3mVGGuVwAH+3w/RVoXmWUA0B1R1qBeYg1WbsT6gHwIu+ZD2DU/wrb5EHatH/mS\nmfUSX++0tZmdV1U46DKSrgBuNrMnGq1Ld6K00+GfzOzeRuuytpD0U9L28D+rIGM0rz2dNhiWk2+a\nYDOsz9/n/wgtLS2RwpIDYdd8CLvmR9g2H8Ku5ZGEmXXKUFrfHaxhpEjOG0W/hRUEQRGSZpPWDZ5g\nZi9WkFt/vxQ2YAYOHsiKF1ZUFwyCIAiCoFvokQ5WEATdjySL74UgCIIgCILKlHOw6lmDFQRBEHSR\nyGHPj7BtPoRd8yHsmh9h23wIu9ZPOFhBEARBEARBEATdRKQIBkHQgUgRDIIgCIIgqE6kCAZBEARB\nEARBEORMOFhBEARrgchhz4+wbT6EXfMh7JofYdt8CLvWTzhYQRAEQRAEQRAE3USswQqCoAPxO1hB\nEAQwcOBQVqx4ttFqBEGwDhO/gxUEQU0kByu+F4Ig2NAR8YwUBEElYpOLIAiChtLSaAV6MC2NVqCH\n0tJoBXoksZ4lP8K2+RB2rZ+aHCxJQyU9XqZulqTh3atWbUgaImmhpOmZsuWN0KUckvaXNKEGueUZ\n+XvKyUjaqhv1KjlOpr6i3n5fzKoi0+33R7bPWq63pIskveD3ykJJn66hTcV+Ja2qXePVbX4o6XFJ\nV0jaRtJcSQsk7VfLta1xrj+QtFTSIkm/krRFpu63kuZJ2rZe3YMgCIIgCILaqCeCtS7GyY8BZpjZ\nYZmydVHPWnSyMsf19lMP1fqrV+9GUOv4V5nZcP/8phv67cq8TwZ2M7NvAwcBi81shJn9rsb+apGZ\nAXzczPYAnga+s7qx2ShgAXBE3ZoH3UBToxXowTQ1WoEeSlOjFeiRNDU1NVqFHkvYNh/CrvVTj4O1\nkaRJkpZIul1S32IBSWMkLfbP5V7WS9IEL2uTdKaXD5N0v79pf0zSjl3Qf0vglaKyVzP6nOhjtkqa\n6GUTJF0t6WFJz0g61ss3kzTTdWmTdJSXD/WIwARJyyRNlnSwt18maS+X21TS+ExU4khX413grzXM\n5dXMcX9J90p6StK1mfLVOZ6SzvJoyOKMTTf1dq1ePtrLR7q+i1y/zbIDS5rmkZ1WSSslfbFGvT8A\nXvM+emUiNIsknVYs7Hab4za+zfU9VNLtGZnVkTVJhxTLV7FbJTrlx1bhVddhkKTZbp/FkvZdo6ou\n8bnOkfRhL5xQuKf8fJX/vRvoByyQdC5wBXCM99uXjtd2rKRHve46SYW6qnM1s5lm1u6nc4GPFIms\nIP27CYIgCIIgCPLAzKp+gKFAO/BJPx8PnOXHs4DhwHbAn4CtSI7bA8BRXjcj09cW/ncucJQf9wH6\n1qJLkV7jgG+WqdsFeAoY4Odb+t8JwG1+vDPwtB/3Bvr58daZ8qEkZ2MXP38MGO/HRwFT/PhS4At+\n3B9YBmxSpNMI4IYqc9of+LuPK1JE4livW+72HQ60AX2BzYAngN2BY4GfZfraHNgI+AMw3Mv6+fXZ\nH5haNPZwYBGweReuxdeA21mzcUrB3oX7Y2tgdsEmwLnAd93uz2bKrwXGlJPP9llCh2nAoBLlF7nt\nFgE3Av3rmNdZwHf8WMBmftwOHO7HVwDnZ+6vYzPt/1bm+CTgvzPnhWv7L8BUoLeX/xQ4oda5FslM\nLdyTmbL/Av6zSjuDizKfWQYWn3/4E3YM265vnw3drlgezJo1K5d+g7BtXoRd1zBr1iy76KKLVn/8\ne4Liz4eonefMbK4fTwJOB67K1I8EZplZIaIxGRgFXALsKOlq4NfADEn9gO3NbCpJs3fr0APvXySn\nYlIZkQOBO8zsdR9jZabuLi9bmlmPIuAySaNID8/bZ+qWm9kSP34SmOnHjwM7+PEhwJGSzvHzPsAQ\nkqOFj7cA+GoN05tnZn/yed4K7AdMydTvB9xpZm+7zBTg34D7gB9JugyYZma/k/QJ4CUzW+g6vOFt\nOgwoaRvgZuA4M6t7fREp5e06MzMfZ2VR/SdJTu/Dfu02AuaY2QeSfkOy3a9I6WvnkHJTOslXUsDM\nyqW+XQt8z8xM0iWk+/YrNc5rPjBe0kbA3WbW5uXvmNmv/XgBaf6lqDVyZv73UySHdL7Puy/wcifh\n8nNNg0oXAO+Z2S1FVS9SU95Pc3WRIAiCIAiCDYimpqYOKZPjxo0rKVePg2VVzqHEw6SZrZS0O3Ao\nKcoxGvhmKdkOHUmnktasGClSsCJT1wv4I/AO6U1+vbxTQuexwDbAnmbWrrShQN8S8u2Z83bW2FDA\nZ83s6S7oU0wttu7cyOxppc0fDgculvQAyZmsZutewK1As5kt7YK+tSBSJHNsibrbgG8ArwPzzexN\ndy7KydeFmWVT634OlN3co0Tbh9zpPgK4SdKVZjYJeC8j9gFr7oP38dTbjGNYDwImmtkFdbZb04H0\nJdI9cGCJ6inAhZKWmNkuXR0j6ApNjVagB9PUaAV6KE2NVqBHEutZ8iNsmw9h1/qpZw3WUEl7+/EX\ngIeK6ucBoyRtJak3Kc1rtqStSelOd5JSwoZ7FOV5SUcDSOojaZNsZ2Z2rZntaWlTghVFde1mtgMp\nXe/4Mvo+CIyW78wmaUAZuYLz0R94xZ2rA0gpesUylbgPOGN1A2mPGtqUY2+ltV+9SPMrtvVDpPU7\nfX091b8DD0naDnjLoxY/IkVClgGDJI1wvfr59clyBdBmZneUUsbXcE2sovP9wCmFvkvYey6wr6Rh\nXr+ppI953WzX9WTgFzXI14WkQZnTY0kplUjaXtLM0q1Wtx1Cui/Gk9ILCzsilrsnngX28uOj6ehg\nVbqPCnUPAMdl1nQNcB1qQmmHxHNI6bfvlBA5EZgezlUQBEEQBEE+1ONgPQWcJmkJaZH89V5eSAlb\nAZxH+uGMVlIk4h5gMNAiqZWUgnaetzsROENSG/AwMLAL+v+etG6lE57SdynJyWsFrszqmxX1v5OB\nka7PCcDSEjKl2he4mLQRyGKlLe2/VywgaYSkGyrMp8A84BpSOuIfzOyu7Nhm1grcREpfe4S0rqsN\n2BWY5/O9ELjEzN4jOWnXSFpEWtO1cdF4ZwOHKG1ysVDSZ4rqh5DWhVXiRuB5YLGPP6ZI5z8DXwJu\ndRvPAXbyunbgXuDT/reiPGWugdJmHYNKVP3Ar8si0tqzb3n5dnSMRJWiCWiTtBD4HPDjSjqQImT7\nuw0+CbyZqasUiSzYaSnpRcQMn/cMoNOcKsz1J6R1dvf7tby2qH4AaXfBYK3T0mgFejAtjVagh9LS\naAV6JPGbQvkRts2HsGv9FDYkWC/x9U5bm9l5VYWDLiPpCuBmM3ui0bp0J0o7Hf7JzO5ttC5rC0k/\nJW0P/7MKMlZjVmpQFy1EylVetBC2zYMWNmy7ijyekVpaWiLlKifCtvkQdi2PJMysU4bS+u5gDSNF\nct6wjr+FFQRBEZJmk9YNnmBmL1aQW3+/FIIgCLqJgQOHsmLFs41WIwiCdZge6WAFQdD9SLL4XgiC\nIAiCIKhMOQernjVYQRAEQReJHPb8CNvmQ9g1H8Ku+RG2zYewa/2EgxUEQRAEQRAEQdBNRIpgEAQd\niBTBIAiCIAiC6kSKYBAEQRAEQRAEQc6EgxUEQbAWiBz2/Ajb5kPYNR/CrvkRts2HsGv9hIMVBEEQ\nBEEQBEHQTcQarCAIOhC/gxUEQbDuMXDwQFa8sKLRagRBkCF+BysIgpqQZDQ3WosgCIKgA80Qz2xB\nsG4Rm1wEQRA0kuWNVqAHE7bNh7BrPoRdcyPWCuVD2LV+anKwJA2V9HiZulmShnevWrUhaYikhZKm\nZ8rWqa8uSftLmlCD3PKM/D3lZCRt1Y16lRwnU19Rb78vZlWR6fb7I9tnLddb0kWSXvB7ZaGkT9fQ\npmK/klbVrvHqNj+U9LikKyRtI2mupAWS9qvl2tY41wGSZkhaJuk+Sf0zdb+VNE/StvXqHgRBEARB\nENRGPRGsdTEufQwww8wOy5Sti3rWopOVOa63n3qo1l+9ejeCWse/ysyG++c33dBvV+Z9MrCbmX0b\nOAhYbGYjzOx3NfZXi8x5wEwz2wl4EPjO6sZmo4AFwBF1ax784+zYaAV6MGHbfAi75kPYNTeampoa\nrUKPJOxaP/U4WBtJmiRpiaTbJfUtFpA0RtJi/1zuZb0kTfCyNklnevkwSfdLWiTpMUld+crZEnil\nqOzVjD4n+pitkiZ62QRJV0t6WNIzko718s0kzXRd2iQd5eVDJS31dsskTZZ0sLdfJmkvl9tU0vhM\nVOJIV+Nd4K81zOXVzHF/SfdKekrStZny1Tmeks7yaMjijE039XatXj7ay0e6votcv82yA0ua5pGd\nVkkrJX2xRr0/AF7zPnplIjSLJJ1WLOx2m+M2vs31PVTS7RmZ1ZE1SYcUy1exWyU65cdW4VXXYZCk\n2W6fxZL2XaOqLvG5zpH0YS+cULin/HyV/70b6AcskHQucAVwjPfbl47XdqykR73uOkmFulrmejQw\n0Y8nkl5CZFlB+ncTBEEQBEEQ5MCH6pDdCfiymc2VNB44FbiqUClpO+ByYE9gJXC/OykvAIPNbDeX\n28KbTAa+b2ZTJfWha+vBegPt2QIz29vH2QU4H9jHzF6XlH2oHGRm+0raGZgKTAHeBo4xszckbQ3M\n9TqAYcBnzWyJpMeAz3v7o3yMY4ELgAfM7CueljVP0kwzewR4xHUaAZxiZl8tnkhBb2cksDPwHHCf\npGPNbEqhUik97iSX6w08KqnF9XzRzD7jcptL2gj4BTDazBZK6ge8VTT2EZl+/x9wl5mtKuhdDjN7\nATjOT78KDCVFaKzI3rhNvwt8yszecifjLOAy4GeSNjGzt4DjgVtc/oIS8peUs5ukacBXzKzUNkvf\ncMfxMeBsM6voPGb6/QLwGzO7zB2dgpO3GTDHzL4r6QpSdOr7pbry/o6W9DczK6Q2vgyMMLMz/Lww\nh39xG/yrmX0g6afAWGBSjXPd1sxe9jFXqHM6YDvpnqlMNvFzB+KNa3ewnLBjXoRt8yHsmg9h19xo\naWmJaEsOhF3X0NLSUtOatHocrOfMbK4fTwJOJ+NgkR72Z5lZIaIxGRhFeiDeUdLVwK+BGf6Qv72Z\nTQUws3fr0APvX8DurkspDgTuMLPXfYyVmbq7vGxp5gFUwGWSRpEeQrfP1C03syV+/CQw048fJz1+\nAhwCHCnpHD/vAwwBlhUGNbMFJEekGvPM7E8+z1uB/UhOYIH9gDvN7G2XmQL8G3Af8CNJlwHTzOx3\nkj4BvGRmC12HN7xNhwElbQPcDBznzlW9HARcZ77FUZG9AT4J7AI87NduI5KD8oGk35Bs9ytS+to5\nQFMp+UoKFBzFElwLfM8dv0tI9+1XapzXfGC8O6p3m1mbl79jZr/24wWk+Zei1shZIf3vU8BwYL7P\nuy/wcifh8nMt12+BF0m2rcwBNfYeBEEQBEGwgdDU1NTB2Rw3blxJuXocrOIHtVLrQTo9TJrZSkm7\nA4cCXwNGA98sJduhI+lUUlTAgMOzb+ol9QL+CLwDTKtjDgXeKaHzWGAbYE8za1faUKBvCfn2zHk7\na2woUpTr6S7oU0wttu7cyOxpj0IdDlws6QGSM1nN1r2AW4FmM1vaBX1rQaT1cmNL1N0GfAN4HZhv\nZm+6c1FOvi7MLJta93Og7OYeJdo+5E73EcBNkq40s0nAexmxD1hzH7yPR2MzjmE9CJhoZhfU2a7A\ny5IGmtnLkgbROYV2CnChpCVmtksXxwi6Qryxzo+wbT6EXfMh7JobEWXJh7Br/dSTljdUUjZt6qGi\n+nnAKElbSeoNjAFme6pXbzO7k5QiNtyjKM9LOhpAUh9Jm2Q7M7NrzWxP35RgRVFdu5ntQEr3Or6M\nvg8Co+U7s0kaUEau4Hz0B15x5+oAUrpbsUwl7gPOWN1A2qOGNuXYW2ntVy/S/Ipt/RBp/U5fpfVU\n/w485Gmab5nZLcCPSJGQZcAgT09EUj+/PlmuANrM7I5Syiit4ZpYqi7D/cAphb5L2HsusK+kYV6/\nqaSPed1s1/VkUjpjNfm6cEejwLHAE16+vaSZpVutbjuEdF+MB250PaH8PfEssJcfH01HB6vSfVSo\newA4TmvWdA1wHWplKvAlPz4JuLuo/kRgejhXQRAEQRAE+VCPg/UUcJqkJaRF8td7eSElbAVpB7MW\noJUUibgHGAy0SGolpaCd5+1OBM6Q1AY8DAzsgv6/B0pube0pfZeSnLxW4MqsvllR/zsZGOn6nAAs\nLSFTqn2Bi0kbgSxW2tL+e8UCkkZIuqHCfArMA64hpSP+wczuyo5tZq3ATaT0tUeAGzx1bVfS2q9W\n4ELgEjN7j+SkXSNpETAD2LhovLOBQ5Q2uVgo6TNF9UOAv1fR+UbgeWCxjz+mSOc/kx78b3UbzyGt\n68PM2oF7gU/734rylLkGSpt1DCpR9QO/LouA/YFvefl2dIxElaIJaJO0EPgc8ONKOpAiZPu7DT4J\nvJmpqxSJLNhpKelFxAyf9wyg05wqzPUK4GBJy0jphpcX1Q8AuiPKGtTLOvUDEj2MsG0+hF3zIeya\nG/F7TfkQdq0frc+/Cu7rnbY2s/OqCgddxjdxuNnMnmi0Lt2J0k6HfzKzexuty9rCN81YbGY/qyBj\nNK89nTYYYmF7foRt8yHsmg9dtWszrM/PbGuD2IwhH8Ku5ZGEmXXKUFrfHaxhpEjOG0W/hRUEQRGS\nZpPWDZ5gZi9WkFt/vxSCIAh6KAMHD2TFC6U2yQ2CoFH0SAcrCILuR5LF90IQBEEQBEFlyjlYXfnt\nqSAIgqBOIoc9P8K2+RB2zYewa36EbfMh7Fo/4WAFQRAEQRAEQRB0E5EiGARBByJFMAiCIAiCoDqR\nIhgE/5+9O4+zo6rzPv75JoBsQ4CoCSAJTEZ5RIUQQFEwacSdVQw7A8OAy0tHVFyGEZUEQUAFjSAq\nYyagiQjMRHYkLOkMBENCCAmQEEHC6hPgUeIERxHTv+ePc25SffuuTVe603zfr9d9dd1Tp6p+9atK\n554+59Q1MzMzMyuZG1hmZuuBx7CXx7kth/NaDue1PM5tOZzX9rmBZWZmZmZm1kc8B8vMuvH3YJmZ\nlW/EiNGsXPl4f4dhZq+AvwfLzFqSGlj+vWBmVi7hz2BmGzY/5MLMrF919ncAg1hnfwcwSHX2dwCD\nVGd/BzBoea5QOZzX9rXUwJI0WtIDddbNljSub8NqjaRRku6TdHOhbEV/xFKPpAmSprVQb0Wh/vX1\n6kjatg/jqnmcwvqGcef7YnaTOn1+fxT32cr1ljRR0oOS1rQaS7P9SlrdWrTdtvm2pAcknS/ptZLm\nSVooab9Wrm2L5/otScsk3S/pvyRtVVj335LmS3p9u7GbmZmZWWva6cEaiP3YhwGzIuJDhbKBGGcr\nMUWd5Xb3045m+2s37v7QyvEfAD4CzOnD/fbmvD8G7BYR/wq8F1gSEXtGxF0t7q+VOrOAt0TEWOAR\n4N/WbhwxHlgIHNh25NYHOvo7gEGso78DGKQ6+juAQaqjvwMYtDo6Ovo7hEHJeW1fOw2sjSVNl7RU\n0lWSNq2uIOkYSUvy67xcNkTStFy2WNJnc/kYSbfmv7TfK2nnXsS/NfBcVdnzhXhOyMdcJOnyXDZN\n0hRJcyU9KunwXL6FpNtyLIslHZLLR+cegWmSlkuaIel9efvlkvbK9TaXNLXQK3FwDuOvwB9bOJfn\nC8vDJN0g6WFJlxTK147xlHRa7g1ZUsjp5nm7Rbn8iFy+d473/hzfFsUDS7ox9wQukrRK0j+2GPca\n4A95H0MKPTT3S/p0deWct7tzjq/M8X5A0lWFOmt71iS9v7p+k7zVFBHLI+KRYv5a8HyOYaSkOTk/\nSyTtuy5UnZ3P9W5Jr8uF0yr3VH6/Ov+8FtgSWCjpy8D5wGF5v5vS/doeJ+mevO6HkirrWjnX2yKi\nK7+dB7yhqspK0r8bMzMzMyvBRm3U3QU4KSLmSZoKfAq4sLJS0nbAecAewCrg1txIeRrYISJ2y/Uq\nQ5ZmAN+MiOskbULv5oMNBbqKBRHxjnycXYGvAO+MiBckFT9UjoyIfSW9GbgOmAn8BTgsIl6UNJz0\n4fS6XH8M8NGIWCrpXuDovP0h+RiHA2cAt0fEyZKGAfMl3RYRvwZ+nWPaE/hERHy8+kQqcWd7A28G\nngRukXR4RMysrFQa5nZirjcUuEdSZ47zmYg4KNf7O0kbA78AjoiI+yRtCfy56tgHFvb7H8A1EbG6\nEnc9EfE0MDG//TgwmtRDE1X5Juf0q8ABEfHn3Mg4DTgX+LGkzSLiz8BRwM9z/TNq1D+7Xt4k3Qic\nHBErG8XdisJ+jwV+FRHn5oZOpZG3BXB3RHxV0vmk3qlv1tpV3t+hkv4nIipDG58F9oyIU/P7yjn8\nn5yDd0XEGkk/AI4DpvfiXP+ZdO2Lukj3TBOTCssd+C+ufaET57EsnTi3ZejEeS1DJ85rOTo7O93b\nUgLndZ3Ozs6W5qS108B6MiLm5eXpwGcoNLBIH/ZnR0SlR2MGMJ70gXhnSVOAm4BZ+UP+9hFxHUBE\n/LWNOMj7F7B7jqWW9wBXR8QL+RirCuuuyWXLtG4+ioBzJY0nfQjdvrBuRUQszcsPAbfl5QeAnfLy\n+4GDJX0pv98EGAUsrxw0IhaSGiLNzI+IJ/J5XgHsR2oEVuwH/DIi/pLrzATeDdwCfEfSucCNEXGX\npLcCv4uI+3IML+Ztuh1Q0muBnwETc+OqXe8Ffhj5kUhV+QbYB9gVmJuv3cakBsoaSb8i5e6/SMPX\nvkT636dH/UYBVBqKfWwBMDU3VK+NiMW5/KWIuCkvLySdfy2t9ppVhv8dAIwDFuTz3hR4tkflJucq\n6Qzg5Yj4edWqZ2jpf/ZJzauYmZmZvYp0dHR0a2xOnjy5ZuJYhU4AACAASURBVL12GljV8z9qzQfp\n8WEyIlZJ2h34APBJ4Ajgc7XqdtuR9ClSr0AAHy7+pV7SEOAx4CXgxjbOoeKlGjEfB7wW2CMiupQe\nKLBpjfpdhfddrMuhSL1cj/Qinmqt5LrnRhGP5F6oDwPfkHQ7qTHZLNdDgCuASRGxrBfxtkKk+XLH\n1Vh3JfAvwAvAgoj4U25c1Ku/3kTEnbnRfSBwmaQLImI68HKh2hrW3Qd/I/fGFhqG7RBweUSc0duY\nJf0T6R54T43VM4GvS1oaEbv29hjWGx39HcAg1tHfAQxSHf0dwCDV0d8BDFruZSmH89q+dobljZZU\nHDZ1Z9X6+cB4SdtKGgocA8zJQ72GRsQvSUPExuVelKckHQogaRNJmxV3FhGXRMQeETGuehhURHRF\nxE7AvaThVLXcARyh/GQ2SdvUqVdpfAwDnsuNq/1Jw92q6zRyC3Dq2g2ksS1sU887lOZ+DSGdX3Wu\n7yTN39lUaT7VR4A78zDNP+dei++QekKWAyPz8EQkbZmvT9H5wOKIuLpWMEpzuC5vEvOtwCcq+66R\n73nAvpLG5PWbS3pjXjcnx/ox1g1pa1T/lSjOddpe0m0NK0ujSPfFVOAnOc5u+6nyOLBXXj6U7g2s\nRvdRZd3twMTCnK5tcgwtkfRBUg/gIRHxUo0qJwA3u3FlZmZmVo52GlgPA5+WtJQ0Sf5HubwyJGwl\ncDppcPEiUk/E9cAOQKekRaQhaKfn7U4ATpW0GJgLjOhF/L8Baj7aOg/pO4fUyFsEXFCMt1g1/5wB\n7J3jOR5YVqNOre0rvkF6EMgSpUfan1VdQdKeki5tcD4V84GLScMRfxsR1xSPHRGLgMtIw9d+DVya\nh669jTT3axHwdeDsiHiZ1Ei7WNL9pKfMvabqeF8A3q/0kIv7JB1UtX4U8L9NYv4J8BSwJB//mKqY\n/x/wT8AVOcd3k+b1kR/KcAPwwfyzYX3qXAOlh3WMrFF+mKSnSMMUb9C6x/pvR/eeqFo6gMWS7gOO\nBL7XKAbg34EJOQf7AH8qrGvUE1nJ0zLSHyJm5fOeBdQ6p5rnClxEepjGrflaXlK1fhvS0wVtvevs\n7wAGsc7+DmCQ6uzvAAapzv4OYNDy9zWVw3ltnzbkbxHP852GR8TpTStbr+WHOPwsIh7s71j6ktKT\nDp+IiBv6O5b1JT80Y0lE/LhBnej/p+8PRp14aFBZOnFuy9CJ81qGTlJexYb8GWwg8sMYyuG81ieJ\niOgxQmlDb2CNIfXkvFj1XVhmVkXSHNK8weMj4pkG9dzAMjMrnRtYZhu6QdnAMrO+lxpYZmZWphEj\nRrNy5eP9HYaZvQL1Gli9+e4pMxvkIsKvPn7Nnj2732MYrC/n1nndkF6VvLpx1fc8V6gczmv73MAy\nMzMzMzPrIx4iaGbdSAr/XjAzMzNrzEMEzczMzMzMSuYGlpnZeuAx7OVxbsvhvJbDeS2Pc1sO57V9\nbmCZmZmZmZn1Ec/BMrNuPAfLzMzMrLl6c7A26o9gzGxgk3r8rjAzs0FixA4jWPn0yv4Ow2zQcg+W\nmXUjKZjU31EMQiuAnfs7iEHKuS2H81qOgZDXSen7Dgebzs5OOjo6+juMQcd5rc9PETQzMzMzMytZ\nSw0sSaMlPVBn3WxJ4/o2rNZIGiXpPkk3F8pW9Ecs9UiaIGlaC/VWFOpfX6+OpG37MK6axymsbxh3\nvi9mN6nT5/dHcZ+tXG9JEyU9KGlNq7E026+k1a1F222bb0t6QNL5kl4raZ6khZL2a+Xatniu20ia\nJWm5pFskDSus+29J8yW9vt3YrQ/091+sBzPnthzOazmc19K4l6Uczmv72unBGoh9yYcBsyLiQ4Wy\ngRhnKzFFneV299OOZvtrN+7+0MrxHwA+Aszpw/325rw/BuwWEf8KvBdYEhF7RsRdLe6vlTqnA7dF\nxC7AHcC/rd04YjywEDiw7cjNzMzMrCXtNLA2ljRd0lJJV0natLqCpGMkLcmv83LZEEnTctliSZ/N\n5WMk3Srpfkn3SurN33S2Bp6rKnu+EM8J+ZiLJF2ey6ZJmiJprqRHJR2ey7eQdFuOZbGkQ3L5aEnL\n8nbLJc2Q9L68/XJJe+V6m0uaWuiVODiH8Vfgjy2cy/OF5WGSbpD0sKRLCuVrx3hKOi33hiwp5HTz\nvN2iXH5ELt87x3t/jm+L4oEl3Zh7AhdJWiXpH1uMew3wh7yPIYUemvslfbq6cs7b3TnHV+Z4PyDp\nqkKdtT1rkt5fXb9J3mqKiOUR8Ugxfy14PscwUtKcnJ8lkvZdF6rOzud6t6TX5cJplXsqv1+df14L\nbAkslPRl4HzgsLzfTel+bY+TdE9e90Np7RMnmp4rcChweV6+nPRHiKKVpH83tr4NqL71Qca5LYfz\nWg7ntTT+vqZyOK/ta+cpgrsAJ0XEPElTgU8BF1ZWStoOOA/YA1gF3JobKU8DO0TEbrneVnmTGcA3\nI+I6SZvQu/lgQ4GuYkFEvCMfZ1fgK8A7I+IFScUPlSMjYl9JbwauA2YCfwEOi4gXJQ0H5uV1AGOA\nj0bEUkn3Akfn7Q/JxzgcOAO4PSJOzsOy5ku6LSJ+Dfw6x7Qn8ImI+Hj1iVTizvYG3gw8Cdwi6fCI\nmFlZqTTM7cRcbyhwj6TOHOczEXFQrvd3kjYGfgEcERH3SdoS+HPVsQ8s7Pc/gGsiYnUl7noi4mlg\nYn77cWA0qYcmqvJNzulXgQMi4s+5kXEacC7wY0mbRcSfgaOAn+f6Z9Sof3a9vEm6ETg5Il7xo5EK\n+z0W+FVEnJsbOpVG3hbA3RHxVUnnk3qnvllrV3l/h0r6n4ioDG18FtgzIk7N7yvn8H9yDt4VEWsk\n/QA4Dpje4rm+PiKezcdcqZ7DAbtI90xjxYGfO+EhLWZmZvaq19nZ2VKDs50G1pMRMS8vTwc+Q6GB\nRfqwPzsiKj0aM4DxpA/EO0uaAtwEzMof8rePiOsAIuKvbcRB3r+A3XMstbwHuDoiXsjHWFVYd00u\nW1b4ACrgXEnjSR9Cty+sWxERS/PyQ8BtefkB0sdPgPcDB0v6Un6/CTAKWF45aEQsJDVEmpkfEU/k\n87wC2I/UCKzYD/hlRPwl15kJvBu4BfiOpHOBGyPiLklvBX4XEfflGF7M23Q7oKTXAj8DJubGVbve\nC/yw8gVKVfkG2AfYFZibr93GpAbKGkm/IuXuv0jD174EdNSq3yiASkOxjy0ApuaG6rURsTiXvxQR\nN+XlhaTzr6XVXrPK8L8DgHHAgnzemwLP9qjc+rlWDyt8hpTbxvZvce/WOjdSy+PclsN5LYfzWhrP\nFSqH87pOR0dHt3xMnjy5Zr12GljVH9RqzQfp8WEyIlZJ2h34APBJ4Ajgc7XqdtuR9ClSr0AAHy7+\npV7SEOAx4CXgxjbOoeKlGjEfB7wW2CMiupQeKLBpjfpdhfddrMuhSL1cj/Qinmqt5LrnRhGP5F6o\nDwPfkHQ7qTHZLNdDgCuASRGxrBfxtkKk+XLH1Vh3JfAvwAvAgoj4U25c1Ku/3kTEnbnRfSBwmaQL\nImI68HKh2hrW3Qd/I/fGFhqG7RBweUSc0cuQn5U0IiKelTSSnkNoZwJfl7Q0Inbt5THMzMzMrI52\nhuWNllQcNnVn1fr5wHhJ20oaChwDzMlDvYZGxC9JQ8TG5V6UpyQdCiBpE0mbFXcWEZdExB4RMa56\nGFREdEXETsC9pOFUtdwBHKH8ZDZJ29SpV2l8DAOey42r/UnD3arrNHILcOraDaSxLWxTzzuU5n4N\nIZ1fda7vJM3f2VRpPtVHgDvzMM0/R8TPge+QekKWAyPz8EQkbZmvT9H5wOKIuLpWMEpzuC6vta7g\nVuATlX3XyPc8YF9JY/L6zSW9Ma+bk2P9GGk4Y7P6r0RxrtP2km5rWFkaRbovpgI/yXF220+Vx4G9\n8vKhdG9gNbqPKutuByYW5nRtk2No1XXAP+XlE4Frq9afANzsxlU/8LyL8ji35XBey+G8lsZzhcrh\nvLavnQbWw8CnJS0lTZL/US6vDAlbSXqCWSewiNQTcT2wA9ApaRFpCNrpebsTgFMlLQbmAiN6Ef9v\ngJqPts5D+s4hNfIWARcU4y1WzT9nAHvneI4HltWoU2v7im+QHgSyROmR9mdVV5C0p6RLG5xPxXzg\nYtJwxN9GxDXFY0fEIuAy0vC1XwOX5qFrbyPN/VoEfB04OyJeJjXSLpZ0PzALeE3V8b4AvF/pIRf3\nSTqoav0o4H+bxPwT4ClgST7+MVUx/z/SB/8rco7vJs3rIyK6gBuAD+afDetT5xooPaxjZI3ywyQ9\nRRqmeIPWPdZ/O7r3RNXSASyWdB9wJPC9RjEA/w5MyDnYB/hTYV2jnshKnpaR/hAxK5/3LKDWOdU8\nV1Jj+X2SlpOGG55XtX4boC96Wc3MzMysBm3I3+Sd5zsNj4jTm1a2XssPcfhZRDzY37H0JaUnHT4R\nETf0dyzrS35oxpKI+HGDOsGk9ReTmZmtZ5NgQ/78ZzZQSCIieoxQ2tAbWGNIPTkvVn0XlplVkTSH\nNG/w+Ih4pkG9DfeXgpmZNTVihxGsfPoVP3DX7FVvUDawzKzvSQr/Xuh7nZ2dfhJTSZzbcjiv5XBe\ny+PclsN5ra9eA6s33z1lZmZmZmZmNbgHy8y6cQ+WmZmZWXPuwTIzMzMzMyuZG1hmZuuBv0ekPM5t\nOZzXcjiv5XFuy+G8ts8NLDMzMzMzsz7iOVhm1o3nYJmZmZk1V28O1kb9EYyZDWxSj98V9gqNGDGa\nlSsf7+8wzMzMrGQeImhmNYRfffx69tkn2rsE1jLPDyiH81oO57U8zm05nNf2NWxgSRot6YE662ZL\nGldOWI1JGiXpPkk3F8pW9Ecs9UiaIGlaC/UGVNxFrcTWrI6kMyWd1ndRdd+npGmSxjepv7WkmZIW\nS5onadcWjjFb0qgm69u6/yVNlLRU0u35/RWS7pf02XwehzfZvpVzPTaf52JJd0narbDuAkkPSZrQ\nTtxmZmZm1rpWerAG4mSMw4BZEfGhQtlAjLOVmAZi3BUbevwVXwEWRcTuwInA9/spjpOBUyLiAEkj\ngb0iYmxETOnDYzwGjM/nejZwaWVFRHwBOAv45z48nlm/6+jo6O8QBiXntRzOa3mc23I4r+1rpYG1\nsaTp+S/vV0natLqCpGMkLcmv83LZkPwX9yX5r+mfzeVjJN2a/3J/r6SdexH31sBzVWXPF+I5IR9z\nkaTLc9k0SVMkzZX0aKW3QNIWkm7LsSyWdEguHy1pWd5uuaQZkt6Xt18uaa9cb3NJU3PPyEJJB+cw\n/gr8sYVzeT7vZ6SkOblnbomkfXP5akln53zdLel1ufygwjFnFcrPlPTTXHe5pFNy+YS8/xskPSzp\nEiUnSfpuIXenSLqgOqfN4q+X9yJJfy/pZkkLcixvkrSVpMcLdTaX9KSkobXq1zj+KlKuG9kVuAMg\nIpYDO1Xy1cDvgTX17uPsSEn35HxWrteJki4qnM/1ksZL+hqwHzBV0reAW4Ad8vXerypP4yR15vO+\nWdKIVs81IuZFROW+mwfsUFVlJenfj5mZmZmVISLqvoDRQBewT34/FTgtL88GxgHbAU8A25IabLcD\nh+R1swr72ir/nAcckpc3ATZtFEOduCYDn6uzblfgYWCb/H7r/HMacGVefjPwSF4eCmyZl4cXykeT\nPszumt/fC0zNy4cAM/PyOcCxeXkYsBzYrCqmPYFLm5zTacC/5WUBW+TlLuDDefl84CuVYxW2PRn4\ndl4+E1iUczsceBIYCUwA/jefl4BZwOHAFsCjwNC8/VzgLb24JvXyfmbhnrkNGJOX3w7cnpd/CUzI\ny0dWctWg/tp91rgvDqpRfg5wQWE/fwX2aPG86t3Hsws5/xBwa14+Efh+of71pB6lyjZ7FO6vJYV6\n0/L12Chfg+GFfExt9Vyr6nyx+r4D3g3c0GS7gPCrz1+ElWP27Nn9HcKg5LyWw3ktj3NbDue1vvx/\ne4/PUq08RfDJiJiXl6cDnwEuLKzfG5gdEX8AkDQDGE8anrSzpCnATcAsSVsC20fEdaSImvU89CBJ\nwO45llreA1wdES/kY6wqrLsmly2T9PrKLoFzlea2dAHbF9atiIilefkh0gd+gAeAnfLy+4GDJX0p\nv98EGEVqaJGPtxD4eJNTW0Dq3dgYuDYiFufylyLipry8EHhvXt5R0lWkBu7GwIrCvq7Nuf29pDtI\njYo/AvMj4glI83+A/SJiptKcoIMkPQxsFBEPNYm1lkZ5R9IWwLuAq/M1JMcNcBVwFDAHOBr4QZP6\nNUXEmXVWnQdMkXQf6dotAta0eF6PUXUfF9bNzD8XkhpMrWj2eL5dgLcCt+bzHgL8rrpSg3NNB5H2\nB04i9ZoVPQO8SdJrIuKl+nuYVFjuyC8zMzOzV6/Ozs6WHvrRSgMrmryHGh8aI2KVpN2BDwCfBI4A\nPlerbrcdSZ8CPpaP8+GIWFlYN4T0gfcl4MYWYq9W/EBZieM44LWknoUupYc2bFqjflfhfRfrcifg\noxHxSC/iWSsi7syNvAOByyRdEBHTgZcL1dYUjnsR8J2IuFHpoQXFD9zFayRqX7NivamkeUoPk3pS\nyjAEeCEiaj0Y4jrgHEnbkHqM7gC2bFC/LRGxmsK8o3yNH2tx21r38Sl5deV+KF6Xv9F96G2PIbVN\nCHgwIvZtc7t1O0gPtrgU+GClwVsREY9JWgY8IemA+o3pSb09vNl65/kB5XBey+G8lse5LYfzuk5H\nR0e3fEyePLlmvVbmYI2W9I68fCxwZ9X6+cB4SdtKGgocA8yRNJw07OyXwFeBcRHxIvCUpEMBJG0i\nabPiziLikojYIyLGFRtXeV1XROxEGq53VJ147wCOkLRtPsY2depVGljDgOdy42p/uvdEtPJlQLcA\np67dQBrbwjY9g0lPrHsuIqYCPyE1NBrFsBXrejZOrFp3aM7tcNLQwAW5fG+luWVDSPm7CyAi5gM7\nkq7dFXXiW9bkFBrmPTdyVkiaWNjnbnndn0jXdApp+Fo0qt8uScNyzyCSPgbMyfciSvPvtmuwbY/7\nuF7V/PNxYGye37Yjqfew7u5rlC0HXidpn3z8jdTCUw8L8Y4C/gv4x4j4bY31uwE7k3qSe9NTaWZm\nZmYNtNLAehj4tKSlpMnxP8rlabJGagSdDnSShl4tiIjrSZPrOyUtAn6W6wCcAJwqaTFprkllAn87\nfkOa89VDHtJ3DqmRtwioPLChXk/cDFLDYzFwPLCsRp1a21d8g/QgkCVKj7Q/q7qCpD0lXdpz0246\ngMV5GNuRwPeaHHcy8J+SFtDzYRRLSNfjbuCsQkP1XuBi0nDH3+ZGQ8VVwNxY94CEYvzDm8TeKO9F\nxwMnKz2w40HSXLaKK0m9ib8olB3XoH4PkiZLOqjGqjcDD+ZG4geAygNXBIwB/tBgt/Xu45r3U0TM\nJTWyHiJdw4XVdeq8r2z/MjAROF/S/aR/U+9s41y/Rvq3cYnSw0bmV63fBng8IrpqbGu2QfJ3tJTD\neS2H81oe57Yczmv7lOZnbVjyfKfhEXF608qvMpLOBFZHxIVV5ROAL0REzUaKpOuBCyNido11BwI7\nR8TFZcTcXyS9BTgpIr7Y37GsL5KOBD4SEcc0qBP12/XWe2JD/H27Iejs7PQQlhI4r+VwXsvj3JbD\nea1PEhHRY0TShtrAGgNcBrwY3b8L61Wv3QaWpGGkYZ6LIuLo9ReprW9Kj99/N+lplbc3qOcGVinc\nwDIzMxtMBlUDy8zKkxpY1tdGjBjNypWP93cYZmZm1kfqNbBamYNlZq8ytb7Twa9X9vrFLy7r78s6\naHl+QDmc13I4r+VxbsvhvLbPDSwzMzMzM7M+4iGCZtaNpPDvBTMzM7PGPETQzMzMzMysZG5gmZmt\nBx7DXh7nthzOazmc1/I4t+VwXtvnBpaZmZmZmVkf8RwsM+vGc7DMzMzMmqs3B2uj/gjGzAY2qcfv\nChugRuwwgpVPr+zvMMzMzCxzD5aZdSMpmNTfUQxCK4CdS9jvpPS9Za9mnZ2ddHR09HcYg47zWg7n\ntTzObTmc1/p69RRBSaMlPVBn3WxJ4/oqwHZIGiXpPkk3F8pW9Ecs9UiaIGlaC/UGVNxFrcTWrI6k\nMyWd1ndRdd+npGmSxjepv7WkmZIWS5onadcWjjFb0qgm69u6/yVNlLRU0u35/RWS7pf02XwehzfZ\nvum55nrfl/RI3vfYQvkFkh6SNKGduM3MzMysda085GIg/mn0MGBWRHyoUDYQ42wlpoEYd8WGHn/F\nV4BFEbE7cCLw/X6K42TglIg4QNJIYK+IGBsRU/rqAJI+BIyJiDcCnwB+VFkXEV8AzgL+ua+OZ20o\no/fKAPyX1ZI4r+VwXsvj3JbDeW1fKw2sjSVNz395v0rSptUVJB0jaUl+nZfLhuS/uC/JPQefzeVj\nJN2a/7p+r6TefOzYGniuquz5Qjwn5GMuknR5LpsmaYqkuZIerfQWSNpC0m05lsWSDsnloyUty9st\nlzRD0vvy9ssl7ZXrbS5pau4ZWSjp4BzGX4E/tnAuz+f9jJQ0J/fMLZG0by5fLensnK+7Jb0ulx9U\nOOasQvmZkn6a6y6XdEoun5D3f4OkhyVdouQkSd8t5O4USRdU57RZ/PXyXiTp7yXdLGlBjuVNkraS\n9HihzuaSnpQ0tFb9GsdfRcp1I7sCdwBExHJgp0q+Gvg9sKbefZwdKemenM/K9TpR0kWF87le0nhJ\nXwP2A6ZK+hZwC7BDvt77VeVpnKTOfN43SxrRxrkeCvw0n+s9wLDC9gArSf9+zMzMzKwErTSwdgEu\njohdgdXAp4orJW0HnAd0AGOBvXMjZSywQ0TslnsOKsPlZgAXRcRY4F3A/+1F3EOBrmJBRLwjx7Mr\nqceiIyL2AIofiEdGxL7AwcD5uewvwGERsRfwHuCCQv0xwLcjYpech6Pz9l/KxwA4A7g9IvbJ239H\n0mYR8euI+HyOaU9Jl9Y6kUrcwLHAryJiHLA7cH8u3wK4O+frTuBjufzOiNgnIvYErgS+XNjt20jX\n413A15V6SwD2Bj4NvBn4B+AjwFXAwZKG5jonAf9RFVtdLea94lLgXyJib1IOfxgR/wMs0rphawfl\nPKypVb/G8T8fEfNyDJMlHVTjuIuBSoP67cAo4A1NzmtiRDxD/fsYYGg+/89Dt1lLPXr1IuIbwL3A\nsRHxZeAQ4NGIGBcRd1XqSdoIuAj4aD7vacA32zjXHYCnCu+fyWUVXaR/P7a+DdjBwBs+f0dLOZzX\ncjiv5XFuy+G8tq+Vpwg+WflQB0wHPgNcWFi/NzA7Iv4AIGkGMB44G9hZ0hTgJmCWpC2B7SPiOoCI\naPbX+B4kidQAmV6nynuAqyPihXyMVYV11+SyZZJeX9klcK7S3JYuYPvCuhURsTQvPwTclpcfAHbK\ny+8nNVC+lN9vQvoAv7xy0IhYCHy8yaktIPVubAxcGxGLc/lLEXFTXl4IvDcv7yjpKmA7YGO6f3y7\nNuf295LuAN5O6k2bHxFPQJr/A+wXETOV5gQdJOlhYKOIeKhJrLU0yjuStiA1+K7O15AcN6RG3lHA\nHOBo4AdN6tcUEWfWWXUeMEXSfaRrtwhY0+J5PUbVfVxYNzP/XAiMbnF/zR7PtwvwVuDWfN5DgN9V\nV2pwrs08A7xJ0msi4qW6tWYXlnfCw9vMzMzsVa+zs7OlBmcrDazqv8bXmnPT40NjRKyStDvwAeCT\nwBHA52rV7bYj6VOkXpoAPhwRKwvrhpA+8L4E3NhC7NWKHygrcRwHvBbYIyK6lB7asGmN+l2F912s\ny51IvQ2P9CKetSLiztzIOxC4TNIFETEdeLlQbU3huBcB34mIG3PvT/EDd/EaifrzpCrlU0m9Tw/T\nvYemLw0BXsg9dNWuA86RtA0wjjScb8sG9dsSEaspzDvK1/ixFretdR+fkldX7ofidfkb3XuGewyp\nbULAg7mntDeeAXYsvH9DLgMgIh6TtAx4QtIBdRvT+/fy6FafG6ml8fyAcjiv5XBey+PclsN5Xaej\no6NbPiZPnlyzXitDBEdLKg5ju7Nq/XxgvKRt8zCzY4A5koaThlD9EvgqMC4iXgSeknQogKRNJG1W\n3FlEXBIRe+ShUyur1nVFxE6koVZH1Yn3DuAISdvmY2xTp16lgTUMeC43rvane09EK18GdAtw6toN\nCk9ta4fSE+uei4ipwE9IDY1GMWzFup6NE6vWHZpzOxyYQOodgzR8c3RuqB4F3AUQEfNJH8qPAa6o\nE9+yJqfQMO+5kbNC0sTCPnfL6/5EuqZTgBsiqVu/XZKG5Z5BJH0MmJPvRZTm323XYNse93G9qvnn\n48BYJTuSeg/r7r5G2XLgdZL2ycffSC089bDgOuCEvO0+wKqIeLZwPruRPupv38ueSjMzMzNroJUG\n1sPApyUtJU2OrzyVLAByI+h0oJM09GpBRFxPmvfRKWkR8LNcB9KHv1MlLQbmAsUJ+K36DbBtrRV5\nSN85pEbeItbNqarXEzeD1PBYDBwPLKtRp9b2Fd8gPQhkidIj7c+qrtBoDlZBB7A4D2M7Evhek+NO\nBv5T0gJ6PoxiCel63A2cVWio3gtcTBru+NvcaKi4CpgbET0ezJEbGQ01yHvR8cDJSg/seJA0D6ni\nSlJv4i8KZcc1qN9Dg3lJbwYezI3ED5Dnh+UheGOAPzTYbb37uOb9FBFzSY2sh0jXcGF1nTrvK9u/\nDEwEzpd0P+nf1DtbPdc8nHSFpEeBH1M1ZxLYBng8Irqqt7WSeQ5WaTw/oBzOazmc1/I4t+VwXtu3\nQX7RcJ7vNDwiTm9a+VVG0pnA6oi4sKp8AvCFiKjZSJF0PXBhRMyuse5AYOeIuLiMmPuLpLcAJ0XE\nF/s7lvVF0pHARyLimAZ1/EXDZfAXDZfGX4JZDue1HM5reZzbcjiv9anOFw1vqA2sMcBlwItV34X1\nqtduA0vSMNIwz0URcfT6i9TWN6XH778b+LeIuL1BvGGqlgAAIABJREFUPTewNiST3MAyMzPrD4Oq\ngWVm5ZHkXwobkBE7jGDl0yubVzQzM7M+Va+B1cocLDN7lYkIv/r4NXv27FL268aV5weUxXkth/Na\nHue2HM5r+9zAMjMzMzMz6yMeImhm3UgK/14wMzMza8xDBM3MzMzMzErmBpaZ2XrgMezlcW7L4byW\nw3ktj3NbDue1fW5gmZmZmZmZ9RHPwTKzbjwHy8zMzKy5enOwNuqPYMxsYJN6/K4wszpGjBjNypWP\n93cYZmY2QHiIoJnVEH71+Wv2AIhhsL76N7fPPvsEg5HnXZTDeS2Pc1sO57V9DRtYkkZLeqDOutmS\nxpUTVmOSRkm6T9LNhbIV/RFLPZImSJrWQr0BFXdRK7E1qyPpTEmn9V1U3fcpaZqk8S1s831Jj0i6\nX9LYFurPljSqyfq27n9JEyUtlXR7fn9Fjuez+TwOb7J903OVdKykxfl1l6TdCusukPSQpAntxG1m\nZmZmrWtliGCUHkX7DgNmRcTphbKBGGcrMQ3EuCs29PgBkPQhYExEvFHSO4AfAfv0QygnA6dExN2S\nRgJ7RcQbc4xNG+MtegwYHxF/lPRB4FLyuUbEFyTNB/4ZmNNHx7OWdfR3AINYR38HMCh1dHT0dwiD\nkvNaHue2HM5r+1oZIrixpOn5L+9XSdq0uoKkYyQtya/zctmQ/Bf3Jfmv6Z/N5WMk3Zr/cn+vpJ17\nEffWwHNVZc8X4jkhH3ORpMtz2TRJUyTNlfRopbdA0haSbsuxLJZ0SC4fLWlZ3m65pBmS3pe3Xy5p\nr1xvc0lTJc2TtFDSwTmMvwJ/bOFcns/7GSlpTu6ZWyJp31y+WtLZOV93S3pdLj+ocMxZhfIzJf00\n110u6ZRcPiHv/wZJD0u6RMlJkr5byN0pki6ozmmz+OvlvUjS30u6WdKCHMubJG0l6fFCnc0lPSlp\naK36NY6/ipTrRg4FfgoQEfcAwySNaLLN74E19e7j7EhJ9+R8Vq7XiZIuKpzP9ZLGS/oasB8wVdK3\ngFuAHfL13q8qT+MkdebzvrkQa9NzjYh5EVG57+YBO1RVWUn692NmZmZmZYiIui9gNNAF7JPfTwVO\ny8uzgXHAdsATwLakBtvtwCF53azCvrbKP+cBh+TlTYBNG8VQJ67JwOfqrNsVeBjYJr/fOv+cBlyZ\nl98MPJKXhwJb5uXhhfLRpA+zu+b39wJT8/IhwMy8fA5wbF4eBiwHNquKaU/g0ibndBrwb3lZwBZ5\nuQv4cF4+H/hK5ViFbU8Gvp2XzwQW5dwOB54ERgITgP/N5yVgFnA4sAXwKDA0bz8XeEsvrkm9vJ9Z\nuGduI/UkAbwduD0v/xKYkJePrOSqQf21+6xxXxxUo/x64F2F97cB41o8r3r38exCzj8E3JqXTwS+\nX3Xs8YVt9ijcX0sK9abl67FRvgbDC/mY2uq5VtX5YvV9B7wbuKHJdgHhV5+/Zg+AGAbrq79zSwxG\ns2fP7u8QBiXntTzObTmc1/ry73+qX60MEXwyIubl5enAZ4ALC+v3BmZHxB8AJM0AxgNnAztLmgLc\nBMyStCWwfURcR4qoWc9DD5IE7J5jqeU9wNUR8UI+xqrCumty2TJJr6/sEjhXaW5LF7B9Yd2KiFia\nlx8ifTAHeADYKS+/HzhY0pfy+02AUaSGFvl4C4GPNzm1BaTejY2BayNicS5/KSJuyssLgffm5R0l\nXUVq4G4MrCjs69qc299LuoPUOPkjMD8inoA0/wfYLyJmKs0JOkjSw8BGEfFQk1hraZR3JG0BvAu4\nOl9DctwAVwFHkYatHQ38oEn9miLizF7E3cxjVN3HhXUz88+FpAZTK5o9nm8X4K3Arfm8hwC/q67U\n7Fwl7Q+cROo1K3oGeJOk10TES/X3MKmw3IGHYJmZmdmrXWdnZ0sP/ejNHKzq91DjQ2NErJK0O/AB\n4JPAEcDnatXttiPpU8DH8nE+HBErC+uGkD7wvgTc2ELs1YofKCtxHAe8ltSz0KX00IZNa9TvKrzv\nYl3uBHw0Ih7pRTxrRcSduZF3IHCZpAsiYjrwcqHamsJxLwK+ExE3Kj20oPiBu3iNRO1rVqw3FfgK\nqQdq2is5jwaGAC9ERK0HQ1wHnCNpG1KP0R3Alg3qt+sZYMfC+zfksqbq3Men5NWV+6F4Xf5G96G3\nPYbUNiHgwYjYt83t1u0gPdjiUuCDlQZvRUQ8JmkZ8ISkA+o3pif19vBWV0d/BzCIdfR3AIOS512U\nw3ktj3NbDud1nY6Ojm75mDx5cs16rczBGq30YACAY4E7q9bPB8ZL2lbSUOAYYI6k4aRhZ78Evkoa\nkvUi8JSkQwEkbSJps+LOIuKSiNgjIsYVG1d5XVdE7EQarndUnXjvAI6QtG0+xjZ16lUaWMOA53Lj\nan+690S08mVAtwCnrt2ghSfU1QwmPbHuuYiYCvyE1NBoFMNWrOvZOLFq3aE5t8NJQwMX5PK9leaW\nDSHl7y6AiJhPaoAcA1xRJ75lTU6hYd4jYjWwQtLEwj53y+v+RLqmU0jD16JR/V64Djgh72MfYFVE\nPJvf3yZpu3ob1rqP61XNPx8Hxub5bTuSeg/r7r5G2XLgdTlOJG0kadcG+6iOdxTwX8A/RsRva6zf\nDdiZ1JPcm55KMzMzM2uglQbWw8CnJS0lTY7/US4PgNwIOh3oJM39WRAR15Mm13dKWgT8LNeB9EH3\nVEmLSXNNmj1soJbfkOZ89ZCH9J1DauQtAioPbKjXEzeD1PBYDBwPLKtRp9b2Fd8gPQhkidIj7c+q\nriBpT0mXNjgfSH+CXSzpPtK8m+81Oe5k4D8lLaDnwyiWkK7H3cBZhYbqvcDFpOGOv82NhoqrgLmx\n7gEJxfiHN4m9Ud6LjgdOVnpgx4OkuWwVV5J6E39RKDuuQf0eJE2WdFCN2G4iNdYeBX4MfCrXFzAG\n+EOD3da7j2veTxExl9TIeoh0DRdW16nzvrL9y8BE4HxJ95P+Tb2z1XMFvkb6t3GJ0sNG5let3wZ4\nPCK6amxrpers7wAGsc7+DmBQ8nfflMN5LY9zWw7ntX1K87M2LHm+0/Do/ph2Iz1FEFgdERdWlU8A\nvhARNRspkq4HLoyI2TXWHQjsHBEXlxFzf5H0FuCkiPhif8eyvkg6EvhIRBzToE7Ub9db73XioWxl\n6aR/cys2xP9Lm+ns7PTQoBI4r+VxbsvhvNYniYjoMSJpQ21gjQEuA16MiA/1czgDSrsNLEnDSMM8\nF0XE0esvUlvflB6//27S0ypvb1DPDSyztgzOBpaZmTU2qBpYZlYeN7DM2uUGlpnZq1G9BlYrc7DM\n7FVHfvnlV4uvESNa/ZaGDYvnXZTDeS2Pc1sO57V9rTym3cxeZfzX+L7nMezlcW7NzGwg8RBBM+tG\nUvj3gpmZmVljHiJoZmZmZmZWMjewzMzWA49hL49zWw7ntRzOa3mc23I4r+1zA8vMzMzMzKyPeA6W\nmXXjOVhmZmZmzXkOlpmZmZmZWcn8mHYz60Hq8ccYMzMzsw3SiB1GsPLplevteB4iaGbdSAom9XcU\ng9AKYOf+DmKQcm7L4byWw3ktj3NbjsGQ10nlfMdnr4YIShot6YE662ZLGtdXAbZD0ihJ90m6uVC2\noj9iqUfSBEnTWqg3oOIuaiW2ZnUknSnptL6Lqvs+JU2TNL6Fbb4v6RFJ90sa20L92ZJGNVnf1v0v\naaKkpZJuz++vyPF8Np/H4U22f0XnKukCSQ9JmtBO3NZHNvT/nAYy57Yczms5nNfyOLflcF7b1soc\nrIHYxXUYMCsiPlQoG4hxthLTQIy7YkOPHwBJHwLGRMQbgU8AP+qnUE4GTomIAySNBPaKiLERMaWv\nDtDoXCPiC8BZwD/31fHMzMzMrLtWGlgbS5qe//J+laRNqytIOkbSkvw6L5cNyX9xXyJpsaTP5vIx\nkm7Nf12/V1Jv2sVbA89VlT1fiOeEfMxFki7PZdMkTZE0V9Kjld4CSVtIui3HsljSIbl8tKRlebvl\nkmZIel/efrmkvXK9zSVNlTRP0kJJB+cw/gr8sYVzeT7vZ6SkOblnbomkfXP5akln53zdLel1ufyg\nwjFnFcrPlPTTXHe5pFNy+YS8/xskPSzpEiUnSfpuIXenSLqgOqfN4q+X9yJJfy/pZkkLcixvkrSV\npMcLdTaX9KSkobXq1zj+KlKuGzkU+ClARNwDDJM0osk2vwfW1LuPsyMl3ZPzWbleJ0q6qHA+10sa\nL+lrwH7AVEnfAm4BdsjXe7+qPI2T1JnP++ZCrH1xritJ/35sfRuwfdWDgHNbDue1HM5reZzbcjiv\nbWulgbULcHFE7AqsBj5VXClpO+A8oAMYC+ydGyljgR0iYreI2B2oDJebAVwUEWOBdwH/txdxDwW6\nigUR8Y4cz67AV4COiNgDKH4gHhkR+wIHA+fnsr8Ah0XEXsB7gAsK9ccA346IXXIejs7bfykfA+AM\n4PaI2Cdv/x1Jm0XEryPi8zmmPSVdWutEKnEDxwK/iohxwO7A/bl8C+DunK87gY/l8jsjYp+I2BO4\nEvhyYbdvI12PdwFfV+otAdgb+DTwZuAfgI8AVwEHSxqa65wE/EdVbHW1mPeKS4F/iYi9STn8YUT8\nD7BI64atHZTzsKZW/RrH/3xEzMsxTJZ0UI3j7gA8VXj/TC5rdF4TI+IZ6t/HAEPz+X8eus1a6tGr\nFxHfAO4Fjo2ILwOHAI9GxLiIuKtST9JGwEXAR/N5TwO+2Yfn2kX692NmZmZmJWjlKYJPVj7UAdOB\nzwAXFtbvDcyOiD8ASJoBjAfOBnaWNAW4CZglaUtg+4i4DiAimv01vgdJIjVAptep8h7g6oh4IR9j\nVWHdNblsmaTXV3YJnKs0t6UL2L6wbkVELM3LDwG35eUHgJ3y8vtJDZQv5febAKOA5ZWDRsRC4ONN\nTm0BqXdjY+DaiFicy1+KiJvy8kLgvXl5R0lXAdsBG9P97wvX5tz+XtIdwNtJvWnzI+IJSPN/gP0i\nYqbSnKCDJD0MbBQRDzWJtZZGeUfSFqQG39X5GpLjhtTIOwqYAxwN/KBJ/Zoi4sxexN3MY1Tdx4V1\nM/PPhcDoFvfX7PF8uwBvBW7N5z0E+F11pVdwrs8Ab5L0moh4qW6t2YXlnfD4677gHJbHuS2H81oO\n57U8zm05nNe1Ojs76ezsbFqvlQZW9V/ja8256fGhMSJWSdod+ADwSeAI4HO16nbbkfQpUi9NAB+O\niJWFdUNIH3hfAm5sIfZqxQ+UlTiOA14L7BERXUoPbdi0Rv2uwvsu1uVOpN6GR3oRz1oRcWdu5B0I\nXCbpgoiYDrxcqLamcNyLgO9ExI2596f4gbt4jUT9eVKV8qmk3qeH6d5D05eGAC/kHrpq1wHnSNoG\nGAfcAWzZoH67ngF2LLx/Qy5rqs59fEpeXbkfitflb3TvGe4xpLYJAQ/mntLeaHiuEfGYpGXAE5IO\nqNuY3r+XRzczMzMbpDo6Oujo6Fj7fvLkyTXrtTJEcLSk4jC2O6vWzwfGS9o2DzM7BpgjaThpCNUv\nga8C4yLiReApSYcCSNpE0mbFnUXEJRGxRx46tbJqXVdE7EQaanVUnXjvAI6QtG0+xjZ16lUaWMOA\n53Ljan+690S08mVAtwCnrt2ghSfU1QwmPbHuuYiYCvyE1NBoFMNWrOvZOLFq3aE5t8OBCaTeMUjD\nN0fnhupRwF0AETGf9KH8GOCKOvEta3IKDfMeEauBFZImFva5W173J9I1nQLcEEnd+r1wHXBC3sc+\nwKqIeDa/vy0Pc62p1n1cr2r++TgwVsmOpN7DuruvUbYceF2OE0kb5eGXrap7rrlsN9LforbvZU+l\n9ZbHsJfHuS2H81oO57U8zm05nNe2tdLAehj4tKSlpMnxlaeSBUBuBJ0OdAKLgAURcT1p3kenpEXA\nz3IdSB/+TpW0GJgLNHvYQC2/AbattSIP6TuH1MhbxLo5VfV64maQGh6LgeOBZTXq1Nq+4hukB4Es\nUXqk/VnVFRrNwSroABZLug84Evhek+NOBv5T0gJ6PoxiCel63A2cVWio3gtcTBru+NvcaKi4Cpgb\nET0ezJEbGQ01yHvR8cDJSg/seJA0D6niSlJv4i8KZcc1qN9DvXlJeYjlCkmPAj8mzyPMQ/DGAH9o\nsNt693HN+yki5pIaWQ+RruHC6jp13le2fxmYCJwv6X7Sv6l3vtJzLdgGeDwiuqq3NTMzM7NXboP8\nouE832l4RJzetPKrjKQzgdURcWFV+QTgCxFRs5Ei6XrgwoiYXWPdgcDOEXFxGTH3F0lvAU6KiC/2\ndyzri6QjgY9ExDEN6viLhs3MzGzwmDSAvmh4AJsJ7KvCFw1b70gaJmk58KdajSuAiLhxsDWuACLi\noVdZ4+oC4IukIahmZmZmVoINsgfLzMojyb8UzMzMbNAYscMIVj69snnFNtXrwWrlKYJm9irjP7z0\nvc7Ozm5PHrK+49yWw3kth/NaHue2HM5r+9yDZWbdSAr/XjAzMzNrbLDNwTIzMzMzMxtw3MAyM1sP\nWvnmd+sd57Yczms5nNfyOLflcF7b5waWmZmZmZlZH/EcLDPrxnOwzMzMzJrzHCwzMzMzM7OSuYFl\nZj1I6vYa+YaR/R3SBs9j2Mvj3JbDeS2H81oe57Yczmv7/D1YZtbTpO5vn530bL+EYWZmZrah8Rws\n63eSVkfE3/V3HACSxgGXA/Mj4uRctiIidu6HWHYHto+Im/P7E4GdImJyk+1uBvYB7oyIQwrlxwBn\nAj+OiO822D6qG1hM8pcPm5mZmRV5DpYNZAPpk/vxwA8qjausrfgk9dW/q7HAh6vKWonlW6Tz6L5h\nxBXABODzrzw0MzMzM6vFDSwbMCRNlrRI0n2SnpY0VdJoScskTZO0XNIMSe+TNDe/3ytvu7ekuyUt\nlHSXpDf2Moytgeeqyp7Px5ggaY6kGyQ9LOmSQuyrJX1H0iJgH0njJHVKWiDpZkkjcr1TJT0k6X5J\nP89lm+dznZfjP1jSxsBZwJE5H0cA/wu82OwEImJ2vXoR8SwwrO2s2CvmMezlcW7L4byWw3ktj3Nb\nDue1fZ6DZQNGRJwJnClpGPDfwEV51RjgoxGxVNK9wNERsa+kQ4AzgI8Ay4D9IqJL0gHAucDEXoQx\nFOiqiusdhbd7A28GngRukXR4RMwEtgB+HRFflLQRMAc4JCJ+L+lI4JvAycC/kob5vSxpq7zPM4Db\nI+LkfO7zgduArwN7RsSp1UFKOjivm9SLc/QfVszMzMxK4gaWDUTTgQsi4n5Jo4EVEbE0r3uI1PgA\neAAYnZe3Bn6ae66CXtzbuWH0FtY17GqZHxFP5PpXAPsBM4E1+SfALsBbgVslidSg+V1etxj4uaRr\ngGty2fuBgyV9Kb/fBBjVKNaIuB64vvWz6+YPksZExG/r1phdWN6pl0exbjo6Ovo7hEHLuS2H81oO\n57U8zm05nNd1Ojs7W+rRcwPLBhRJk4AnI+KnheKXCstdhfddrLuHvwHcERGH50ZZsYlQ2ffZwIFA\nRMS4qnVvIPUcPRoR9zYIsXoOVOX9nwvfzivgwYjYt8b2BwLjgUOAMyS9Ldf/aEQ8UhXTPg3ieCWm\nAPdL+kxEXFazxv4lHdnMzMxsA9XR0dGtwTl5cu3njnmokA0EgrXD3t4LfLbW+iaGAc/k5ZNqVYiI\nr0bEHtWNq7zuaWCHFIY6Ghzn7Xle2BDgKODOGjEuB15XaSBJ2kjSrnndqIiYA5wObEUaWngLsHYY\noKSxeXF1rtMbon7evgL8Q93GlZXCY9jL49yWw3kth/NaHue2HM5r+9zAsoGg0vPzeWB7YEF+sMOk\nqvXVy0XfAs6TtJBe3te5B+pRYNsG1e4FLiYNVfxtRFSG+a2NKyJeJs3/Ol/S/cAi4J15COJ0SYuB\nhcCUiPgfUu/bxpKWSHqA9HALSL1wuxYecrFWfhDGpFoBSvpv4ErgPZKelPS+qiqb5IddmJmZmVkf\n8/dgmRVI+gHwQET8qMa6CcAXit8ttaGR9HpgcURs16COvwfLzMzMrAl/D5ZZa34KnCRpan8H0tfy\nFw3PIvX2mZmZmVkJ3MAyK4iIeyLiHVVfNFxZN2dD7r2KiCsiYmxEfLdp5UndXyN2GFFmaK8KHsNe\nHue2HM5rOZzX8ji35XBe2+enCJpZDx4OaGZmZtY7noNlZt1ICv9eMDMzM2vMc7DMzMzMzMxK5gaW\nmdl64DHs5XFuy+G8lsN5LY9zWw7ntX1uYJmZmZmZmfURz8Eys248B8vMzMysOc/BMjMzMzMzK5kb\nWGbWg6Rur5FvGNnfIW3wPIa9PM5tOZzXcjiv5XFuy+G8ts/fg2VmPU3q/vbZSc/2SxhmZmZmGxrP\nwTLrJ5JGAzdExNtarP8t4GDgJeD/t3fnQZaV9RnHvw8gIgjExDiYGVmMIooiM+CgAWJjAi4ERIyC\nQRHUbEyEimK5JTBdRaKE0kgBEhcyIehIobggZRREhgQVWYZNFiWyCMYZF1DAKBH45Y97Gk/3dPfM\n7XvP9CzfT1VXn/Oec+5971M9XfPr933P+R5wdFXd38f7PQtYAiwA3lNVH5zivJpYYLHYhw9LkiS1\nuQZLWjf1U7VcBOxaVbsDtwHv7vO9fgq8FTilz+skSZK0hiywpNn1uCSfSHJzkvOSbJFkjyTXJlme\n5IYkjwBU1Ver6tHmuiuAef28UVX9pKquAR4e8mfQGnAOe3fMthvm2g1z7Y7ZdsNc+2eBJc2uZwGn\nV9VzgAeAY6rqmqqaX1ULgC8z+YjTm4D/WIv9lCRJ0hpwDZY0S5o1WJdV1Y7N/n7AW6vq0Gb/MOAt\nwAHtB1MleS+woKpePcP3PRF4YNo1WC9uNewInO0aLEmStHFbtmzZuBG90dHRSddgeRdBaXZNrFoK\nIMlzgROAfScUV0cBrwBeMtmLJTkJOBCoZgRsZvab8ZWSJEkbpJGREUZGRh7bHx0dnfQ8pwhKs2uH\nJHs1238GXJ5kW2ApcGRV3Tt2YpKXAe8ADq6qhyZ7sar6u9b0wums8tcWdcs57N0x226YazfMtTtm\n2w1z7Z8jWNLsuhVYlGQJ8G3gTOC1wPbAx5KE34xGnQZsDlzca+aKqjpmTd8oyRzgamBr4NEkxwHP\nqaoHh/mBJEmSNmauwZI0js/BkiRJWj2fgyVJkiRJHbPAkrSqxeO/5sydM4ud2TA4h707ZtsNc+2G\nuXbHbLthrv1zDZakVTgdUJIkaWZcgyVpnCTl7wVJkqTpuQZLkiRJkjpmgSVJa4Fz2Ltjtt0w126Y\na3fMthvm2j8LLEmSJEkaEtdgSRrHNViSJEmr5xosSZIkSeqYBZakVSQZ6Gu7edvN9kdY5ziHvTtm\n2w1z7Ya5dsdsu2Gu/fM5WJJWtXiwy1cuXjmUbkiSJK1vXIMlrWOS3AHsUVX3Jrm8qvZJ8mLg+Ko6\naIDXPQv4E2BlVe02zXk1aIHFYh9WLEmSNmyuwZLWH49VJlW1z2TtM7QEeOmAryFJkqRpWGBJsyTJ\nXya5NsnyJLcnuWTsUOucB1qXbJvkwiS3Jvlwv+9XVZcD9w3Ybc2Qc9i7Y7bdMNdumGt3zLYb5to/\nCyxpllTVR6pqPrAQuBv4wGSntbZfACwCng08I8mh3fdSkiRJ/XANljTLmtGolVU12uy312DdX1Xb\nNGuwRqtqpDnnaOB5VfW2Pt9rB+CLq12D9eJWw47ATn19JNdgSZKkDc6yZcvGjeiNjo5OugbLuwhK\nsyjJUcDTquqYNTh9YsUybj/JQuAjTfsJVXXhjDu234yvlCRJ2iCNjIwwMjLy2P7o6Oik5zlFUJol\nSfYA3g68frrTWtt7JdkhySbAYcDl7ROr6sqqml9VC6YprjLhNbWWOIe9O2bbDXPthrl2x2y7Ya79\ns8CSZs8i4EnApc2NLj7atLdHptrbVwKnAzcB36uqz/XzZkmWAt8Adk7y/WaaoSRJkobINViSxvE5\nWJIkSavnc7AkSZIkqWMWWJK0FjiHvTtm2w1z7Ya5dsdsu2Gu/fMugpJWtXiwy+fMnTOUbkiSJK1v\nXIMlaZwk5e8FSZKk6bkGS5IkSZI6ZoElSWuBc9i7Y7bdMNdumGt3zLYb5to/CyxJkiRJGhLXYEka\nxzVYkiRJq+caLEmSJEnqmAWWJK0FzmHvjtl2w1y7Ya7dMdtumGv/fA6WpFUkq4x2S5IkbTTmzJ3D\nintWzOha12BJGidJDfqgYUmSpPXaYlhdneQaLGkNJHk0ySmt/bcnOWGW+rJD059FrbbTkhw5G/2R\nJEnS6llgSeM9BBya5LdnuyONHwHHJXE67/rujtnuwAbMbLthrt0w1+6YbTfMtW8WWNJ4DwMfBd42\n8UAzonRJkuuSXJxkXtO+JMmpSb6e5L+THNq65vgkVzbXnDiD/vwYuAQ4apL+7J7km81rn59k26b9\n0iTvT/KtJLcm2btp3yTJPzXt1yX58xn0R5IkSdOwwJLGK+AM4IgkW084dhqwpKp2B5Y2+2O2q6q9\ngYOAkwGS7A88s6oWAvOBPZPsM4P+nAwcn1XvPHE28I6mP98G2gXcplW1F/C38NiKqjcDP2vaFwJ/\nkWSHPvujmdpptjuwATPbbphrN8y1O2bbDXPtm9OOpAmq6sEkZwPHAb9sHXoR8Kpm+xyaQqrx+eba\nW5I8pWk7ANg/yXIgwFbAM4HL++zPnUmuAI4Ya0uyDbBtVY291tnAea3LPtt8vwYYK6IOAJ6X5DXN\n/jZNf+5a5U0vbW3viL9cJUnSRm/ZsmVrdNt6CyxpcqcCy4ElrbbpbiXzUGs7re/vq6qPTXVRkkPo\njTwV8JaqWj7Fqe8DPgMsm+R9puvPI/zm33mAt1bVxdNc17Pfas9Qv+7AQrUrZtsNc+2GuXbHbLth\nro8ZGRlhZGTksf3R0dFJz3OKoDReAKrqPnojQm9uHfsG8Lpm+/XAf033GsBXgDcl2Qogye8l+d32\niVX1+aqaX1ULpiiuxvrzHeBm4OBm/37g3rF0qtlUAAAIw0lEQVT1VcAbgMvWoD/HjN0wI8kzkzxh\nimskSZI0A45gSeO1R6k+ACxqtR0LLElyPL2bTxw9yTWP7VfVxUl2Ab7ZLJ96gF5h9uMZ9ucf6I2q\njTkK+JemSLp9df0BPk5vwt/yZj3Xj4BD+uiLBuFf/7pjtt0w126Ya3fMthvm2jcfNCxpHB80LEmS\nNnqLfdCwJK3bfI5Id8y2G+baDXPtjtl2w1z75giWpHGS+EtBkiRt1ObMncOKe1ZMe44jWJLWWFX5\nNeSvE088cdb7sKF+ma25rk9f5mq269vXxprr6oqr6VhgSZIkSdKQWGBJ0lpw5513znYXNlhm2w1z\n7Ya5dsdsu2Gu/XMNlqRxXIMlSZK0ZmqSNVgWWJIkSZI0JE4RlCRJkqQhscCSJEmSpCGxwJIkSZKk\nIbHAkgRAkpcluTXJd5O8c7b7s65LclaSlUluaLU9KclFSb6T5CtJtm0de3eS25LckuSAVvuCJDc0\nuX9obX+OdVGSeUm+luSmJDcmObZpN98BJHl8km8lubbJ9h+bdnMdgiSbJFme5IJm31yHIMmdSa5v\nfm6vbNrMdkBJtk3y6Sanm5LsZa7DY4EliSSbAKcDLwV2BV6XZJfZ7dU6bwm9vNreBXy1qp4FfA14\nN0CS5wCvBZ4NvBz4cJKxuw6dCby5qnYGdk4y8TU3Rg8Db6uqXYEXAYuan0fzHUBVPQTsV1Xzgd2A\nlyTZG3MdluOAm1v75jocjwIjVTW/qhY2bWY7uFOBL1XVs4HnA7dirkNjgSUJYCFwW1XdVVW/Bs4F\nXjnLfVqnVdXlwH0Tml8JnN1snw0c0mwfDJxbVQ9X1Z3AbcDCJNsBW1fVVc15/966ZqNVVSuq6rpm\n+0HgFmAe5juwqvrfZvPx9P4PcB/mOrAk84BXAB9vNZvrcIRV/79qtgNIsg2wb1UtAWjy+jnmOjQW\nWJIA5gJ3t/bvadrUn6dU1UroFQnAU5r2ifn+oGmbSy/rMeY+QZIdgd2BK4A55juYZhrbtcAKYFlV\n3Yy5DsM/A+8A2s++MdfhKODiJFcleUvTZraD2Qn4SZIlzbTWjybZEnMdGgssSeqODxocQJInAp8B\njmtGsibmab59qqpHmymC84B9k4xgrgNJciCwshl1XeWBoy3mOjN7V9UCeiOEi5Lsiz+zg9oMWACc\n0WT7C3rTA811SCywJEHvr1Hbt/bnNW3qz8okcwCaqRM/atp/ADytdd5YvlO1b/SSbEavuDqnqr7Q\nNJvvkFTV/cCXgD0x10HtDRyc5HbgU/TWtp0DrDDXwVXVD5vvPwY+T29Kuz+zg7kHuLuqrm72z6dX\ncJnrkFhgSQK4CnhGkh2SbA4cDlwwy31aH4Txf7G+ADiq2X4j8IVW++FJNk+yE/AM4MpmCsbPkyxs\nFgwf2bpmY/evwM1VdWqrzXwHkOTJY3cFS/IEYH/gWsx1IFX1nqravqqeTu9359eq6g3AFzHXgSTZ\nshnJJslWwAHAjfgzO5BmGuDdSXZumv4IuAlzHZrNZrsDkmZfVT2S5G+Ai+j94eWsqrpllru1Tkuy\nFBgBfifJ94ETgfcDn07yJuAuenddoqpuTnIevTuM/Ro4pqrGpl4sAv4N2ILeHZ2+vDY/x7qoubPd\nEcCNzXqhAt4DnAycZ74z9lTg7OY/QpvQGx28pMnYXIfv/ZjroOYAn0tS9P7P+smquijJ1ZjtoI4F\nPpnkccDtwNHAppjrUOQ3+UiSJEmSBuEUQUmSJEkaEgssSZIkSRoSCyxJkiRJGhILLEmSJEkaEgss\nSZIkSRoSCyxJkiRJGhILLEmStE5JcmmSBVMcOzfJ05vtO5NcNuH4dUluaLbfmOS0ad7nzCQvmuLY\nwUn+fuafQtLGygJLkiStF5L8PrBVVd3eNBWwdZK5zfFdmra26R74uRdwxRTHvgi8OslmA3RZ0kbI\nAkuSJE0ryZZJLkxybZIbkrymab8jyclN2xWtkaUnJ/lMkm81X3/Qep2zmnOvSXJw075Fkk8luSnJ\nZ4EtpujK4fQKn7bzmnaA1wFLJxzfvhkR+06SE1qfaRfgu1VVSY5t3vu6JEsBqqqAbwAHzDA2SRsp\nCyxJkrQ6LwN+UFXzq2o34MutY/c1bWcApzZtpwIfrKq9gD8FPt60vxe4pKpeCLwEOCXJE4C/Bn5R\nVbsCJwJ7TtGPfYCrW/sFnA+8qtk/iFULsBc0x58PvKY19fDlrc/xTmD3qtod+KvWtVcBfzhFXyRp\nUhZYkiRpdW4E9k/yviT7VNUDrWPnNt8/Bbyw2f5j4PQk1wIXAE9MsiW90aB3Ne3LgM2B7ekVMZ8A\nqKobgeun6McOwA8ntP0UuC/JYcDNwC8nHL+4qn5WVb8CPkuvSAN4Kb8psK4HliY5Anikde3/ADtO\n0RdJmpTziiVJ0rSq6rZm5OcVwElJvlpVJ40dbp/afN8E2Kuqft1+nSQAr66q2yZpH9c0VVemOHYe\nvRG0I6e4Ztx+M2q2bVWtaNoOpFfkHQy8N8lzq+rR5r2mW8MlSatwBEuSJE0ryVOBX1bVUuAUoH2H\nv8Oa74cD32y2vwIc17r++a32Y1vtuzeb/wkc0bQ9F9htiq7cBWzX7lrz/XPAycBFk1yzf5Lfaoqq\nQ4CvA/sBlzbvF2D7qroMeBewDfDE5tqnNu8pSWvMESxJkrQ6z6O3XupR4P8Yv07pSUmuB35F7yYT\n0CuuzmjaN6VXQB0DnAR8qLmNeoA76I0anQksSXITcAvj11m1XU5vfdbyZr8AqupBeoXfZKNhV9Kb\nGjgXOKeqlje3bv90c3xT4BNJtmn6dGpV3d8cWwhcuNp0JKklvZvkSJIk9SfJHcAeVXXvWnq/pwOn\nVdWBA77O1fSmMD4yzTmhV8i9oKoeHuT9JG1cnCIoSZJmaq3+lbZ5/tX9Y7eDH+B19pyuuGocBJxv\ncSWpX45gSZIkSdKQOIIlSZIkSUNigSVJkiRJQ2KBJUmSJElDYoElSZIkSUNigSVJkiRJQ/L/uDxL\nkuJrs2cAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "co_t, de_t = compression_decompression_times()\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression speed', fontsize=14, y=1.01)\n", - "\n", - "\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "xlim = (0, np.max((nbytes / 1000000) / np.array(co_t)) + 100)\n", - "ax.set_xlim(*xlim)\n", - "ax.set_ylim(0, len(co_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcVMW9///Xm00CiIAaVlkk6nUJyLgLARQ1eqOEGEQU\nlxjUeM034pqH1yWAC2oi/uJN1MQEUaNZ9F7iBQwqCIPRiKhsCkiUTUFxYXO7YmQ+vz+qGs709DbD\nzPQ0fJ6PRz+m+1Sdqs85dcRTXVWnZWY455xzzjnnnNtxjYodgHPOOeecc87tLLyD5ZxzzjnnnHO1\nxDtYzjnnnHPOOVdLvIPlnHPOOeecc7XEO1jOOeecc845V0u8g+Wcc84555xztcQ7WM4553Y6ks6X\n9HGx4ygWSXtKqpDUv9ixOOfcrsY7WM45VySSJsab4K2SvpT0vqSZki6V1KTY8ZW4PwP7FjuIIvMf\nunTOuSLwDpZzzhXXdKAD0A04EZgMjAX+LulrxQysrkhqWtd1mNkWM/uorutp4FTsAJxzblfkHSzn\nnCuuLWb2oZm9Z2aLzOyXwECgDPhpKpOkppLukPSOpM8kvSTppGRBkg6Q9L+SNkn6RNILkg6OaZJ0\no6S3JX0haZGkwYl9u8XRtDMllUv6XNI8Sd+Mr39I+lTSbEn7JPYbLek1SSMlrY77/VXSnok8EyVN\nkfRTSe8A7xRyTJKaSPovSWtjzKsljUukny5pYaxzvaRZkvaOaT+Q9Ena+fmRpDclbYl/L0xLr5B0\nkaTH4rEulzQiV+NJOkTSDEmb4zmfL2lATBsQy/xO3P5/kl6RVJZWxrHxnH8maY2keyXtnpbnp5Le\nise6MD0uSUfEsv9P0qvAUbnids45V3e8g+Wccw2MmS0GngK+n9j8IPAtYDhwMPAQMFnSNwEkdQSe\nB7YCg4DewH8BjeP+lwNXAdcAhwB/BSZJ6pVW/RjgNuBQYBPwR+BXwLXAEUCLWG5Sd2AEcFqsez9g\nQlqeAcA3gW/HPHmPCRgFfBcYBnwDOBNYFo+3PfAnYCLwb7GcPyTqMxJT5CR9Lx7HXbGuu4F7JX0n\nLc4b47npBfwFeEBSF7L7I/AucDjhnI8BvkjL8wvCeT8MWAFMkdQ8xvVN4GngiXh+vhfLeSAR+63A\nBcB/AAcS2uc3kk6J6S2BqcBbhI75tcCd+BRB55wrDjPzl7/85S9/FeFF6BxMzpJ2G/BpfN+T0HHq\nkpbnr8Cv4/tbgZVA4yzlrQGuT9s2C3g4vu8GVAAXJtK/E7d9N7HtfGBz4vNo4F9A58S2vnG/nonj\nfB9oksizbwHHdDcwPcvx9In775Ml/Xzg48Tn54HfZTj/zyU+VwC3JD43Bj4Dzs7RhpuBc7OkDYhl\nDk9sawlsBH4YPz+UIa5D4357ETq0nwN90/L8f8DU+P5iYAPwtUT6iHh++hf7OveXv/zlr13t5Yuo\nnXOuYRLbRyD6xM9LJCXX1TQDno3vDwWeN7OtVQoK0806Af9IS3oeOCVt22uJ9+/HGF5P29ZKUnMz\nS43UrDWztYk8LxE6CAcCy+O2183sq0SeshzHNDO+fxCYLumfwDPA34BpZmbAwnjsiyU9A8wA/tuy\nr7s6kKqjas8TRt0yHr+ZbZX0IfD1LGVCGBGbIOkHMZ7/MbNliXQD5iTK/EzSa8BBcdNhQE9JwxP7\npNo+1bFuDjxV+TTRhNChhjCCt8jM/i+R/iK+Bss554rCO1jOOdcwHUSYTgZhOncFYRraV2n5/o8d\nkz6N7F8Z0jJtq+4U88/SPuc9JjObL6kb26cVPgQsAE40swrgJElHAScBI4HbJPU3s9coXK7jT6Vn\nPVYzGyvpEUJH9WRgtKQfmdmDBdbfCPg9oaOW3iFaS5guCHAqce1ajlidc841AL4GyznnGhhJhxBu\n1h+Pm+YTbr47mtmKtNd7iTz9lOHx7mb2CWGdUN+0pH7AkloIubOkzonPR8V4c5VdyDFhZp+Z2SQz\n+zFhyuIgSd9IpL9kZjeb2RGEYzwzS31LqXr838oTY0HMbLmZ/drMTiWMkiUfniHg6G0fwnqpQxL1\nzgMONrOVGc7DlphvC9A9Q3qqw7UU+KYqP3XyGHwNlnPOFYWPYDnnXHHtFh/Y0AjYGzgB+E/gZWA8\ngJm9KemPwIOSribclLcjPG1wuZk9AdwL/Ah4PD4UYSPhoRRLzGwR4UELYyW9BbwKnEvoYPXJE18h\n08y+AB6SdBVhzdB9hPVBK7LtUMgxSboCeI8wavUVYV3RZmBNHLk6gfCAiPcJUw67AIuzVPkL4DFJ\n8wjTDU8BziI8VKJG4oMq7iR0hFcRHrffjzA9L+kGSR/FY/kZocP0p5h2B/CipPuA3wKfEKYznmpm\nl5jZp5LuBO6U1Ah4DmhF6LRtNbPfEx60cSswUdJNQGfgupoel3POuR3jHSznnCuuEwgjL1sJT+17\nnXAT/ru0NUs/AK4n3JB3ITzUYC5xvZKZvSupP6EjMZMwevEa4QEIEJ781yru357wNL7TzSy5virT\niEchoyArCT/sOwXYk9DpuaiA/XIeE6GzcQ3hCYJGGPU62cy+kLSZMCL1/4A2hOlzN5nZn8jAzP5X\n0k+AqwkPiFgN/IeZ/S3PseY6/q1AW8LDMjoC6wnn4Jq0/a8ldJb3J3QAv5NaL2Vmr8V2uwUoJzxY\nYwXhYR+p2G+UtI7wFMh7gY8Jnc6fx/TP4tMQ7yN0nt8gPOJ/co7YnXPO1RGFtcLOOedc9UkaDXzf\nzNIf977Li7+HNRPY28w2FDse55xz9cPXYDnnnHN1x5/k55xzuxjvYDnnnHN1x6eJOOfcLsanCDrn\nnHPOOedcLfERLOecc84555yrJd7Bcs4555xzzrla4h0s55xzzjnnnKsl/jtYzrlKJPnCTOecc865\nAphZlafFegfLOVeFP/ym9PzgBz/gwQcfLHYYrpq83UqTt1tp8nYrTQ253aTMv8ThUwSdc24n0L17\n92KH4GrA2600ebuVJm+30lSK7eYdLOecc84555yrJT5F0DlXRbYh711R+/bdWLduVbHDyKtNmzbF\nDsHVgLdbafJ2K03ebqWpFNvNO1jOuQx8DVbK+++XRmfz0EMPLXYIrga83UqTt1tp8nYrTaXYbvLF\n7M65pPAUQf93YTv5Qz+cc87Vqe7du7N69epih+Gy6NatG6tWraqyXVLGpwjW2xosSd0kvZYlbZak\nsvqKJa3urpLmSZqW2LayGLFkI2mApIkF5MsZt6RP4t+Okh6L78+X9KualFdgnaMlXZmvnOpIlilp\noqT+efIPkLQptvM8STcUUMcsSV3zpFfrmpU0VNISSc/Gz3+StEDSqHgcp+fZv5BjPVvSwvh6XlKv\nRNp4SYslDahO3M4555yrW6tXr8bM/NVAX9Xt/Nb3Qy4a4tfAQ4BnzOyUxLaGGGchMeXLYwBm9p6Z\nDStgv9qos6F4zszK4uuWIsUwErjQzAZJ6gAcbmaHmtndtVjHCqC/mfUGbgHuTyWY2VXATcAPa7E+\n10CUl5cXOwRXA95upcnbrTR5u7n6Ut8drKaSHonf4j8mqXl6BklnSVoUX7fHbY3it/eL4jfzo+L2\nnpKmx1GAVyT1qEFMbYAP0rZ9mIjnvFjnfEkPxW0TJd0t6QVJb6VGHiS1lDQjxrJQ0uC4vZukpXG/\nZZIelXRi3H+ZpMNjvhaSJkiaI+lVSafFML4ENhdwLB/GcsbGeOdJWiNpQupwEvEkRxO7xhGZZZJ+\nluk85Ksz27lKkrSvpGmSXpY0W9L+klpLWpXI00LS25IaZ8qfof5NhPOTT3UX0qwHtma79qJhkl6S\n9IakvjH+SiOCkqZI6i/pRqAfMEHSz4Gngc6xjfpVClQqk1Qej3uapPaFHquZzTGz1LUyB+iclmUd\n4Zp3zjnnnHN1oL4fcnEAcIGZzYk3/ZcCd6USJXUEbgf6EG4mp8dOyhqgs5n1ivlax10eBcaZ2WRJ\nzahZh7ExUJHcYGZHxXoOAq4DjjGzjZKSN6YdzKyvpAOBycAk4AtgiJl9KmlPwg3u5Ji/J/B9M1si\n6RVgeNx/cKzjdOB64FkzGylpD2CupBlm9iLwYozpMOBHZnZx+oGk4jaz0cDoWMZzQOqGPznalHx/\nBHBwjP9lSVPNbF6qvFwKPFcp98fYl0s6ErgvjubMlzTAzGYDpwJPmdlWSVXyA4PS6r8i9V7SWOBl\nM5uaoe5jJC0A1gLXmNmSPMc1NJZZRuZrD6CxmR0l6RRgDHBiavcM5d0s6XjgSjObL+keYIqZlcVy\nR8a/TQjtNdjM1ksaBowDRlbjWFMuBKalbasgXPN5jEm8HxhfriEbOHBgsUNwNeDtVpq83UqTt5vb\nUeXl5QWNhNZ3B+ttM5sT3z8C/IREB4twoz/LzDYASHoU6E+Y6tRD0t3A34BnJLUCOpnZZAAzK2QU\noxJJAnrHWDI5HnjczDbGOjYl0p6I25ZK+nqqSOA2hXUyFUCnRNrKxE39YmBGfP8a0D2+Pwk4TdI1\n8XMzoCuwLFWpmb0KVOlcZfEIcJeZLciTb3rq2CRNIoy0zCuwjpRc5wpJLYFjgcfjeQdoGv8+BpwJ\nzAaGA/fkyZ9R7Fhm8irQ1cw+j52hJ4BMo2GZrCDt2kukTUqU363A8vKNpB0AHEL4ckGELw3eTc+U\n41hDJdJxwAWEtkxaC+wvaTcz25K9hDF5wnTOOeec27UMHDiwUkd97NixGfMVew1WpvU7VW5A4816\nb6AcuAT4Xba8lQqSLk1MleuQltYIWAkcCDxZUPSVJW9OU3GMAPYC+phZH8LUw+YZ8lckPlewvaMr\nwihXn/jqYWbLqAFJYwgd2ipT9TIopF12VCNgY1wDlTq+Q2LaZOBkSW2BMmBmnvzVYmafmtnn8f00\nwlTVdgXum+3ag+1tuJXtbfgVlf+7qjINNg8BryeOu3fa+sD8BYQHW9xPGAXbmEwzsxXAUmC1pIOr\nGZtrwHxtQWnyditN3m6lqdTarUOH7kiqs1eHDt0LiqNHjx7MnDkzY9rzzz/PgQceWCvHm6uefL74\n4gtOO+002rRpw5lnngnADTfcwN57702nTp1YvXo1jRo1oqKiIk9JtaO+O1jdJKWmnZ0N/D0tfS7Q\nX1I7SY2Bs4DZcbpdYzP7K3ADUGZmnwLvSPougKRmkr6WLMzM7o03qWVmti4trcLMugOvEEZPMpkJ\nnJG6GY8dgExSHaw9gA/MrCKOIHTLkCeXp4HLtu0g1ejB/wprt04ARqUnZdnlRElt4vkbAryQocyl\nearNea7M7BNgpaShiTJ7xbTPCO1wNzDVgqz5qyuxhok41VCJUdIZcWpqtn2rXHvZssa/q4BDFewD\nHJkrtAzblgF7Szo61t8kTr8siMKTD/8HONfMlmdI7wX0IIz+Li60XOecc87Vr/ffX034zrtuXqH8\nHdOvXz+WLt1+i7gjnaQd8d///d98+OGHbNy4kb/85S+888473HXXXbzxxhu8+26YCLR9QlTdq+8O\n1hvAjyUtISy0/03cnnq63TrgWsJowXzCGpMphIX65ZLmA3+IeQDOAy6TtJDQKdh2I10N/wQyjmbE\nKX23Ejp584HxyXiTWePfR4EjYjznEEYK0vNk2j/lZsLoyiKFh1DclJ5B0mFxbVIuVwCdCOup5sXR\nrFz1ziVMd1tAmOZXaXpg7GTklONcJZ0DjFR4KMnrwOBE2l8II4B/TmwbkSN/FQoP9zg1Q9JQSa/H\nuH5JmIaYmiLaE9iQo9hs117Ga8DMXiB0shbHul5Nz5Plc2r/fwFDgTsU1ozNB46pxrHeSLie742j\nt3PT0tsCq8ysfr7CcfXG1xaUJm+30uTtVpq83XZeq1evZv/999/WiVq9ejV77bUXe+6Z9xa2buR6\n5vuu8AKuAW4vdhwN+QV8B/h/xY6jDo7rYODOYsdRz8c8DPhTnjwG5q9tL8w555yrS5n+X1P3/z8u\n7P9v3bt3t9tuu80OOugga9eunf3whz+0LVu2mJlZeXm5denSxczMzj33XGvUqJG1aNHCdt99d/vF\nL35RpayPPvrITj31VGvTpo21a9fO+vfvX6meO++803r16mVt2rSx4cOHb6vnwQcftH79+lUqS5It\nX77cRo8ebc2aNbOmTZva7rvvbr/97W/ta1/7mjVu3Nh23313u+CCC2zVqlXWqFEj27p1q5mZbd68\n2UaOHGkdO3a0Ll262A033GAVFRXVap/E9ir3UvU9gtUQTQL6KvFDw64yM3vSzH5d7Dhqm5ktNrOr\nix1HfZE0Hrga+H0Buf0VX+3bd8t/uhqAUltb4AJvt9Lk7VaavN1q7o9//CPTp09n+fLlLFu2jFtu\n2f6ToqlRo4cffpiuXbsydepUPv74Y66+uuot1vjx49lnn31Yv349H3zwAePGjauU/vjjj/PMM8+w\ncuVKFi5cyIMPPlilnvTPY8aM4brrrmP48OF8/PHHXHzxxUybNo1OnTrx8ccf88ADD1SJ4/zzz6dZ\ns2asWLGC+fPnM336dH7/+wJujwpU308RbHAsrFP5VrHjcK6uWfih4ULz1mUozjnnnCshP/nJT+jU\nqRMA119/PZdddhk33VRlJQuQ+x6iadOmvPfee6xcuZKePXvSt2/fSumjRo2iffuw4ue0005jwYLs\nD8Ku6b3K+++/z7Rp09i8eTO77bYbzZs35/LLL+f+++/noosuqlGZ6XwEyznndgK+tqA0ebuVJm+3\n0uTtVnNdunTZ9r5bt27bHhxRXT/96U/p2bMnJ510Et/4xje44447KqWnOlcALVq04NNPP61ZwDm8\n/fbb/Otf/6Jjx460a9eOtm3bcskll/DRRx/VWh27/AiWc84555xzLrt33nln2/vVq1dvG81Kl+9J\nfS1btuTOO+/kzjvvZMmSJRx33HEceeSRHHfccXn3+/zzz7d9XrduXY2fCrjPPvvQvHlz1q9fX2dP\nFvQRLOec2wn42oLS5O1WmrzdSpO3W83dc889rF27lg0bNjBu3DiGDx+eMV+HDh1YsWJF1nKefPJJ\nli8PvyKz++6706RJExo3bpy3/t69e7N48WIWLVrEli1bsv7Aby6pKYUdOnTgpJNO4oorruCTTz7B\nzFixYgXPPfdctcvMxjtYzjnnnHPONTDhIUvFf4iTJM4+++xt0/r2228/rr/++ox5r732Wm6++Wba\ntWvHXXfdVSX9zTff5IQTTmD33Xenb9++/PjHP6Z///7b6slmv/3242c/+xmDBg1i//3351vfqv7j\nE5LlP/zww3z55ZccdNBBtGvXjjPOOIN169bl2LuadflidudckiTzfxecc865+iPJHzDVgGVrn7i9\nSs/QR7Ccc84555xzrpZ4B8s5V4Ukf+0Crw5dOhT7Utvl+ZqQ0uTtVpq83Vx98acIOueqGlPsAFy1\nrQR6VG+X98e8XyehOOecc7syX4PlnKtEknkHaxcxxn9U2jnnGgJfg9WwNdg1WJK6SXotS9osSWX1\nFUta3V0lzZM0LbFtZTFiyUbSAEkTC8iXM25Jn8S/HSU9Ft+fL+lXNSmvwDpHS7oyXznVkSxT0kRJ\n/fPkHyBpU2zneZJuKKCOWZK65kmv1jUraaikJZKejZ//JGmBpFHxOE7Ps3/eY435/kvSm7HsQxPb\nx0taLGlAdeJ2zjnnnHOFq+81WA2xaz4EeMbMTklsa4hxFhJTvjwGYGbvmdmwAvarjTobiufMrCy+\nbilSDCOBC81skKQOwOFmdqiZ3V1bFUg6BehpZvsBPwJ+k0ozs6uAm4Af1lZ9rgFpUF8LuUL5mpDS\n5O1WmrzdXH2p7w5WU0mPxG/xH5PUPD2DpLMkLYqv2+O2RvHb+0WSFkoaFbf3lDQ9flP/iqRqrkAA\noA3wQdq2DxPxnBfrnC/pobhtoqS7Jb0g6a3UyIOklpJmxFgWShoct3eTtDTut0zSo5JOjPsvk3R4\nzNdC0gRJcyS9Kum0GMaXwOYCjuXDWM7YGO88SWskTUgdTiKe5Ghi1zgis0zSzzKdh3x1ZjtXSZL2\nlTRN0suSZkvaX1JrSasSeVpIeltS40z5M9S/iXB+8qnuT3WvB7Zmu/aiYZJekvSGpL4x/kojgpKm\nSOov6UagHzBB0s+Bp4HOsY36VQpUKpNUHo97mqT21TjW7wIPA5jZS8Aeif0B1hGueeecc845Vwfq\n+yEXBwAXmNmceNN/KbDtV8gkdQRuB/oQbianx07KGqCzmfWK+VrHXR4FxpnZZEnNqFmHsTFQkdxg\nZkfFeg4CrgOOMbONkpI3ph3MrK+kA4HJwCTgC2CImX0qaU9gTkwD6Al838yWSHoFGB73HxzrOB24\nHnjWzEZK2gOYK2mGmb0IvBhjOgz4kZldnH4gqbjNbDQwOpbxHJC64U+ONiXfHwEcHON/WdJUM5uX\nKi+XAs9Vyv0x9uWSjgTui6M58yUNMLPZwKnAU2a2VVKV/MCgtPqvSL2XNBZ42cymZqj7GEkLgLXA\nNWa2JM9xDY1llpH52gNobGZHxVGjMcCJqd0zlHezpOOBK81svqR7gClmVhbLHRn/NiG012AzWy9p\nGDAOGFngsXYG3kl8Xhu3pZ5mUEG45nOblXjfnWo/PMEVgbdRSRo4cGCxQ3A14O1Wmrzd3I4qLy8v\naCS0vkew3jazOfH9I4Rv9JOOAGaZ2QYzqyB0oPoDK4AecdTo28AnkloBncxsMoCZfWlmX1QnGEkC\nehM6cJkcDzxuZhtjHZsSaU/EbUuBr6eKBG6TtBCYAXSSlEpbmbipXxzTAV4j3MICnARcK2k+UA40\nAyqtAzKzVzN1rrJ4BLjLzBbkyTfdzDbF8zeJqu1SiFznCkktgWOBx+Px/RZIjaw8BpwZ3w8H/pIn\nf0ZmNjpL5+pVoKuZHQr8mth2Bapy7SXSJiXKL+zn0POPpB0AHEL4cmE+odPdKT1TjmPNZy2wv6Td\ncuY6LvHyG3fnnHOu3nXo0mGX+amO4447jgceeKDG+19wwQW0a9eOo48+GoD77ruPDh060Lp1azZs\n2ECjRo1YsWLFDsc5cOBAxowZs+2VTX2PYKV/s59p/U6VG1Az2ySpN/Bt4BLgDODyTHkrFSRdClwU\n6/l3M1uXSGtEuHneAjxZjWNI2ZIh5hHAXkAfM6tQeABE8wz5KxKfK9jeDiKMcr1Zg3gqkTSG0KGt\nMlUvg0LaZUc1AjamRmzSTAZuldQWKANmAq1y5K8WM/s08X6apHsltTOzDQXsm+nauzAmp9pwK9vb\n8Csqf3FRZRpsHgJeN7O+1dwvZS2wT+Jzl7gNADNbIWkpsFrSIDNbXMN6XENTg8e0u+IrLy/3b9VL\nkLdbaSq1dnt/7ft1+rMpO8tPdTz//PM8++yzvPvuuzRv3pyvvvqKq666irlz53LIIYcA4Wl/9am+\nR7C6SUpNOzsb+Hta+lygv6R2khoDZwGz43S7xmb2V+AGoCzeNL8j6bsAkppJ+lqyMDO718z6xAcb\nrEtLqzCz7sArbB89STcTOENSu1hH2yz5Uq22B/BB7FwdR+VRjUJa9mngsm07JJ4AVx0Ka7dOAEal\nJ2XZ5URJbeL5GwK8kKHMpXmqzXmuzOwTYKWkoYkye8W0zwjtcDcw1YKs+asruQYpTjVUqnOlsGau\nY459q1x72bLGv6uAQxXsAxyZK7QM25YBe0s6OtbfJE6/LNRk4Ly479HAJjPb9i9oPIc9CKO/3rly\nzjnnXElbtWoV3bt3p3nz8J32unXr2LJlCwceeOC2PPX9CPz67mC9AfxY0hLCQvvUE85ST7dbB1xL\nmB43n7DGZAphDUl5nDL1h5gHwo3kZXFK3gvkmUKWxT+BdpkS4pS+WwmdvPnA+GS8yazx76PAETGe\nc4ClGfJk2j/lZsKDQBYpPITipvQMkg6La5NyuYIwrexlhYcojMlT71zCdLcFhGl+89Lq3DNPfbnO\nVdI5wEiFh5K8DgxOpP2FMAL458S2ETnyV6HwcI9TMyQNlfR6jOuXhGmIqSmiPYFcI1nZrr2M14CZ\nvUDoZC2Odb2anifL59T+/wKGAncorBmbDxxT6LGa2d8IHdO3CNMqL03L0hZYFafgup2Jj16VpFL6\nNt1t5+1WmrzdaqZHjx6MHz+e3r1707ZtW8466yy+/HL7M7d+97vfsd9++7HXXnsxZMgQ3nvvvYzl\nbNmyhXPPPZe99tqLtm3bctRRR/Hhh9ufp7Zq1Sr69etH69atOfnkk9mwIdyezZ49m3322adSWT16\n9GDmzJk88MADXHTRRbz44ou0bt2aESNG8G//9m8AtG3blhNOOKFKHF9++SVXX3013bp1o2PHjlx6\n6aVs2bKlSr4dscv/0LCka4A9zezavJl3UZK+A/Qws18XO5baJOlgwkNXri52LPVF4aEZ3zOzs3Lk\n8R8a3lWM8R8ads65hkAZfshWUp1OESz0/wE9evSgffv2/O///i+77bYbxx57LJdffjkXX3wxM2fO\n5Mwzz2TGjBkcdNBBXHXVVSxcuJDZs2dXKef+++/nySef5LHHHqNZs2YsWLCA/fbbj1atWnHcccex\nZs0annrqKbp06cLJJ5/MMcccw7hx45g9ezbnnnsub7/9dqWYJkyYwPHHH89DDz3EhAkTeO655wBY\nvXo1++67L1999dW2qYGNGjXirbfeYt999+WKK65g5cqVPPTQQzRp0oSzzz6bQw45hFtvvTXrOcjU\nPontVWYk1fcarIZoEvCgpGlpv4XlIjOryRq1Bi9OkduVOlfjgW8B/5k385i6jsY1BO0712TQ39Wm\nUlsT4gJvt9Lk7VZzo0aNon378P+M0047jQULwvPT/vjHPzJy5Eh69+4NwG233Ubbtm15++236dq1\n0nPaaNq0KevXr+ef//wn3/zmN+nTp0+l9AsuuICePXsCMGzYMKZMmbJDMZtZxrVXv/vd73jttdfY\nY489ALj22msZMWJEzg5Wde3yHSwzW0646XRupxZ/aLjQvHUZiqsDfuPgnHOurqQ6VwAtWrTYNg3w\n3Xff5bDDDtuW1rJlS/bcc0/Wrl1bpYN13nnnsWbNGoYPH87mzZsZMWIE48aNo3Hj8OsxHTp0qFTH\np59+Sm1wALmwAAAgAElEQVT78MMP+fzzzyvFXFFRUev3PfW9Bss551wd8M5VafJ2K03ebqXJ2632\nderUidWrV2/7/Nlnn7F+/Xo6d+5cJW/jxo258cYbWbx4Mf/4xz+YOnUqDz/8cN46WrZsyeeff77t\n89atWyut3aqOvfbaixYtWrB48WI2bNjAhg0b2LRpE5s3b65Redl4B8s555xzzjlXbWeddRYTJ05k\n0aJFbNmyheuuu46jjz66yugVhJkWr7/+OhUVFbRq1YqmTZtuG73KZf/99+eLL75g2rRpfPXVV9xy\nyy2VHrKRSbYRKUlcdNFFXH755ds6aWvXruWZZ54p4GgLt8tPEXTOuZ2BTxEsTd5upcnbrTSVWru1\n79y+Tn+rqtB1uLl+Q2rQoEHcfPPNnH766WzatIljjz2WP//5zxnzrlu3jksuuYS1a9fSqlUrhg8f\nzjnnnJO3jtatW3PvvfcycuRIKioq+OlPf0qXLl2qFXPy8x133MHYsWM5+uijt422/cd//AcnnXRS\nzjKrY5d/iqBzrjJJ5v8ulJ5Su3FwgbdbafJ2K00Nud2yPaXONQzVfYqgd7Ccc5V4B8s555yrX97B\natiq28HyNVjOOeecc845V0u8g+Wcq0KSv3bBV4cO3Yt96e1yysvLix2CqwFvt9Lk7ebqiz/kwjmX\ngU9TKD3lwMAdKuH997MvMnbOOedcYXwNlnOuEknmHaxdla8BcM65YlCWNT6uYcjWPnF78dZgSeom\n6bUsabMkldVXLGl1d5U0T9K0xLaVxYglG0kDJE0sIF/OuCV9Ev92lPRYfH++pF/VpLwC6xwt6cp8\n5VRHskxJEyX1z5N/sKSFkuZLekXS8QXUMUtS1R9xqJxerWtW0lBJSyQ9Gz//SdICSaPicZyeZ/9C\njvXseKwLJT0vqVcibbykxZIGVCdu55xzztWtbt26FX2auL+yv7p161at9qzvNVgNsWs+BHjGzE5J\nbGuIcRYSU748BmBm75nZsAL2q406G4IZZtbbzPoAFwD3FymOkcCFZjZIUgfgcDM71MzursU6VgD9\nzaw3cAuJYzWzq4CbgB/WYn2uwSgvdgCuBnxNSGnyditNDbndVq1ahZn5K8Nr1qxZRY9h1apV1WrP\n+u5gNZX0SPwW/zFJzdMzSDpL0qL4uj1uaxS/vV8Uv5kfFbf3lDRdYRTgFUk9ahBTG+CDtG0fJuI5\nT9tHPx6K2yZKulvSC5LeUhx5kNRS0owYy0JJg+P2bpKWxv2WSXpU0olx/2WSDo/5WkiaIGmOpFcl\nnRbD+BLYXMCxfBjLGRvjnSdpjaQJqcNJxJMcTeyqMCKzTNLPMp2HfHVmO1dJkvaVNE3Sy5JmS9pf\nUmtJqxJ5Wkh6W1LjTPkz1L+JcH6yMrPPEx9bAR8VcFzrga3Zrr1omKSXJL0hqW+Mv9KIoKQpkvpL\nuhHoB0yQ9HPgaaBzbKN+aeepTFJ5PO5pklK/BFjIsc4xs9S1MgfonJZlHeGad84555xzdaC+H3Jx\nAHCBmc2JN/2XAnelEiV1BG4H+hBuJqfHTsoaoLOZ9Yr5WsddHgXGmdlkSc2oWYexMVCR3GBmR8V6\nDgKuA44xs42SkjemHcysr6QDgcnAJOALYIiZfSppT8IN7uSYvyfwfTNbIukVYHjcf3Cs43TgeuBZ\nMxspaQ9grqQZZvYi8GKM6TDgR2Z2cfqBpOI2s9HA6FjGc0Dqhj852pR8fwRwcIz/ZUlTzWxeqrxc\nCjxXKffH2JdLOhK4z8JoznxJA8xsNnAq8JSZbZVUJT8wKK3+K1LvJY0FXjazqekVSxoC3AZ0AL5d\nwHENjfuVkfnaA2hsZkdJOgUYA5yY2j1DeTcrTE280szmS7oHmGJmZbHckfFvE0J7DTaz9ZKGAeOA\nkYUea8KFwLS0bRWEaz6PMYn3A9nRhye4+jCw2AG4GmioP3rqcvN2K03ebqWpIbVbeXl5QSOh9d3B\netvM5sT3jwA/IdHBItzozzKzDQCSHgX6E6Y69ZB0N/A34BlJrYBOZjYZwMxyfrOfiSQBvWMsmRwP\nPG5mG2MdmxJpT8RtSyV9PVUkcJvCOpkKoFMibaWZLYnvFwMz4vvXgO7x/UnAaZKuiZ+bAV2BZalK\nzexVoErnKotHgLvMbEGefNNTxyZpEmGkZV6BdaTkOldIagkcCzwezztA0/j3MeBMYDYwHLgnT/6M\nYscyW9oTwBNxtOgPhM5+IVaQdu0l0ibFv68ChU7OzfeYtgOAQwhfLojwpcG76ZlyHSuApOMI0yH7\npSWtBfaXtJuZbclewpg8YTrnnHPO7VoGDhxYqcM3duzYjPmKvQYr0/qdKjeg8Wa9N2GRwSXA77Ll\nrVSQdGliqlyHtLRGwErgQODJgqKvLHlzmopjBLAX0MfCep8PgOYZ8lckPlewvaMrwihXn/jqYWbL\nqAFJYwgd2ipT9TIopF12VCNgo5mVJY7vkJg2GThZUlugDJiZJ3+NmdnzQJM4wlhI/mzXHmxvw61s\nb8OvqPzfVZVpsHkIeD1x3L2t8vrA/AWEB1vcTxgF25hMM7MVwFJgtaSDqxmba9DKix2Aq4GGvCbE\nZeftVpq83UpTKbZbfXewuklKTTs7G/h7WvpcoL+kdpIaA2cBs+PNcGMz+ytwA1BmZp8C70j6LoCk\nZpK+lizMzO6NN6llZrYuLa3CzLoDrxBGTzKZCZwhqV2so22WfKkO1h7AB2ZWEUcQumXIk8vTwGXb\ndpAOLWCfqsGEtVsnAKPSk7LscqKkNvH8DQFeyFDm0jzV5jxXZvYJsFLS0ESZvWLaZ4R2uBuYakHW\n/NUlqWfifVmsc338PCNOTc22b5VrL1vW+HcVcKiCfYAjc4WWYdsyYG9JR8f6m8TplwVRePLh/wDn\nmtnyDOm9gB6E0d/FhZbrnHPOOecKU98drDeAH0taQlho/5u4PfV0u3XAtYSvYucT1phMISzUL5c0\nnzC969q433nAZZIWEjoFqYcBVMc/gXaZEuKUvlsJnbz5wPhkvMms8e+jwBExnnMIIwXpeTLtn3Iz\n4UEgixQeQnFTegZJh8W1SblcAXQirKeaF0ezctU7lzDdbQFhml+l6YGFjPbkOFdJ5wAjFR5K8jow\nOJH2F8II4J8T20bkyF+FwsM9Ts2Q9H1Jr0uaR+jEDY/5RVgbtyFHsdmuvYzXgJm9QOhkLQZ+SZg+\nSK590vb/FzAUuEPSAsJ/B8dU41hvJFzP98bR27lp6W2BVWZWUXVXV9oGFjsAVwMNaW2BK5y3W2ny\nditNpdhuu/wPDcf1Tnua2bV5M++iJH0H6GFmvy52LLUpTpG7wMyuLnYs9SU+NON7ZnZWjjz+Q8O7\nLP+hS+ecc65QKvYPDTdgk4C+SvzQsKvMzJ7c2TpXAGa2eBfrXI0HrgZ+X+xYXF0oL3YArgZKcW2B\n83YrVd5upakU262+nyLY4MR1Kt8qdhzO1TULPzRcoEKWDLqdTfv21fuleuecc85VtctPEXTOVSbJ\n/N8F55xzzrncfIqgc84555xzztUx72A559xOoBTnqDtvt1Ll7VaavN1KUym2m3ewnHPOOeecc66W\n+Bos51wlvgbLOeeccy4/X4PlnHPOOeecc3XMO1jOObcTKMU56s7brVR5u5Umb7fSVIrttsv/DpZz\nrirJfwfLuZT2nduzbs26YofhnHOuRPgaLOdcJZKMMcWOwrkGZAz4/yudc86l8zVYzjnnnHPOOVfH\n6q2DJambpNeypM2SVFZfsaTV3VXSPEnTEttWFiOWbCQNkDSxgHw545b0SfzbUdJj8f35kn5Vk/IK\nrHO0pCvzlVMdyTIlTZTUP0/+wZIWSpov6RVJxxdQxyxJXfOkV+ualTRU0hJJz8bPf5K0QNKoeByn\n59k/77HGfP8l6c1Y9qGJ7eMlLZY0oDpxuxLRoP7VcoUqxbUFztutVHm7laZSbLf6XoPVEOdYDAGe\nMbNrE9saYpyFxJQvjwGY2XvAsAL2q406G4IZZjYZQNI3gb8C3yhCHCOBC83sH5I6AIeb2X4xrrwd\n6EJIOgXoaWb7SToK+A1wNICZXSVpLvBDYHZt1Oecc8455yqr7ymCTSU9Er/Ff0xS8/QMks6StCi+\nbo/bGsVv7xfFkYhRcXtPSdPjN/WvSOpRg5jaAB+kbfswEc95idGPh+K2iZLulvSCpLdSIw+SWkqa\nEWNZKGlw3N5N0tK43zJJj0o6Me6/TNLhMV8LSRMkzZH0qqTTYhhfApsLOJYPYzljY7zzJK2RNCF1\nOIl4kqOJXeOIzDJJP8t0HvLVme1cJUnaV9I0SS9Lmi1pf0mtJa1K5Gkh6W1JjTPlz1D/JsL5ycrM\nPk98bAV8VMBxrQe2Zrv2omGSXpL0hqS+Mf5KI4KSpkjqL+lGoB8wQdLPgaeBzrGN+qWdpzJJ5fG4\np0lqX+ixAt8FHo7H/RKwR2J/gHWEa97tbGryr58ruoEDBxY7BFcD3m6lydutNJViu9X3CNYBwAVm\nNife9F8K3JVKlNQRuB3oQ7iZnB47KWuAzmbWK+ZrHXd5FBhnZpMlNaNmHcbGQEVyg5kdFes5CLgO\nOMbMNkpK3ph2MLO+kg4EJgOTgC+AIWb2qaQ9gTkxDaAn8H0zWyLpFWB43H9wrON04HrgWTMbKWkP\nYK6kGWb2IvBijOkw4EdmdnH6gaTiNrPRwOhYxnNA6oY/OdqUfH8EcHCM/2VJU81sXqq8XAo8Vyn3\nx9iXSzoSuM/MBsUO2QAzmw2cCjxlZlslVckPDEqr/4rUe0ljgZfNbGp6xZKGALcBHYBvF3BcQ+N+\nZWS+9gAam9lRCqNGY4ATU7tnKO9mhamJV5rZfEn3AFPMrCyWOzL+bUJor8Fmtl7SMGAcMLLAY+0M\nvJP4vDZuez9+riBc87nNSrzvjt+8O+ecc26XV15eXtCUxfruYL1tZnPi+0eAn5DoYBFu9GeZ2QYA\nSY8C/YFbgB6S7gb+BjwjqRXQKTX1y8zyfbNfhSQBvWMsmRwPPG5mG2MdmxJpT8RtSyV9PVUkcJvC\nOpkKoFMibaWZLYnvFwMz4vvXCLewACcBp0m6Jn5uBnQFlqUqNbNXgSqdqyweAe4yswV58k1PHZuk\nSYSRlnkF1pGS61whqSVwLPB4PO8ATePfx4AzCdPWhgP35MmfUexYZkt7Angijhb9gdDZL8QK0q69\nRNqk+PdVoFuB5eV7/vkBwCGELxdE+NLg3fRMuY41j7XA/pJ2M7MtWXMdV8PSXfGsxDvCJai8vLwk\nv53d1Xm7lSZvt9LUkNpt4MCBlWIZO3ZsxnzFXoOVaf1OlRtQM9skqTdh5OES4Azg8kx5KxUkXQpc\nFOv5dzNbl0hrRLh53gI8WY1jSEnenKbiGAHsBfQxswqFB0A0z5C/IvG5gu3tIMIo15s1iKcSSWMI\nHdoqU/UyKKRddlQjYGNqxCbNZOBWSW2BMmAmYSpftvw1ZmbPS2oiaU8zW19A/kzX3oUxOdWGW9ne\nhl9ReSS1yjTYPAS8bmZ9q7lfylpgn8TnLnEbAGa2QtJSYLWkQWa2uIb1OOecc865DOp7DVY3hYX3\nAGcDf09Lnwv0l9ROUmPgLGB2nG7X2Mz+CtwAlJnZp8A7kr4LIKmZpK8lCzOze82sj5mVJTtXMa3C\nzLoDrxBGTzKZCZwhqV2so22WfKkO1h7AB7FzdRyVRzUK+eXWp4HLtu2QeAJcdSis3ToBGJWelGWX\nEyW1iedvCPBChjKX5qk257kys0+AlZKGJsrsFdM+I7TD3cBUC7Lmry5JPRPvy2Kd6+PnGXFqarZ9\nq1x72bLGv6uAQxXsAxyZK7QM25YBe0s6OtbfJE6/LNRk4Ly479HAJjNLTQ9MncMehNFf71ztTHz0\nqiQ1lG9lXfV4u5Umb7fSVIrtVt8drDeAH0taQlho/5u4PfV0u3XAtUA5MJ+wxmQKYQ1JuaT5hOld\nqSf+nQdcJmkhoVOQXMxfqH8C7TIlxCl9txI6efOB8cl4k1nj30eBI2I85wBLM+TJtH/KzYQHgSxS\neAjFTekZJB0W1yblcgXQibCeal4czcpV71zCdLcFhGl+laYHxk5GTjnOVdI5wEiFh5K8DgxOpP2F\nMAL458S2ETnyV6HwcI9TMyR9X9LrkuYROnHDY34R1sZtyFFstmsv4zVgZi8QOlmLgV8Spg+Sa5+0\n/f8FDAXukLSA8N/BMYUeq5n9jdAxfQv4LWGdY1JbYJWZVaTv65xzzjnndpx29V+nj+ud9kx7TLtL\nkPQdoIeZ/brYsdQmSQcTHrpydbFjqS/xoRnfM7OzcuQxxtRfTK6W+BqsujMG6ur/lQ1pbYErnLdb\nafJ2K00Nud0kYWZVZiTV9xqshmgS8KCkaWZ2SrGDaYjMrCZr1Bq8OEVuV+pcjQe+Bfxn3sxj6joa\n50pH+841mRzhnHNuV7XLj2A55yqTZP7vgnPOOedcbtlGsOp7DZZzzjnnnHPO7bS8g+WcczuBQn74\n0DU83m6lydutNHm7laZSbDfvYDnnnHPOOedcLfE1WM65SnwNlnPOOedcfr4GyznnnHPOOefqmHew\nnHNuJ1CKc9Sdt1up8nYrTd5upakU281/B8s5V4VUZbTb7eTat+/GunWrih2Gc845V/J8DZZzrhJJ\nBv7vwq5H+P8PnHPOucL5GiznnHPOOeecq2P11sGS1E3Sa1nSZkkqq69Y0uruKmmepGmJbSuLEUs2\nkgZImlhAvpxxS/ok/u0o6bH4/nxJv6pJeQXWOVrSlfnKqY5kmZImSuqfJ/8Bkv4h6YtCY4nXZNc8\n6dW6ZiUNlbRE0rPx858kLZA0Kh7H6Xn2L+RYz5a0ML6el9QrkTZe0mJJA6oTtysV5cUOwNVAKa4t\ncN5upcrbrTSVYrvV9xqshjj/ZAjwjJldm9jWEOMsJKZ8eQzAzN4DhhWwX23U2RCsB35CaOtiGglc\naGb/kNQBONzM9oPQeaqlOlYA/c1ss6STgfuBowHM7CpJc4EfArNrqT7nnHPOOZdQ31MEm0p6JH6L\n/5ik5ukZJJ0laVF83R63NYrf3i+K38yPitt7SpoeRwFekdSjBjG1AT5I2/ZhIp7zYp3zJT0Ut02U\ndLekFyS9lRp5kNRS0owYy0JJg+P2bpKWxv2WSXpU0olx/2WSDo/5WkiaIGmOpFclnRbD+BLYXMCx\nfBjLGRvjnSdpjaQJqcNJxJMcTewaR2SWSfpZpvOQr85s5ypJ0r6Spkl6WdJsSftLai1pVSJPC0lv\nS2qcKX+G+jcRzk9WZvaRmb0KfFXA8aSsB7Zmu/aiYZJekvSGpL4x/kojgpKmSOov6UagHzBB0s+B\np4HOsY36pZ2nMknl8binSWpfjWOdY2apa2UO0DktyzrCNe92OgOLHYCrgYEDBxY7BFcD3m6lydut\nNJViu9X3CNYBwAVmNife9F8K3JVKlNQRuB3oQ7iZnB47KWuAzmbWK+ZrHXd5FBhnZpMlNaNmHcbG\nQEVyg5kdFes5CLgOOMbMNkpK3ph2MLO+kg4EJgOTgC+AIWb2qaQ9CTe4k2P+nsD3zWyJpFeA4XH/\nwbGO04HrgWfNbKSkPYC5kmaY2YvAizGmw4AfmdnF6QeSitvMRgOjYxnPAakb/uRoU/L9EcDBMf6X\nJU01s3mp8nIp8Fyl3B9jXy7pSOA+MxsUO2QDzGw2cCrwlJltlVQlPzAorf4rUu8ljQVeNrOp+eIu\n4LiGxjLLyHztATQ2s6MknQKMAU5M7Z6hvJslHQ9caWbzJd0DTDGzsljuyPi3CaG9BpvZeknDgHHA\nyBoc64XAtLRtFYRrPo8xifcD8Zt355xzzu3qysvLC5qyWN8drLfNbE58/whh2tZdifQjgFlmtgFA\n0qNAf+AWoIeku4G/Ac9IagV0MrPJAGaW85v9TCQJ6B1jyeR44HEz2xjr2JRIeyJuWyrp66kigdsU\n1slUAJ0SaSvNbEl8vxiYEd+/BnSP708CTpN0TfzcDOgKLEtVGkdiqnSusngEuMvMFuTJNz11bJIm\nEUZa5hVYR0quc4WklsCxwOPxvAM0jX8fA84kTFsbDtyTJ39GsWNZ21aQdu0l0ibFv68C3QosL9/z\nzw8ADiF8uSDClwbvpmfKd6ySjgMuILRl0lpgf0m7mdmW7CWMyROma3jK8Y5w6SkvLy/Jb2d3dd5u\npcnbrTQ1pHYbOHBgpVjGjh2bMV+x12BlWr9T5QbUzDZJ6g18G7gEOAO4PFPeSgVJlwIXxXr+3czW\nJdIaEW6etwBPVuMYUpI3p6k4RgB7AX3MrELhARDNM+SvSHyuYHs7iDDK9WYN4qlE0hhCh7bKVL0M\nCmmXHdUI2JgasUkzGbhVUlugDJgJtMqRv95kufYujMmpNtzK9jb8isojqVWmweYh4HUz61uziEHh\nwRb3AyenOrwpZrZC0lJgtaRBZra4pvU455xzzrmq6nsNVjdJqWlnZwN/T0ufC/SX1E5SY+AsYHac\nbtfYzP4K3ACUmdmnwDuSvgsgqZmkryULM7N7zayPmZUlO1cxrcLMugOvEEZPMpkJnCGpXayjbZZ8\nqQ7WHsAHsXN1HJVHNQr55dangcu27SAdWsA+VYMJa7dOAEalJ2XZ5URJbeL5GwK8kKHMpXmqzXmu\nzOwTYKWkoYkye8W0zwjtcDcw1YKs+XdQpXOgsGauY9bMGa69POWuAg5VsA9wZKGxRMuAvSUdHetv\nEqdfFkThyYf/A5xrZsszpPcCehBGf71ztVMZWOwAXA00lG9lXfV4u5Umb7fSVIrtVt8drDeAH0ta\nQlho/5u4PfV0u3XAtYS5LvMJa0ymEBbql0uaD/wh5gE4D7hM0kJCpyD1MIDq+CfQLlNCnNJ3K6GT\nNx8Yn4w3mTX+fRQ4IsZzDrA0Q55M+6fcTHgQyCKFh1DclJ5B0mFxbVIuVwCdCOup5sXRrFz1ziVM\nd1tAmOZXaXpg7GTklONcJZ0DjFR4KMnrwOBE2l8II4B/TmwbkSN/FQoP9zg1w/b2kt4hnJfrFR6i\n0SpOwesJbMhRbLZrL+M1YGYvEDpZi4FfEqYPkmuftP3/BQwF7pC0gPDfwTGFHitwI+F6vjeubZub\nlt4WWGVmFVV3dc4555xzO0pmpfCU7boT1zvtmfaYdpcg6TtADzP7dbFjqU2SDiY8dOXqYsdSX+JD\nM75nZmflyGOl8fR9V1k5OzaKJXb1/x8UQ0NaW+AK5+1WmrzdSlNDbjdJmFmVGUn1vQarIZoEPChp\nmpmdUuxgGiIzq8katQYvTpHblTpX44FvAf9ZQO66Dsc1MO3bF/qcFuecc87lssuPYDnnKpNk/u+C\nc84551xu2Uaw6nsNlnPOOeecc87ttLyD5ZxzO4FCfvjQNTzebqXJ2600ebuVplJsN+9gOeecc845\n51wt8TVYzrlKfA2Wc84551x+vgbLOeecc8455+qYd7Ccc24nUIpz1J23W6nyditN3m6lqRTbzX8H\nyzlXheS/g+V2Xu07t2fdmnXFDsM559xOytdgOecqkWSMKXYUztWhMeD/73POObejfA2Wc84555xz\nztWxeutgSeom6bUsabMkldVXLGl1d5U0T9K0xLaVxYglG0kDJE0sIF/OuCV9Ev92lPRYfH++pF/V\npLwC6xwt6cp85VRHskxJEyX1z5P/AEn/kPRFobHEa7JrnvRqXbOShkpaIunZ+PlPkhZIGhWP4/Q8\n++c91pjvvyS9Gcs+NLF9vKTFkgZUJ25XIhrUv1quUKW4tsB5u5Uqb7fSVIrtVt9rsBrinIwhwDNm\ndm1iW0OMs5CY8uUxADN7DxhWwH61UWdDsB74CaGti2kkcKGZ/UNSB+BwM9sPQuepNiqQdArQ08z2\nk3QU8BvgaAAzu0rSXOCHwOzaqM8555xzzlVW31MEm0p6JH6L/5ik5ukZJJ0laVF83R63NYrf3i+S\ntFDSqLi9p6Tp8Zv6VyT1qEFMbYAP0rZ9mIjnvFjnfEkPxW0TJd0t6QVJb6VGHiS1lDQjxrJQ0uC4\nvZukpXG/ZZIelXRi3H+ZpMNjvhaSJkiaI+lVSafFML4ENhdwLB/GcsbGeOdJWiNpQupwEvEkRxO7\nxhGZZZJ+luk85Ksz27lKkrSvpGmSXpY0W9L+klpLWpXI00LS25IaZ8qfof5NhPOTlZl9ZGavAl8V\ncDwp64Gt2a69aJiklyS9IalvjL/SiKCkKZL6S7oR6AdMkPRz4Gmgc2yjfmnnqUxSeTzuaZLaF3qs\nwHeBh+NxvwTskdgfYB3hmnc7m5r86+eKbuDAgcUOwdWAt1tp8nYrTaXYbvU9gnUAcIGZzYk3/ZcC\nd6USJXUEbgf6EG4mp8dOyhqgs5n1ivlax10eBcaZ2WRJzahZh7ExUJHcYGZHxXoOAq4DjjGzjZKS\nN6YdzKyvpAOBycAk4AtgiJl9KmlPYE5MA+gJfN/Mlkh6BRge9x8c6zgduB541sxGStoDmCtphpm9\nCLwYYzoM+JGZXZx+IKm4zWw0MDqW8RyQuuFPjjYl3x8BHBzjf1nSVDOblyovlwLPVcr9Mfblko4E\n7jOzQbFDNsDMZgOnAk+Z2VZJVfIDg9LqvyL1XtJY4GUzm5ov7gKOa2gss4zM1x5AYzM7SmHUaAxw\nYmr3DOXdLOl44Eozmy/pHmCKmZXFckfGv00I7TXYzNZLGgaMA0YWeKydgXcSn9fGbe/HzxWEa/7/\nZ+/to6wsrnz/zxeUURQVfOHFjK1haQw3RsWoMTra+ouYmGi8KMaXDC7jaLJwqZiYtbyZ/AIoEaMx\nUWd+jtE4JAquQe4lGdQoAul2FEUQmhcFUW/QiDet3igqs4iO9v79UfvA06fPWx+b7n6692ets049\nVbuqdtV+TvezT9WuU5mmTPpA4uE9CIIgCIJ+T3Nzc01bFrvbwfqTmS319CzStq2fZ8qPBprM7G0A\nSbOBE4HpwEGSbgN+DzwmaXdglJnNBzCzat/sd0CSgMNdl1KcAsw1s3e8j82Zst953npJ+xWaBGYo\nxTnXmyQAACAASURBVMm0AaMyZRvNbJ2nnwcWeXot6REWYBxwhqQf+PUg4ABgQ6FTX4np4FyVYRbw\nczNbVUVuYWFskuaRVlpW1thHgUpzhaTdgC8Bc33eAXb29weAb5K2rZ0H/H9V5EvijmVX80eK7r1M\n2Tx/XwE01NhetfPPPwN8jvTlgkhfGvyfYqFPMNbXgUMk/Y2ZfVBW6uQ6Ww96jo2EI5xDmpubc/nt\nbH8n7JZPwm75pDfZrbGxsZ0u06ZNKynX0zFYpeJ3OjyAmtlmSYcDpwHfBSYAk0vJtmtImgRc6v2c\nbmatmbIBpIfnD4CHOzGGAtmH04IeFwL7AEeaWZvSARC7lJBvy1y3sd0OIq1yvVSHPu2QNJXk0HbY\nqleCWuzySRkAvFNYsSliPvATSUOBscAfgN0ryHcbZe69f/Digg0/ZrsNP6L9SmqHbbBVEPCcmR1f\nn8a8Dvxt5vpTngeAmf1R0nrgVUn/j5k9X2c/QRAEQRAEQQm6OwarQSnwHuAC4Imi8mXAiZKGSRoI\nnA887tvtBprZb4EfAWPNbAvwmqRvAEgaJGnXbGNmdoeZHWlmY7POlZe1mdmBwLOk1ZNS/AGYIGmY\n9zG0jFzBwdoTeNOdq5Npv6pRyy+3LgCu3FYhcwJcZ1CK3foycFVxUZkqp0ray+fvLGBJiTbXV+m2\n4lyZ2fvARknnZNr8vJf9J8kOtwEPWaKs/Cek3RwoxcyNLCtc4t6r0u4rwBFK/C1wTK26OBuAfSV9\n0fvfybdf1sp8YKLX/SKw2cwK2wMLc3gQafU3nKu+RKxe5ZLe8q1s0DnCbvkk7JZP8mi37nawXgAu\nl7SOFGh/p+cXTrdrBa4FmoEWUozJg6QYkmZJLcB9LgPpQfJKSatJTkE2mL9WXgSGlSrwLX0/ITl5\nLcAtWX2zov4+Gzja9fkWsL6ETKn6Ba4nHQSyRukQiuuKBSQd5bFJlbgaGEWKp1rpq1mV+l1G2u62\nirTNr932QHcyKlJhrrJ8C7hE6VCS54AzM2VzSCuA/5bJu7CCfAeUDvf4eon84ZJeI83LPyodorG7\nb8EbDbxdodly917Je8DMlpCcrOeBW0nbB6lUp6j+fwHnAD+VtIr0OTiu1rGa2e9JjunLwC9JcY5Z\nhgKvmFlbcd0gCIIgCILgk6P+/mv2Hu+0d9Ex7UEGSV8DDjKzf+5pXboSSf+NdOjKNT2tS3fhh2b8\ndzM7v4KMMbX7dAq6iIjBqp2p0Fv+9/Wm2IKgdsJu+STslk96s90kYWYddiR1dwxWb2Qe8GtJj5jZ\nV3tamd6ImdUTo9br8S1y/cm5ugX4O+B/VBWeuqO1CYKeY/j+9Wx2CIIgCILa6PcrWEEQtEeSxd+F\nIAiCIAiCypRbweruGKwgCIIgCIIgCII+SzhYQRAEfYBafvgw6H2E3fJJ2C2fhN3ySR7tFg5WEARB\nEARBEARBFxExWEEQtCNisIIgCIIgCKoTMVhBEARBEARBEAQ7mHCwgiAI+gB53KMehN3yStgtn4Td\n8kke7Ra/gxUEQQekDqvdQcDw4Q20tr7S02oEQRAEQa8mYrCCIGiHJIP4uxCUQsT/jCAIgiBIRAxW\nEARBEARBEATBDmaHOliSGiStLVPWJGnsjuy/HJIOkLRS0iOZvI09oUs5JJ0kaWYNcp3WW9JVknYp\nU3aRpNs9PUXSxCptXSRpShWZ9zurYzUKbfo91lSDfJOkFyS1uO33qSJfcf69/MFO6jxI0kLvf4Kk\nEyQ959eHlvusZOpXHaukXSU9JGm9pLWSbsiUHeL9zemM3kFeaO5pBYI6yGNsQRB2yytht3ySR7t1\nxwpWb9xPchbwmJl9NZPXG/WsRad69J4MDK6jXr067Ii5tTLpSpxvZkea2Vgz+7+d7KOe8mLGAub9\nzwUuBG4ws7HA1hrbq0XmZjP7LHAkcIKk00gdv2hmnwMOk3RQJ3UPgiAIgiAIaqA7HKydJc2StE7S\nA6VWTiSdL2mNv270vAGSZnreaklXef5oXwVYJenZOh8U9wLeLMp7K6PPRO+zRdJvPG+mpNskLZH0\nsqTxnr+bpEWuy2pJZ3p+g68izJS0QdJsSad6/Q2SvuBygyXdI2mppBWSznA1PgTerWEsb3k70zKr\nM5u8zcG+mtHi8zhB0hXAKKBJ0mKve7HrtBQ4PtP2FtKDfyW2uhyS9pM0z23TIumLhSnNzO01kpa5\nzBTPmyFpUkZmiqTvlZMv4mPg7RrmCTp3v2+bf1+tKsztCkm7ucwQSXPdzvdl9N8oaZinj/LVs32B\n+4CjvZ3LgHOB67N1vc4ASTdJesbHfWmtYzWzrWb2uKc/AlYCnyoSe4P0GQj6FI09rUBQB42NjT2t\nQlAHYbd8EnbLJ3m0W3ecIvgZ4GIzWyrpHmAS8PNCoaSRwI2kb9s3AwvdSdkE7G9mn3e5PbzKbNK3\n/vMlDaI+J3Eg0JbNMLNjvZ8xwA+B48zsHUnZB9ERZna8pM8C84F5wF+Bs8xsi6S9gaVeBjAaONvM\n1kl6FjjP65/pfYwH/hFYbGaXSNoTWCZpkZk9DTztOh0FfMfMLiseSEFvM5sCTPE2/gP4Z+ArwOtm\n9nVvZ4iZvS/paqDRxzcCmEqa//dI+4xWepu3VJtIM3sgc3k70Gxm4yUJ2L0g5v2fChxsZsd4+XxJ\nJwBzgFuBO1z+XGBcOXkzexJ32sxsE3COtz8SuLsw3hL8WtJ/AfPMbHqVcW2bf+D7wCQze1rSYJLN\nAY4AxgCtwBJJXzKzp+i4ymRm9pakfwC+b2YFJ/w44EEzmyepISN/CbDZzI71e3yJpMfM7NVOjBW/\nd88gzW2WNtJnoAJTM+lG4uE9CIIgCIL+TnNzc01bFrtjBetPZrbU07OAE4rKjwaazOxtM2sjOVAn\nAn8EDvJVo9OA9yXtDowys/kAZvahmf2VTuAP6oeTHLhSnALMNbN3vI/NmbLfed56YL9Ck8AMSauB\nRcAoSYWyjWa2ztPPeznAWuBAT48DrpXUQnJuBgEHZBUysxWlnKsyzAJuMbMW7+dUXyE6wcwKsVBi\n+6rSsWyf/49Izk69nAL8i+tsmf4KjHN9VpKcuM+QHKhVwL6SRkj6PPC2mb1eTr5c52b25woOxwVm\ndhjwd8DfSfpWJ8a1BPiFr/4N9fsUYJn3acAqttv0k55xPg6Y6PfEM8AwisZdZaxIGgjcD9xqZq8U\nFW8ifQYqMDXzaqxd86AHae5pBYI6yGNsQRB2yytht3zSm+zW2NjI1KlTt73K0R0rWB2+zS8h0+GB\n1Mw2SzocOA34LjCBFDtU8eHVt5pd6v2cbmatmbIBJMftA+DhToyhwAcldL4Q2Ac40szalA6d2KWE\nfFvmuo3tcy/SKtdLdejTDklTSQ7tvQBm9pLSQSKnA9N9ZazUyk1X/ehRtfggATPM7O4SZXNJNh7B\ndievknyn4p/M7M/+/p+S7geOITmjtdT9qaSHgK+RVpPGeVHWvh+z3aYfsf3Li5KHiVRBwBVmtrCO\nugXuAjaY2T+VKPslsEDSMWb2nU/QRxAEQRAEQVBEd6xgNUg61tMXAE8UlS8DTpQ0zL91Px943Lfb\nDTSz3wI/Asaa2RbgNUnfgG2nsu2abczM7sgcZNBaVNZmZgcCzwLfLKPvH4AJmRiaoWXkCk7JnsCb\n7lydDDSUkKnEAuDKbRWkI2qo01GZFLv1ZeCqTN5IYKuZ3Q/cTDpkAdJWwMKWy2dI8z9U0s4kJ6dU\n+5dn46TKsJi0BbQQRzSkUN3fFwDfLsQwSRrlsUkADwDnAWeTnK1y8vsUtVkVSQP9fsLH+HXgOb8+\nS5mT9srU/7SZPW9mNwHLgUOrdLkROMrTZ9eqZ4YFwCRJO3n/Bxff51X0nQ7sYWZXlxG5BrgknKu+\nRmNPKxDUQR5jC4KwW14Ju+WTPNqtOxysF4DLJa0jBdbf6fkG4E7QtaT9LS3AcjN7ENgfaPZtUve5\nDMBE4ErfkrcEGF6HTi+Stl11wLf0/YTk5LUAhTikcitxs0kHF6wGvgWsLyFTqn6B60kHgaxROqb7\numIBPyjhrgrjAbiadHjFcj9EYSpwGCmmqwX4MVBYvbobeFTSYp//aaTYsSeAdR1aThwK/KWKDpOB\nkyWtITmxYzy/YOuFpG1rT7vMXDxOy+d9CLDJzN6oID8k22YWSSN9pamYvyGt2KwibTXc5HMAKU6u\n2mEik5WOPF9NOvzikRIyWX2uA26XtIy0mlWOcvfEr0h2WOn3xJ0UrTaXG6uk/UnxfWO0/WCObxeJ\nDQVerqBXEARBEARBUCdK4SP9C0k/APY2s2urCgcASJoPjPc4rT6DpHuBq82smvPYJ/AYxDXAOWa2\noYyM9c5fLQgq08yOX8US/fF/xo6kubk5l9/O9nfCbvkk7JZPerPdJGFmHXZVdccKVm9kHnC8Mj80\nHFTGzM7sa84VgJlN7EfO1SGkVeIW0ipuEARBEARB0MX0yxWsIAjKk1awgqAjw4c30Nr6Sk+rEQRB\nEAS9gnIrWN1ximAQBDkjvngJgiAIgiCoj/66RTAIgqBP0Zt+JySonbBbPgm75ZOwWz7Jo93CwQqC\nIAiCIAiCIOgiIgYrCIJ2SLL4uxAEQRAEQVCZOEUwCIIgCIIgCIJgBxMOVhAEQR8gj3vUg7BbXgm7\n5ZOwWz7Jo93CwQqCIAiCIAiCIOgiIgYrCIJ2xO9gBUHQ1xi+/3BaN7X2tBpBEPQxysVghYMVBEE7\nJBlTe1qLIAiCLmRq/L5fEARdTxxyEQRB0JfZ2NMKBHURdssleYwJCcJueSWPdtuhDpakBklry5Q1\nSRq7I/svh6QDJK2U9Egmr1f9m5N0kqSZNch1Wm9JV0napUzZRZJu9/QUSROrtHWRpClVZN7vrI7V\nKLTp91hTDfJNkl6Q1OK236eKfMX59/IHO6nzIEkLvf8Jkk6Q9JxfH1rus5KpX+tYx0paI+lFSbdm\n8g/x/uZ0Ru8gCIIgCIKgdrpjBas3rsmfBTxmZl/N5PVGPWvRqR69JwOD66hXrw47Ym6tTLoS55vZ\nkWY21sz+byf7qKe8mLGAef9zgQuBG8xsLLC1xvZqkfkX4BIzOwQ4RNJppI5fNLPPAYdJOqiTuge9\nnbBoPgm75ZLGxsaeViGog7BbPsmj3brDwdpZ0ixJ6yQ9UGrlRNL5/o37Gkk3et4ASTM9b7Wkqzx/\ntK8CrJL0bJ0PinsBbxblvZXRZ6L32SLpN543U9JtkpZIelnSeM/fTdIi12W1pDM9v0HSeq+3QdJs\nSad6/Q2SvuBygyXdI2mppBWSznA1PgTerWEsb3k70zKrM5u8zcGSHvL8Nb5qcgUwCmiStNjrXuw6\nLQWOz7S9hfTgX4mtLoek/STNc9u0SPpiYUozc3uNpGUuM8XzZkialJGZIul75eSL+Bh4u4Z5gs7d\n79vm31erCnO7QtJuLjNE0ly3830Z/TdKGubpo3z1bF/gPuBob+cy4Fzg+mxdrzNA0k2SnvFxX1rr\nWCWNAIaY2XLPupf0hUKWN0ifgSAIgiAIgqCL2akb+vgMcLGZLZV0DzAJ+HmhUNJI4EbgSGAzsNCd\nlE3A/mb2eZfbw6vMJn3rP1/SIOpzEgcCbdkMMzvW+xkD/BA4zszekZR9EB1hZsdL+iwwH5gH/BU4\ny8y2SNobWOplAKOBs81snaRngfO8/pnex3jgH4HFZnaJpD2BZZIWmdnTwNOu01HAd8zssuKBFPQ2\nsynAFG/jP4B/Br4CvG5mX/d2hpjZ+5KuBhp9fCOAqaT5fw9oBlZ6m7dUm0gzeyBzeTvQbGbjJQnY\nvSDm/Z8KHGxmx3j5fEknAHOAW4E7XP5cYFw5eTN7EnfazGwTcI63PxK4uzDeEvxa0n8B88xsepVx\nbZt/4PvAJDN7WtJgks0BjgDGAK3AEklfMrOn6LjKZGb2lqR/AL5vZgUn/DjgQTObJ6khI38JsNnM\njvV7fImkx8zs1RrGuj/ps1Ngk+dlaSN9BsqT3Yh4IPEtex7YSNgpj4Tdcklzc3Muv1Xv74Td8klv\nsltzc3NNMWHd4WD9ycyWenoWcAUZBws4Gmgys7cBJM0GTgSmAwdJug34PfCYpN2BUWY2H8DMPuys\nMv6gfrjrUopTgLlm9o73sTlT9jvPWy9pv0KTwAxJJ5IeXEdlyjaa2TpPPw8s8vRa0mMrwDjgDEk/\n8OtBwAHAhkKnZrYC6OBclWEWcIuZtUjaAvxM0gzgYXdMCjoXVpWOpf38zwEOrrGvYk4B/t51NqA4\n9moccKqkld7/biQHaqakfd3Z2w9428xelzS5lDzwJCUwsz8D5ZyrC8zsz776NE/St8ys3D1QzBLg\nF35vznPdAJZ5n0haRbLpU2RW7OpkHGkb3wS/3oM07lcLAlXGWo1NpM/As2UlTq6z5SAIgiAIgj5K\nY2NjO2dv2rRpJeW6w8Hq8G1+CZkOD6RmtlnS4cBpwHeBCaTYoYoPr77V7FLv53Qza82UDQD+CHwA\nPNyJMRT4oITOFwL7AEeaWZvSoRO7lJBvy1y3sX3uRVrleqkOfdohaSrJob0XwMxeUjpI5HRguq+M\nlVq5+aQOQYFq8UECZpjZ3SXK5pJsPIK0olVNvlPxTwVHyMz+U9L9wDGUd7KL6/5U0kPA10irSeO8\nKGvfj9lu04/YvrJa8jCRKgi4wswW1lH3deBvM9ef8rwsvwQWSDrGzL5TRx9BbyRWQfJJ2C2X9JZv\n04POEXbLJ3m0W3fEYDVIOtbTFwBPFJUvA06UNEzSQOB84HHfbjfQzH4L/AgYa2ZbgNckfQO2ncq2\na7YxM7sjc5BBa1FZm5kdSPrm/ptl9P0DMCETQzO0jFzBKdkTeNOdq5OBhhIylVgAXLmtgnREDXU6\nKpNit74MXJXJGwlsNbP7gZtJhyxA2gpY2HL5DGn+h0rameTklGr/8mycVBkWk7aAFuKIhhSq+/sC\n4NuFGCZJozw2CeAB4DzgbJKzVU5+n6I2qyJpoN9P+Bi/Djzn12dJuqFK/U+b2fNmdhOwHDi0Spcb\ngaM8fXatemZYAEyStJP3f3DxfV4Ov+fflVTYVjkR+PcisWtIh2CEcxUEQRAEQdDFdIeD9QJwuaR1\npMD6Oz3fYNsD4bWk2J8WYLmZPUiKG2mW1EI6HOBarzcRuFLSatLWreF16PQiMKxUgW/p+wnJyWsB\nCnFI5VbiZpMOLlgNfAtYX0KmVP0C15MOAlmjdEz3dcUCSgcl3FVhPABXkw6vWK50iMJU4DBSTFcL\n8GPStkuAu4FHJS32+Z9Gih17AljXoeXEocBfqugwGThZ0hqSEzvG8wu2XgjcDzztMnPxOC2f9yHA\nJjN7o4L8kGybWSSN9JWmYv6GtGKzihRftsnnAFKcXLXDRCZLWus2/hB4pIRMVp/rgNslLSOtZpWj\n3D3xK5IdVvo9cSdFq80VxgpwOXAP6T5/ycweLSofCrxcQa8gj/SqH5oIaibslkvy+Ls8Qdgtr+TR\nbuqPv2zu8U57m9m1VYUDACTNB8abWSWHIXdIuhe42syqOY99Al/VWgOcY2YbysgYU7tVraAriMMS\n8knYrXuYCl35vNObgu6D2gm75ZPebDdJmFmHXVX91cEaDfwa2FL0W1hB0GeRdAhpK+Ya4CIr8+GX\n1P/+KARB0KcZvv9wWje1VhcMgiDoBOFgBUFQE5LK+V5BEARBEASBU87B6o4YrCAIgmAHk8c96kHY\nLa+E3fJJ2C2f5NFu4WAFQRAEQRAEQRB0EbFFMAiCdsQWwSAIgiAIgurEFsEgCIIgCIIgCIIdTDhY\nQRAEfYA87lEPwm55JeyWT8Ju+SSPdgsHKwiCIAiCIAiCoIuIGKwgCNoRv4MV5JHhwxtobX2lp9UI\ngiAI+hHxO1hBENREcrDi70KQN0T8PwuCIAi6kzjkIgiCoE/T3NMKBHWQx9iCIOyWV8Ju+SSPdtuh\nDpakBklry5Q1SRq7I/svh6QDJK2U9Egmb2NP6FIOSSdJmlmDXKf1lnSVpF3KlF0k6XZPT5E0sUpb\nF0maUkXm/c7qWI1Cm36PNdUg/4ikFknPSfqVpJ2qyFecfy9/sJM6D5K00O+9CZJOcH1WSjq03Gcl\nU7/qWCXtKukhSeslrZV0Q6bsEO9vTmf0DoIgCIIgCGqnO1aweuOejbOAx8zsq5m83qhnLTrVo/dk\nYHAd9erVYUfMrZVJl2OCmR1pZp8D9gK+2ck+6ikvZixgZjbWzOYCFwI3mNlYYGuN7dUic7OZfRY4\nEjhB0mmkjl/08R8m6aBO6h70ehp7WoGgDhobG3tahaAOwm75JOyWT/Jot+5wsHaWNEvSOkkPlFo5\nkXS+pDX+utHzBkia6XmrJV3l+aN9FWCVpGfrfFDcC3izKO+tjD4Tvc8WSb/xvJmSbpO0RNLLksZ7\n/m6SFrkuqyWd6fkNvoowU9IGSbMlner1N0j6gssNlnSPpKWSVkg6w9X4EHi3hrG85e1Mc31XStrk\nbQ721YwWn8cJkq4ARgFNkhZ73Ytdp6XA8Zm2t5Ae/Cux1eWQtJ+keW6bFklfLExpZm6vkbTMZaZ4\n3gxJkzIyUyR9r5x8ER8Db1ebJDMr6LgzMAj4S5Uq2+bfV6sKc7tC0m4uM0TSXLfzfRn9N0oa5umj\nlFZr9wXuA472di4DzgWuz9b1OgMk3STpGR/3pbWO1cy2mtnjnv4IWAl8qkjsDdJnIAiCIAiCIOhq\nzGyHvYAGoA34ol/fA3zP002kb/RHAq8Cw0gO32LgTC97LNPWHv6+FDjT04OAXerQaxowuUzZGOAF\nYKhf7+XvM4E5nv4s8JKnBwK7e3rvTH4D6SF9jF8/C9zj6TOBeZ7+CXCBp/cENgC7Ful0FHBXjWPb\nE1hNWr0YD/wyUzbE3/+YGd+IzPzvBDwJ3F6nvf8NuNLTyvT3nr+fWtDHyx8ETgCOAJoz7TwP7F9O\n3q/fL9H/SOChCvo9SnKs5nRyXPOB4zw92O/Tk4B3vE8BTwFfyszvsIzt/uDpk4D5mXZnAuMz98sa\nT18K/DBzjy8HGjoz1sK9C/xv4MCi/MXAFyrUM5iSeTUZWLx6/au/2wnLI01NTT2tQlAHYbd8EnbL\nJ73Jbk1NTTZlypRtL//fQ/GrYhxKF/EnM1vq6VnAFcDPM+VHA01m9jaApNnAicB04CBJtwG/Bx6T\ntDswyszmk0b0YWeVkSTgcNelFKcAc83sHe9jc6bsd563XtJ+hSaBGZJOJDmTozJlG81snaefBxZ5\nei1woKfHAWdI+oFfDwIOIDlaeH8rgMtqHOIs4BYza5G0BfiZpBnAw2b2ZEbnwqrSsbSf/znAwTX2\nVcwpwN+7zgYUx16NA06VtNL73w042MxmStpX0ghgP+BtM3td0uRS8iQnsANm9mfg6+WUM7OvSBoE\nPCBpopndW+O4lgC/8HtznusGsMz7RNIqkk2fIrNiVyfjSNv4Jvj1HqRxv5oZS8WxShoI3A/camav\nFBVvIn0Gni2vwtTOax0EQRAEQdCHaWxsbLdlcdq0aSXlusPBsirXUOKB1Mw2SzocOA34LjCBFDtU\n8eHVt5pd6v2cbmatmbIBpNWFD4CHOzGGAh+U0PlCYB/gSDNrUzp0YpcS8m2Z6za2z72As83spTr0\naYekqSSH9l4AM3tJ6SCR04HpkhaZ2fRSVT9p304p2xb3M8PM7i5RNpdk4xHAnBrkq/VVWkGzDyX9\nL+AYoCYHy8x+Kukh4GvAEknjvChr34/ZbtOP2L79tuRhIlUQcIWZLayjboG7gA1m9k8lyn4JLJB0\njJl95xP0EfQqGntagaAO8hhbEITd8krYLZ/k0W7dEYPVIOlYT18APFFUvgw4UdIw/9b9fOBxSXsD\nA83st8CPgLGW4mhek/QN2HYq267ZxszsDkuHGYzNOlde1mZmB5K+uS93yMEfgAmZGJqhZeQKTsme\nwJvuXJ1M2upVLFOJBcCV2ypIR9RQp6MyKXbry8BVmbyRwFYzux+4mbTtEuA90qoIwDOk+R/q8UkT\nKIGky7NxUmVYDExy+QGShhSq+/sC4NuFGCZJozw2CeAB4DzgbJKzVU5+n6I2q6IUJzfC0zuRHKVV\nfn2WMiftlan/aTN73sxuIm3XO7RKlxtJWwPx8XSWBcAk1xVJBxff51X0nU7aUnt1GZFrgEvCuQqC\nIAiCIOh6usPBegG4XNI6UkzInZ5vAO4EXUv6EZcWYLmZPUiKwWmW1EI6HOBarzcRuFLSatLWreF1\n6PQiKeaoA76l7yckJ68FuCWrb1bU32eTDi5YDXwLWF9CplT9AteTDgJZo3RM93XFAn5Qwl0VxgNw\nNenwiuV+iMJU4DBgmY/jx6RtlwB3A49KWuzzP40U2/YEsK5Dy4lDqX4wxGTgZElrSE7sGM8v2Hoh\nadva0y4zF9jdy9YBQ4BNZvZGBfkh2TazSBrpK03F7AbM9218K4DXgH/1stFUP0xkstKR56tJcXWP\nlJDJ6nMdcLukZaTVrHKUuyd+RbLDSr8n7qRotbncWCXtD/wQGJM5mOPbRWJDgZcr6BXkkuaeViCo\ngzz+vksQdssrYbd8kke7KYXK9C883mlvM7u2qnAAgKT5pAMZKjkMuUPSvcDVZlbNeewTeAziGuAc\nM9tQRsbq3IEZ9CjN9O9tgiKP/8+am5tzuf2lvxN2yydht3zSm+0mCTPrsKuqvzpYo4FfA1us/W9h\nBUGfRdIhpK2Ya4CLrMyHPzlYQZAvhg9voLX1lZ5WIwiCIOhHhIMVBEFNSCrnewVBEARBEAROOQer\nO2KwgiAIgh1MHveoB2G3vBJ2yydht3ySR7uFgxUEQRAEQRAEQdBFxBbBIAjaEVsEgyAIgiAIqhNb\nBIMgCIIgCIIgCHYw4WAFQRD0AfK4Rz0Iu+WVsFs+CbvlkzzaLRysIAiCIAiCIAiCLiJisIIgaEf8\nDlYQBL2B4fsPp3VTa0+rEQRBUJb4HawgCGpCkjG1p7UIgqDfMxXiGSUIgt5MHHIRBEHQl9nYsMjF\nvgAAIABJREFU0woEdRF2yyV5jAkJwm55JY9226EOlqQGSWvLlDVJGrsj+y+HpAMkrZT0SCavV/2b\nk3SSpJk1yHVab0lXSdqlTNlFkm739BRJE6u0dZGkKVVk3u+sjtUotOn3WFMN8o9IapH0nKRfSdqp\ninzF+ffyBzup8yBJC/3emyDpBNdnpaRDy31WMvVrHetYSWskvSjp1kz+Id7fnM7oHQRBEARBENRO\nd6xg9cb1/bOAx8zsq5m83qhnLTrVo/dkYHAd9erVYUfMrZVJl2OCmR1pZp8D9gK+2ck+6ikvZixg\nZjbWzOYCFwI3mNlYYGuN7dUi8y/AJWZ2CHCIpNNIHb/o4z9M0kGd1D3o7YRF80nYLZc0Njb2tApB\nHYTd8kke7dYdDtbOkmZJWifpgVIrJ5LO92/c10i60fMGSJrpeaslXeX5o30VYJWkZ+t8UNwLeLMo\n762MPhO9zxZJv/G8mZJuk7RE0suSxnv+bpIWuS6rJZ3p+Q2S1nu9DZJmSzrV62+Q9AWXGyzpHklL\nJa2QdIar8SHwbg1jecvbmeb6rpS0ydscLOkhz1/jqyZXAKOAJkmLve7FrtNS4PhM21tID/6V2Opy\nSNpP0jy3TYukLxamNDO310ha5jJTPG+GpEkZmSmSvldOvoiPgberTZKZFXTcGRgE/KVKlW3z76tV\nhbldIWk3lxkiaa7b+b6M/hslDfP0UUqrtfsC9wFHezuXAecC12frep0Bkm6S9IyP+9JaxyppBDDE\nzJZ71r2kLxSyvEH6DARBEARBEARdTMVtUl3EZ4CLzWyppHuAScDPC4WSRgI3AkcCm4GF7qRsAvY3\ns8+73B5eZTbpW//5kgZRn5M4EGjLZpjZsd7PGOCHwHFm9o6k7IPoCDM7XtJngfnAPOCvwFlmtkXS\n3sBSLwMYDZxtZuskPQuc5/XP9D7GA/8ILDazSyTtCSyTtMjMngaedp2OAr5jZpcVD6Sgt5lNAaZ4\nG/8B/DPwFeB1M/u6tzPEzN6XdDXQ6OMbAUwlzf97QDOw0tu8pdpEmtkDmcvbgWYzGy9JwO4FMe//\nVOBgMzvGy+dLOgGYA9wK3OHy5wLjysmb2ZO402Zmm4BzvP2RwN2F8RYj6VHgaGCRmT1aZVzb5h/4\nPjDJzJ6WNJhkc4AjgDFAK7BE0pfM7Ck6rjKZmb0l6R+A75tZwQk/DnjQzOZJasjIXwJsNrNj/R5f\nIukxM3u1hrHuT/rsFNjkeVnaSJ+B8mQ3Ih5IfMueBzYSdsojYbdc0tzcnMtv1fs7Ybd80pvs1tzc\nXFNMWHc4WH8ys6WengVcQcbBIj3wNpnZ2wCSZgMnAtOBgyTdBvweeEzS7sAoM5sPYGYfdlYZf1A/\n3HUpxSnAXDN7x/vYnCn7neetl7RfoUlghqQTSQ+uozJlG81snaefBxZ5ei3psRVgHHCGpB/49SDg\nAGBDoVMzWwF0cK7KMAu4xcxaJG0BfiZpBvCwOyYFnQurSsfSfv7nAAfX2FcxpwB/7zobUBx7NQ44\nVdJK7383kgM1U9K+7uztB7xtZq9LmlxKHniSEpjZn4GSzpWXf8UdlgckTTSze2sc1xLgF35vznPd\nAJZ5n0haRbLpU2RW7OpkHGkb3wS/3oM07lczY6k41ipsIn0Gni0rcXKdLQdBEARBEPRRGhsb2zl7\n06ZNKynXHQ5Wh2/zS8h0eCA1s82SDgdOA74LTCDFDlV8ePWtZpd6P6ebWWumbADwR+AD4OFOjKHA\nByV0vhDYBzjSzNqUDp3YpYR8W+a6je1zL9Iq10t16NMOSVNJDu29AGb2ktJBIqcD031lbHqpqp+0\nb6dafJCAGWZ2d4myuSQbjyCtaFWTryuuy8w+lPS/gGNI2+dqqfNTSQ8BXyOtJo3zoqx9P2a7TT9i\n+8pqycNEqiDgCjNbWEfd14G/zVx/yvOy/BJYIOkYM/tOHX0EvZFYBcknYbdc0lu+TQ86R9gtn+TR\nbt0Rg9Ug6VhPXwA8UVS+DDhR0jBJA4Hzgcd9u91AM/st8CNgrMfRvCbpG7DtVLZds42Z2R1+mMHY\nrHPlZW1mdiDpm/tyhxz8AZiQiaEZWkau4JTsCbzpztXJQEMJmUosAK7cVkE6ooY6HZVJsVtfBq7K\n5I0EtprZ/cDNpEMWIG0FLGy5fIY0/0M9PmkCJZB0eTZOqgyLSVtAC3FEQwrV/X0B8O1CDJOkUR6b\nBPAAcB5wNsnZKie/T1GbVVGKkxvh6Z1IjtIqvz5L0g1V6n/azJ43s5uA5cChVbrcCBzl6bNr1TPD\nAmCS64qkg4vv83L4Pf+upMK2yonAvxeJXUM6BCOcqyAIgiAIgi6mOxysF4DLJa0jBdbf6fkG2x4I\nryXF/rQAy83sQVLcSLOkFtLhANd6vYnAlZJWk7ZuDa9DpxeBYaUKfEvfT0hOXgtQiEMqtxI3m3Rw\nwWrgW8D6EjKl6he4nnQQyBqlY7qvKxZQOijhrgrjAbiadHjFcqVDFKYCh5FiulqAH5O2XQLcDTwq\nabHP/zRS7NgTwLoOLScOpfrBEJOBkyWtITmxYzy/YOuFwP3A0y4zF4/T8nkfAmwyszcqyA/JtplF\n0khfaSpmN1L81ipgBfAa8K9eNprqh4lMlrTWbfwh8EgJmaw+1wG3S1pGWs0qR7l74lckO6z0e+JO\nilabK4wV4HLgHtJ9/lKJeLOhwMsV9ArySK/6oYmgZsJuuSSPv8sThN3ySh7tpv74K+ke77S3mV1b\nVTgAQNJ8YLyZVXIYcoeke4Grzaya89gn8FWtNcA5ZrahjIwxtVvVCrqCOCwhn4TdyjMVeuszSm8K\nug9qJ+yWT3qz3SRhZh12VfVXB2s08GtgS9FvYQVBn0XSIaStmGuAi6zMh19S//ujEARBr2P4/sNp\n3dRaXTAIgqCHCAcrCIKakFTO9wqCIAiCIAiccg5Wd8RgBUEQBDuYPO5RD8JueSXslk/Cbvkkj3YL\nBysIgiAIgiAIgqCLiC2CQRC0I7YIBkEQBEEQVCe2CAZBEARBEARBEOxgwsEKgiDoA+Rxj3oQdssr\nYbd8EnbLJ3m0WzhYQRAEQRAEQRAEXUTEYAVB0I74HaygtzN8eAOtra/0tBpBEARBPyd+BysIgppI\nDlb8XQh6MyL+dwVBEAQ9TRxyEQRB0Kdp7mkFgjrIY2xBEHbLK2G3fJJHu+1QB0tSg6S1ZcqaJI3d\nkf2XQ9IBklZKeiSTt7EndCmHpJMkzaxBrtN6S7pK0i5lyi6SdLunp0iaWKWtiyRNqSLzfmd1rEah\nTb/HmmqQny7pT5Leq7H9ivPv5Q/WrjFIGiRpod97EySdIOk5vz603GclU7/qWCXtKukhSeslrZV0\nQ6bsEO9vTmf0DoIgCIIgCGqnO1aweuM+jrOAx8zsq5m83qhnLTrVo/dkYHAd9erVYUfMrZVJl2M+\ncPQn6KOe8mLGAmZmY81sLnAhcIOZjQW21theLTI3m9lngSOBEySdRur4RTP7HHCYpIM6qXvQ62ns\naQWCOmhsbOxpFYI6CLvlk7BbPsmj3brDwdpZ0ixJ6yQ9UGrlRNL5ktb460bPGyBppuetlnSV54/2\nVYBVkp6t80FxL+DNory3MvpM9D5bJP3G82ZKuk3SEkkvSxrv+btJWuS6rJZ0puc3+CrCTEkbJM2W\ndKrX3yDpCy43WNI9kpZKWiHpDFfjQ+DdGsbylrczzfVdKWmTtznYVzNafB4nSLoCGAU0SVrsdS92\nnZYCx2fa3kJ68K/EVpdD0n6S5rltWiR9sTClmbm9RtIyl5nieTMkTcrITJH0vXLyRXwMvF1tksxs\nmZm9UU0uw7b599WqwtyukLSbywyRNNftfF9G/42Shnn6KKXV2n2B+4CjvZ3LgHOB67N1vc4ASTdJ\nesbHfWmtYzWzrWb2uKc/AlYCnyoSe4P0GQiCIAiCIAi6GjPbYS+gAWgDvujX9wDf83QT6Rv9kcCr\nwDCSw7cYONPLHsu0tYe/LwXO9PQgYJc69JoGTC5TNgZ4ARjq13v5+0xgjqc/C7zk6YHA7p7eO5Pf\nQHpIH+PXzwL3ePpMYJ6nfwJc4Ok9gQ3ArkU6HQXcVePY9gRWk1YvxgO/zJQN8fc/ZsY3IjP/OwFP\nArfXae9/A670tDL9vefvpxb08fIHgROAI4DmTDvPA/uXk/fr90v0PxJ4qIqO79UxrvnAcZ4e7Pfp\nScA73qeAp4AvZeZ3WMZ2f/D0ScD8TLszgfGZ+2WNpy8Ffpi5x5cDDXWMdS/gfwMHFuUvBr5QoZ7B\nlMyrycDi1etf/clOWF+hqampp1UI6iDslk/CbvmkN9mtqanJpkyZsu3l/48ofu3EjudPZrbU07OA\nK4CfZ8qPBprM7G0ASbOBE4HpwEGSbgN+DzwmaXdglJnNJ43ow84qI0nA4a5LKU4B5prZO97H5kzZ\n7zxvvaT9Ck0CMySdSHImR2XKNprZOk8/Dyzy9FrgQE+PA86Q9AO/HgQcQHK08P5WAJfVOMRZwC1m\n1iJpC/AzSTOAh83syYzOhVWlY2k//3OAg2vsq5hTgL93nQ0ojr0aB5wqaaX3vxtwsJnNlLSvpBHA\nfsDbZva6pMml5ElOYAfM7M/A1+vUvRJLgF/4vTnPdQNY5n0iaRXJpk+RWbGrk3GkbXwT/HoP0rhf\nLQhUG6ukgcD9wK1m9kpR8SbSZ+DZ8ipM7bzWQRAEQRAEfZjGxsZ2WxanTZtWUq47HCyrcg0lHkjN\nbLOkw4HTgO8CE0ixQxUfXn2r2aXez+lm1popG0BaXfgAeLgTYyjwQQmdLwT2AY40szalQyd2KSHf\nlrluY/vcCzjbzF6qQ592SJpKcmjvBTCzl5QOEjkdmC5pkZlNL1X1k/btlLJtcT8zzOzuEmVzSTYe\nAcypQb5aX12Gmf1U0kPA14AlksZ5Uda+H7Pdph+xffttycNEqiDgCjNbWI++zl3ABjP7pxJlvwQW\nSDrGzL7zCfoIehWNPa1AUAd5jC0Iwm55JeyWT/Jot+6IwWqQdKynLwCeKCpfBpwoaZh/634+8Lik\nvYGBZvZb4EfAWDPbArwm6Ruw7VS2XbONmdkdZnakpYMEWovK2szsQNI3998so+8fgAmZGJqhZeQK\nTsmewJvuXJ1M2upVLFOJBcCV2ypIR9RQp6MyKXbry8BVmbyRwFYzux+4mbTtEuA90qoIwDOk+R8q\naWeSk1Oq/cuzcVJlWAxMcvkBkoYUqvv7AuDbhRgmSaM8NgngAeA84GySs1VOfp+iNjtLu3qSzlLm\npL2SFaRPm9nzZnYTabveoVX62EjaGghpPJ1lATBJ0k7e/8HF93kVfaeTttReXUbkGuCScK6CIAiC\nIAi6nu5wsF4ALpe0jhQTcqfnG4A7QdeSfsSlBVhuZg+SYnCaJbWQDge41utNBK6UtJq0dWt4HTq9\nSIo56oBv6fsJyclrAW7J6psV9ffZpIMLVgPfAtaXkClVv8D1pINA1igd031dsYAflHBXhfEAXE06\nvGK5H6IwFTgMWObj+DFp2yXA3cCjkhb7/E8jxbY9Aazr0HLiUOAvVXSYDJwsaQ3JiR3j+QVbLyRt\nW3vaZeYCu3vZOmAIsMn8MIoy8kOybWaRNNJXmjog6aeSXgN2VTqu/cdeNJrqh4lMVjryfDUpru6R\nEjJZfa4Dbpe0jLSaVY5y98SvSHZY6ffEnRStNpcbq6T9gR8CYzIHc3y7SGwo8HIFvYJc0tzTCgR1\nkMffdwnCbnkl7JZP8mg3pVCZ/oXHO+1tZtdWFQ4AkDSfdCBDJYchd0i6F7jazKo5j30Cj0FcA5xj\nZhvKyFg37sAMuoxm+s82QdFX/nc1NzfncvtLfyfslk/CbvmkN9tNEmbWYVdVf3WwRgO/BrZY+9/C\nCoI+i6RDSFsx1wAXWZkPfzhYQe+n7zhYQRAEQX4JBysIgppIDlYQ9F6GD2+gtfWVnlYjCIIg6OeU\nc7C6IwYrCIKcUeo3HeLVu19NTU09rkN3vfqSc5XH2IIg7JZXwm75JI92CwcrCIIgCIIgCIKgi4gt\ngkEQtEOSxd+FIAiCIAiCysQWwSAIgiAIgiAIgh1MOFhBEAR9gDzuUQ/Cbnkl7JZPwm75JI92Cwcr\nCIIgCIIgCIKgi4gYrCAI2hExWEEQBEEQBNUpF4O1U08oEwRB70bq8LciCIKgJMP3H07rptaeViMI\ngqDXECtYQRC0Q5Ixtae1CDrNRuCgnlYi6DR9wW5T02/n9Seam5tpbGzsaTWCThJ2yye92W5ximAQ\nBEEQBEEQBMEOZoc6WJIaJK0tU9YkaeyO7L8ckg6QtFLSI5m8jT2hSzkknSRpZg1yndZb0lWSdilT\ndpGk2z09RdLEKm1dJGlKFZn3O6tjNQpt+j3WVIP8dEl/kvReje1XnH8vf7B2jUHSIEkL/d6bIOkE\nSc/59aHlPiuZ+rWOdaykNZJelHRrJv8Q729OZ/QOckLeV0H6K2G3XNJbv00PKhN2yyd5tFt3rGD1\nxn0DZwGPmdlXM3m9Uc9adKpH78nA4Drq1avDjphbK5Mux3zg6E/QRz3lxYwFzMzGmtlc4ELgBjMb\nC2ytsb1aZP4FuMTMDgEOkXQaqeMXzexzwGGS4rEuCIIgCIJgB9AdDtbOkmZJWifpgVIrJ5LO92/c\n10i60fMGSJrpeaslXeX5o30VYJWkZ+t8UNwLeLMo762MPhO9zxZJv/G8mZJuk7RE0suSxnv+bpIW\nuS6rJZ3p+Q2S1nu9DZJmSzrV62+Q9AWXGyzpHklLJa2QdIar8SHwbg1jecvbmeb6rpS0ydscLOkh\nz1/jqyZXAKOAJkmLve7FrtNS4PhM21tID/6V2OpySNpP0jy3TYukLxamNDO310ha5jJTPG+GpEkZ\nmSmSvldOvoiPgberTZKZLTOzN6rJZdg2/75aVZjbFZJ2c5khkua6ne/L6L9R0jBPH6W0WrsvcB9w\ntLdzGXAucH22rtcZIOkmSc/4uC+tdaySRgBDzGy5Z91L+kIhyxukz0DQl+hVa/BBzYTdckkef5cn\nCLvllTzarTtOEfwMcLGZLZV0DzAJ+HmhUNJI4EbgSGAzsNCdlE3A/mb2eZfbw6vMJn3rP1/SIOpz\nEgcCbdkMMzvW+xkD/BA4zszekZR9EB1hZsdL+ixpRWQe8FfgLDPbImlvYKmXAYwGzjazdZKeBc7z\n+md6H+OBfwQWm9klkvYElklaZGZPA0+7TkcB3zGzy4oHUtDbzKYAU7yN/wD+GfgK8LqZfd3bGWJm\n70u6Gmj08Y0AppLm/z2gGVjpbd5SbSLN7IHM5e1As5mNlyRg94KY938qcLCZHePl8yWdAMwBbgXu\ncPlzgXHl5M3sSdxpM7NNwDne/kjg7sJ4PwnZ+Qe+D0wys6clDSbZHOAIYAzQCiyR9CUze4qOq0xm\nZm9J+gfg+2ZWcMKPAx40s3mSGjLylwCbzexYv8eXSHrMzF6tYaz7kz47BTZ5XpY20megPNmNiAcS\n25iCIAiCIOj3NDc31+TwdYeD9SczW+rpWcAVZBws0ratJjN7G0DSbOBEYDpwkKTbgN8Dj0naHRhl\nZvMBzOzDzirjD+qHuy6lOAWYa2bveB+bM2W/87z1kvYrNAnMkHQi6cF1VKZso5mt8/TzwCJPryU9\ntgKMA86Q9AO/HgQcAGwodGpmK4AOzlUZZgG3mFmLpC3AzyTNAB52x6Sgc2FV6Vjaz/8c4OAa+yrm\nFODvXWcDimOvxgGnSlrp/e9GcqBmStrXnb39gLfN7HVJk0vJA09SAjP7M/CJnasSLAF+4ffmPNcN\nYJn3iaRVJJs+RWbFrk7GkbbxTfDrPUjjfrUg8AnHuon0GXi2rMTJdbYc9BzhBOeTsFsuyWNMSBB2\nyyu9yW6NjY3t9Jk2bVpJue5wsDp8m19CpsMDqZltlnQ4cBrwXWACKXao4sOrbzW71Ps53cxaM2UD\ngD8CHwAPd2IMBT4oofOFwD7AkWbWpnToxC4l5Nsy121sn3uRVrleqkOfdkiaSnJo7wUws5eUDhI5\nHZjuK2PTS1X9pH071eKDBMwws7tLlM0l2XgEaUWrmny3xcyZ2U8lPQR8jbSaNM6Lsvb9mO02/Yjt\nK6slDxOpgoArzGxhHXVfB/42c/0pz8vyS2CBpGPM7Dt19BEEQRAEQRCUoTtisBokHevpC4AnisqX\nASdKGiZpIHA+8LhvtxtoZr8FfgSMNbMtwGuSvgHbTmXbNduYmd1hZkf6QQKtRWVtZnYg6Zv7b5bR\n9w/AhEwMzdAycgWnZE/gTXeuTgYaSshUYgFw5bYK0hE11OmoTIrd+jJwVSZvJLDVzO4HbiYdsgBp\nK2Bhy+UzpPkfKmlnkpNTqv3Ls3FSZVhM2gJaiCMaUqju7wuAbxdimCSN8tgkgAeA84CzSc5WOfl9\nitrsLO3qSTpL0g0VK0ifNrPnzewmYDlwaJU+NgJHefrsOnRcAEyStJP3f3DxfV4Ov+fflVTYVjkR\n+PcisWtIh2CEc9WXiFiefBJ2yyV5jAkJwm55JY926w4H6wXgcknrSIH1d3q+wbYHwmtJsT8twHIz\ne5AUN9IsqYV0OMC1Xm8icKWk1aStW8Pr0OlFYFipAt/S9xOSk9cCFOKQyq3EzSYdXLAa+BawvoRM\nqfoFricdBLJG6Zju64oF/KCEuyqMB+Bq0uEVy/0QhanAYaSYrhbgx6RtlwB3A49KWuzzP40UO/YE\nsK5Dy4lDgb9U0WEycLKkNSQndoznF2y9ELgfeNpl5uJxWj7vQ4BNhcMoysgPybaZRdJIX2nqgKSf\nSnoN2FXpuPYfe9Foqh8mMlnSWrfxh8AjJWSy+lwH3C5pGWk1qxzl7olfkeyw0u+JOylaba40VuBy\n4B7Sff6SmT1aVD4UeLmCXkEQBEEQBEGdqL/9+jqAxzvtbWbXVhUOAJA0HxhvZpUchtwh6V7gajOr\n5jz2CXxVaw1wjpltKCNjTO1WtYIgyDNToT8+SwRBEEjCzDrsquqvDtZo4NfAlqLfwgqCPoukQ0hb\nMdcAF1mZD7+k/vdHIQiCuhm+/3BaN7VWFwyCIOhjhIMVBEFNSCrnewW9mObm5l510lJQG2G3fBJ2\nyydht3zSm+1WzsHqjhisIAiCIAiCIAiCfkGsYAVB0I5YwQqCIAiCIKhOrGAFQRAEQRAEQRDsYMLB\nCoIg6APk8XdCgrBbXgm75ZOwWz7Jo93CwQqCIAiCIAiCIOgiIgYrCIJ2RAxWEARBEARBdcrFYO3U\nE8oEQdC7Sb9HHATBJ2X48AZaW1/paTWCIAiCbiS2CAZBUAKLV+5eTb1Ah3gVv95441UqkcfYgiDs\nllfCbvkkj3YLBysIgiAIgiAIgqCL6BIHS1KDpLVlypokje2KfjqLpAMkrZT0SCZvY0/oUg5JJ0ma\nWYPcRn8vO9dF8kMkvSbp9mwbkoZ1Qreqc+X2PaBC+UWS/qnWPmvU66LCuCRNkTSxivzRklr8tVrS\nN2voY6akE6uUj++k3idIes7vyb+RdLOktZJ+6uP4XpX6tYz1y5Ke9XEul3Rypuz7kl6oZfxBHmns\naQWCOmhsbOxpFYI6CLvlk7BbPsmj3boyBsu6sK2u4izgMTO7NpPXG/WsRScrky7H9cDjdfTzSeR3\ndDv1shY4yszaJI0AnpP0P83s427W40LgBjO7H0DSpcBQMzNJU7qoj7eAr5tZq6T/BiwAPgVgZrdI\nehK4GZjTRf0FQRAEQRAEGbpyi+DOkmZJWifpAUm7FAtIOl/SGn/d6HkDfDVgjX/rfpXnj5a0UNIq\n/0b+oDp02gt4syjvrYw+E73PFkm/8byZkm6TtETSy4VVCkm7SVqUWR040/MbJK33ehskzZZ0qtff\nIOkLLjdY0j2SlkpaIekMV+ND4N0axvJWcYakuzMrM29K+n89/yhgP+Cx4irAld7/akmHZMb2r26D\nVZL+e7k+S/AX4GNv5yve9ipJC0vou4+k/ynpGX8dp8RGSXtk5F6UtG8p+RL9bwG2VlLQzP5qZm1+\nuSvwbg3O1WaSbZB0o688rZJ0U0bmpBL3yUmSHsyM5Z/8PrsEOBe4XtJ9kv4d2B1YIWlC0Tx9WtIj\nvgL1eMFOwPs1jHW1mbV6+nlgF0k7Z0RagT2rjD3IJc09rUBQB3mMLQjCbnkl7JZP8mi3rlzB+gxw\nsZktlfT/s3fvUXZUZf7/3x8yiQiBhBAIkDEX4ncwcUhMAiIjQiOgw8yAiIAgN10sQC6CXBwZvBAE\nEUX4fqP+wOEyMUIECQNMEDMkQDq6IBFyJyQEgTgQmXARgmQGQejP74/aB6tP17l006G7Os9rrV59\nTtVTVc+uXZ2cffbeVTcApwNXVVZK2hm4HJhI9uF1bmqkrAOG2x6f4ioftGeQfds/S9IAutYY7Ae0\n5RfY3isdZxxwIbC37ZclDc6F7WT7o5LGArOA24E/AYfZ3ihpe2BhWgcwBviM7VWSFgFHp+0PTcc4\nHPgacJ/tkyQNAh6SdK/tBcCClNNk4FTbp1QXpJJ31bKT03YjgNnANEkCvk/WW3JQwTl53vZkSacB\n5wOnAN8ANuTqYFCtYxbkcETaZihwLbCP7aerzmfFVOAq2w9Keh9wj+1xku4EPg1Ml/Rh4He2X5A0\nozoeGFd1/CsrryWdmi3ytdUHTvv9N2A08LkmynVO2m4IWb1/IL3fNhdWdJ1AQY+d7Rsk7QPcZfv2\ntK8/2p6UXud7sK4luw6eTHlfAxxgO//3VLOsuZgjgCW2/5xb3EZTf/dTcq9biOFnIYQQQtjctba2\nNtXg684G1tO2F6bXNwFfItfAAvYE5tl+CSB9eN4XuBQYLWkq8EtgjqSBwC62ZwHYfqOzyaSGxoSU\nS5GPAzNtv5yOsSG37s60bLWkHSu7BL6jbF5OG7BLbt1a26vS60eBe9PrR4BR6fUngEMkfSW9HwCM\nANZUDmp7MVmDpzPl3BKYCZxpe52kM4C7bT+bnQKq77d9R/q9mKxRA3Ag8Pa8HNvN9KhEg1wQAAAg\nAElEQVRV+wgw3/bTaR8bCmIOBMamugEYKGkr4Fbgm8B04Gj+MnytVnwh2/9aZ91DwN9K2g24R9I8\n239solyvAK9Juh64G/hFbl3RdfKOSNoa+DtgZq7c/avj6pU17eeDwHfo2Mh+EdhB0uAadZRMaT7p\n0Eu09HQCoQvKOLcgRL2VVdRbOfWmemtpaWmXz8UXX1wYtynnYBXNu+nwcB3bGyRNAD4JfBE4Evhy\nUWy7HUmnAyen4/xDZVhUWrcF8BTwOtmH4s56vSDnY4GhwMQ0l2ctsGVBfFvufb63QGS9XL/tQj71\nXAPcZnteer83sE86P9uQDd181faFVbm+Rfc/B63Rw5ME7FXVowKwQNmQ0KFk8+a+VS9e7+AZTbbX\nSHoS+D9kjcxG8W+lXqQDyK7NM9NrKL5O3qR9b2uHobINbAG8XOnZ6gpJf03Wm3a87d/l19l+TdIt\nwFOSPmu7w1DOEEIIIYTQdd05B2ukpMqQss8Bv65a/xCwr6QhkvoBxwDz03C7frbvAL4OTLK9EXhG\n0qcAJA2Q9N78zmxfbXui7Un5xlVa12Z7FLCIXM9MlfuBI9MQMCRtVyOu8sF5ENnwujZld2YbWRBT\nzz3AWW9vIH2oiW3qSr1VA21fUVlm+zjbo2zvSjYE8Ke5xlUtc4EzcvvtMLxP2fyznevsYyHwMUkj\nU3zR+ZwDnJ3b54TcujvIejxX5XpW6sU3TdKodM2R8ns/8Nv0frrSPLka224NDLb9n8C5wPhaoen3\nfwHjJPVP5/GAGvH5bd5m+1VgbRreV8mh1jGL8h1E1sv21VyPcn79YLK/ieHRuOprWns6gdAFZZxb\nEKLeyirqrZzKWG/d2cB6DDhD0iqym0v8OC03QGoEXUD2KWAp8LDtu4DhQKukpcCNKQbgBLIbMiwH\nHgCGdSGnx4HC25KnIX3fJmvkLQUqc3lq9cTNAPZM+RwHrC6IKdq+4hKy3qQVym6z/q3qAEmTJdWc\nU1PgPGB3ZTe5WCKp0fDCWrldCgxRdsvwpVSNNUpD1cYAL9Xcsf0i2fDGO9I+bikIOxvYQ9kNNlYC\np+bW3UrWS3hLk/EdSDq1xjnYB1guaUk6zim54YHjgWfr7HYb4Bep3n8FnJOWF14nttelY6xMZVlS\nHVPnfcVxwEnKbqqxEji0OqBOWc8kq6tv5q6Lobn1g4DnbNe9WUYIIYQQQuga2T19B+1NJ8132r7q\nNu2hk9J8ni/YPr+nc+lOkrYBrre92TwXKg13nGq76I6MlRj3/J31Q+grRF/+fzaEEDZnkrDdYURS\nX29gjQF+Amy0fXAPpxNCj5J0Hlkv4RW2b64T13f/UQjhXTZs2EjWr/9dT6cRQghhE9gsG1ghhM6T\n5Ph3oXxaW1t71Z2WQnOi3sop6q2cot7KqTfXW60GVnfOwQohhBBCCCGEzVr0YIUQ2okerBBCCCGE\nxqIHK4QQQgghhBA2sWhghRBCH1DG54SEqLeyinorp6i3cipjvUUDK4QQQgghhBC6SczBCiG0E3Ow\nQgghhBAaqzUH6696IpkQQu8mdfi3IoQQQgibyLDhw1i/bn1PpxG6SfRghRDakWSm9HQWodPWAqN7\nOonQaVFv5RT1Vk69ud6mQHwmLxbPwQohhBBCCCGEzVi3NLAkjZT0SI118yRN6o7jdJakEZKWSJqd\nW7a2J3KpRdJ+kqY1Ebc2/a55rqvit5H0jKQf5PchaUgncmt4rlL9jqiz/kRJP2z2mE3mdWKlXJIu\nknRCg/g9JS1NP8slfbaJY0yTtG+D9Yd3Mu99JK1M1+R7JF0h6RFJ303lOLfB9g3LmuL+RdJvJa2W\n9Inc8vMkPdZM+UMJ9dZvZUN9UW/lFPVWTlFvpdRbe6/q6c4erN7Yr3kYMMf2wbllvTHPZnJyjde1\nXALM78Jx3kn8pt5PVz0CTLY9Efgk8P9J6tcDeRwLXGZ7ku3XgZOB8ba/2l0HkDQWOAoYCxwMXK00\nocr2lcCJwBnddbwQQgghhNBedzaw+ku6SdIqSbdK2rI6QNIxklakn8vTsi1Sb8CK1Ltwdlo+RtJc\nScskLZLUle8dBgPPVy17IZfPCemYSyVNT8umSZoq6QFJT1R6KSRtLenelMtySYem5SNTT8E0SWsk\nzZB0UNp+jaQ9UtxWkm6QtFDSYkmHpDTeAF5poiwvVC+QdF2uZ+Z5Sd9IyycDOwJzqjcBzkrHXy7p\nb3Jl+7dUB8skfbrWMQv8AXgr7efv076XSZpbkO9QSbdJ+k362VuZtZK2zcU9LmmHoviC428EXquX\noO0/2W5Lb98LvGL7rQbl2kBWN0i6PPU8LZP0vVzMfgXXyX6S7sqV5YfpOjuJrOFziaQbJf0HMBBY\nLOnIqvO0q6TZkh6WNL9ST8CrjcoKfAq4xfabtn8H/Bb4cG79emBQg32EMupVffOhaVFv5RT1Vk5R\nb6VUxudgdeddBHcDvmB7oaQbgNOBqyorJe0MXA5MJPvwOjc1UtYBw22PT3GVD9ozyL7tnyVpAF1r\nDPYD2vILbO+VjjMOuBDY2/bLkgbnwnay/dHUGzALuB34E3CY7Y2StgcWpnUAY4DP2F4laRFwdNr+\n0HSMw4GvAffZPknSIOAhSffaXgAsSDlNBk61fUp1QSp5Vy07OW03ApgNTEu9Fd8n6y05qOCcPG97\nsqTTgPOBU4BvABtydTCo1jELcjgibTMUuBbYx/bTVeezYipwle0HJb0PuMf2OEl3Ap8Gpkv6MPA7\n2y9ImlEdD4yrOv6VldeSTs0W+drqA6f9/hvZAIHPNVGuc9J2Q8jq/QPp/ba5sKLrBAp67GzfIGkf\n4C7bt6d9/dH2pPT6olz4tWTXwZMp72uAA2zn/55qlXU46XpKfp+WVbTRzN/9vNzrUcSwihBCCCFs\n9lpbW5tq8HVnA+tp2wvT65uAL5FrYAF7AvNsvwSQPjzvC1wKjJY0FfglMEfSQGAX27MAbL/R2WRS\nQ2NCyqXIx4GZtl9Ox9iQW3dnWrZa0o6VXQLfUTYvpw3YJbdure1V6fWjwL3p9SNkH08BPgEcIukr\n6f0AYASwpnJQ24vJGjydKeeWwEzgTNvrJJ0B3G372ewUUH1nkzvS78VkjRqAA4G35+XYbqZHrdpH\ngPm2n0772FAQcyAwNtUNwEBJWwG3At8EpgNHAz9vEF/I9r/WWfcQ8LeSdgPukTTP9h+bKNcrwGuS\nrgfuBn6RW1d0nbwjkrYG/g6YmSt3/+q4emVt4EVgB0mDa9RRZv8u7j30nGgEl1PUWzlFvZVT1Fsp\n9aY5WC0tLe3yufjiiwvjurOBVf2tfdG8mw63MbS9QdIEsrkxXwSOBL5cFNtuR9LpZHNYDPyD7fW5\ndVsATwGvk30o7qzXC3I+FhgKTLTdpuwGEFsWxLfl3ud7C0TWy/XbLuRTzzXAbbYrfQ57A/uk87MN\n2dDNV21fWJXrW3T/c9AaPTxJwF62/1y1fIGyIaFDyebNfatevN7BM5psr5H0JPB/yBqZjeLfSr1I\nB5Bdm2em11B8nbxJ+97WDkNlG9gCeLnSs9UFvwfel3v/12kZALZfk3QL8JSkz9ruMJQzhBBCCCF0\nXXfOwRopqTKk7HPAr6vWPwTsK2mIshsMHAPMT8Pt+tm+A/g6MMn2RuAZSZ8CkDRA0nvzO7N9te2J\n6YYB66vWtdkeBSwi1zNT5X7gyDQEDEnb1YirfHAeRDa8rk3S/sDIgph67gHOensD6UNNbFNX6q0a\naPuKyjLbx9keZXtXsiGAP801rmqZS+7GB0XD+5TNP9u5zj4WAh+TNDLFF53POcDZuX1OyK27g6zH\nc1WuZ6VefNMkjUrXHCm/95PNTULSdKV5cjW23RoYbPs/gXOB8bVC0+//AsZJ6p/O4wE14vPbvM32\nq8BaSUfkcqh1zCKzgKPT38xosrI+lNvXYLK/ieHRuOpjYm5BOUW9lVPUWzlFvZVSGedgdWcD6zHg\nDEmryG4u8eO03ACpEXQB0AosBR62fRfZ/JBWSUuBG1MMwAlkN2RYDjwADOtCTo8DhbclT0P6vk3W\nyFsKVOby1OqJmwHsmfI5DlhdEFO0fcUlZL1JK5TdZv1b1QGSJkvqMH+ojvOA3ZXd5GKJpEbDC2vl\ndikwRNktw5cCLVV5iWye2Us1d2y/SDa88Y60j1sKws4G9lB2g42VwKm5dbeS9RLe0mR8B5JOrXEO\n9gGWS1qSjnNKbnjgeODZOrvdBvhFqvdfAeek5YXXie116RgrU1mWVMfUeV9xHHCSsptqrAQOrQ6o\nVdZ0Xd8KrCIbcnu62z+5cBDwnO1GN8sIIYQQQghdoL781Og032l72xc0DA41Sfog2Q1Mzu/pXLqT\npG2A621vNs+FSsMdp9ouuiNjJcZMefdyCiGEEDZ7U6AvfybvqyRhu8OIpL7ewBoD/ATYWPUsrBA2\nO5LOI+slvML2zXXi+u4/CiGEEEIvNGz4MNavW984MPQqm2UDK4TQeZIc/y6UT2tra6+601JoTtRb\nOUW9lVPUWzn15nqr1cDqzjlYIYQQQgghhLBZix6sEEI70YMVQgghhNBY9GCFEEIIIYQQwiYWDawQ\nQugDyvickBD1VlZRb+UU9VZOZay3aGCFEEIIIYQQQjeJOVghhHZiDlYIIYQQQmO15mD9VU8kE0Lo\n3aQO/1aEsFkaNmwk69f/rqfTCCGEUCIxRDCEUMDxU7qfeb0gh77389xz/8WmVMa5BSHqrayi3sqp\njPUWDawQQgghhBBC6Cbd0sCSNFLSIzXWzZM0qTuO01mSRkhaIml2btnansilFkn7SZrWRNza9Lvm\nua6K30bSM5J+kN+HpCGdyK3huUr1O6LO+hMl/bDZYzaZ14mVckm6SNIJDeKHSLpf0qv589Fgm2mS\n9m2w/vBO5r2PpJXpmnyPpCskPSLpu6kc5zbYvpmyHihpkaTlkh6WtH9u3XmSHpP02c7kHcqipacT\nCF3Q0tLS0ymELoh6K6eot3IqY711Zw+Wu3Ff3eUwYI7tg3PLemOezeTkGq9ruQSY34XjvJP4Tb2f\nrvoT8HXgvB7O41jgMtuTbL8OnAyMt/3VbjzGC8A/2Z4AfB64sbLC9pXAicAZ3Xi8EEIIIYSQ050N\nrP6SbpK0StKtkrasDpB0jKQV6efytGyL1BuwIn3rfnZaPkbSXEnL0jfyo7uQ02Dg+aplL+TyOSEd\nc6mk6WnZNElTJT0g6YlKL4WkrSXdm+sdODQtHylpddpujaQZkg5K26+RtEeK20rSDZIWSlos6ZCU\nxhvAK02U5YXqBZKuS7kvlfS8pG+k5ZOBHYE51ZsAZ6XjL5f0N7my/Vuqg2WSPl3rmAX+ALyV9vP3\nad/LJM0tyHeopNsk/Sb97K3MWknb5uIel7RDUXzB8TcCr9VL0Pb/2n4QeL2J8lRsIKsbJF2eep6W\nSfpeLma/gutkP0l35cryw3SdnQQcBVwi6UZJ/wEMBBZLOrLqPO0qaXbqgZpfqSfg1SbKutz2+vT6\nUWBLSf1zIeuBQZ04D6E0Wns6gdAFZZxbEKLeyirqrZzKWG/deRfB3YAv2F4o6QbgdOCqykpJOwOX\nAxPJPrzOTY2UdcBw2+NTXOWD9gyyb/tnSRpA1xqD/YC2/ALbe6XjjAMuBPa2/bKkwbmwnWx/VNJY\nYBZwO1kvyGG2N0raHliY1gGMAT5je5WkRcDRaftD0zEOB74G3Gf7JEmDgIck3Wt7AbAg5TQZONX2\nKdUFqeRdtezktN0IYDYwTZKA75P1lhxUcE6etz1Z0mnA+cApwDeADbk6GFTrmAU5HJG2GQpcC+xj\n++mq81kxFbjK9oOS3gfcY3ucpDuBTwPTJX0Y+J3tFyTNqI4HxlUd/8rKa0mnZot8baO8myjXOWmf\nQ8jq/QPp/ba5sKLrBAp67GzfIGkf4C7bt6d9/dH2pPT6olz4tWTXwZPpfFwDHGA7//fUsKySjgCW\n2P5zbnEbTf3dT8m9biGGn4UQQghhc9fa2tpUg687G1hP216YXt8EfIlcAwvYE5hn+yWA9OF5X+BS\nYLSkqcAvgTmSBgK72J4FYPuNziaTGhoTUi5FPg7MtP1yOsaG3Lo707LVknas7BL4jrJ5OW3ALrl1\na22vSq8fBe5Nrx8BRqXXnwAOkfSV9H4AMAJYUzmo7cVkDZ7OlHNLYCZwpu11ks4A7rb9bHYKqL7f\n9h3p92KyRg3AgcDb83JsN9OjVu0jwHzbT6d9bCiIORAYm+oGYKCkrYBbgW8C04GjgZ83iC9k+1+7\nkHcjrwCvSboeuBv4RW5d0XXyjkjaGvg7YGau3P2r4xqVVdIHge/QsZH9IrCDpME16iiZ0nzSoZdo\n6ekEQheUcW5BiHorq6i3cupN9dbS0tIun4svvrgwrjsbWNXf2hfNu+nwcB3bGyRNAD4JfBE4Evhy\nUWy7HUmnk81hMfAPlWFRad0WwFNkQ8Lu7kQZKvJDySp5HAsMBSbablN2A4gtC+Lbcu/zvQUi6+X6\nbRfyqeca4Dbb89L7vYF90vnZhmzo5qu2L6zK9S26/zlojR6eJGCvqh4VgAXKhoQOJZs396168XoX\nn9Fk+63Ui3QA2bV5ZnoNxdfJm7Tvbe0wVLaBLYCXKz1bXSHpr8l60463/bv8OtuvSboFeErSZ213\nGMoZQgghhBC6rjvnYI2UVBlS9jng11XrHwL2VXZHt37AMcD8NNyun+07yG5EMMn2RuAZSZ8CkDRA\n0nvzO7N9te2J6YYB66vWtdkeBSwi1zNT5X7gyDQEDEnb1YirfHAeRDa8rk3ZndlGFsTUcw9w1tsb\nSB9qYpu6Um/VQNtXVJbZPs72KNu7kg0B/GmucVXLXHI3Piga3qds/tnOdfaxEPiYpJEpvuh8zgHO\nzu1zQm7dHWQ9nqtyPSv14ruqXV1Jmq40T64wOOtRGmz7P4FzgfEN9vtfwDhJ/dN5PKBGfIdcAGy/\nCqxNw/sqOdQ6ZlG+g8h62b6a61HOrx9M9jcxPBpXfU1rTycQuqCMcwtC1FtZRb2VUxnrrTsbWI8B\nZ0haRXZziR+n5QZIjaALyD4FLAUetn0XMBxolbSU7I5nF6TtTiC7IcNy4AFgWBdyehwovC15GtL3\nbbJG3lKgMpenVk/cDGDPlM9xwOqCmKLtKy4h601aoew269+qDpA0WVJn5g+dB+yu7CYXSyQ1Gl5Y\nK7dLgSHKbhm+lKqxRmmo2hjgpZo7tl8kG954R9rHLQVhZwN7KLvBxkrg1Ny6W8l6CW9pMr4DSafW\nOgepx/FK4ERJT0v6QFo1Hni2zm63AX6R6v1XwDlpeeF1YntdKsvKVJYl1TF13lccB5yk7KYaK4FD\nC8pTq6xnktXVN3PXxdDc+kHAc7br3iwjhBBCCCF0jeyevoP2ppPmO21v+4KGwaGmNJ/nC7bP7+lc\nupOkbYDrbW82z4VKwx2n2i66I2Mlxj1/Z/0QegvRl/+fDCGE0HWSsN1hRFJfb2CNAX4CbKx6FlYI\nmx1J55H1El5h++Y6cdHACuFt0cAKIYRQrFYDqzuHCPY6tp+0/bFoXIWQ3dI+zVms2bj6C8VP/MQP\nYtiw/HTb7lfGuQUh6q2sot7KqYz11t13kQsh9AHxjX35tLa29qpb2YYQQgibqz49RDCE0HmSHP8u\nhBBCCCHUt1kOEQwhhBBCCCGEd1M0sEIIoQ8o4xj1EPVWVlFv5RT1Vk5lrLdoYIUQQgghhBBCN4k5\nWCGEdmIOVgghhBBCYzEHK4QQQgghhBA2sbhNewihA6nDlzEhhBBCqQwbPoz169a//T4eZ1FOZay3\naGCFEDqa0tMJhE5bC4zu6SRCp0W9lVPUWyk8N+W5nk4hbKZiDlYIoR1JjgZWCCGE0psC8Tk3bEqb\ndA6WpJGSHqmxbp6kSd1xnM6SNELSEkmzc8vW9kQutUjaT9K0JuLWpt81z3VV/DaSnpH0g/w+JA3p\nRG4Nz1Wq3xF11p8o6YfNHrPJvE6slEvSRZJOaBA/RNL9kl7Nn48G20yTtG+D9Yd3Mu99JK1M1+R7\nJF0h6RFJ303lOLfB9g3LmuL+RdJvJa2W9Inc8vMkPSbps53JO4QQQgghNK87b3LRG78iOAyYY/vg\n3LLemGczObnG61ouAeZ34TjvJH5T76er/gR8HTivh/M4FrjM9iTbrwMnA+Ntf7W7DiBpLHAUMBY4\nGLhaaUKV7SuBE4Ezuut4oRfpVV8dhaZFvZVT1FsplfF5SqGc9dadDaz+km6StErSrZK2rA6QdIyk\nFenn8rRsi9QbsELScklnp+VjJM2VtEzSIkldGe08GHi+atkLuXxOSMdcKml6WjZN0lRJD0h6otJL\nIWlrSfemXJZLOjQtH5l6CqZJWiNphqSD0vZrJO2R4raSdIOkhZIWSzokpfEG8EoTZXmheoGk61Lu\nSyU9L+kbaflkYEdgTvUmwFnp+Msl/U2ubP+W6mCZpE/XOmaBPwBvpf38fdr3MklzC/IdKuk2Sb9J\nP3srs1bStrm4xyXtUBRfcPyNwGv1ErT9v7YfBF5vojwVG8jqBkmXp56nZZK+l4vZr+A62U/SXbmy\n/DBdZyeRNXwukXSjpP8ABgKLJR1ZdZ52lTRb0sOS5lfqCXi1UVmBTwG32H7T9u+A3wIfzq1fDwzq\nxHkIIYQQQgid0J03udgN+ILthZJuAE4HrqqslLQzcDkwkezD69zUSFkHDLc9PsVVPmjPIPu2f5ak\nAXStMdgPaMsvsL1XOs444EJgb9svSxqcC9vJ9kdTb8As4HayXpDDbG+UtD2wMK0DGAN8xvYqSYuA\no9P2h6ZjHA58DbjP9kmSBgEPSbrX9gJgQcppMnCq7VOqC1LJu2rZyWm7EcBsYFrqrfg+WW/JQQXn\n5HnbkyWdBpwPnAJ8A9iQq4NBtY5ZkMMRaZuhwLXAPrafrjqfFVOBq2w/KOl9wD22x0m6E/g0MF3S\nh4Hf2X5B0ozqeGBc1fGvrLyWdGq2yNc2yruJcp2T9jmErN4/kN5vmwsruk6goMfO9g2S9gHusn17\n2tcfbU9Kry/KhV9Ldh08mc7HNcABtvN/T7XKOpx0PSW/T8sq2mjm735e7vUoYjJ3GUQdlVPUWzlF\nvZVS2e5EFzK9qd5aW1ub6lHrzgbW07YXptc3AV8i18AC9gTm2X4JIH143he4FBgtaSrwS2COpIHA\nLrZnAdh+o7PJpIbGhJRLkY8DM22/nI6xIbfuzrRstaQdK7sEvqNsXk4bsEtu3Vrbq9LrR4F70+tH\nyD6eAnwCOETSV9L7AcAIYE3loLYXkzV4OlPOLYGZwJm210k6A7jb9rPZKaB64t0d6fdiskYNwIHA\n2/NybDfTo1btI8B820+nfWwoiDkQGJvqBmCgpK2AW4FvAtOBo4GfN4gvZPtfu5B3I68Ar0m6Hrgb\n+EVuXdF18o5I2hr4O2Bmrtz9q+PeQVlfBHaQNLhGHWX27+LeQwghhBD6qJaWlnYNvosvvrgwblPO\nwSqad9PhLhvpQ94EoBX4InBdrdh2O5JOT0PjlkjaqWrdFmQjpMeSfSjurPxQskoexwJDgYm2J5IN\nPdyyIL4t9z7fWyCyXq6J6We07TW8c9cAt9mu9DnsDZwp6SmynqzjJV1WULa36P7b9Dd6eJKAvXLn\nYEQavrcAGJN6wQ4D/r1efDfnXJftt8iG2N0G/BPwn7nVRdfJm7T/u+owVLaBLYCX0zytSrn/thPb\n/x54X+79X6dlANh+DbgFeEpSUQ9nKKuYE1JOUW/lFPVWSmWcyxPKWW/d2cAaKakypOxzwK+r1j8E\n7Kvsjm79gGOA+Wm4XT/bd5DdiGCS7Y3AM5I+BSBpgKT35ndm++r04XOS7fVV69psjwIWkeuZqXI/\ncGQaAoak7WrEVT44DyIbXtcmaX9gZEFMPfcAZ729gfShJrapK/VWDbR9RWWZ7eNsj7K9K9kQwJ/a\nvrDBruaSu/FB0fA+ZfPPdq6zj4XAxySNTPFF53MOcHZunxNy6+4g6/FcletZqRffVe3qStJ0pXly\nhcFZj9Jg2/8JnAuMb7Df/wLGSeqfzuMBzeYCYPtVYK2kI3I51DpmkVnA0elvZjTwfrK/vcq+BpP9\nTQy33WGeXAghhBBCeGe6s4H1GHCGpFVkN5f4cVpugNQIuoCsp2op8LDtu8jmh7RKWgrcmGIATiC7\nIcNy4AFgWBdyehwovC15GtL3bbJG3lKgMpenVk/cDGDPlM9xwOqCmKLtKy4huxHICmW3Wf9WdYCk\nyZI6M3/oPGD3XE9eo+GFtXK7FBii7JbhS4GWqrxENs/spZo7tl8kG954R9rHLQVhZwN7KLvBxkrg\n1Ny6W8l6CW9pMr4DSafWOgfKbjl/JXCipKclfSCtGg88W2e32wC/SPX+K+CctLzwOrG9LpVlZSrL\nkuqYOu8rjgNOUnZTjZXAoQXlKSxruq5vBVaRDbk93e0fAjIIeC71ZIW+JOaElFPUWzlFvZVSb5rL\nE5pXxnrr0w8aTvOdtrd9QcPgUJOkD5LdwOT8ns6lO0naBrje9mbzXKh004yptovuyFiJiQcNhxBC\nKL8p8aDhsGmpxoOG+3oDawzwE2Bj1bOwQtjsSDqPrJfwCts314nru/8ohBBC2GwMGz6M9ev+Mouk\ntbW1lL0hm7veXG+1GljdfZODXsX2k8DHejqPEHqDdEv7KxsGEt/4lVFv/g8o1Bb1Vk5RbyGEevp0\nD1YIofMkOf5dCCGEEEKor1YPVnfe5CKEEEIIIYQQNmvRwAohhD6gjM8JCVFvZRX1Vk5Rb+VUxnqL\nBlYIIYQQQgghdJOYgxVCaCfmYIUQQgghNBZzsEIIIYQQQghhE4sGVgihA0nxU9KfnXYa1dOXT+iE\nMs4tCFFvZRX1Vk5lrLc+/RysEEJXxRDB8mkFWnjuuQ4jFUIIIYTwLoo5WCGEdiQ5GlhlpnhQdAgh\nhPAu6FVzsCSNlPRIjXXzJE16t3NKxx4haYmk2blla3sil1ok7SdpWhNxa9Pvmh8kNx0AACAASURB\nVOe6Kn4bSc9I+kFu2TxJIxpsN03Svg3yvavR8Tsjv09JJ0q6qIltvivpEUkrJB3VRPxFkk5osP7c\nTua9m6SlkhZLGi3pLEmrJN2YyvHDBts3LKukCZIeTGVdli+rpGMkPSbpnM7kHUIIIYQQmteTc7B6\n41eshwFzbB+cW9Yb82wmJ9d4XcslwPyupdOpXDbFPuvuX9I/AB8CxgMfAc6XNHAT5NTIYcBM25Nt\nrwVOAw60fXxa39l6LfI/wPG2dwcOBv6fpG0BbN8M7AdEA6tPau3pBEIXlHFuQYh6K6uot3IqY731\nZAOrv6Sb0jf4t0rasjogfeO+Iv1cnpZtkXpNVkhaLunstHyMpLnpW/tFkkZ3IafBwPNVy17I5XNC\nOuZSSdPTsmmSpkp6QNITkg5Py7eWdG/KZbmkQ9PykZJWp+3WSJoh6aC0/RpJe6S4rSTdIGlh6vE4\nJKXxBvBKE2V5oXqBpOtS7kslPS/pG2n5ZGBHYE7VJn8A3mpwnA0pJyTtmcqxLOW9ddXxC8skaYGk\nsbm4eZIm1TkHea8BGxvkOA74lTP/C6wA/r7BNq+mfZN6mh5N5fpZLuaDKdcnJH0pxbbrMZR0Xurt\nOhj4MnCapPskXQPsCsyuXMO5bYZKuk3Sb9LP3s2W1fYTtp9Mr/+b7HreIbf+OWBQg7KHEEIIIYQu\n6smbXOwGfMH2Qkk3AKcDV1VWStoZuByYSPYhfm5qpKwDhtsen+K2TZvMAC6zPUvSALrWeOwHtOUX\n2N4rHWcccCGwt+2XJQ3Ohe1k+6OpkTALuB34E3CY7Y2StgcWpnUAY4DP2F4laRFwdNr+0HSMw4Gv\nAffZPknSIOAhSffaXgAsSDlNBk61fUp1QSp5Vy07OW03ApgNTJMk4PvAscBBVfFHNDphts9J++wP\n3AIcaXtJ6iF6rSq8sExpu88CUyTtlM7nEknfrhGfP/6tldepATbZ9pSq4y4HvinpKmBrYH/g0Qbl\nuir39qvAKNt/zl1vkF3DLWQNljWSrq5s3nF3ni3px8CrlX1L+iTQkq6nE3PxU4GrbD8o6X3APcC4\nJstKLubDQP9Kgyunib+N/G5b0k/o3Vp6OoHQBS0tLT2dQuiCqLdyinorp95Ub62trU31qPVkA+tp\n2wvT65uAL5FrYAF7AvNsvwQgaQawL3ApMFrSVOCXwJz0YX4X27MAbL/R2WRSQ2NCyqXIx8mGd72c\njrEht+7OtGy1pB0ruwS+o2x+UhuwS27dWtur0utHgUqj4RFgVHr9CeAQSV9J7wcAI4A1lYPaXgx0\naFw1KOeWwEzgTNvrJJ0B3G372ewU0NVbkO0GPGt7ScptYzpePqZWmWaS9Z5NAY4CbmsQX8j2XUCH\n+V6250raE3iQrEfnQRr3zOUtB34m6U5SXSd3234T+IOk54BhndgnZOe66HwfCIzVX07eQElbpd43\noHZZ395x9gXFT4HjC1a/JGlMQcMrZ0rD5EMIIYQQNictLS3tGnwXX3xxYVxvmoNVNLekw4fP1LCZ\nQDbh4IvAdbVi2+1IOj0NjVuSekny67YA1gJjgbubyr691wtyPhYYCky0PZHsg/2WBfFtufdt/KXR\nK7JeronpZ7TtNbxz1wC32Z6X3u8NnCnpKbKerOMlXdbFfTdqnBWWyfazwIuSdifryfp5bptuOQe2\nL0v7+CTZdf94Jzb/R+BHwCTg4XS9QMd6/CvgTbKe0IoOQ1+bIGCvXLlH5BtXDTeWtgF+AfyL7YcL\nQqYCyyR9vgu5hV6rtacTCF1QxrkFIeqtrKLeyqmM9daTDayRkirD2D4H/Lpq/UPAvpKGSOoHHAPM\nT8Pt+tm+A/g6MCn1ljwj6VMAkgZIem9+Z7avTh9WJ9leX7WuzfYoYBHZB/wi9wNHShqSjrFdjbhK\nI2MQ8LztNkn7AyMLYuq5Bzjr7Q2kDzWxTV2pt2qg7Ssqy2wfZ3uU7V2B84Gf2r6wYNvpSvPDalgD\n7JSGLSJpYKq3vHpl+jnwz8C2tlc2Ed80ZfP2KvU2HtidNN9M0mWV66bGtgJG2J4PXABsC9S7QcZz\nwA6StpP0HuCfupDyHODteVmSJjS7YRqqeScwPf2NFLkQeL/tn3QhtxBCCCGEUEdPNrAeA86QtIrs\n5hI/TssNkBpBF5B9LbsUeDgNixoOtEpaCtyYYgBOAM6StBx4gM4P14KsV2NI0Yo0pO/bZI28pcCV\n+Xzzoen3DGDPlM9xwOqCmKLtKy4huxHIinTThG9VB0iaLOnaOuWpdh6we64nrzPDC8cDz9ZaafvP\nZI3TH0laRtZIeE9VWL0y/Tsde68urRPfgaRDJE0pWNUf+LWklWTX2XG2K3PtdgfWF2xT0Q+4KdXj\nYmCq7T8WxFWu2zdTng+TNRBXF8S226bA2cAeym6OshI4tTqgTlmPAvYBPp+r5/FVMQPSzS5Cn9LS\n0wmELuhNcwtC86LeyinqrZzKWG/xoOGcNNdne9sXNAzejKQhZ9fbrtW7V1qSZlfdlr9PS/MAl9ve\nuU5MPGi41OJBwyGEEMK7Qb3pQcO92O3AR5V70HAA26/2xcYVwGbWuDqGrGfxe01Ex09Jf4YNy49G\nDr1dGecWhKi3sop6K6cy1ltP3kWw10l3VftYT+cRwqaQHjR8c5Oxmzib0N1aW1tLOYwihBBC6Gti\niGAIoR1Jjn8XQgghhBDqiyGCIYQQQgghhLCJRQMrhBD6gDKOUQ9Rb2UV9VZOUW/lVMZ6iwZWCCGE\nEEIIIXSTmIMVQmgn5mCFEEIIITQWc7BCCCGEEEIIYROL27SHEDqQOnwZE0IouWHDh7F+3fqeTqNP\niMcilFPUWzmVsd6igRVC6GhKTycQOm0tMLqnkwid9i7W23NTnnt3DhRCCJu5mIMVQmhHkqOBFUIf\nNCUeIh5CCN2pV83BkjRS0iM11s2TNOndzikde4SkJZJm55at7YlcapG0n6RpTcStTb9rnuuq+G0k\nPSPpB7ll8ySNaLDdNEn7Nsj3rkbH74z8PiWdKOmiJrb5rqRHJK2QdFQT8RdJOqHB+nM7mfdukpZK\nWixptKSzJK2SdGMqxw8bbN9sWU+U9LikNfkySDpG0mOSzulM3iGEEEIIoXk9eZOL3vg12mHAHNsH\n55b1xjybyck1XtdyCTC/a+l0KpdNsc+6+5f0D8CHgPHAR4DzJQ3cBDk1chgw0/Zk22uB04ADbR+f\n1ne2XjuQtB3wTWBPYC/gIkmDAGzfDOwHRAOrL+pVXwWFpkW9lVIZn8sTot7Kqoz11pMNrP6Sbkrf\n4N8qacvqgPSN+4r0c3latkXqNVkhabmks9PyMZLmSlomaZGkroxqHww8X7XshVw+J6RjLpU0PS2b\nJmmqpAckPSHp8LR8a0n3plyWSzo0LR8paXXabo2kGZIOStuvkbRHittK0g2SFqYej0NSGm8ArzRR\nlheqF0i6LuW+VNLzkr6Rlk8GdgTmVG3yB+CtBsfZkHJC0p6pHMtS3ltXHb+wTJIWSBqbi5snaVKd\nc5D3GrCxQY7jgF8587/ACuDvG2zzato3qafp0VSun+ViPphyfULSl1Jsux5DSeel3q6DgS8Dp0m6\nT9I1wK7A7Mo1nNtmqKTbJP0m/ezdibJ+kuxLgldsbyCr07fLavs5YFCDfYQQQgghhC7qyZtc7AZ8\nwfZCSTcApwNXVVZK2hm4HJhI9iF+bmqkrAOG2x6f4rZNm8wALrM9S9IAutZ47Ae05RfY3isdZxxw\nIbC37ZclDc6F7WT7o6mRMAu4HfgTcJjtjZK2BxamdQBjgM/YXiVpEXB02v7QdIzDga8B99k+KfVA\nPCTpXtsLgAUpp8nAqbZPqS5IJe+qZSen7UYAs4FpkgR8HzgWOKgq/ohGJ8z2OWmf/YFbgCNtL0k9\nRK9VhReWKW33WWCKpJ3S+Vwi6ds14vPHv7XyOjXAJtueUnXc5cA3JV0FbA3sDzzaoFxX5d5+FRhl\n+8+56w2ya7iFrMGyRtLVlc077s6zJf0YeLWyb0mfBFrS9XRiLn4qcJXtByW9D7gHGNdkWYcDz+Te\n/z4ty2v8tzEv93oUcfOEMog6Kqeot1Iq2x3NQibqrZx6U721trY21aPWkw2sp20vTK9vAr5EroFF\nNsRpnu2XACTNAPYFLgVGS5oK/BKYkz7M72J7FoDtNzqbTGpoTEi5FPk42fCul9MxNuTW3ZmWrZa0\nY2WXwHeUzU9qA3bJrVtre1V6/ShQaTQ8QvZxFuATwCGSvpLeDwBGAGsqB7W9GOjQuGpQzi2BmcCZ\nttdJOgO42/az2Smgq/fn3g141vaSlNvGdLx8TK0yzSTraZkCHAXc1iC+kO27gA7zvWzPlbQn8CBZ\nD+WDNO6Zy1sO/EzSnaS6Tu62/SbwB0nPAcM6sU/IznXR+T4QGKu/nLyBkrZKvW9A7bI26SVJY2w/\nWTNi/y7uOYQQQgihj2ppaWnX4Lv44osL43rTHKyiuSUdPnymhs0EoBX4InBdrdh2O5JOT0PjlqRe\nkvy6LchGwo8F7m4q+/ZeL8j5WGAoMNH2RLIP9lsWxLfl3rfxl0avyHq5Jqaf0bbX8M5dA9xmu9JH\nsTdwpqSnyHqyjpd0WRf33ahxVlgm288CL0ranawn6+e5bbrlHNi+LO3jk2TX/eOd2PwfgR8Bk4CH\n0/UCHevxr4A3yXpCKzoMfW2CgL1y5R6Rb1w18HvaN0L/Oi3Lmwosk/T5LuQWequYy1NOUW+lVMY5\nISHqrazKWG892cAaKakyjO1zwK+r1j8E7CtpiKR+wDHA/DTcrp/tO4CvA5NSb8kzkj4FIGmApPfm\nd2b76vRhdZLt9VXr2myPAhaRfcAvcj9wpKQh6Rjb1YirNDIGAc/bbpO0PzCyIKaee4Cz3t5A+lAT\n29SVeqsG2r6issz2cbZH2d4VOB/4qe0LC7adrjQ/rIY1wE5p2CKSBqZ6y6tXpp8D/wxsa3tlE/FN\nUzZvr1Jv44HdSfPNJF1WuW5qbCtghO35wAXAtkC9G2Q8B+wgaTtJ7wH+qQspzwHenpclaUIntr0H\nOEjSoHSNHpSW5V0IvN/2T7qQWwghhBBCqKMnG1iPAWdIWkV2c4kfp+UGSI2gC8h6qpYCD6dhUcOB\nVklLgRtTDMAJwFmSlgMP0PnhWpD1agwpWpGG9H2brJG3FLgyn28+NP2eAeyZ8jkOWF0QU7R9xSVk\nNwJZkW6a8K3qAEmTJV1bpzzVzgN2z/XkdWZ44Xjg2Vorbf+ZrHH6I0nLyBoJ76kKq1emf6dj79Wl\ndeI7kHSIpCkFq/oDv5a0kuw6O852Za7d7sD6gm0q+gE3pXpcDEy1/ceCuMp1+2bK82Gyhs3qgth2\n2xQ4G9hD2c1RVgKnVgfUKmsawnoJ2ZcFvwEurhrOCjAg3ewi9CUxl6ecot5KqTfNCQnNi3orpzLW\nWzxoOCfN9dne9gUNgzcjkrYBrrddq3evtCTNrrotf5+W5gEut71znZh40HAIfdGUeNBwCCF0J9V4\n0HA0sHIkjQF+AmzcnD50h82DpGPI7og43fb/rRMX/yiE0AcNGz6M9evqddiHZrW2tpbyW/XNXdRb\nOfXmeqvVwOrJuwj2Oumuah/r6TxC2BTSg4ZvbjJ2E2cTultv/g8o1Bb1FkIIfU/0YIUQ2pHk+Hch\nhBBCCKG+Wj1YPXmTixBCCCGEEELoU6KBFUIIfUAZnxMSot7KKuqtnKLeyqmM9RYNrBBCCCGEEELo\nJjEHK4TQTszBCiGEEEJoLOZghRBCCCGEEMImFrdpDyF0IHX4MiaEsAlst90wXnopnk1VNnF7/XKK\neiunMtZbNLBCCAViiGD5tAItPZxD6KyXX44vM0IIoa+JOVghhHYkORpYIbxbFA/2DiGEkupVc7Ak\njZT0SI118yRNerdzSsceIWmJpNm5ZWt7IpdaJO0naVoTcWvT75rnuip+G0nPSPpBbtk8SSMabDdN\n0r4N8r2r0fE7I79PSSdKuqiJbd5KdbtU0p1NxF8k6YQG68/tZN67peMvljRa0lmSVkm6MZXjhw22\nb1hWSRMkPSjpEUnLJB2VW3eMpMckndOZvEMIIYQQQvN68iYXvfEru8OAObYPzi3rjXk2k5NrvK7l\nEmB+19LpVC6bYp/N7P9/bE+yPdH2YZsgn2YcBsy0Pdn2WuA04EDbx6f1na3XIv8DHG97d+Bg4P9J\n2hbA9s3AfkA0sPqk1p5OIITNRhmfyxOi3sqqjPXWkw2s/pJuSt/g3yppy+qA9I37ivRzeVq2Reo1\nWSFpuaSz0/Ixkuamb+0XSRrdhZwGA89XLXshl88J6ZhLJU1Py6ZJmirpAUlPSDo8Ld9a0r0pl+WS\nDk3LR0panbZbI2mGpIPS9msk7ZHitpJ0g6SFqcfjkJTGG8ArTZTlheoFkq5LuS+V9Lykb6Tlk4Ed\ngTlVm/wBeKvBcTaknJC0ZyrHspT31lXHLyyTpAWSxubi5kmaVOcc5L0GbGyQI0BnJzq8mvZN6ml6\nNJXrZ7mYD6Zcn5D0pRTbrsdQ0nmpt+tg4MvAaZLuk3QNsCswu3IN57YZKuk2Sb9JP3s3W1bbT9h+\nMr3+b7LreYfc+ueAQZ08FyGEEEIIoUk9eZOL3YAv2F4o6QbgdOCqykpJOwOXAxPJPsTPTY2UdcBw\n2+NT3LZpkxnAZbZnSRpA1xqP/YC2/ALbe6XjjAMuBPa2/bKkwbmwnWx/NDUSZgG3A38CDrO9UdL2\nwMK0DmAM8BnbqyQtAo5O2x+ajnE48DXgPtsnSRoEPCTpXtsLgAUpp8nAqbZPqS5IJe+qZSen7UYA\ns4FpkgR8HzgWOKgq/ohGJ8z2OWmf/YFbgCNtL5E0kNRAySksU9rus8AUSTul87lE0rdrxOePf2vl\ndWqATbY9pSDV96Rz/QbwXdv/0aBcV+XefhUYZfvPuesNsmu4hazBskbS1ZXNO+7OsyX9GHi1sm9J\nnwRa0vV0Yi5+KnCV7QclvQ+4BxjXibJWYj4M9K80uHKa+NvI77aFuHlCGbT0dAIhbDbKdkezkIl6\nK6feVG+tra1N9aj1ZAPradsL0+ubgC+Ra2ABewLzbL8EIGkGsC9wKTBa0lTgl8Cc9GF+F9uzAGy/\n0dlkUkNjQsqlyMfJhne9nI6xIbfuzrRstaQdK7sEvqNsflIbsEtu3Vrbq9LrR4FKo+ERYFR6/Qng\nEElfSe8HACOANZWD2l4MdGhcNSjnlsBM4Ezb6ySdAdxt+9nsFHS6p6diN+BZ20tSbhvT8fIxtco0\nk6z3bApwFHBbg/hCtu8Cas33Gmn7v1PP5v2SVqRhes1YDvxM2dyt/Pytu22/CfxB0nPAsCb3VyGK\nz/eBwFj95eQNlLSV7f+tBDQoa+ULip8CxxesfknSmIKGV86UhsmHEEIIIWxOWlpa2jX4Lr744sK4\n3jQHq2huSYcPn6lhM4FswsEXgetqxbbbkXR6Ghq3JPWS5NdtAawFxgJ3N5V9e68X5HwsMBSYaHsi\n2VCtLQvi23Lv2/hLo1dkvVwT089o22t4564BbrM9L73fGzhT0lNkPVnHS7qsi/tu1DgrLJPtZ4EX\nJe1O1pP189w23XIO0nA5UqOqlaxntFn/CPwImAQ8nK4X6FiPfwW8SdYTWtFh6GsTBOyVK/eIfOOq\n4cbSNsAvgH+x/XBByFRgmaTPdyG30Gu19nQCIWw2yjgnJES9lVUZ660nG1gjJVWGsX0O+HXV+oeA\nfSUNkdQPOAaYn4bb9bN9B/B1YFLqLXlG0qcAJA2Q9N78zmxfnT6sTrK9vmpdm+1RwCKyD/hF7geO\nlDQkHWO7GnGVRsYg4HnbbZL2B0YWxNRzD3DW2xtIH2pim7pSb9VA21dUltk+zvYo27sC5wM/tX1h\nwbbTleaH1bAG2CkNW0TSwFRvefXK9HPgn4Ftba9sIr5pkganYaNIGgp8FFiV3l9WuW5qbCtghO35\nwAXAtsDAOod7DthB0naS3gP8UxdSngO8PS9L0oRmN0xDNe8Epqe/kSIXAu+3/ZMu5BZCCCGEEOro\nyQbWY8AZklaR3Vzix2m5AVIj6AKyr2WXAg+nYVHDgVZJS4EbUwzACcBZkpYDD9D54VoAjwNDilak\nIX3fJmvkLQWuzOebD02/ZwB7pnyOA1YXxBRtX3EJ2Y1AVqSbJnyrOkDSZEnX1ilPtfOA3XM9eZ0Z\nXjgeeLbWStt/Jmuc/kjSMrJGwnuqwuqV6d/p2Ht1aZ34DiQdImlKwaqxwKJUb/eRzdV7LK3bHVhf\nsE1FP+CmVI+Lgam2/1gQV7lu30x5PkzWQFxdENtumwJnA3souznKSuDU6oA6ZT0K2Af4fK6ex1fF\nDEg3uwh9SktPJxDCZqM3zQkJzYt6K6cy1ls8aDgnzfXZ3vYFDYM3I2nI2fW2a/XulZak2VW35e/T\n0jzA5bZ3rhMTDxoO4V0TDxoOIYSyUm960HAvdjvwUeUeNBzA9qt9sXEFsJk1ro4h61n8Xk/nEjaF\n1p5OIITNRhnnhISot7IqY7315F0Ee510V7WP9XQeIWwK6UHDNzcX3dWbSYYQOmO77boymj2EEEJv\nFkMEQwjtSHL8uxBCCCGEUF8MEQwhhBBCCCGETSwaWCGE0AeUcYx6iHorq6i3cop6K6cy1ls0sEII\nIYQQQgihm8QcrBBCOzEHK4QQQgihsZiDFUIIIYQQQgibWDSwQgihDyjjGPUQ9VZWUW/lFPVWTmWs\nt3gOVgihAymegxVCCGHzNWz4MNavW9/TaYSSijlYIYR2JJkpPZ1FCCGE0IOmQHxGDo3EHKwQQggh\nhBBC2MR6pIElaaSkR2qsmydp0rudUzr2CElLJM3OLVvbE7nUImk/SdOaiFubftc811Xx20h6RtIP\ncsvmSRrRYLtpkvZtkO9djY7fGfl9SjpR0kVNbPNWqtulku5sIv4iSSc0WH9uJ/PeLR1/saTRks6S\ntErSjakcP2ywfbNlPVHS45LW5Msg6RhJj0k6pzN5h5LoVf9ShaZFvZVT1Fs5Rb2VUszB6pze2O96\nGDDH9gW5Zb0xz2Zyco3XtVwCzO9aOp3KZVPss5n9/4/tHmm45xwGzLR9GYCk04ADbD8r6UQ6X68d\nSNoO+CYwCRCwWNJ/2H7F9s2S7gceBv7vOylICCGEEEIo1pNDBPtLuil9g3+rpC2rA9I37ivSz+Vp\n2Rap12SFpOWSzk7Lx0iaK2mZpEWSRnchp8HA81XLXsjlc0I65lJJ09OyaZKmSnpA0hOSDk/Lt5Z0\nb8pluaRD0/KRklan7dZImiHpoLT9Gkl7pLitJN0gaWHq8TgkpfEG8EoTZXmheoGk61LuSyU9L+kb\naflkYEdgTtUmfwDeanCcDSknJO2ZyrEs5b111fELyyRpgaSxubh5kibVOQd5rwEbG+QIWWOjM15N\n+yb1ND2ayvWzXMwHU65PSPr/2Tv3cC2rMv9/vpCIipKHBGUGNKbL0UlN0J85Org1D6l5GJOMNK3x\nMhtNOziao05hmodKr/E3jlrpoCNoanlAiUSMTaYiChvQQLKiPA1ooxSa5/39/fGsF5797vf82/ju\nB+7Pde1rP+9a91rrXut+2Dz3c99rvacn2R4RQ0lnpmjXIcBXgH+W9ICka4APAtNL93CuzVaSfizp\n0fSzVxNzPZjsJcGfbK8ks+nHS5W2VwBDm1yLoAi08hcvaD9ht2ISdismYbdC0tHR0W4VmqadEawd\ngM/bniPpeuBU4IpSpaRtgEuB3cge4u9PTspzwAjbuyS5zVKTKcDFtqdKGkRrzuNAoDtfYHvPNM5O\nwLnAXrZfkfT+nNhw23snJ2EqcAfwBnCU7VclbQnMSXUAo4FP2l4s6XHg06n9EWmMo4HzgAdsnyRp\nKDBX0kzbjwCPJJ3GAqfY/kL5REp6l5WdnNqNBKYDkyQJ+B5wHHBgmfwx9RbM9ldTnxsAPwLG254v\naQjJQclRcU6p3bHAREnD03rOl/TtKvL58W8rXScHbKztiRVU3TCt9VvAZbbvrjOvK3Ifvw5sZ/vt\n3P0G2T3cQeawLJV0dal57+48XdK1wKpS35IOBjrS/XRiTv5K4ArbD0v6a+A+YKcG5zoCeDb3+flU\nlqf+v41ZuevtiP+UgiAIgiBY7+ns7GwoZbGdDtYztuek68nA6eQcLGAPYJbtlwEkTQHGARcB20u6\nEvgpMCM9zG9reyqA7beaVSY5GrsmXSqxP1l61ytpjJW5urtS2RJJW5e6BC5Rtj+pG9g2V7fM9uJ0\n/Sug5DQ8QfY4C3AQcLiks9LnQcBIYGlpUNvzgF7OVZ15DgZuB75k+zlJpwHTUppaSe9W2AF4wfb8\npNuraby8TLU53U4WaZkIfAr4cR35iti+B6i232uU7f9Jkc2fS1pku9Fs7IXAzcr2buX3b02z/Q7w\nv5JWAMMa7K+EqLzeBwA7as3iDZG0se2/lATqzLUeL0sabfu3VSX2a7HnoH0sIxzhIhJ2KyZht2IS\ndisknZ2d/SaK1dHR0UOXCy64oKJcf9qDVWlvSa+HT9srJe1Klgr1RWA8WepVTcdA0qnAyWmcQ20v\nz9UNAH4HvAlMa2IOJd6soPNxwFbAbra7lR06MbiCfHfuczdrbCKyKNfTLehTi2uAH9suxSj2AvZJ\n67MpWermKtvnttB3Pees6pwk/VHSzmSRrFNyVb3kU5SrKWz/T/q9TFInWWS0UQfrMDLn/gjgPEkf\nTuXldnwf8A5ZJLREr9TXBhCwp+23W2j7PFlUrcRf0TMeBVmEbIGk023f0MIYQRAEQRAEQRXauQdr\nlKRSGttngAfL6ucC4yRtIWkgMAGYndLtBtq+EzgfGJOiJc9KOhJA0iBJG+U7s3217d1sj8k7V6mu\n2/Z2wONkD/iV+DkwXtIWaYzNq8iVnIyhwIvJudoPGFVBphb3AWesbiB9pIE2NUnRqiG2v1sqs328\n7e1sfxD4F+C/KzlXkm5U2h9WhaXA8JS2iKQhyW55as3pVuBsYDPbTzYg70Ac1gAAIABJREFU3zCS\n3p/SRpG0FbA3sDh9vrh031RpK2Ck7dnAOcBmwJAaw60APiBpc0kbAp9oQeUZwOp9WemFQqPcBxwo\naWi6Rw9MZXnOBf4mnKt1jHgrW0zCbsUk7FZMwm6FpL9Er5qhnQ7WU8BpkhaTHS5xbSo3QHKCzgE6\ngS7gsZQWNQLolNQF3JRkAE4AzpC0EHiI5tO1AH4NbFGpIqX0fZvMyesCLs/rmxdNv6cAeyR9jgeW\nVJCp1L7EhWTRpEXp0IRvlQtIGivpBzXmU86ZwM7KDrmYL6mZ9MJdgBeqVaZoy7HAVZIWkDkJG5aJ\n1ZrTT1L7W3NlF9WQ74WkwyVNrFC1I/B4stsDZHv1nkp1OwO1vqp9IDA52XEecKXtP1eQK9237yQ9\nHyNzbJZUkO3RpgJfBnZXdjjKk/SM6AHV55pSWC8ke1nwKHBBWTorwKB02EUQBEEQBEHQxyi+pXoN\naa/PlmXHtK/3SNoUuM52teheYZE03fYh7dbjvSLtA1xoe5saMmbie6dT0EfE3oJiEnYrJmG3YtKM\n3SZCPCP3D/rTHqxyJGG7V2ZaO/dg9UfuAG5Y3x6662F7FdVTJwvN+mRnSRPITkT8Tl3hiWtbmyAI\ngiDovwwb0UoiVBBkRAQrCIIeSHL8XQiCIAiCIKhNtQhWO/dgBUEQBEEQBEEQrFOEgxUEQbAO0MgX\nHwb9j7BbMQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9iD1YQRAEQRAE9Yk9WEEQBEEQBEEQ\nBGuZcLCCIAjWAYqYox6E3YpK2K2YhN2KSRHtFt+DFQRBL6Re0e4gCNrIsGGjWL789+1WIwiCIGiA\n2IMVBEEPJBni70IQ9C9E/H8dBEHQv4g9WEEQBEEQBEEQBGuZtjhYkkZJeqJK3SxJY95rndLYIyXN\nlzQ9V7asHbpUQ9K+kiY1ILcs/a661mXym0p6VtL/zZXNkjSyTrtJksbV0feeeuM3Q75PSSdK+mYD\nbaZLekXS1AbH+KakE+rUf61xrUHSDpK6JM2TtL2kMyQtlnRTmsd/1Glfd66SdpX0sKQnJC2Q9Klc\n3QRJT0n6ajN6B0Whs90KBC3R2W4FghYo4p6QIOxWVIpot3ZGsPpjrsNRwAzbh+TK+qOejejkKtfV\nuBCY3Zo6TemyNvpspP/vAMevBT2a4SjgdttjbS8D/hk4wPZnU32zdq3Ea8Bnbe8MHAL8u6TNAGzf\nAuwLhIMVBEEQBEGwlming7WBpMnpDf5tkgaXC6Q37ovSz6WpbECKmiyStFDSl1P5aEn3p7f2j0va\nvgWd3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0dCrfRNLMpMtCSUek8lGSlqR2SyVNkXRgar9U\n0u5JbmNJ10uakyIehyc13gL+1MBcXiovkPTDpHuXpBcl/VsqHwtsDcwoa/K/wLt1xlmZdELSHmke\nC5Lem5SNX3FOkh6RtGNObpakMTXWIM/rwKt1dMT2rEbkcqxKfZMiTb9K87o5J/N3SdffSDo9yfaI\nGEo6M0W7DgG+AvyzpAckXQN8EJheuodzbbaS9GNJj6afvRqdq+3f2P5tuv4fsvv5A7n6FcDQJtYh\nKAwd7VYgaImOdisQtEBHR0e7VQhaIOxWTIpot3aeIrgD8HnbcyRdD5wKXFGqlLQNcCmwG9lD/P3J\nSXkOGGF7lyS3WWoyBbjY9lRJg2jNeRwIdOcLbO+ZxtkJOBfYy/Yrkt6fExtue+/kJEwF7gDeAI6y\n/aqkLYE5qQ5gNPBJ24slPQ58OrU/Io1xNHAe8IDtkyQNBeZKmmn7EeCRpNNY4BTbXyifSEnvsrKT\nU7uRwHRgkiQB3wOOAw4skz+m3oLZ/mrqcwPgR8B42/MlDSE5KDkqzim1OxaYKGl4Ws/5kr5dRT4/\n/m2l6+SAjbU9sZ7eDczritzHrwPb2X47d79Bdg93kDksSyVdXWreuztPl3QtsKrUt6SDgY50P52Y\nk78SuML2w5L+GrgP2KnZuUr6P8AGJYcrRwP/NvLddhAPgUEQBEEQrO90dnY2lLLYTgfrGdtz0vVk\n4HRyDhawBzDL9ssAkqYA44CLgO0lXQn8FJiRHua3tT0VwPZbzSqTHI1dky6V2J8sveuVNMbKXN1d\nqWyJpK1LXQKXKNuf1A1sm6tbZntxuv4VUHIangC2S9cHAYdLOit9HgSMBJaWBrU9D+jlXNWZ52Dg\nduBLtp+TdBowzfYL2RLQ6vncOwAv2J6fdHs1jZeXqTan28miZxOBTwE/riNfEdv3AH263yuxELhZ\n0l0kWyem2X4H+F9JK4BhTfYrKq/3AcCOWrN4QyRtbPsvJYF6c00vKP4b+GyF6pclja7geOWYWFf5\noL/RSTjCRaSTsFvx6OzsLORb9fWdsFsx6U926+jo6KHLBRdcUFGunQ5Wr7f8FWR6PXzaXilpV+Bg\n4IvAeLLUq5qOgaRTgZPTOIfaXp6rGwD8DngTmNbEHEq8WUHn44CtgN1sdys7dGJwBfnu3Odu1thE\nZFGup1vQpxbXAD9OKXMAewH7pPXZlCx1c5Xtc1vou55zVnVOkv4oaWeySNYpuape8inK9V5yGJlz\nfwRwnqQPp/JyO74PeIcsElqiV+prAwjY0/bbLbRF0qbAvcC/2n6sgsiVwAJJp9u+oZUxgiAIgiAI\ngsq0cw/WKEmlNLbPAA+W1c8FxknaQtJAYAIwO6XbDbR9J3A+MCZFS56VdCSApEGSNsp3Zvtq27vZ\nHpN3rlJdt+3tgMfJHvAr8XNgvKQt0hibV5ErORlDgReTc7UfMKqCTC3uA85Y3UD6SANtapKiVUNs\nf7dUZvt429vZ/iDwL8B/V3KuJN2otD+sCkuB4SltEUlDkt3y1JrTrcDZwGa2n2xAvhV6RYwkXVy6\nbyo2yKJII23PBs4BNgOG1BhjBfABSZtL2hD4RAt6zgBW78tKLxQaIqVq3gXcmP6NVOJc4G/CuVrX\n6Gi3AkFLdLRbgaAF+svb9KA5wm7FpIh2a6eD9RRwmqTFZIdLXJvKDZCcoHPI8ie6gMdSWtQIoFNS\nF3BTkgE4AThD0kLgIZpP1wL4NbBFpYqU0vdtMievC7g8r29eNP2eAuyR9DkeWFJBplL7EheSRZMW\npUMTvlUuIGmspB/UmE85ZwI7KzvkYr6kZtILdwFeqFaZoi3HAldJWkDmJGxYJlZrTj9J7W/NlV1U\nQ74Xkg6XNLFK3S9S3/tLekZSab/ZzsDySm0SA4HJyY7zgCtt/7mCXOm+fSfp+RiZg7ikgmyPNhX4\nMrC7ssNRnqRnRK80n2pz/RSwD/C5nJ13KZMZlA67CIIgCIIgCPoYxTfDryHt9dnS9jl1hdcjUsrZ\ndbarRfcKi6TpZcfyr9OkfYALbW9TQ8b989sJgtp0EtGQItJJY3YT8f91/6E/7QkJGifsVkz6s90k\nYbtXZlo792D1R+4AbljfHrrrYXsV1VMnC836ZGdJE8hORPxOA9JrW50gCJpg2LBR9YWCIAiCfkFE\nsIIg6IEkx9+FIAiCIAiC2lSLYLVzD1YQBEEQBEEQBME6RThYQRAE6wCNfPFh0P8IuxWTsFsxCbsV\nkyLaLRysIAiCIAiCIAiCPiL2YAVB0IPYgxUEQRAEQVCf2IMVBEEQBEEQBEGwlgkHKwiCYB2giDnq\nQditqITdiknYrZgU0W7xPVhBEPRCiu/BCoIgCIK+ZNiIYSx/bnm71QjeA2IPVhAEPZBkJrZbiyAI\ngiBYx5gI8dy9bhF7sIIgCIIgCIIgCNYybXGwJI2S9ESVulmSxrzXOqWxR0qaL2l6rmxZO3SphqR9\nJU1qQG5Z+l11rcvkN5X0rKT/myubJWlknXaTJI2ro+899cZvhnyfkk6U9M0G2kyX9IqkqQ2O8U1J\nJ9Sp/1rjWoOkHSR1SZonaXtJZ0haLOmmNI//qNO+0bmeKOnXkpbm5yBpgqSnJH21Gb2DgtCv/lIF\nDRN2KyZht2ISdiskRdyD1c4IVn+MkR4FzLB9SK6sP+rZiE6ucl2NC4HZranTlC5ro89G+v8OcPxa\n0KMZjgJutz3W9jLgn4EDbH821Tdr115I2hz4BrAHsCfwTUlDAWzfAuwLhIMVBEEQBEGwlming7WB\npMnpDf5tkgaXC6Q37ovSz6WpbECKmiyStFDSl1P5aEn3S1og6XFJ27eg0/uBF8vKXsrpc0Ias0vS\njalskqQrJT0k6TeSjk7lm0iamXRZKOmIVD5K0pLUbqmkKZIOTO2XSto9yW0s6XpJc1LE4/CkxlvA\nnxqYy0vlBZJ+mHTvkvSipH9L5WOBrYEZZU3+F3i3zjgrk05I2iPNY0HSe5Oy8SvOSdIjknbMyc2S\nNKbGGuR5HXi1jo7YntWIXI5VqW9SpOlXaV4352T+Lun6G0mnJ9keEUNJZ6Zo1yHAV4B/lvSApGuA\nDwLTS/dwrs1Wkn4s6dH0s1cTcz2Y7CXBn2yvJLPpx3PrsAIY2sQ6BEWhlb94QfsJuxWTsFsxCbsV\nko6Ojnar0DTtPEVwB+DztudIuh44FbiiVClpG+BSYDeyh/j7k5PyHDDC9i5JbrPUZApwse2pkgbR\nmvM4EOjOF9jeM42zE3AusJftVyS9Pyc23PbeyUmYCtwBvAEcZftVSVsCc1IdwGjgk7YXS3oc+HRq\nf0Qa42jgPOAB2yelCMRcSTNtPwI8knQaC5xi+wvlEynpXVZ2cmo3EpgOTJIk4HvAccCBZfLH1Fsw\n219NfW4A/AgYb3u+pCEkByVHxTmldscCEyUNT+s5X9K3q8jnx7+tdJ0csLG2J9bTu4F5XZH7+HVg\nO9tv5+43yO7hDjKHZamkq0vNe3fn6ZKuBVaV+pZ0MNCR7qcTc/JXAlfYfljSXwP3ATs1ONcRwLO5\nz8+nsjz1/23Myl1vR/ynFARBEATBek9nZ2dDKYvtdLCesT0nXU8GTifnYJGlOM2y/TKApCnAOOAi\nYHtJVwI/BWakh/ltbU8FsP1Ws8okR2PXpEsl9idL73oljbEyV3dXKlsiaetSl8AlyvYndQPb5uqW\n2V6crn8FlJyGJ8geZwEOAg6XdFb6PAgYCSwtDWp7HtDLuaozz8HA7cCXbD8n6TRgmu0XsiWg1fO5\ndwBesD0/6fZqGi8vU21Ot5NFWiYCnwJ+XEe+IrbvAfp0v1diIXCzpLtItk5Ms/0O8L+SVgDDmuxX\nVF7vA4AdtWbxhkja2PZfSgL/n3N9WdJo27+tKrFfiz0H7WMZ4QgXkbBbMQm7FZOwWyHp7OzsN1Gs\njo6OHrpccMEFFeXa6WD1estfQabXw6ftlZJ2JUuF+iIwniz1qqZjIOlU4OQ0zqG2l+fqBgC/A94E\npjUxhxJvVtD5OGArYDfb3coOnRhcQb4797mbNTYRWZTr6Rb0qcU1wI9TyhzAXsA+aX02JUvdXGX7\n3Bb6ruecVZ2TpD9K2pksknVKrqqXfIpyvZccRubcHwGcJ+nDqbzcju8D3iGLhJbolfraAAL2tP12\nC22fJ4uqlfgresajIIuQLZB0uu0bWhgjCIIgCIIgqEI792CNklRKY/sM8GBZ/VxgnKQtJA0EJgCz\nU7rdQNt3AucDY1K05FlJRwJIGiRpo3xntq+2vZvtMXnnKtV1294OeJzsAb8SPwfGS9oijbF5FbmS\nkzEUeDE5V/sBoyrI1OI+4IzVDaSPNNCmJilaNcT2d0tlto+3vZ3tDwL/Avx3JedK0o1K+8OqsBQY\nntIWkTQk2S1PrTndCpwNbGb7yQbkW6FXxEjSxaX7pmKDLIo00vZs4BxgM2BIjTFWAB+QtLmkDYFP\ntKDnDGD1vqz0QqFR7gMOlDQ03aMHprI85wJ/E87VOka8lS0mYbdiEnYrJmG3QtJfolfN0E4H6yng\nNEmLyQ6XuDaVGyA5QecAnUAX8FhKixoBdErqAm5KMgAnAGdIWgg8RPPpWgC/BraoVJFS+r5N5uR1\nAZfn9c2Lpt9TgD2SPscDSyrIVGpf4kKyaNKidGjCt8oFJI2V9IMa8ynnTGBnZYdczJfUTHrhLsAL\n1SpTtOVY4CpJC8ichA3LxGrN6Sep/a25sotqyPdC0uGSJlap+0Xqe39Jz0gq7TfbGaj1teoDgcnJ\njvOAK23/uYJc6b59J+n5GJljs6SCbI82FfgysLuyw1GepGdErzSfinNNKawXkr0seBS4oCydFWBQ\nOuwiCIIgCIIg6GMU3yi9hrTXZ0vb59QVXo+QtClwne1q0b3CIml62bH86zRpH+BC29vUkDET3zud\ngj4i9hYUk7BbMQm7FZN2220ixHN38/SnPVjlSMJ2r8y0du7B6o/cAdywvj1018P2KqqnThaa9cnO\nkiaQnYj4nbrCE9e2NkEQBEGwfjFsRCvJVUERiQhWEAQ9kOT4uxAEQRAEQVCbahGsdu7BCoIgCIIg\nCIIgWKcIBysIgmAdoJEvPgz6H2G3YhJ2KyZht2JSRLuFgxUEQRAEQRAEQdBHxB6sIAh6EHuwgiAI\ngiAI6hN7sIIgCIIgCIIgCNYy4WAFQRCsAxQxRz0IuxWVsFsxCbsVkyLaLb4HKwiCXki9ot3BOsaw\nYaNYvvz37VYjCIIgCNY5Yg9WEAQ9kGSIvwvrPiL+/gdBEARB68QerCAIgiAIgiAIgrVMQw6WpFGS\nnqhSN0vSmL5VqzEkjZQ0X9L0XNmyduhSDUn7SprUgNyynPw91WQkbdGHelUcJ1dfU+90X8yqI9Pn\n90e+z0bsLWkXSQ9LWijpbklDGmhTs19JqxrXeHWb70p6QtJlkraSNEfSPEn7NGLbBuf6HUlLJC2Q\n9BNJm+XqfiFprqStm9U9KAKd7VYgaIEi7i0Iwm5FJexWTIpot2YiWP0xl+QoYIbtQ3Jl/VHPRnRy\nletm+2mGev01q3c7aGT864Czbe8K3Amc3Qf9tjLvk4FdbH8dOABYZHus7V822F8jMjOAv7P9EeBp\n4F9XN7bHAfOAw5rWPAiCIAiCIGiIZhysDSRNlrRY0m2SBpcLSJogaVH6uTSVDZA0KZUtlPTlVD5a\n0v3pTfvjkrZvQf/3Ay+Wlb2U0+eENGaXpBtT2SRJV0p6SNJvJB2dyjeRNDPpslDSEal8VIoITJK0\nVNIUSQem9ksl7Z7kNpZ0fS4qcXhS4y3gTw3M5aXc9VBJ90p6StLVufLVOZ6SvpaiIYtya7pxateV\nysen8j2SvguSfpvkB5Y0LUUCuyStlPTZBvV+F3g59TEgF6FZIOm0cuG0bg+nNb416XuwpNtyMqsj\na5IOKpevs27V+FByYgBmAp9soM1LSYfhkman9Vkkae81quqiNNeHJX0gFU4q3VPp86r0+25gCDBP\n0tnAZcBRqd/B9LTtcZIeTXXXSKtPnKg7V9szbXenj3OAvyoTWU727yZY5+hotwJBC3R0dLRbhaAF\nwm7FJOxWTIpot2ZOEdwB+LztOZKuB04FrihVStoGuBTYDVgJ3J+clOeAEbZ3SXKllKUpwMW2p0oa\nRGv7wQYC3fkC23umcXYCzgX2sv2KpPxD5XDbe0vaEZgK3AG8ARxl+1VJW5I9nE5N8qOBT9peLOlx\n4NOp/RFpjKOB84AHbJ8kaSgwV9JM248AjySdxgKn2P5C+URKeif2AHYEngHuk3S07TtKlcrS405M\ncgOBRyV1Jj2ft/2JJLeppA2AHwHjbc9Xlh73etnYh+X6/S/gLturSnpXw/ZzwDHp4xeAUWQRGpet\nN2lNzwc+Zvv15GR8DbgE+L6kjWy/DhwL3Jzkz6sgf1G1dZM0DTjJ9vIyVX8l6QjbU4FP0dvpqDS3\nUr+fAX5m+5Lk6JScvE2Ah22fL+kysujUxZW6Sv0dKenPtkupjSuAsbbPSJ9Lc/jbtAZ/b/tdSf8J\nHAdMbnCuef6JzPZ5usnumTpMzF13EA/vQRAEQRCs73R2djaUstiMU/OM7TnpejKwT1n9HsAs2y+n\nN+hTgHHA74DtU9ToYGBVesjfNj3wYvst2280oQvpYXdXMgeuEvsDt9t+JY2xMld3VypbApT2owi4\nRNJCsijHtlqzV2WZ7cXp+lepHuAJYLt0fRBwjqQuss0Qg4CReYVsz6vkXFVgru0/ODvi6xZ6r/U+\nwJ2237D9GpmD+A9JnwMlXSJpn+Qk7QC8YHt+0uHVXIRjNZK2Am4CJqR2zXIA8P2kc/l6A3wU2Al4\nKK3RCcBI2+8CPwMOlzSQLH1tajX5WgrYPqyKw/FPwGmSHiNzjN5qYl6PAZ+X9A0y5/G1VP6m7Z+m\n63msuQ/KafS881L638eAMcBjad77Ax/sJVx9rtmg0nnA27ZvLqt6HtilvjoTcz8d9cWDfkBnuxUI\nWqCIewuCsFtRCbsVk/5kt46ODiZOnLj6pxrNRLDK939U2g/S62HS9kpJuwIHA18ExgNfqSTboyPp\nVLKogIFD8w+TkgaQOW5vAtOamEOJNyvofBywFbCb7W5lBwoMriDfnfvczZo1FFmU6+kW9CmnkbXu\n3ch+OkWhDgUulPQAmTNZb60HkDlyE5PTuTYQ2X654yrU3Qp8CXgFeMz2a8mBribfFLZ/TXb/IelD\nNLEHyfaDksalNjdIutz2ZODtnNi7rLkP3iG9uEhz2KBJdQXcaPu8Jtut6UD6HNk9sH+F6juAb0ha\nbHunVscIgiAIgiAIKtNMBGuUpHza1INl9XOBcZK2SJGICcDslOo10PadZCliY2y/Cjwr6UgASYMk\nbZTvzPbVtnezPab8Tb3tbtvbAY+TpVNV4ufAeKWT2SRtXkWu5HwMBV5MztV+ZOlu5TK1uA84Y3UD\n6SMNtKnGnsr2fg0gm1/5Wj9Itn9nsLL9VP8IPJjSNF9PUYvvkUVClgLDU3oikoYk++S5DFho+/ZK\nyijbw3VjHZ3vB04p9V1hvecAe0saneo3Ts4OwOyk68msSWmrJd8Uuf1RA8juwWvT520lzazTdiTZ\nfXE92WEZpRMRq90Tvwd2T9dH0tPBqnUfleoeAI7J6bx50qEhJH0cOAs4wvabFUROAKaHc7Uu0tFu\nBYIWKOLegiDsVlTCbsWkiHZrxsF6iizNajHZJvlrU3kpJWw5cA5ZnkoXWSTiHmAE0JnSnW5KMpA9\n6J2RUvIeAoa1oP+vgYpHW6eUvm+TOXldwOV5ffOi6fcUYI+kz/HAkgoyldqXuJDsIJBFyo60/1a5\ngKSxkn5QYz4l5gJXkaUj/tb2XfmxbXcBN5Clrz0C/MD2QmBnsr1fXcA3gItsv03mpF0laQHZKXMb\nlo13JnCQskMu5kv6RFn9SOAvdXS+DngWWJTGn1Cm8x+BzwG3pDV+mCx9kZSyeC/w8fS7pjxVbKDs\nsI7hFaomSFoKLCbbo3ZDKt+GnpGoSnQACyXNJ9u/9e+1dAB+COyb1uCjwGu5ulqRyNI6LSFzAmek\nec8Aes2pxlz/g+wwjfuTLa8uq9+c7HTBIAiCIAiCYC2gtGWmkEg6C9jS9jl1hYOWSYc43GT7yXbr\n0pcoO+nwD7bvbbcu7xXp0IxFtr9fQ8btP30/aJ5OmotiiSL//V9X6OzsLOTb2fWdsFsxCbsVk/5s\nN0nY7pWh1MwerP7IHWT7YqaXfRdW0Iek721a57D9n+3W4b1E0myyfYOVTjsMgiAIgiAI+oBCR7CC\nIOh7sghWsK4zbNgoli//fbvVCIIgCILCsq5GsIIgWAvEi5cgCIIgCILWaOXLfYMgCIJ+Rn/6npCg\nccJuxSTsVkzCbsWkiHYLBysIgiAIgiAIgqCPiD1YQRD0QJLj70IQBEEQBEFtqu3BighWEARBEARB\nEARBHxEOVhAEwTpAEXPUg7BbUQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9iO/BCoKgGsNG\nDGP5c8vbrUYQBEG/oNoerHCwgiDogSQzsd1aBEHQL5kY35MXBEFQIg65CIIgWJdZ1m4FgpYIuxWS\nIu4JCcJuRaWIdmvIwZI0StITVepmSRrTt2o1hqSRkuZLmp4r61f/XUnaV9KkBuSW5eTvqSYjaYs+\n1KviOLn6mnqn+2JWHZk+vz/yfTZib0m7SHpY0kJJd0sa0kCbmv1KWtW4xqvbfFfSE5Iuk7SVpDmS\n5knapxHbNjjXzSXNkLRU0n2ShubqfiFprqStm9U9CIIgCIIgaIxmIlj9MSfgKGCG7UNyZf1Rz0Z0\ncpXrZvtphnr9Nat3O2hk/OuAs23vCtwJnN0H/bYy75OBXWx/HTgAWGR7rO1fNthfIzLnADNt7wD8\nHPjX1Y3tccA84LCmNQ/6P9u3W4GgJcJuhaSjo6PdKgQtEHYrJkW0WzMO1gaSJktaLOk2SYPLBSRN\nkLQo/VyaygZImpTKFkr6ciofLel+SQskPS6plf9m3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0\ndCrfRNLMpMtCSUek8lGSlqR2SyVNkXRgar9U0u5JbmNJ1+eiEocnNd4C/tTAXF7KXQ+VdK+kpyRd\nnStfneMp6WspGrIot6Ybp3ZdqXx8Kt8j6bsg6bdJfmBJ01IksEvSSkmfbVDvd4GXUx8DchGaBZJO\nKxdO6/ZwWuNbk74HS7otJ7M6sibpoHL5OutWjQ8lJwZgJvDJBtq8lHQYLml2Wp9FkvZeo6ouSnN9\nWNIHUuGk0j2VPq9Kv+8GhgDzJJ0NXAYclfodTE/bHifp0VR3jaRSXSNzPRK4MV3fSPYSIs9ysn83\nQRAEQRAEwVrgfU3I7gB83vYcSdcDpwJXlColbQNcCuwGrATuT07Kc8AI27skuc1SkynAxbanShpE\na/vBBgLd+QLbe6ZxdgLOBfay/Yqk/EPlcNt7S9oRmArcAbwBHGX7VUlbAnNSHcBo4JO2F0t6HPh0\nan9EGuNo4DzgAdsnpbSsuZJm2n4EeCTpNBY4xfYXyidS0juxB7Aj8Axwn6Sjbd9RqlSWHndikhsI\nPCqpM+n5vO1PJLlNJW0A/AgYb3u+svS418vGPizX738Bd9leVdKW2VSCAAAgAElEQVS7GrafA45J\nH78AjCKL0LhsvUlrej7wMduvJyfja8AlwPclbWT7deBY4OYkf14F+YuqrZukacBJtsuPuPqVpCNs\nTwU+BfxVrXmV9fsZ4Ge2L0mOTsnJ2wR42Pb5ki4ji05dXKmr1N+Rkv5su5TauAIYa/uM9Lk0h79N\na/D3tt+V9J/AccDkBue6te0Vaczl6p0O2E12z9Qmn/i5HfGWvQgsI+xURMJuhaSzs7OQb9XXd8Ju\nxaQ/2a2zs7OhPWHNOFjP2J6TricDp5NzsMge9mfZLkU0pgDjyB6It5d0JfBTYEZ6yN82PfBi+60m\n9CD1L2DXpEsl9gdut/1KGmNlru6uVLYk9wAq4BJJ48geQrfN1S2zvThd/4osCgLwBNnjJ8BBwOGS\nzkqfBwEjgaWlQW3PI3NE6jHX9h/SPG8B9iFzAkvsA9xp+40kcwfwD8B9wPckXQJMs/1LSR8GXrA9\nP+nwamrTY0BJWwE3Acck56pZDgCucTpeqmy9AT4K7AQ8lGy3AZmD8q6kn5Gt3U/I0tfOAjoqyddS\noOQoVuCfgP+Q9G9kTnMz99tjwPXJUb3b9sJU/qbtn6breWTzr0Svk2WqUEr/+xgwBngszXswsKKX\ncPW5Vuu3xPNka1ub/RrsPQiCIAiCYD2ho6Ojh7N3wQUXVJRrxsEqf1CrtB+k18Ok7ZWSdgUOBr4I\njAe+Ukm2R0fSqWRRAQOH5t/USxoA/A54E5jWxBxKvFlB5+OArYDdbHcrO1BgcAX57tznbtasocii\nXE+3oE85jax170b20ykKdShwoaQHyJzJems9ALgFmGh7SQv6NoLI9ssdV6HuVuBLwCvAY7ZfS85F\nNfmmsP1rsvsPSR+iiT1Ith9MTvdhwA2SLrc9GXg7J/Yua+6Dd0jR2Jxj2AwCbrR9XpPtSqyQNMz2\nCknD6Z1CewfwDUmLbe/U4hhBfySiIMUk7FZI+svb9KA5wm7FpIh2ayYtb5SkfNrUg2X1c4FxkraQ\nNBCYAMxOqV4Dbd9JliI2JkVRnpV0JICkQZI2yndm+2rbu9keU54GZbvb9nbA42TpVJX4OTBe6WQ2\nSZtXkSs5H0OBF5NztR9Zulu5TC3uA85Y3UD6SANtqrGnsr1fA8jmV77WD5Lt3xmsbD/VPwIPpjTN\n123fDHyPLBKyFBie0hORNCTZJ89lwELbt1dSRtkerhsr1eW4Hzil1HeF9Z4D7C1pdKrfODk7ALOT\nrieTpTPWk2+K3P6oAWT34LXp87aSZtZpO5Lsvrie7LCM0omI1e6J3wO7p+sj6elg1bqPSnUPAMfk\ndN486dAoU4HPpesTgbvL6k8ApodzFQRBEARBsHZoxsF6CjhN0mKyTfLXpvJSSthyshPMOoEuskjE\nPcAIoFNSF1kK2jmp3QnAGZIWAg8Bw1rQ/9dAxaOtU0rft8mcvC7g8ry+edH0ewqwR9LneGBJBZlK\n7UtcSHYQyCJlR9p/q1xA0lhJP6gxnxJzgavI0hF/a/uu/Ni2u4AbyNLXHgF+kFLXdibb+9UFfAO4\nyPbbZE7aVZIWADOADcvGOxM4SNkhF/MlfaKsfiTwlzo6Xwc8CyxK408o0/mPZA/+t6Q1fphsXx+2\nu4F7gY+n3zXlqWIDZYd1DK9QNUHSUmAx2R61G1L5NvSMRFWiA1goaT7Z/q1/r6UD8ENg37QGHwVe\ny9XVikSW1mkJmRM4I817BtBrTjXmehlwYJrvx8j2RebZHOiLKGvQ3+hXX1ARNEzYrZAU8Xt5grBb\nUSmi3VTkb2RP+522tH1OXeGgZdIhDjfZfrLduvQlyk46/IPte9uty3tFOjRjke3v15AxE987nYI+\nIg5LKCZFs9tEKPJzQ1/RnzbdB40Tdism/dlukrDdK0Op6A7WaLJIzqtl34UVBEEZkmaT7Rs83vbz\nNeSK+0chCIK1yrARw1j+XPnhpUEQBOsn66SDFQRB3yPJ8XchCIIgCIKgNtUcrFa+eyoIgiDoZxQx\nRz0IuxWVsFsxCbsVkyLaLRysIAiCIAiCIAiCPiJSBIMg6EGkCAZBEARBENQnUgSDIAiCIAiCIAjW\nMuFgBUEQrAMUMUc9CLsVlbBbMQm7FZMi2i0crCAIgiAIgiAIgj4i9mAFQdCD+B6sIOjfDBs2iuXL\nf99uNYIgCNZ74nuwgiBoiMzBir8LQdB/EfF/dxAEQfuJQy6CIAjWaTrbrUDQEp3tViBogSLuCQnC\nbkWliHZryMGSNErSE1XqZkka07dqNYakkZLmS5qeK1vWDl2qIWlfSZMakFuWk7+nmoykLfpQr4rj\n5Opr6p3ui1l1ZPr8/sj32Yi9JX1T0nPpXpkv6eMNtKnZr6RVjWu8us13JT0h6TJJW0maI2mepH0a\nsW2Dc/2OpCWSFkj6iaTNcnW/kDRX0tbN6h4EQRAEQRA0RjMRrP6Yj3AUMMP2Ibmy/qhnIzq5ynWz\n/TRDvf6a1bsdNDr+FbbHpJ+f9UG/rcz7ZGAX218HDgAW2R5r+5cN9teIzAzg72x/BHga+NfVje1x\nwDzgsKY1DwpAR7sVCFqio90KBC3Q0dHRbhWCFgi7FZMi2q0ZB2sDSZMlLZZ0m6TB5QKSJkhalH4u\nTWUDJE1KZQslfTmVj5Z0f3rT/rik7VvQ//3Ai2VlL+X0OSGN2SXpxlQ2SdKVkh6S9BtJR6fyTSTN\nTLoslHREKh+VIgKTJC2VNEXSgan9Ukm7J7mNJV2fi0ocntR4C/hTA3N5KXc9VNK9kp6SdHWufHWO\np6SvpWjIotyabpzadaXy8al8j6TvgqTfJvmBJU1LkZ0uSSslfbZBvd8FXk59DMhFaBZIOq1cOK3b\nw2mNb036HizptpzM6siapIPK5eusWy165cfW4aWkw3BJs9P6LJK09xpVdVGa68OSPpAKJ5XuqfR5\nVfp9NzAEmCfpbOAy4KjU72B62vY4SY+mumsklerqztX2TNvd6eMc4K/KRJaT/bsJgiAIgiAI1gLv\na0J2B+DztudIuh44FbiiVClpG+BSYDdgJXB/clKeA0bY3iXJlVKWpgAX254qaRCt7QcbCHTnC2zv\nmcbZCTgX2Mv2K5LyD5XDbe8taUdgKnAH8AZwlO1XJW1J9nA6NcmPBj5pe7Gkx4FPp/ZHpDGOBs4D\nHrB9kqShwFxJM20/AjySdBoLnGL7C+UTKemd2APYEXgGuE/S0bbvKFUqS487MckNBB6V1Jn0fN72\nJ5LcppI2AH4EjLc9X9IQ4PWysQ/L9ftfwF22V5X0robt54Bj0scvAKPIIjQuW2/Smp4PfMz268nJ\n+BpwCfB9SRvZfh04Frg5yZ9XQf6iausmaRpwku3lFdT9UnIcHwfOtF3Tecz1+xngZ7YvSY5Oycnb\nBHjY9vmSLiOLTl1cqavU35GS/my7lNq4Ahhr+4z0uTSHv01r8Pe235X0n8BxwOQm5lrin8hsn6eb\n7J6pw8TcdQfxlr0IdBJ2KiKdhN2KR2dnZyHfqq/vhN2KSX+yW2dnZ0N7wppxsJ6xPSddTwZOJ+dg\nkT3sz7JdimhMAcaRPRBvL+lK4KfAjPSQv63tqQC232pCD1L/AnZNulRif+B226+kMVbm6u5KZUu0\nZj+KgEskjSN7CN02V7fM9uJ0/StgZrp+AtguXR8EHC7prPR5EDASWFoa1PY8MkekHnNt/yHN8xZg\nHzInsMQ+wJ2230gydwD/ANwHfE/SJcA027+U9GHgBdvzkw6vpjY9BpS0FXATcExyrprlAOAap6Ot\nytYb4KPATsBDyXYbkDko70r6Gdna/YQsfe0ssieOXvK1FCg5ihW4GvhWcvwuIrtvT2pwXo8B1ydH\n9W7bC1P5m7Z/mq7nkc2/Eo1Gzkrpfx8DxgCPpXkPBlb0Eq4+12xQ6Tzgbds3l1U9T0NPcxPriwRB\nEARBEKxHdHR09HD2LrjggopyzThY5fs/Ku0H6fUwaXulpF2Bg4EvAuOBr1SS7dGRdCpZVMDAofk3\n9ZIGAL8D3gSmNTGHEm9W0Pk4YCtgN9vdyg4UGFxBvjv3uZs1ayiyKNfTLehTTiNr3buR/XSKQh0K\nXCjpATJnst5aDwBuASbaXtKCvo0gsv1yx1WouxX4EvAK8Jjt15JzUU2+KWznU+t+CFQ93KNC2weT\n030YcIOky21PBt7Oib3LmvvgHVI0NucYNoOAG22f12S7NR1InyO7B/avUH0H8A1Ji23v1OoYQX+k\no90KBC3R0W4FghboL2/Tg+YIuxWTItqtmbS8UZLyaVMPltXPBcZJ2kLSQGACMDuleg20fSdZitiY\nFEV5VtKRAJIGSdoo35ntq23vlg4lWF5W1217O7J0r2Or6PtzYLzSyWySNq8iV3I+hgIvJudqP7J0\nt3KZWtwHnLG6gfSRBtpUY09le78GkM2vfK0fJNu/M1jZfqp/BB5MaZqvp6jF98giIUuB4Sk9EUlD\nkn3yXAYstH17JWWU7eG6sY7O9wOnlPqusN5zgL0ljU71G0v6UKqbnXQ9mTUpbbXkm0LS8NzHo4En\nU/m2kmZWbrW67Uiy++J64LqkJ1S/J34P7J6uj6Sng1XrPirVPQAcozV7ujZPOjSEshMSzwKOsP1m\nBZETgOnhXAVBEARBEKwdmnGwngJOk7SYbJP8tam8lBK2HDiHLKG8iywScQ8wAuiU1EWWgnZOancC\ncIakhcBDwLAW9P81UPFo65TS920yJ68LuDyvb140/Z4C7JH0OR5YUkGmUvsSF5IdBLJI2ZH23yoX\nkDRW0g9qzKfEXOAqsnTE39q+Kz+27S7gBrL0tUeAH6TUtZ3J9n51Ad8ALrL9NpmTdpWkBWSnzG1Y\nNt6ZwEHKDrmYL+kTZfUjgb/U0fk64FlgURp/QpnOfwQ+B9yS1vhhsn19pEMZ7gU+nn7XlKeKDZQd\n1jG8QtV3kl0WAPsCX03l29AzElWJDmChpPnAp4B/r6UDWYRs37QGHwVey9XVikSW1mkJ2YuIGWne\nM4Bec6ox1/8gO0zj/mTLq8vqNyc7XTBY5+hstwJBS3S2W4GgBYr4vTxB2K2oFNFuKvK3waf9Tlva\nPqeucNAy6RCHm2w/2W5d+hJlJx3+wfa97dblvSIdmrHI9vdryLj9p+8HzdNJpJsVkU6at5so8v/d\n6wL9adN90Dhht2LSn+0mCdu9MpSK7mCNJovkvFr2XVhBEJQhaTbZvsHjbT9fQ664fxSCYD1g2LBR\nLF/++3arEQRBsN6zTjpYQRD0PZIcfxeCIAiCIAhqU83BauW7p4IgCIJ+RhFz1IOwW1EJuxWTsFsx\nKaLdwsEKgiAIgiAIgiDoIyJFMAiCHkSKYBAEQRAEQX0iRTAIgiAIgiAIgmAtEw5WEATBOkARc9SD\nsFtRCbsVk7BbMSmi3cLBCoIgCIIgCIIg6CNiD1YQBD2I78EKgiAIho0YxvLnlrdbjSDo18T3YAVB\n0BCSzMR2axEEQRC0lYkQz4hBUJs45CIIgmBdZlm7FQhaIuxWTMJuhaSIe3mCYtqtIQdL0ihJT1Sp\nmyVpTN+q1RiSRkqaL2l6rqxf/dmTtK+kSQ3ILcvJ31NNRtIWfahXxXFy9TX1TvfFrDoyfX5/5Pts\nxN6SvinpuXSvzJf08Qba1OxX0qrGNV7d5ruSnpB0maStJM2RNE/SPo3YtsG5bi5phqSlku6TNDRX\n9wtJcyVt3azuQRAEQRAEQWM0E8Hqj3Hio4AZtg/JlfVHPRvRyVWum+2nGer116ze7aDR8a+wPSb9\n/KwP+m1l3icDu9j+OnAAsMj2WNu/bLC/RmTOAWba3gH4OfCvqxvb44B5wGFNax70f7ZvtwJBS4Td\niknYrZB0dHS0W4WgBYpot2YcrA0kTZa0WNJtkgaXC0iaIGlR+rk0lQ2QNCmVLZT05VQ+WtL9khZI\nelxSK3+u3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0dCrfRNLMpMtCSUek8lGSlqR2SyVNkXRg\nar9U0u5JbmNJ1+eiEocnNd4C/tTAXF7KXQ+VdK+kpyRdnStfneMp6WspGrIot6Ybp3ZdqXx8Kt8j\n6bsg6bdJfmBJ01Jkp0vSSkmfbVDvd4GXUx8DchGaBZJOKxdO6/ZwWuNbk74HS7otJ7M6sibpoHL5\nOutWi175sXV4KekwXNLstD6LJO29RlVdlOb6sKQPpMJJpXsqfV6Vft8NDAHmSTobuAw4KvU7mJ62\nPU7So6nuGkmlukbmeiRwY7q+kewlRJ7lZP9ugiAIgiAIgrXA+5qQ3QH4vO05kq4HTgWuKFVK2ga4\nFNgNWAncn5yU54ARtndJcpulJlOAi21PlTSI1vaDDQS68wW290zj7AScC+xl+xVJ+YfK4bb3lrQj\nMBW4A3gDOMr2q5K2BOakOoDRwCdtL5b0OPDp1P6INMbRwHnAA7ZPSmlZcyXNtP0I8EjSaSxwiu0v\nlE+kpHdiD2BH4BngPklH276jVKksPe7EJDcQeFRSZ9LzedufSHKbStoA+BEw3vZ8SUOA18vGPizX\n738Bd9leVdK7GrafA45JH78AjCKL0LhsvUlrej7wMduvJyfja8AlwPclbWT7deBY4OYkf14F+Yuq\nrZukacBJtisde/Sl5Dg+Dpxpu6bzmOv3M8DPbF+SHJ2Sk7cJ8LDt8yVdRhadurhSV6m/IyX92XYp\ntXEFMNb2GelzaQ5/m9bg722/K+k/geOAyQ3OdWvbK9KYy9U7HbCb7J6pTT7xczvibW0RWEbYqYiE\n3YpJ2K2QdHZ2FjIasr7Tn+zW2dnZ0J6wZhysZ2zPSdeTgdPJOVhkD/uzbJciGlOAcWQPxNtLuhL4\nKTAjPeRva3sqgO23mtCD1L+AXZMuldgfuN32K2mMlbm6u1LZktwDqIBLJI0jewjdNle3zPbidP0r\nYGa6foLs8RPgIOBwSWelz4OAkcDS0qC255E5IvWYa/sPaZ63APuQOYEl9gHutP1GkrkD+AfgPuB7\nki4Bptn+paQPAy/Ynp90eDW16TGgpK2Am4BjknPVLAcA1zgdOVS23gAfBXYCHkq224DMQXlX0s/I\n1u4nZOlrZwEdleRrKVByFCtwNfCt5PhdRHbfntTgvB4Drk+O6t22F6byN23/NF3PI5t/JRqNnJXS\n/z4GjAEeS/MeDKzoJVx9rtX6LfE82drWZr8Gew+CIAiCIFhP6Ojo6OHsXXDBBRXlmnGwyh/UKu0H\n6fUwaXulpF2Bg4EvAuOBr1SS7dGRdCpZVMDAofk39ZIGAL8D3gSmNTGHEm9W0Pk4YCtgN9vdyg4U\nGFxBvjv3uZs1ayiyKNfTLehTTiNr3buR/XSKQh0KXCjpATJnst5aDwBuASbaXtKCvo0gsv1yx1Wo\nuxX4EvAK8Jjt15JzUU2+KWznU+t+CFQ93KNC2weT030YcIOky21PBt7Oib3LmvvgHVI0NucYNoOA\nG22f12S7EiskDbO9QtJweqfQ3gF8Q9Ji2zu1OEbQH4m36cUk7FZMwm6FpL9EQYLmKKLdmknLGyUp\nnzb1YFn9XGCcpC0kDQQmALNTqtdA23eSpYiNSVGUZyUdCSBpkKSN8p3Zvtr2bulQguVldd22tyNL\n9zq2ir4/B8YrncwmafMqciXnYyjwYnKu9iNLdyuXqcV9wBmrG0gfaaBNNfZUtvdrANn8ytf6QbL9\nO4OV7af6R+DBlKb5uu2bge+RRUKWAsNTeiKShiT75LkMWGj79krKKNvDdWOluhz3A6eU+q6w3nOA\nvSWNTvUbS/pQqpuddD2ZLJ2xnnxTJEejxNHAk6l8W0kzK7da3XYk2X1xPXBd0hOq3xO/B3ZP10fS\n08GqdR+V6h4AjtGaPV2bJx0aZSrwuXR9InB3Wf0JwPRwroIgCIIgCNYOzThYTwGnSVpMtkn+2lRe\nSglbTnaCWSfQRRaJuAcYAXRK6iJLQTsntTsBOEPSQuAhYFgL+v8aqHi0dUrp+zaZk9cFXJ7XNy+a\nfk8B9kj6HA8sqSBTqX2JC8kOAlmk7Ej7b5ULSBor6Qc15lNiLnAVWTrib23flR/bdhdwA1n62iPA\nD1Lq2s5ke7+6gG8AF9l+m8xJu0rSAmAGsGHZeGcCByk75GK+pE+U1Y8E/lJH5+uAZ4FFafwJZTr/\nkezB/5a0xg+T7evDdjdwL/Dx9LumPFVsoOywjuEVqr6T7LIA2Bf4airfhp6RqEp0AAslzQc+Bfx7\nLR3IImT7pjX4KPBarq5WJLK0TkvIXkTMSPOeAfSaU425XgYcKGkpWbrhpWX1mwN9EWUN+hv96gsq\ngoYJuxWTsFshKeL3KQXFtJuK/C3dab/TlrbPqSsctEw6xOEm20+2W5e+RNlJh3+wfW+7dXmvSIdm\nLLL9/RoyZuJ7p1PQR8Sm+2ISdism64PdJkKRnxEr0Z8OSwgapz/bTRK2e2UoFd3BGk0WyXm17Luw\ngiAoQ9Jssn2Dx9t+voZccf8oBEEQBH3CsBHDWP5cpUN5gyAosU46WEEQ9D2SHH8XgiAIgiAIalPN\nwWrlu6eCIAiCfkYRc9SDsFtRCbsVk7BbMSmi3cLBCoIgCIIgCIIg6CMiRTAIgh5EimAQBEEQBEF9\nIkUwCIIgCIIgCIJgLRMOVhAEwTpAEXPUg7BbUQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9\niO/BCoJ1g2HDRrF8+e/brUYQBME6S3wPVhAEDZE5WPF3IQiKj4j/44MgCNYecchFEATBOk1nuxUI\nWqKz3QoELVDEPSFB2K2oFNFuDTlYkkZJeqJK3SxJY/pWrcaQNFLSfEnTc2XL2qFLNSTtK2lSA3LL\ncvL3VJORtEUf6lVxnFx9Tb3TfTGrjkyf3x/5Phuxt6RjJD0p6d1GdanXr6RVjWnbo813JT0h6TJJ\nW0maI2mepH0asW2Dc/2OpCWSFkj6iaTNcnW/kDRX0tbN6v7/2Lv3eKuqeu/jny+oeTuiUoGaoIdT\nPlkpopalwTbLynvmXY8e0y4vO1nZ5VhWgJfUSk+kWfnEQQsy9RzybiLK4ihGICKoIGnitQf1lBR2\nyoz9e/4YY8Hca6/b3m7ce+L3/Xrt155rzDHn/M0xJps11rgsMzMzM2tPT3qwBuI4g8OAGRHx4ULa\nQIyznZiiwXZPz9MTrc7X07j7QzvXfwD4CDC7D8/bm/v+OLBLRPwb8H5gcUTsHhF3t3m+dvLMAN4W\nEaOBR4CvrDk4YiywADiwx5FbCXT0dwDWKx39HYD1QkdHR3+HYL3geiunMtZbTxpYG0qaKmmJpGsk\nbVybQdKxkhbnnwty2iBJU3LaIkmfzemjJN2eP2m/V9KOvYh/S+C5mrTnC/GcmK+5UNKVOW2KpEmS\n5kh6VNLhOX0zSTNzLIskHZLTR+YegSmSlkmaJukD+fhlkvbI+TaVNLnQK3FwDuNvwB/buJfnC9tD\nJN0k6WFJlxXS14zxlHRG7g1ZXCjTTfNxC3P6kTl9zxzv/Tm+zYoXlnRz7glcKGmlpH9uM+7VwB/y\nOQYVemjul/Tp2sy53O7JZXx1jveDkq4p5FnTsyZp/9r8LcqtrohYFhGPFMuvDc/nGIZLmp3LZ7Gk\nvdeGqnPzvd4j6Q05cUr1mcqvV+Xf1wObAwskfRm4EDgsn3djutbt8ZJ+nff9QFJ1Xzv3OjMiOvPL\nucCbarKsIP27MTMzM7N1YIMe5N0JODki5kqaDJwGXFzdKWkb4AJgN2AlcHtupDwNbBcRu+R81SFL\n04BvRsQNkjaid/PBBgOdxYSIeFe+zs7AV4F3R8QLkopvKodHxN6S3grcAEwH/gocFhEvShpKenN6\nQ84/CvhoRCyRdC9wTD7+kHyNw4GzgDsi4hRJQ4B5kmZGxK+AX+WYdgc+GRGfqL2RatzZnsBbgSeB\n2yQdHhHTqzuVhrmdlPMNBn4tqZLjfCYiDsr5/kHShsDPgSMj4j5JmwN/qbn2gYXz/gdwXUSsqsbd\nSEQ8DRyRX34CGEnqoYma8iaX6deA/SLiL7mRcQZwPvAjSZtExF+Ao4Gf5fxn1cl/bqNyk3QzcEpE\nrGgWdzsK5z0O+GVEnJ8bOtVG3mbAPRHxNUkXknqnvlnvVPl8h0r6U0RUhzY+C+weEafn19V7+D+5\nDN4TEaslfR84Hpjai3v9GKnuizpJz0wLEwrbHfhT9jKo4Hoqowqut/KpVCql/FT9tc71Vk4Dqd4q\nlUpbc8J60sB6MiLm5u2pwGcoNLBIb/ZnRUS1R2MaMJb0hnhHSZOAW4AZ+U3+thFxA0BE/K0HcZDP\nL2DXHEs97wOujYgX8jVWFvZdl9OWau18FAHnSxpLehO6bWHf8ohYkrcfAmbm7QeAHfL2/sDBkr6U\nX28EjACWVS8aEQtIDZFW5kXEE/k+rwL2ITUCq/YBfhERf815pgPvBW4DviPpfODmiLhb0tuB30XE\nfTmGF/MxXS4o6fXAT4EjcuOqp94P/CDyklU15Q2wF7AzMCfX3YakBspqSb8kld1/kYavfYn0jqNb\n/mYBVBuKfWw+MDk3VK+PiEU5/aWIuCVvLyDdfz3t9ppVh//tB4wB5uf73hh4tlvmFvcq6Szg5Yj4\nWc2uZ2jr3dyE1lnMzMzMXkM6Ojq6NPYmTpxYN19PGli18z/qzQfp9mYyIlZK2hX4IPAp4Ejgc/Xy\ndjmRdBqpVyCAA4qf1EsaBDwGvATc3IN7qHqpTszHA68HdouITqUFBTauk7+z8LqTtWUoUi/XI72I\np1Y7Zd39oIhHci/UAcA5ku4gNSZblfUg4CpgQkQs7UW87RBpvtzxdfZdDfwr8AIwPyL+nBsXjfK/\naiLirtzoPhC4QtJFETEVeLmQbTVrn4O/k3tjCw3DnhBwZUSc1duYJf0L6Rl4X53d04FvSFoSETv3\n9ho2EHX0dwDWKx39HYD1wkD5NN16xvVWTmWst54MyxspqThs6q6a/fOAsZK2ljQYOBaYnYd6DY6I\nX5CGiI3JvShPSToUQNJGkjYpniwiLouI3SJiTO0wqIjojDHZmKEAACAASURBVIgdgHtJw6nquRM4\nUnllNklbNchXbXwMAZ7Ljat9ScPdavM0cxtw+poDpNFtHNPIu5Tmfg0i3V9tWd9Fmr+zsdJ8qo8A\nd+Vhmn/JvRbfIfWELAOG5+GJSNo810/RhcCiiLi2XjBKc7iubBHz7cAnq+euU95zgb0ljcr7N5X0\n5rxvdo7146wd0tYs/ytRnOu0raSZTTNLI0jPxWTgxznOLuep8TiwR94+lK4NrGbPUXXfHcARhTld\nW+UY2iLpQ6QewEMi4qU6WU4EbnXjyszMzGzd6EkD62Hg05KWkCbJ/zCnV4eErQDOJA0oX0jqibgR\n2A6oSFpIGoJ2Zj7uROB0SYuAOcCwXsT/G6Du0tZ5SN95pEbeQuCiYrzFrPn3NGDPHM8JwNI6eeod\nX3UOaSGQxUpL2p9dm0HS7pIub3I/VfOAS0nDEX8bEdcVrx0RC4ErSMPXfgVcnoeuvYM092sh8A3g\n3Ih4mdRIu1TS/aRV5l5Xc70vAPsrLXJxn6SDavaPAP63Rcw/Bp4CFufrH1sT8/8A/wJclcv4HtK8\nPvKiDDcBH8q/m+anQR0oLdYxvE76YZKeIg1TvElrl/Xfhq49UfV0AIsk3QccBXy3WQzA/wXG5TLY\nC/hzYV+znshqOS0lfRAxI9/3DKDePdW9V+AS0mIat+e6vKxm/1ak1QVtvVPp7wCsVyr9HYD1Qhm/\nl8dcb2VVxnpTmb/lPc93GhoRZ7bMbL2WF3H4aUQ82N+x9CWllQ6fiIib+juWV0teNGNxRPyoSZ7o\n/9X3recqeLhZGVVYd/Umyvx//EA2kCbdW/tcb+U0kOtNEhHRbYRS2RtYo0g9OS/WfBeWmdWQNJs0\nb/CEiHimST43sMzWC25gmZmtS+tlA8vM+l5qYJlZ2Q0bNpIVKx7v7zDMzNZbjRpYvfnuKTNbz0WE\nf0r2M2vWrH6PwT8Dq97cuFp3yjgnxFxvZVXGenMDy8zMzMzMrI94iKCZdSEp/HfBzMzMrDkPETQz\nMzMzM1vH3MAyM1sPlHGMurneysr1Vk6ut3IqY725gWVmZmZmZtZHPAfLzLrwHCwzMzOz1hrNwdqg\nP4Ixs4FN6va3wszMrEeGbTeMFU+v6O8wzF517sEysy4kBRP6OwrrseXAjv0dhPWY662cXG/tmZC+\nV3GgqFQqdHR09HcY1kMDud68iqCZmZmZmdk61lYDS9JISQ802DdL0pi+Das9kkZIuk/SrYW05f0R\nSyOSxkma0ka+5YX8NzbKI2nrPoyr7nUK+5vGnZ+LWS3y9PnzUTxnO/Ut6QhJD0pa3W4src4raVV7\n0XY55tuSHpB0oaTXS5oraYGkfdqp2zbvdStJMyQtk3SbpCGFff8taZ6kN/Y0disBf5peTq63cnK9\nldJA7QWx5spYbz3pwRo4fbxrHQbMiIgPF9IGYpztxBQNtnt6np5odb6ext0f2rn+A8BHgNl9eN7e\n3PfHgV0i4t+A9wOLI2L3iLi7zfO1k+dMYGZE7ATcCXxlzcERY4EFwIE9jtzMzMzM2tKTBtaGkqZK\nWiLpGkkb12aQdKykxfnngpw2SNKUnLZI0mdz+ihJt0u6X9K9knrzedCWwHM1ac8X4jkxX3OhpCtz\n2hRJkyTNkfSopMNz+maSZuZYFkk6JKePlLQ0H7dM0jRJH8jHL5O0R863qaTJhV6Jg3MYfwP+2Ma9\nPF/YHiLpJkkPS7qskL5mjKekM3JvyOJCmW6aj1uY04/M6XvmeO/P8W1WvLCkm3NP4EJJKyX9c5tx\nrwb+kM8xqNBDc7+kT9dmzuV2Ty7jq3O8H5R0TSHPmp41SfvX5m9RbnVFxLKIeKRYfm14PscwXNLs\nXD6LJe29NlSdm+/1HklvyIlTqs9Ufr0q/74e2BxYIOnLwIXAYfm8G9O1bo+X9Ou87wfSmhUnWt4r\ncChwZd6+kvQhRNEK0r8bW98MqL57a5vrrZxcb6VUxu9TsnLWW09WEdwJODki5kqaDJwGXFzdKWkb\n4AJgN2AlcHtupDwNbBcRu+R8W+RDpgHfjIgbJG1E7+aDDQY6iwkR8a58nZ2BrwLvjogXJBXfVA6P\niL0lvRW4AZgO/BU4LCJelDQUmJv3AYwCPhoRSyTdCxyTjz8kX+Nw4Czgjog4JQ/LmidpZkT8CvhV\njml34JMR8YnaG6nGne0JvBV4ErhN0uERMb26U2mY20k532Dg15IqOc5nIuKgnO8fJG0I/Bw4MiLu\nk7Q58Jeaax9YOO9/ANdFxKpq3I1ExNPAEfnlJ4CRpB6aqClvcpl+DdgvIv6SGxlnAOcDP5K0SUT8\nBTga+FnOf1ad/Oc2KjdJNwOnRMQrXrKocN7jgF9GxPm5oVNt5G0G3BMRX5N0Ial36pv1TpXPd6ik\nP0VEdWjjs8DuEXF6fl29h/+Ty+A9EbFa0veB44Gpbd7rGyPi2XzNFeo+HLCT9Mw0Vxz4uQMeDmNm\nZmaveZVKpa0GX08aWE9GxNy8PRX4DIUGFunN/qyIqPZoTAPGkt4Q7yhpEnALMCO/yd82Im4AiIi/\n9SAO8vkF7Jpjqed9wLUR8UK+xsrCvuty2tLCG1AB50saS3oTum1h3/KIWJK3HwJm5u0HSG8/AfYH\nDpb0pfx6I2AEsKx60YhYQGqItDIvIp7I93kVsA+pEVi1D/CLiPhrzjMdeC9wG/AdSecDN0fE3ZLe\nDvwuIu7LMbyYj+lyQUmvB34KHJEbVz31fuAH1S9QqilvgL2AnYE5ue42JDVQVkv6Jans/os0fO1L\nQEe9/M0CqDYU+9h8YHJuqF4fEYty+ksRcUveXkC6/3ra7TWrDv/bDxgDzM/3vTHwbLfM7d9r7bDC\nZ0hl29y+bZ7dBg43gsvJ9VZOrrdSKuNcHhtY9dbR0dElnokTJ9bN15MGVu0btXrzQbq9mYyIlZJ2\nBT4IfAo4EvhcvbxdTiSdRuoVCOCA4if1kgYBjwEvATf34B6qXqoT8/HA64HdIqJTaUGBjevk7yy8\n7mRtGYrUy/VIL+Kp1U5Zdz8o4pHcC3UAcI6kO0iNyVZlPQi4CpgQEUt7EW87RJovd3ydfVcD/wq8\nAMyPiD/nxkWj/K+aiLgrN7oPBK6QdFFETAVeLmRbzdrn4O/k3thCw7AnBFwZEWf1MuRnJQ2LiGcl\nDaf7ENrpwDckLYmInXt5DTMzMzNroCfD8kZKKg6buqtm/zxgrKStJQ0GjgVm56FegyPiF6QhYmNy\nL8pTkg4FkLSRpE2KJ4uIyyJit4gYUzsMKiI6I2IH4F7ScKp67gSOVF6ZTdJWDfJVGx9DgOdy42pf\n0nC32jzN3AacvuYAaXQbxzTyLqW5X4NI91db1neR5u9srDSf6iPAXXmY5l8i4mfAd0g9IcuA4Xl4\nIpI2z/VTdCGwKCKurReM0hyuK+vtK7gd+GT13HXKey6wt6RRef+mkt6c983OsX6cNJyxVf5XojjX\naVtJM5tmlkaQnovJwI9znF3OU+NxYI+8fShdG1jNnqPqvjuAIwpzurbKMbTrBuBf8vZJwPU1+08E\nbnXjaj3kOSHl5HorJ9dbKZVxLo+Vs9560sB6GPi0pCWkSfI/zOnVIWErSCuYVYCFpJ6IG4HtgIqk\nhaQhaGfm404ETpe0CJgDDOtF/L8B6i5tnYf0nUdq5C0ELirGW8yaf08D9szxnAAsrZOn3vFV55AW\nAlmstKT92bUZJO0u6fIm91M1D7iUNBzxtxFxXfHaEbEQuII0fO1XwOV56No7SHO/FgLfAM6NiJdJ\njbRLJd0PzABeV3O9LwD7Ky1ycZ+kg2r2jwD+t0XMPwaeAhbn6x9bE/P/kN74X5XL+B7SvD4iohO4\nCfhQ/t00Pw3qQGmxjuF10g+T9BRpmOJNWrus/zZ07YmqpwNYJOk+4Cjgu81iAP4vMC6XwV7Anwv7\nmvVEVstpKemDiBn5vmcA9e6p7r2SGssfkLSMNNzwgpr9WwF90ctqZmZmZnVoIH3Ddk/l+U5DI+LM\nlpmt1/IiDj+NiAf7O5a+pLTS4RMRcVN/x/JqyYtmLI6IHzXJE0x49WIyM7P11AQo8/tMs1YkERHd\nRiiVvYE1itST82LNd2GZWQ1Js0nzBk+IiGea5CvvHwUzMxswhm03jBVPv+KFfc0GrPWygWVmfU9S\n+O9C+VQqlQG10pK1x/VWTq63cnK9ldNArrdGDazefPeUmZmZmZmZ1eEeLDPrwj1YZmZmZq25B8vM\nzMzMzGwdcwPLzGw9UMbvCTHXW1m53srJ9VZOZaw3N7DMzMzMzMz6iOdgmVkXnoNlZmZm1lqjOVgb\n9EcwZjawSd3+VlgfGDZsJCtWPN7fYZiZmdk65CGCZlZH+Gcd/Dz77BM9qoWeKOMYdXO9lZXrrZxc\nb+VUxnpr2sCSNFLSAw32zZI0Zt2E1ZykEZLuk3RrIW15f8TSiKRxkqa0kW9AxV3UTmyt8kgaL+mM\nvouq6zklTZE0tkX+LSVNl7RI0lxJO7dxjVmSRrTY36PnX9IRkpZIuiO/vkrS/ZI+m+/j8BbHt3Ov\nx+X7XCTpbkm7FPZdJOkhSeN6EreZmZmZta+dHqyBOBnjMGBGRHy4kDYQ42wnpoEYd1XZ46/6KrAw\nInYFTgK+109xnAKcGhH7SRoO7BERoyNiUh9e4zFgbL7Xc4HLqzsi4gvA2cDH+vB6NkAM1G+5t+Zc\nb+Xkeisn11s5lbHe2mlgbShpav7k/RpJG9dmkHSspMX554KcNih/4r44f5r+2Zw+StLt+ZP7eyXt\n2Iu4twSeq0l7vhDPifmaCyVdmdOmSJokaY6kR6u9BZI2kzQzx7JI0iE5faSkpfm4ZZKmSfpAPn6Z\npD1yvk0lTc49IwskHZzD+Bvwxzbu5fl8nuGSZueeucWS9s7pqySdm8vrHklvyOkHFa45o5A+XtJP\nct5lkk7N6ePy+W+S9LCky5ScLOnfC2V3qqSLasu0VfyNyr1I0j9KulXS/BzLWyRtIenxQp5NJT0p\naXC9/HWuv5JU1s3sDNwJEBHLgB2q5dXE74HVjZ7j7ChJv87lWa2vkyRdUrifGyWNlfR1YB9gsqRv\nAbcB2+X63qemnMZIquT7vlXSsHbvNSLmRkT1uZsLbFeTZQXp34+ZmZmZrQsR0fAHGAl0Anvl15OB\nM/L2LGAMsA3wBLA1qcF2B3BI3jejcK4t8u+5wCF5eyNg42YxNIhrIvC5Bvt2Bh4Gtsqvt8y/pwBX\n5+23Ao/k7cHA5nl7aCF9JOnN7M759b3A5Lx9CDA9b58HHJe3hwDLgE1qYtoduLzFPZ0BfCVvC9gs\nb3cCB+TtC4GvVq9VOPYU4Nt5ezywMJftUOBJYDgwDvjffF8CZgCHA5sBjwKD8/FzgLf1ok4alfv4\nwjMzExiVt98J3JG3fwGMy9tHVcuqSf4156zzXBxUJ/084KLCef4G7NbmfTV6jmcVyvzDwO15+yTg\ne4X8N5J6lKrH7FZ4vhYX8k3J9bFBroOhhfKY3O691uT5Yu1zB7wXuKnFcQHhn3XyQ6wrs2bNWmfn\ntnXH9VZOrrdycr2V00Cut/z/erf3Uu2sIvhkRMzN21OBzwAXF/bvCcyKiD8ASJoGjCUNT9pR0iTg\nFmCGpM2BbSPiBlJErXoeupEkYNccSz3vA66NiBfyNVYW9l2X05ZKemP1lMD5SnNbOoFtC/uWR8SS\nvP0Q6Q0/wAPADnl7f+BgSV/KrzcCRpAaWuTrLQA+0eLW5pN6NzYEro+IRTn9pYi4JW8vAN6ft7eX\ndA2pgbshsLxwrutz2f5e0p2kRsUfgXkR8QSk+T/APhExXWlO0EGSHgY2iIiHWsRaT7NyR9JmwHuA\na3MdkuMGuAY4GpgNHAN8v0X+uiJifINdFwCTJN1HqruFwOo27+sxap7jwr7p+fcCUoOpHa2W59sJ\neDtwe77vQcDvajM1udd0EWlf4GRSr1nRM8BbJL0uIl5qfIYJhe2O/GNmZmb22lWpVNpadKOdBla0\neA113jRGxEpJuwIfBD4FHAl8rl7eLieSTgM+nq9zQESsKOwbRHrD+xJwcxux1yq+oazGcTzwelLP\nQqfSog0b18nfWXjdydqyE/DRiHikF/GsERF35UbegcAVki6KiKnAy4VsqwvXvQT4TkTcrLRoQfEN\nd7GORP06K+abTJqn9DCpJ2VdGAS8EBH1Foa4AThP0lakHqM7gc2b5O+RiFhFYd5RruPH2jy23nN8\nat5dfR6K9fJ3ug697TaktgUBD0bE3j08bu0J0sIWlwMfqjZ4qyLiMUlLgSck7de4MT2ht5e3flLG\nMermeisr11s5ud7KaSDVW0dHR5d4Jk6cWDdfO3OwRkp6V94+DrirZv88YKykrSUNBo4FZksaShp2\n9gvga8CYiHgReErSoQCSNpK0SfFkEXFZROwWEWOKjau8rzMidiAN1zu6Qbx3AkdK2jpfY6sG+aoN\nrCHAc7lxtS9deyLa+TKg24DT1xwgjW7jmO7BpBXrnouIycCPSQ2NZjFswdqejZNq9h2ay3YoaWjg\n/Jy+p9LcskGk8rsbICLmAduT6u6qBvEtbXELTcs9N3KWSzqicM5d8r4/k+p0Emn4WjTL31OShuSe\nQSR9HJidn0WU5t9t0+TYbs9xo6z59+PA6Dy/bXtS72HD09dJWwa8QdJe+fobqI1VDwvxjgD+C/jn\niPhtnf27ADuSepJ701NpZmZmZk2008B6GPi0pCWkyfE/zOlpskZqBJ0JVEhDr+ZHxI2kyfUVSQuB\nn+Y8ACcCp0taRJprUp3A3xO/Ic356iYP6TuP1MhbCFQXbGjUEzeN1PBYBJwALK2Tp97xVeeQFgJZ\nrLSk/dm1GSTtLuny7od20QEsysPYjgK+2+K6E4H/lDSf7otRLCbVxz3A2YWG6r3ApaThjr/NjYaq\na4A5sXaBhGL8Q1vE3qzci04ATlFasONB0ly2qqtJvYk/L6Qd3yR/N5ImSjqozq63Ag/mRuIHgeqC\nKwJGAX9octpGz3Hd5yki5pAaWQ+R6nBBbZ4Gr6vHvwwcAVwo6X7Sv6l39+Bev076t3GZ0mIj82r2\nbwU8HhGddY61Eivj94SY662sXG/l5HorpzLWm9L8rHLJ852GRsSZLTO/xkgaD6yKiItr0scBX4iI\nuo0USTcCF0fErDr7DgR2jIhL10XM/UXS24CTI+KL/R3Lq0XSUcBHIuLYJnmicbveXhmxrv7mViqV\nATWMwtrjeisn11s5ud7KaSDXmyQiotuIpLI2sEYBVwAvRtfvwnrN62kDS9IQ0jDPhRFxzKsXqb3a\nlJbffy9ptco7muRzA2udWXcNLDMzM3t1rVcNLDNbd1IDy9aFYcNGsmLF4/0dhpmZmfWBRg2sduZg\nmdlrTL3vdPDPK/9Zl42rMo5RN9dbWbneysn1Vk5lrDc3sMzMzMzMzPqIhwiaWReSwn8XzMzMzJrz\nEEEzMzMzM7N1zA0sM7P1QBnHqJvrraxcb+XkeiunMtabG1hmZmZmZmZ9xHOwzKwLz8EyMzMza63R\nHKwN+iMYMxvYpG5/K8z61LDthrHi6RX9HYaZmVmfcw+WmXUhKZjQ31FYjy0HduzvIHpgQvq+tde6\nSqVCR0dHf4dhPeR6KyfXWzkN5Hrr1SqCkkZKeqDBvlmSxvRVgD0haYSk+yTdWkhb3h+xNCJpnKQp\nbeQbUHEXtRNbqzySxks6o++i6npOSVMkjW2Rf0tJ0yUtkjRX0s5tXGOWpBEt9vfo+Zd0hKQlku7I\nr6+SdL+kz+b7OLzF8S3vNef7nqRH8rlHF9IvkvSQpHE9idvMzMzM2tfOIhcD8SPGw4AZEfHhQtpA\njLOdmAZi3FVlj7/qq8DCiNgVOAn4Xj/FcQpwakTsJ2k4sEdEjI6ISX11AUkfBkZFxJuBTwI/rO6L\niC8AZwMf66vr2QBSpt4rW2Ogfiprzbneysn1Vk5lrLd2GlgbSpqaP3m/RtLGtRkkHStpcf65IKcN\nyp+4L849B5/N6aMk3Z4/Xb9XUm/eFmwJPFeT9nwhnhPzNRdKujKnTZE0SdIcSY9WewskbSZpZo5l\nkaRDcvpISUvzccskTZP0gXz8Mkl75HybSpqce0YWSDo4h/E34I9t3Mvz+TzDJc3OPXOLJe2d01dJ\nOjeX1z2S3pDTDypcc0Yhfbykn+S8yySdmtPH5fPfJOlhSZcpOVnSvxfK7lRJF9WWaav4G5V7kaR/\nlHSrpPk5lrdI2kLS44U8m0p6UtLgevnrXH8lqayb2Rm4EyAilgE7VMurid8Dqxs9x9lRkn6dy7Na\nXydJuqRwPzdKGivp68A+wGRJ3wJuA7bL9b1PTTmNkVTJ932rpGE9uNdDgZ/ke/01MKRwPMAK0r8f\nMzMzM1sH2mlg7QRcGhE7A6uA04o7JW0DXAB0AKOBPXMjZTSwXUTsknsOqsPlpgGXRMRo4D3A/+tF\n3IOBzmJCRLwrx7MzqceiIyJ2A4pviIdHxN7AwcCFOe2vwGERsQfwPuCiQv5RwLcjYqdcDsfk47+U\nrwFwFnBHROyVj/+OpE0i4lcR8fkc0+6SLq93I9W4geOAX0bEGGBX4P6cvhlwTy6vu4CP5/S7ImKv\niNgduBr4cuG07yDVx3uAbyj1lgDsCXwaeCvwT8BHgGuAgyUNznlOBv6jJraG2iz3qsuBf42IPUll\n+IOI+BOwUGuHrR2Uy2F1vfx1rv/5iJibY5go6aA6110EVBvU7wRGAG9qcV9HRMQzNH6OAQbn+/88\ndJm11K1XLyLOAe4FjouILwOHAI9GxJiIuLuaT9IGwCXAR/N9TwG+2YN73Q54qvD6mZxW1Un692Pr\nmwE72NiaKeP3u5jrraxcb+VUxnprZxXBJ6tv6oCpwGeAiwv79wRmRcQfACRNA8YC5wI7SpoE3ALM\nkLQ5sG1E3AAQEa0+je9GkkgNkKkNsrwPuDYiXsjXWFnYd11OWyrpjdVTAucrzW3pBLYt7FseEUvy\n9kPAzLz9ALBD3t6f1ED5Un69EekN/LLqRSNiAfCJFrc2n9S7sSFwfUQsyukvRcQteXsB8P68vb2k\na4BtgA3p+vbq+ly2v5d0J/BOUm/avIh4AtL8H2CfiJiuNCfoIEkPAxtExEMtYq2nWbkjaTNSg+/a\nXIfkuCE18o4GZgPHAN9vkb+uiBjfYNcFwCRJ95HqbiGwus37eoya57iwb3r+vQAY2eb5Wi3PtxPw\nduD2fN+DgN/VZmpyr608A7xF0usi4qWGuWYVtnfAw8/MzMzsNa9SqbTV4GungVX7aXy9OTfd3jRG\nxEpJuwIfBD4FHAl8rl7eLieSTiP10gRwQESsKOwbRHrD+xJwcxux1yq+oazGcTzwemC3iOhUWrRh\n4zr5OwuvO1lbdiL1NjzSi3jWiIi7ciPvQOAKSRdFxFTg5UK21YXrXgJ8JyJuzr0/xTfcxToSjedJ\nVdMnk3qfHqZrD01fGgS8kHvoat0AnCdpK2AMaTjf5k3y90hErKIw7yjX8WNtHlvvOT41764+D8V6\n+Ttde4a7DaltQcCDuae0N54Bti+8flNOAyAiHpO0FHhC0n4NG9P79vLq1n/cCC6lMs4tMNdbWbne\nymkg1VtHR0eXeCZOnFg3XztDBEdKKg5ju6tm/zxgrKSt8zCzY4HZkoaShlD9AvgaMCYiXgSeknQo\ngKSNJG1SPFlEXBYRu+WhUytq9nVGxA6koVZHN4j3TuBISVvna2zVIF+1gTUEeC43rvala09EO18G\ndBtw+poDCqu29YTSinXPRcRk4MekhkazGLZgbc/GSTX7Ds1lOxQYR+odgzR8c2RuqB4N3A0QEfNI\nb8qPBa5qEN/SFrfQtNxzI2e5pCMK59wl7/szqU4nATdF0jB/T0kaknsGkfRxYHZ+FlGaf7dNk2O7\nPceNsubfjwOjlWxP6j1sePo6acuAN0jaK19/A7Wx6mHBDcCJ+di9gJUR8WzhfnYhvRXftpc9lWZm\nZmbWRDsNrIeBT0taQpocX12VLAByI+hMoEIaejU/Im4kzfuoSFoI/DTngfTm73RJi4A5QHECfrt+\nA2xdb0ce0nceqZG3kLVzqhr1xE0jNTwWAScAS+vkqXd81TmkhUAWKy1pf3ZthmZzsAo6gEV5GNtR\nwHdbXHci8J+S5tN9MYrFpPq4Bzi70FC9F7iUNNzxt7nRUHUNMCciui3MkRsZTTUp96ITgFOUFux4\nkDQPqepqUm/izwtpxzfJ302TeUlvBR7MjcQPkueH5SF4o4A/NDlto+e47vMUEXNIjayHSHW4oDZP\ng9fV418GjgAulHQ/6d/Uu9u91zycdLmkR4EfUTNnEtgKeDwiOmuPtZLzHKxSKuPcAnO9lZXrrZzK\nWG+l/KLhPN9paESc2TLza4yk8cCqiLi4Jn0c8IWIqNtIkXQjcHFEzKqz70Bgx4i4dF3E3F8kvQ04\nOSK+2N+xvFokHQV8JCKObZLHXzRcRv6i4VIayF+gaY253srJ9VZOA7ne1OCLhsvawBoFXAG8WPNd\nWK95PW1gSRpCGua5MCKOefUitVeb0vL77wW+EhF3NMnnBpatexPcwDIzs3JbrxpYZrbuSPIfBVvn\nhm03jBVPr2id0czMbIBq1MBqZw6Wmb3GRIR/SvYza9asfo+hJz9uXCVlnFtgrreycr2VUxnrzQ0s\nMzMzMzOzPuIhgmbWhaTw3wUzMzOz5jxE0MzMzMzMbB1zA8vMbD1QxjHq5norK9dbObneyqmM9eYG\nlpmZmZmZWR/xHCwz68JzsMzMzMxaazQHa4P+CMbMBjap298KMzMzs/XesGEjWbHi8Vd0DvdgmVkX\n6YuG/XehfCpARz/HYD1XwfVWRhVcb2VUwfVWRhVe3XoT7baPerWKoKSRkh5osG+WpDFtXb2PSRoh\n6T5JtxbSlvdHLI1IGidpShv5BlTcRe3E1iqPpPGSzui7qLqeU9IUSWPbOOZ7kh6RdL+k0W3knyVp\nRIv9PXr+JR0haYmkO/Lrq3I8n833cXiL41veq6Tjupj/5QAAIABJREFUJC3KP3dL2qWw7yJJD0ka\n15O4zczMzKx97QwRHIgfZR8GzIiIMwtpAzHOdmIaiHFXlT1+ACR9GBgVEW+W9C7gh8Be/RDKKcCp\nEXGPpOHAHhHx5hxjy8Z4mx4DxkbEHyV9CLicfK8R8QVJ84CPAbP76Ho2YHT0dwDWKx39HYD1Skd/\nB2C90tHfAVivdPR3AD3WziqCG0qamj95v0bSxrUZJB0raXH+uSCnDcqfuC/On6Z/NqePknR7/uT+\nXkk79iLuLYHnatKeL8RzYr7mQklX5rQpkiZJmiPp0WpvgaTNJM3MsSySdEhOHylpaT5umaRpkj6Q\nj18maY+cb1NJkyXNlbRA0sE5jL8Bf2zjXp7P5xkuaXbumVssae+cvkrSubm87pH0hpx+UOGaMwrp\n4yX9JOddJunUnD4un/8mSQ9LukzJyZL+vVB2p0q6qLZMW8XfqNyLJP2jpFslzc+xvEXSFpIeL+TZ\nVNKTkgbXy1/n+itJZd3MocBPACLi18AQScNaHPN7YHWj5zg7StKvc3lW6+skSZcU7udGSWMlfR3Y\nB5gs6VvAbcB2ub73qSmnMZIq+b5vLcTa8l4jYm5EVJ+7ucB2NVlWkP79mJmZmdm6EBENf4CRQCew\nV349GTgjb88CxgDbAE8AW5MabHcAh+R9Mwrn2iL/ngsckrc3AjZuFkODuCYCn2uwb2fgYWCr/HrL\n/HsKcHXefivwSN4eDGyet4cW0keS3szunF/fC0zO24cA0/P2ecBxeXsIsAzYpCam3YHLW9zTGcBX\n8raAzfJ2J3BA3r4Q+Gr1WoVjTwG+nbfHAwtz2Q4FngSGA+OA/833JWAGcDiwGfAoMDgfPwd4Wy/q\npFG5jy88MzNJPUkA7wTuyNu/AMbl7aOqZdUk/5pz1nkuDqqTfiPwnsLrmcCYNu+r0XM8q1DmHwZu\nz9snAd+rufbYwjG7FZ6vxYV8U3J9bJDrYGihPCa3e681eb5Y+9wB7wVuanFcQPindD+zBkAM/nG9\nvVZ+XG/l/HG9lfPn1a43ol05L7U/7QwRfDIi5ubtqcBngIsL+/cEZkXEHwAkTQPGAucCO0qaBNwC\nzJC0ObBtRNxAiqhVz0M3kgTsmmOp533AtRHxQr7GysK+63LaUklvrJ4SOF9pbksnsG1h3/KIWJK3\nHyK9MQd4ANghb+8PHCzpS/n1RsAIUkOLfL0FwCda3Np8Uu/GhsD1EbEop78UEbfk7QXA+/P29pKu\nITVwNwSWF851fS7b30u6k9Q4+SMwLyKegDT/B9gnIqYrzQk6SNLDwAYR8VCLWOtpVu5I2gx4D3Bt\nrkNy3ADXAEeThq0dA3y/Rf66ImJ8L+Ju5TFqnuPCvun59wJSg6kdrZbn2wl4O3B7vu9BwO9qM7W6\nV0n7AieTes2KngHeIul1EfFS4zNMKGx3UMbueTMzM7O+VKlU2vri497Mwap9DXXeNEbESkm7Ah8E\nPgUcCXyuXt4uJ5JOAz6er3NARKwo7BtEesP7EnBzG7HXKr6hrMZxPPB6Us9Cp9KiDRvXyd9ZeN3J\n2rIT8NGIeKQX8awREXflRt6BwBWSLoqIqcDLhWyrC9e9BPhORNystGhB8Q13sY5E/Tor5psMfJXU\nAzXlldxHE4OAFyKi3sIQNwDnSdqK1GN0J7B5k/w99QywfeH1m3JaSw2e41Pz7urzUKyXv9N16G23\nIbUtCHgwIvbu4XFrT5AWtrgc+FC1wVsVEY9JWgo8IWm/xo3pCb29vPWbjv4OwHqlo78DsF7p6O8A\nrFc6+jsA65WO/g5gjY6ODjo6Ota8njhxYt187czBGqm0MADAccBdNfvnAWMlbS1pMHAsMFvSUNKw\ns18AXyMNyXoReErSoQCSNpK0SfFkEXFZROwWEWOKjau8rzMidiAN1zu6Qbx3AkdK2jpfY6sG+aoN\nrCHAc7lxtS9deyLa+TKg24DT1xzQxgp1dYNJK9Y9FxGTgR+TGhrNYtiCtT0bJ9XsOzSX7VDS0MD5\nOX1Ppbllg0jldzdARMwjNUCOBa5qEN/SFrfQtNwjYhWwXNIRhXPukvf9mVSnk0jD16JZ/l64ATgx\nn2MvYGVEPJtfz5S0TaMD6z3HjbLm348Do/P8tu1JvYcNT18nbRnwhhwnkjaQtHOTc9TGOwL4L+Cf\nI+K3dfbvAuxI6knuTU+lmZmZmTXRTgPrYeDTkpaQJsf/MKcHQG4EnUlapH4hMD8ibiRNrq9IWgj8\nNOeB9Eb3dEmLSHNNWi02UM9vSHO+uslD+s4jNfIWAtUFGxr1xE0jNTwWAScAS+vkqXd81TmkhUAW\nKy1pf3ZtBkm7S7q8yf1Aap4vknQfad7Nd1tcdyLwn5Lm030xisWk+rgHOLvQUL0XuJQ03PG3udFQ\ndQ0wJ9YukFCMf2iL2JuVe9EJwClKC3Y8SJrLVnU1qTfx54W045vk70bSREkH1YntFlJj7VHgR8Bp\nOb+AUcAfmpy20XNc93mKiDmkRtZDpDpcUJunwevq8S8DRwAXSrqf9G/q3e3eK/B10r+Ny5QWG5lX\ns38r4PGI6KxzrJVapb8DsF6p9HcA1iuV/g7AeqXS3wFYr1T6O4AeK+UXDef5TkOj6zLtRlpFEFgV\nERfXpI8DvhARdRspkm4ELo6IWXX2HQjsGBGXrouY+4uktwEnR8QX+zuWV4uko4CPRMSxTfJE43a9\nDVwVBtIwCmtXBddbGVVwvZVRBddbGVUo2xcNl7WBNQq4AngxIj7cz+EMKD1tYEkaQhrmuTAijnn1\nIrVXm9Ly++8lrVZ5R5N8bmCZmZnZa9RrtIFlZuuOG1hmZmb22vXKG1jtrCJoZq857azvYmZmZrZ+\nGTas3W/eacwNLDPrxj3b5VOpVLosHWvl4HorJ9dbObneyqmM9eYhgmbWhaTw3wUzMzOz5hoNEWxn\nmXYzMzMzMzNrgxtYZmbrgUql0t8hWC+43srJ9VZOrrdyKmO9uYFlZmZmZmbWRzwHy8y68BwsMzMz\ns9Y8B8vMzMzMzGwd8zLtZtaN5O/BMjMzs4Fj2HbDWPH0iv4Ooy0eImhmXUgKJvR3FNZjy4Ed+zsI\n6zHXWzm53srJ9VZO1XqbMPC+p7NXQwQljZT0QIN9sySN6asAe0LSCEn3Sbq1kLa8P2JpRNI4SVPa\nyDeg4i5qJ7ZWeSSNl3RG30XV9ZySpkga28Yx35P0iKT7JY1uI/8sSSNa7O/R8y/pCElLJN2RX1+V\n4/lsvo/DWxz/iu5V0kWSHpI0ridxW0n4TUM5ud7KyfVWTq63ciphvbUzB2tgNRWTw4AZEfHhQtpA\njLOdmAZi3FVljx8ASR8GRkXEm4FPAj/sp1BOAU6NiP0kDQf2iIjRETGpry7Q7F4j4gvA2cDH+up6\nZmZmZtZVOw2sDSVNzZ+8XyNp49oMko6VtDj/XJDTBuVP3BdLWiTpszl9lKTb86fr90rqTbt0S+C5\nmrTnC/GcmK+5UNKVOW2KpEmS5kh6tNpbIGkzSTNzLIskHZLTR0pamo9bJmmapA/k45dJ2iPn21TS\nZElzJS2QdHAO42/AH9u4l+fzeYZLmp175hZL2junr5J0bi6veyS9IacfVLjmjEL6eEk/yXmXSTo1\np4/L579J0sOSLlNysqR/L5TdqZIuqi3TVvE3KvciSf8o6VZJ83Msb5G0haTHC3k2lfSkpMH18te5\n/kpSWTdzKPATgIj4NTBE0rAWx/weWN3oOc6OkvTrXJ7V+jpJ0iWF+7lR0lhJXwf2ASZL+hZwG7Bd\nru99asppjKRKvu9bC7H2xb2uIP37sfXNgO0Lt6Zcb+Xkeisn11s5lbDe2mlg7QRcGhE7A6uA04o7\nJW0DXAB0AKOBPXMjZTSwXUTsEhG7AtXhctOASyJiNPAe4P/1Iu7BQGcxISLelePZGfgq0BERuwHF\nN8TDI2Jv4GDgwpz2V+CwiNgDeB9wUSH/KODbEbFTLodj8vFfytcAOAu4IyL2ysd/R9ImEfGriPh8\njml3SZfXu5Fq3MBxwC8jYgywK3B/Tt8MuCeX113Ax3P6XRGxV0TsDlwNfLlw2neQ6uM9wDeUeksA\n9gQ+DbwV+CfgI8A1wMGSBuc8JwP/URNbQ22We9XlwL9GxJ6kMvxBRPwJWKi1w9YOyuWwul7+Otf/\nfETMzTFMlHRQnetuBzxVeP1MTmt2X0dExDM0fo4BBuf7/zx0mbXUrVcvIs4B7gWOi4gvA4cAj0bE\nmIi4u5pP0gbAJcBH831PAb7Zh/faSfr3Y2ZmZmbrQDurCD5ZfVMHTAU+A1xc2L8nMCsi/gAgaRow\nFjgX2FHSJOAWYIakzYFtI+IGgIho9Wl8N5JEaoBMbZDlfcC1EfFCvsbKwr7rctpSSW+snhI4X2lu\nSyewbWHf8ohYkrcfAmbm7QeAHfL2/qQGypfy642AEcCy6kUjYgHwiRa3Np/Uu7EhcH1ELMrpL0XE\nLXl7AfD+vL29pGuAbYAN6dq+vz6X7e8l3Qm8k9SbNi8inoA0/wfYJyKmK80JOkjSw8AGEfFQi1jr\naVbuSNqM1OC7NtchOW5IjbyjgdnAMcD3W+SvKyLG9yLuVh6j5jku7Juefy8ARrZ5vlbL8+0EvB24\nPd/3IOB3tZlewb0+A7xF0usi4qWGuWYVtneglOOfX3NcR+Xkeisn11s5ud7KaQDVW6VSoVKptMzX\nTgOr9tP4enNuur1pjIiVknYFPgh8CjgS+Fy9vF1OJJ1G6qUJ4ICIWFHYN4j0hvcl4OY2Yq9VfENZ\njeN44PXAbhHRqbRow8Z18ncWXneytuxE6m14pBfxrBERd+VG3oHAFZIuioipwMuFbKsL170E+E5E\n3Jx7f4pvuIt1JBrPk6qmTyb1Pj1M1x6avjQIeCH30NW6AThP0lbAGOBOYPMm+XvqGWD7wus35bSW\nGjzHp+bd1eehWC9/p2vPcLchtS0IeDD3lPZG03uNiMckLQWekLRfw8b0vr28upmZmdl6qqOjg46O\njjWvJ06cWDdfO0MER0oqDmO7q2b/PGCspK3zMLNjgdmShpKGUP0C+BowJiJeBJ6SdCiApI0kbVI8\nWURcFhG75aFTK2r2dUbEDqShVkc3iPdO4EhJW+drbNUgX7WBNQR4Ljeu9qVrT0Q7XwZ0G3D6mgPa\nWKGubjBpxbrnImIy8GNSQ6NZDFuwtmfjpJp9h+ayHQqMI/WOQRq+OTI3VI8G7gaIiHmkN+XHAlc1\niG9pi1toWu4RsQpYLumIwjl3yfv+TKrTScBNkTTM3ws3ACfmc+wFrIyIZ/PrmXmYa131nuNGWfPv\nx4HRSrYn9R42PH2dtGXAG3KcSNogD79sV8N7zWm7kD4L2raXPZU2UJVwjLrheisr11s5ud7KqYT1\n1k4D62Hg05KWkCbHV1clC4DcCDoTqAALgfkRcSNp3kdF0kLgpzkPpDd/p0taBMwBWi02UM9vgK3r\n7chD+s4jNfIWsnZOVaOeuGmkhsci4ARgaZ089Y6vOoe0EMhipSXtz67N0GwOVkEHsEjSfcBRwHdb\nXHci8J+S5tN9MYrFpPq4Bzi70FC9F7iUNNzxt7nRUHUNMCciui3MkRsZTTUp96ITgFOUFux4kDQP\nqepqUm/izwtpxzfJ302jeUl5iOVySY8CPyLPI8xD8EYBf2hy2kbPcd3nKSLmkBpZD5HqcEFtngav\nq8e/DBwBXCjpftK/qXe/0nst2Ap4PCI6a481MzMzs1eulF80nOc7DY2IM1tmfo2RNB5YFREX16SP\nA74QEXUbKZJuBC6OiFl19h0I7BgRl66LmPuLpLcBJ0fEF/s7lleLpKOAj0TEsU3y+IuGzczMbGCZ\nsJ580fAANh3YW4UvGrbekTRE0jLgz/UaVwARcfP61rgCiIiHXmONq4uAL5KGoJqZmZnZOlDKHiwz\nW3ck+Y+CmZmZDSjDthvGiqdXtM74KmrUg9XOKoJm9hrjD17Kp1KpdFnZyMrB9VZOrrdycr2VUxnr\nzT1YZtaFpPDfBTMzM7Pm1rc5WGZmZmZmZgOOG1hmZuuBdr5Z3gYe11s5ud7KyfVWTmWsNzewzMzM\nzMzM+ojnYJlZF56DZWZmZtaa52CZmZmZmZmtY25gmVk3kgbEz/A3De/voiiNMo5RN9dbWbneysn1\nVk5lrDd/D5aZdTehvwNInp3wbH+HYGZmZtYjnoNl/U7Sqoj4h/6OA0DSGOBKYF5EnJLTlkfEjv0Q\ny67AthFxa359ErBDRExscdytwF7AXRFxSCH9WGA88KOI+Pcmx8dAaWAxwV96bGZmZgOT52DZQDaQ\n3kGfAHy/2rjKehSfpL76dzUaOKAmrZ1YvkW6j64HRlwFjAM+/8pDMzMzM7N63MCyAUPSREkLJd0n\n6WlJkyWNlLRU0hRJyyRNk/QBSXPy6z3ysXtKukfSAkl3S3pzL8PYEniuJu35fI1xkmZLuknSw5Iu\nK8S+StJ3JC0E9pI0RlJF0nxJt0oalvOdLukhSfdL+llO2zTf69wc/8GSNgTOBo7K5XEk8L/Ai61u\nICJmNcoXEc8CQ3pcKjbglXGMurneysr1Vk6ut3IqY715DpYNGBExHhgvaQjw38Aledco4KMRsUTS\nvcAxEbG3pEOAs4CPAEuBfSKiU9J+wPnAEb0IYzDQWRPXuwov9wTeCjwJ3Cbp8IiYDmwG/Coivihp\nA2A2cEhE/F7SUcA3gVOAfyMN83tZ0hb5nGcBd0TEKfne5wEzgW8Au0fE6bVBSjo475vQi3v0Bytm\nZmZm64gbWDYQTQUuioj7JY0ElkfEkrzvIVLjA+ABYGTe3hL4Se65CnrxbOeG0dtY27CrZ15EPJHz\nXwXsA0wHVuffADsBbwdulyRSg+Z3ed8i4GeSrgOuy2n7AwdL+lJ+vREwolmsEXEjcGP7d9fFHySN\niojfNswxq7C9A/Cqz0Cznuro6OjvEKwXXG/l5HorJ9dbOQ2keqtUKm31qLmBZQOKpAnAkxHxk0Ly\nS4XtzsLrTtY+w+cAd0bE4blRVmwiVM99LnAgEBExpmbfm0g9R49GxL1NQqydA1V9/ZfCt/MKeDAi\n9q5z/IHAWOAQ4CxJ78j5PxoRj9TEtFeTOF6JScD9kj4TEVfUzbHvOrqymZmZWUl1dHR0afBNnFh/\n3TEPFbKBQLBm2Nv7gc/W29/CEOCZvH1yvQwR8bWI2K22cZX3PQ1sl8JQR5PrvDPPCxsEHA3cVSfG\nZcAbqg0kSRtI2jnvGxERs4EzgS1IQwtvA9YMA5Q0Om+uynl6QzQut68C/9SwcWWlVMYx6uZ6KyvX\nWzm53sqpjPXmBpYNBNWen88D2wLz88IOE2r2124XfQu4QNICevlc5x6oR4Gtm2S7F7iUNFTxtxFR\nHea3Jq6IeJk0/+tCSfcDC4F35yGIUyUtAhYAkyLiT6Tetw0lLZb0AGlxC0i9cDsXFrlYIy+EMaFe\ngJL+G7gaeJ+kJyV9oCbLRnmxCzMzMzPrY/4eLLMCSd8HHoiIH9bZNw74QvG7pcpG0huBRRGxTZM8\n/h4sMzMzsxb8PVhm7fkJcLKkyf0dSF/LXzQ8g9TbZ2ZmZmbrgHuwzKwLSQPmj8Kw7Yax4ukV/R1G\nKVQqlQG10pK1x/VWTq63cnK9ldNArrdGPVheRdDMuvEHL2ZmZma94x4sM+tCUvjvgpmZmVlznoNl\nZmZmZma2jrmBZWa2Hijj94SY662sXG/l5HorpzLWmxtYZmZmZmZmfcRzsMysC8/BMjMzM2vNc7DM\nzMzMzMzWMTewzKwbSa/oZ/ibhvf3LbzmlHGMurneysr1Vk6ut3IqY735e7DMrLsJr+zwZyc82ydh\nmJmZmZWN52CZ9RNJI4GbIuIdbeb/FnAw8BLwW+DkiPhTD663EzAFGAN8NSIubpAvXmkDiwn+smIz\nMzNbv3kOltnA1JNWyAzgbRExGngE+EoPr/V74DPAt3t4nJmZmZm1yQ0ss/61oaSpkpZIukbSxpJ2\nl7RQ0n2SFktaDRARMyOiMx83F3hTTy4UEf8TEQuAv/fxPdgAUMYx6uZ6KyvXWzm53sqpjPXmBpZZ\n/9oJuDQidgZWAadFxIKI2C0ixgC/pH6P08eAW1/FOM3MzMysDZ6DZdZP8hys2RGxQ369L/CZiDg8\nvz4aOBXYv/jFVJLOAsZExEd7ed3xwKqmc7DGFRJ2AHbs4UUmeA6WmZmZrV8qlUqXHrWJEyfWnYPl\nVQTN+ldtKyQAJL0d+Abw3prG1b8ABwDvq3cySecCBwKRe8B6Z99eH2lmZv+/vTsPsqws7zj+/SGF\nCwIxUQczCGjU4IYM4qCBShor4BYQNSoGRVyyQQIVxXJLdKbKqMQykaASFzIhmIFCRUXKEhBoE1QE\nHAaQTRIWlzhoBQiSKBF48sd9G05PLzPT0829p/v7qZq657xne24/dXv6ue/7niNpURobG2NsbOz+\n9dWrV0+7n0MEpeHaLcm+bfkPgIuS7ASsBY6oqtsmdkzyQuBtwCFVdfd0J6uqv+wML5zNlG9b1G99\nHKMu89ZX5q2fzFs/9TFv9mBJw3UdcHSSNcB3gZOAVwG7Ap9KEh7ojToR2A44b9DMxVV11OZeKMky\n4DJgB+C+JMcCT6uqu+bzDUmSJC1lzsGSNInPwZIkSdo0n4MlSZIkSQvMHixJkyTZ6l8Ky5YvY8MP\nN8xHONpM4+Pjkybeqh/MWz+Zt34yb/00ynmbqQfLOViSpvCLF0mSpLmxB0vSJEnK3wuSJEmzcw6W\nJEmSJC0wCyxJWgT6+JwQmbe+Mm/9ZN76qY95s8CSJEmSpHniHCxJkzgHS5IkadOcgyVJkiRJC8wC\nS9IUSRb9v5132XnYP+Z51ccx6jJvfWXe+sm89VMf8+ZzsCRNtWrYASy8W1fdOuwQJEnSIuQcLGnE\nJLkJeHZV3ZbkoqraP8nvAMdV1cFbcd6Tgd8Dbq2qPWfZr5ZCgcUqH6gsSZLmzjlYUn/c/1d/Ve0/\nXfscrQFesJXnkCRJ0iwssKQhSfLHSS5Psi7JjUnOn9jU2ednnUN2SnJ2kuuSfHxLr1dVFwG3b2XY\nGlF9HKMu89ZX5q2fzFs/9TFvFljSkFTVJ6pqBbAS+AHw4el26yw/BzgaeCrwpCQvX/goJUmStCW8\nyYU0fH8PXFBVX9nEfpdU1S0ASU4D9gfOXJCILuws7w48YUGuonk0NjY27BA0B+atn8xbP5m3fhql\nvI2Pj29Wj5oFljRESY4EHl9VR23G7hvPwZq0nmQl8InW/p6qOnvOgR0w5yMlSZIWpbGxsUkF3+rV\nq6fdzyGC0pAkeTbwVuC1s+3WWd43yW5JtgFeDVzU3bGqLqmqFVW19yzFVTY6pxaJPo5Rl3nrK/PW\nT+atn/qYNwssaXiOBh4FXNhudPHJ1t7tmeouXwJ8FLga+I+q+sKWXCzJWuCbwFOSfD/JG+YeuiRJ\nkqbjc7AkTeJzsCRJkjbN52BJkiRJ0gKzwJKkRaCPY9Rl3vrKvPWTeeunPubNuwhKmmrVsANYeMuW\nLxt2CJIkaRFyDpakSZKUvxckSZJm5xwsSZIkSVpgFliStAj0cYy6zFtfmbd+Mm/91Me8WWBJkiRJ\n0jxxDpakSZyDJUmStGnOwZIkSZKkBWaBJUmLQB/HqMu89ZV56yfz1k99zJvPwZI0RTKlt1uSJPXM\nsuXL2PDDDcMOY8lxDpakSZLUUnjQsCRJi94q8G/9heMcLGkzJLkvyYc6629N8p4hxbJbi+foTtuJ\nSY4YRjySJEnaNAssabK7gZcn+dVhB9L8BDg2icN5Nbubhh2A5sS89ZN56yfz1kt9nINlgSVNdg/w\nSeAtG29oPUrnJ1mf5Lwku7T2NUlOSPKNJP+e5OWdY45Lckk75r1ziOenwPnAkdPEs1eSb7Vzfz7J\nTq39wiQfTPLtJNcl2a+1b5Pkb1r7+iR/OId4JEmSNAsLLGmyAj4GHJ5kh422nQisqaq9gLVtfcLO\nVbUfcDBwPECSA4EnV9VKYAWwT5L95xDP8cBxmXrniVOAt7V4vgt0C7iHVNW+wF/A/TOq3gTc0dpX\nAn+UZLctjEej6gnDDkBzYt76ybz1k3nrpbGxsWGHsMUcdiRtpKruSnIKcCzw886m5wEva8un0gqp\n5ovt2GuTPLa1HQQcmGQdEGB74MnARVsYz81JLgYOn2hLsiOwU1VNnOsU4IzOYWe21+8AE0XUQcAz\nk7yyre/Y4rllykUv7Czvjv8pSZKkJW98fHyzhixaYEnTOwFYB6zptM12G567O8vpvH6gqj4100FJ\nDmXQ81TAm6tq3Qy7fgD4HDA+zXVmi+deHvicB/jzqjpvluMGDtjkHho1N2Eh3EfmrZ/MWz+Zt14a\nHx8fmV6ssbGxSbGsXr162v0cIihNFoCqup1Bj9CbOtu+CbymLb8W+LfZzgGcA7wxyfYASX49yWO6\nO1bVF6tqRVXtPUNxNRHP9cA1wCFt/U7gton5VcDrgK9vRjxHTdwwI8mTkzx8hmMkSZI0B/ZgSZN1\ne6k+DBzdaTsGWJPkOAY3n3jDNMfcv15V5yXZA/hWmz71MwaF2U/nGM9fM+hVm3Ak8A+tSLpxU/EA\nn2Yw4G9dm8/1E+DQLYhFo8xvZfvJvPWTeesn89ZLo9J7tSV80LCkSXzQsCRJi8QqHzS8kHzQsCQt\nZj7fpZ/MWz+Zt34yb73Ux+dgOURQ0lSrhh2AJEnaWsuWLxt2CFtt/fr1vRsmaIElaQqHE/TPqlWr\nWLVq1bDD0BYyb/1k3vrJvPXTHXfcMewQtphDBCVJkiRpnlhgSdIicPPNNw87BM2Beesn89ZP5q2f\n+pg37yIoaZIk/lKQJEnaDNPdRdACS5IkSZLmiUMEJUmSJGmeWGBJkiRJ0jyxwJIkSZKkeWKBJQmA\nJC9Mcl2S7yV5+7DjWeqSnJzk1iRXdtoeleTcJNcnOSfJTp1t70xyQ5JrkxzUad87yZUtrx95sN/H\nUpNklyQXJLk6yVVJjmnt5m6EJXlokm8nubxC8BmbAAAGy0lEQVTl7v2t3bz1QJJtkqxLclZbN28j\nLsnNSa5on7lLWtuiyZsFliSSbAN8FHgB8HTgNUn2GG5US94aBvnoegfwtar6TeAC4J0ASZ4GvAp4\nKvAi4ONJJu5qdBLwpqp6CvCUJBufU/PrHuAtVfV04HnA0e2zZO5GWFXdDRxQVSuAPYHnJ9kP89YX\nxwLXdNbN2+i7DxirqhVVtbK1LZq8WWBJAlgJ3FBVt1TVL4HTgZcOOaYlraouAm7fqPmlwClt+RTg\n0LZ8CHB6Vd1TVTcDNwArk+wM7FBVl7b9/rlzjBZAVW2oqvVt+S7gWmAXzN3Iq6r/bYsPZfD30e2Y\nt5GXZBfgxcCnO83mbfSFqXXIosmbBZYkgOXADzrrP2xtGi2PrapbYfCHPPDY1r5x/n7U2pYzyOUE\n8/ogSrI7sBdwMbDM3I22NszscmADMF5V12De+uDvgLcB3ecOmbfRV8B5SS5N8ubWtmjytu2wA5Ak\nzZkPMhxRSR4JfA44tqrumuYB3uZuxFTVfcCKJDsC5yQZY2qezNsISfIS4NaqWt/yNRPzNnr2q6of\nJ3kMcG6S61lEnzd7sCTB4NugXTvru7Q2jZZbkywDaEMjftLafwQ8vrPfRP5matcCSrItg+Lq1Kr6\nUms2dz1RVXcCXwH2wbyNuv2AQ5LcCJzGYO7cqcAG8zbaqurH7fWnwBcZTFVYNJ83CyxJAJcCT0qy\nW5LtgMOAs4YckwZj1NNZPws4si2/HvhSp/2wJNsleQLwJOCSNsTiv5OsbBOCj+gco4Xzj8A1VXVC\np83cjbAkj564Y1mShwMHApdj3kZaVb2rqnatqicy+H/rgqp6HfBlzNvISvKI1stPku2Bg4CrWESf\nN4cISqKq7k3yZ8C5DL54Obmqrh1yWEtakrXAGPBrSb4PvBf4IPDZJG8EbmFwVyWq6pokZzC4i9Yv\ngaOqamJoxdHAPwEPA75SVV99MN/HUtPuPHc4cFWbz1PAu4DjgTPM3ch6HHBK+yNtGwa9j+e3HJq3\n/vkg5m2ULQO+0IZObwv8S1Wdm+QyFkne8kB8kiRJkqSt4RBBSZIkSZonFliSJEmSNE8ssCRJkiRp\nnlhgSZIkSdI8scCSJEmSpHligSVJkiRJ88QCS5IkjZQkFybZe4Ztpyd5Ylu+OcnXN9q+PsmVbfn1\nSU6c5TonJXneDNsOSfJXc38XkpYqCyxJktQLSX4D2L6qbmxNBeyQZHnbvkdr65rtgZ/7AhfPsO3L\nwCuSbLsVIUtagiywJEnSrJI8IsnZSS5PcmWSV7b2m5Ic39ou7vQsPTrJ55J8u/37rc55Tm77fifJ\nIa39YUlOS3J1kjOBh80QymEMCp+uM1o7wGuAtRtt37X1iF2f5D2d97QH8L2qqiTHtGuvT7IWoKoK\n+CZw0Bx/bJKWKAssSZK0KS8EflRVK6pqT+CrnW23t7aPASe0thOAv62qfYHfBz7d2t8NnF9VzwWe\nD3woycOBPwX+p6qeDrwX2GeGOPYHLuusF/B54GVt/WCmFmDPadufBbyyM/TwRZ338XZgr6raC/iT\nzrGXAr89QyySNC0LLEmStClXAQcm+UCS/avqZ51tp7fX04DntuXfBT6a5HLgLOCRSR7BoDfoHa19\nHNgO2JVBEfMZgKq6Crhihjh2A368Udt/AbcneTVwDfDzjbafV1V3VNUvgDMZFGkAL+CBAusKYG2S\nw4F7O8f+J7D7DLFI0rQcVyxJkmZVVTe0np8XA+9L8rWqet/E5u6u7XUbYN+q+mX3PEkAXlFVN0zT\nPqlpplBm2HYGgx60I2Y4ZtJ66zXbqao2tLaXMCjyDgHeneQZVXVfu9Zsc7gkaQp7sCRJ0qySPA74\neVWtBT4EdO/w9+r2ehjwrbZ8DnBs5/hnddqP6bTv1Rb/FTi8tT0D2HOGUG4Bdu6G1l6/ABwPnDvN\nMQcm+ZVWVB0KfAM4ALiwXS/ArlX1deAdwI7AI9uxj2vXlKTNZg+WJEnalGcymC91H/B/TJ6n9Kgk\nVwC/YHCTCRgUVx9r7Q9hUEAdBbwP+Ei7jXqAmxj0Gp0ErElyNXAtk+dZdV3EYH7WurZeAFV1F4PC\nb7resEsYDA1cDpxaVevards/27Y/BPhMkh1bTCdU1Z1t20rg7E3+dCSpI4Ob5EiSJG2ZJDcBz66q\n2x6k6z0ROLGqXrKV57mMwRDGe2fZJwwKuedU1T1bcz1JS4tDBCVJ0lw9qN/Studf3TlxO/itOM8+\nsxVXzcHA5y2uJG0pe7AkSZIkaZ7YgyVJkiRJ88QCS5IkSZLmiQWWJEmSJM0TCyxJkiRJmicWWJIk\nSZI0T/4f8YECY6U7o3gAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "co_t, de_t = compression_decompression_times()\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Decompression speed', fontsize=14, y=1.01)\n", - "\n", - "\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "xlim = (0, np.max((nbytes / 1000000) / np.array(de_t)) + 100)\n", - "ax.set_xlim(*xlim)\n", - "ax.set_ylim(0, len(de_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import cpuinfo" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Vendor ID: GenuineIntel\n", - "Hardware Raw: \n", - "Brand: Intel(R) Xeon(R) CPU E3-1505M v5 @ 2.80GHz\n", - "Hz Advertised: 2.8000 GHz\n", - "Hz Actual: 1.1000 GHz\n", - "Hz Advertised Raw: (2800000000, 0)\n", - "Hz Actual Raw: (1100000000, 0)\n", - "Arch: X86_64\n", - "Bits: 64\n", - "Count: 8\n", - "Raw Arch String: x86_64\n", - "L2 Cache Size: 8192 KB\n", - "L2 Cache Line Size: 0\n", - "L2 Cache Associativity: 0\n", - "Stepping: 3\n", - "Model: 94\n", - "Family: 6\n", - "Processor Type: 0\n", - "Extended Model: 0\n", - "Extended Family: 0\n", - "Flags: 3dnowprefetch, abm, acpi, adx, aes, aperfmperf, apic, arat, arch_perfmon, avx, avx2, bmi1, bmi2, bts, clflush, clflushopt, cmov, constant_tsc, cx16, cx8, de, ds_cpl, dtes64, dtherm, dts, eagerfpu, epb, ept, erms, est, f16c, flexpriority, fma, fpu, fsgsbase, fxsr, hle, ht, hwp, hwp_act_window, hwp_epp, hwp_noitfy, ida, invpcid, lahf_lm, lm, mca, mce, mmx, monitor, movbe, mpx, msr, mtrr, nonstop_tsc, nopl, nx, pae, pat, pbe, pcid, pclmulqdq, pdcm, pdpe1gb, pebs, pge, pln, pni, popcnt, pse, pse36, pts, rdrand, rdseed, rdtscp, rep_good, rtm, sep, smap, smep, smx, ss, sse, sse2, sse4_1, sse4_2, ssse3, syscall, tm, tm2, tpr_shadow, tsc, tsc_adjust, tsc_deadline_timer, vme, vmx, vnmi, vpid, x2apic, xgetbv1, xsave, xsavec, xsaveopt, xtopology, xtpr\n" - ] - } - ], - "source": [ - "cpuinfo.main()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/object_arrays.ipynb b/notebooks/object_arrays.ipynb deleted file mode 100644 index 714d024907..0000000000 --- a/notebooks/object_arrays.ipynb +++ /dev/null @@ -1,350 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Object arrays\n", - "\n", - "See [#212](https://github.com/alimanfoo/zarr/pull/212) for more information." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.2.0a2.dev82+dirty'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.5.0'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numcodecs\n", - "numcodecs.__version__" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## API changes in Zarr version 2.2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Creation of an object array requires providing new ``object_codec`` argument:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To maintain backwards compatibility with previously-created data, the object codec is treated as a filter and inserted as the first filter in the chain:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeobject
Shape(10,)
Chunk shape(5,)
OrderC
Read-onlyFalse
Filter [0]MsgPack(encoding='utf-8')
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes80
No. bytes stored396
Storage ratio0.2
Chunks initialized0/2
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : object\n", - "Shape : (10,)\n", - "Chunk shape : (5,)\n", - "Order : C\n", - "Read-only : False\n", - "Filter [0] : MsgPack(encoding='utf-8')\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 80\n", - "No. bytes stored : 396\n", - "Storage ratio : 0.2\n", - "Chunks initialized : 0/2" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['foo', 'bar', 1, list([2, 4, 6, 'baz']), {'a': 'b', 'c': 'd'}, None,\n", - " None, None, None, None], dtype=object)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z[0] = 'foo'\n", - "z[1] = b'bar' # msgpack doesn't support bytes objects correctly\n", - "z[2] = 1\n", - "z[3] = [2, 4, 6, 'baz']\n", - "z[4] = {'a': 'b', 'c': 'd'}\n", - "a = z[:]\n", - "a" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If no ``object_codec`` is provided, a ``ValueError`` is raised:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "missing object_codec for object array", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzarr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/creation.py\u001b[0m in \u001b[0;36mempty\u001b[0;34m(shape, **kwargs)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \"\"\"\n\u001b[0;32m--> 206\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/creation.py\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(shape, chunks, dtype, compressor, fill_value, order, store, synchronizer, overwrite, path, chunk_store, filters, cache_metadata, read_only, object_codec, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor,\n\u001b[1;32m 113\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m chunk_store=chunk_store, filters=filters, object_codec=object_codec)\n\u001b[0m\u001b[1;32m 115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0;31m# instantiate array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/storage.py\u001b[0m in \u001b[0;36minit_array\u001b[0;34m(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec)\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[0mchunk_store\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunk_store\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfilters\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m object_codec=object_codec)\n\u001b[0m\u001b[1;32m 292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/storage.py\u001b[0m in \u001b[0;36m_init_array_metadata\u001b[0;34m(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec)\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfilters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[0;31m# there are no filters so we can be sure there is no object codec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 348\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'missing object_codec for object array'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 349\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 350\u001b[0m \u001b[0;31m# one of the filters may be an object codec, issue a warning rather\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mValueError\u001b[0m: missing object_codec for object array" - ] - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For API backward-compatibility, if object codec is provided via filters, issue a warning but don't raise an error." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/aliman/src/github/alimanfoo/zarr/zarr/storage.py:353: FutureWarning: missing object_codec for object array; this will raise a ValueError in version 3.0\n", - " 'ValueError in version 3.0', FutureWarning)\n" - ] - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, filters=[numcodecs.MsgPack()])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If a user tries to subvert the system and create an object array with no object codec, a runtime check is added to ensure no object arrays are passed down to the compressor (which could lead to nasty errors and/or segfaults):" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z._filters = None # try to live dangerously, manually wipe filters" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "cannot write object array without object codec", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'foo'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, selection, value)\u001b[0m\n\u001b[1;32m 1094\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1095\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1096\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1097\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1098\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mset_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mset_basic_selection\u001b[0;34m(self, selection, value, fields)\u001b[0m\n\u001b[1;32m 1189\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1190\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1191\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_basic_selection_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mset_orthogonal_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_set_basic_selection_nd\u001b[0;34m(self, selection, value, fields)\u001b[0m\n\u001b[1;32m 1480\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1482\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1484\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_set_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_set_selection\u001b[0;34m(self, indexer, value, fields)\u001b[0m\n\u001b[1;32m 1528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1529\u001b[0m \u001b[0;31m# put data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1530\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_chunk_setitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_coords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_selection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1532\u001b[0m def _chunk_getitem(self, chunk_coords, chunk_selection, out, out_selection,\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_setitem\u001b[0;34m(self, chunk_coords, chunk_selection, value, fields)\u001b[0m\n\u001b[1;32m 1633\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1634\u001b[0m self._chunk_setitem_nosync(chunk_coords, chunk_selection, value,\n\u001b[0;32m-> 1635\u001b[0;31m fields=fields)\n\u001b[0m\u001b[1;32m 1636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1637\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_chunk_setitem_nosync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_coords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_selection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_setitem_nosync\u001b[0;34m(self, chunk_coords, chunk_selection, value, fields)\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1708\u001b[0m \u001b[0;31m# encode chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1709\u001b[0;31m \u001b[0mcdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_encode_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1711\u001b[0m \u001b[0;31m# store\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_encode_chunk\u001b[0;34m(self, chunk)\u001b[0m\n\u001b[1;32m 1753\u001b[0m \u001b[0;31m# check object encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1754\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1755\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cannot write object array without object codec'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1756\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1757\u001b[0m \u001b[0;31m# compress\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: cannot write object array without object codec" - ] - } - ], - "source": [ - "z[0] = 'foo'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here is another way to subvert the system, wiping filters **after** storing some data. To cover this case a runtime check is added to ensure no object arrays are handled inappropriately during decoding (which could lead to nasty errors and/or segfaults)." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', 'Hei maailma!',\n", - " 'Xin chào thế giới', 'Njatjeta Botë!', 'Γεια σου κόσμε!', 'こんにちは世界',\n", - " '世界，你好！', 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from numcodecs.tests.common import greetings\n", - "z = zarr.array(greetings, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "cannot read object array without object codec", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# try to live dangerously, manually wipe filters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 553\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 555\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEllipsis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mget_basic_selection\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 678\u001b[0m return self._get_basic_selection_nd(selection=selection, out=out,\n\u001b[0;32m--> 679\u001b[0;31m fields=fields)\n\u001b[0m\u001b[1;32m 680\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 681\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_basic_selection_nd\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 719\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 720\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 721\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 722\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 723\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_orthogonal_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_selection\u001b[0;34m(self, indexer, out, fields)\u001b[0m\n\u001b[1;32m 1007\u001b[0m \u001b[0;31m# load chunk selection into output array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1008\u001b[0m self._chunk_getitem(chunk_coords, chunk_selection, out, out_selection,\n\u001b[0;32m-> 1009\u001b[0;31m drop_axes=indexer.drop_axes, fields=fields)\n\u001b[0m\u001b[1;32m 1010\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1011\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_getitem\u001b[0;34m(self, chunk_coords, chunk_selection, out, out_selection, drop_axes, fields)\u001b[0m\n\u001b[1;32m 1597\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1598\u001b[0m \u001b[0;31m# decode chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1599\u001b[0;31m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_decode_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1601\u001b[0m \u001b[0;31m# select data from chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_decode_chunk\u001b[0;34m(self, cdata)\u001b[0m\n\u001b[1;32m 1733\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1734\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1735\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cannot read object array without object codec'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1736\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1737\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: cannot read object array without object codec" - ] - } - ], - "source": [ - "z._filters = [] # try to live dangerously, manually wipe filters\n", - "z[:]" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/repr_info.ipynb b/notebooks/repr_info.ipynb deleted file mode 100644 index 487a4175ba..0000000000 --- a/notebooks/repr_info.ipynb +++ /dev/null @@ -1,365 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import zarr" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root = zarr.group()\n", - "root" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members0
No. arrays0
No. groups0
" - ], - "text/plain": [ - "Name : /\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 0\n", - "No. arrays : 0\n", - "No. groups : 0" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root.info" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "z = root.zeros('foo/bar/baz', shape=1000000)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar/baz
Typezarr.core.Array
Data typefloat64
Shape(1000000,)
Chunk shape(15625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typezarr.storage.DictStore
No. bytes8000000 (7.6M)
No. bytes stored321
Storage ratio24922.1
Chunks initialized0/64
" - ], - "text/plain": [ - "Name : /foo/bar/baz\n", - "Type : zarr.core.Array\n", - "Data type : float64\n", - "Shape : (1000000,)\n", - "Chunk shape : (15625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : zarr.storage.DictStore\n", - "No. bytes : 8000000 (7.6M)\n", - "No. bytes stored : 321\n", - "Storage ratio : 24922.1\n", - "Chunks initialized : 0/64" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "z[:] = 42" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar/baz
Typezarr.core.Array
Data typefloat64
Shape(1000000,)
Chunk shape(15625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typezarr.storage.DictStore
No. bytes8000000 (7.6M)
No. bytes stored39553 (38.6K)
Storage ratio202.3
Chunks initialized64/64
" - ], - "text/plain": [ - "Name : /foo/bar/baz\n", - "Type : zarr.core.Array\n", - "Data type : float64\n", - "Shape : (1000000,)\n", - "Chunk shape : (15625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : zarr.storage.DictStore\n", - "No. bytes : 8000000 (7.6M)\n", - "No. bytes stored : 39553 (38.6K)\n", - "Storage ratio : 202.3\n", - "Chunks initialized : 64/64" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(1000):\n", - " root.create_group(i)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members1001
No. arrays0
No. groups1001
Groups0, 1, 10, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 11, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 12, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 13, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 14, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 15, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 16, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 17, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 18, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 19, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 2, 20, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 21, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 22, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 23, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 24, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 25, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 26, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 27, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 28, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 29, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 3, 30, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 31, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 32, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 33, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 34, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 35, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 36, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 37, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 38, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 39, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 4, 40, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 41, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 42, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 43, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 44, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 45, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 46, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 47, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 48, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 49, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 5, 50, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 51, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 52, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 53, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 54, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 55, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 56, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 57, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 58, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 59, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 6, 60, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 61, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 62, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 63, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 64, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 65, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 66, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 67, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 68, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 69, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 7, 70, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 71, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 72, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 73, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 74, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 75, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 76, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 77, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 78, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 79, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 8, 80, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 81, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 82, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 83, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 84, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 85, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 86, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 87, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 88, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 89, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 9, 90, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 91, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 92, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 93, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 94, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 95, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 96, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 97, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 98, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 99, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, foo
" - ], - "text/plain": [ - "Name : /\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 1001\n", - "No. arrays : 0\n", - "No. groups : 1001\n", - "Groups : 0, 1, 10, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 11,\n", - " : 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 12, 120, 121,\n", - " : 122, 123, 124, 125, 126, 127, 128, 129, 13, 130, 131, 132, 133,\n", - " : 134, 135, 136, 137, 138, 139, 14, 140, 141, 142, 143, 144, 145,\n", - " : 146, 147, 148, 149, 15, 150, 151, 152, 153, 154, 155, 156, 157,\n", - " : 158, 159, 16, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,\n", - " : 17, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 18, 180,\n", - " : 181, 182, 183, 184, 185, 186, 187, 188, 189, 19, 190, 191, 192,\n", - " : 193, 194, 195, 196, 197, 198, 199, 2, 20, 200, 201, 202, 203, 204,\n", - " : 205, 206, 207, 208, 209, 21, 210, 211, 212, 213, 214, 215, 216,\n", - " : 217, 218, 219, 22, 220, 221, 222, 223, 224, 225, 226, 227, 228,\n", - " : 229, 23, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 24,\n", - " : 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 25, 250, 251,\n", - " : 252, 253, 254, 255, 256, 257, 258, 259, 26, 260, 261, 262, 263,\n", - " : 264, 265, 266, 267, 268, 269, 27, 270, 271, 272, 273, 274, 275,\n", - " : 276, 277, 278, 279, 28, 280, 281, 282, 283, 284, 285, 286, 287,\n", - " : 288, 289, 29, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 3,\n", - " : 30, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 31, 310,\n", - " : 311, 312, 313, 314, 315, 316, 317, 318, 319, 32, 320, 321, 322,\n", - " : 323, 324, 325, 326, 327, 328, 329, 33, 330, 331, 332, 333, 334,\n", - " : 335, 336, 337, 338, 339, 34, 340, 341, 342, 343, 344, 345, 346,\n", - " : 347, 348, 349, 35, 350, 351, 352, 353, 354, 355, 356, 357, 358,\n", - " : 359, 36, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 37,\n", - " : 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 38, 380, 381,\n", - " : 382, 383, 384, 385, 386, 387, 388, 389, 39, 390, 391, 392, 393,\n", - " : 394, 395, 396, 397, 398, 399, 4, 40, 400, 401, 402, 403, 404, 405,\n", - " : 406, 407, 408, 409, 41, 410, 411, 412, 413, 414, 415, 416, 417,\n", - " : 418, 419, 42, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429,\n", - " : 43, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 44, 440,\n", - " : 441, 442, 443, 444, 445, 446, 447, 448, 449, 45, 450, 451, 452,\n", - " : 453, 454, 455, 456, 457, 458, 459, 46, 460, 461, 462, 463, 464,\n", - " : 465, 466, 467, 468, 469, 47, 470, 471, 472, 473, 474, 475, 476,\n", - " : 477, 478, 479, 48, 480, 481, 482, 483, 484, 485, 486, 487, 488,\n", - " : 489, 49, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 5, 50,\n", - " : 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 51, 510, 511,\n", - " : 512, 513, 514, 515, 516, 517, 518, 519, 52, 520, 521, 522, 523,\n", - " : 524, 525, 526, 527, 528, 529, 53, 530, 531, 532, 533, 534, 535,\n", - " : 536, 537, 538, 539, 54, 540, 541, 542, 543, 544, 545, 546, 547,\n", - " : 548, 549, 55, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559,\n", - " : 56, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 57, 570,\n", - " : 571, 572, 573, 574, 575, 576, 577, 578, 579, 58, 580, 581, 582,\n", - " : 583, 584, 585, 586, 587, 588, 589, 59, 590, 591, 592, 593, 594,\n", - " : 595, 596, 597, 598, 599, 6, 60, 600, 601, 602, 603, 604, 605, 606,\n", - " : 607, 608, 609, 61, 610, 611, 612, 613, 614, 615, 616, 617, 618,\n", - " : 619, 62, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 63,\n", - " : 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 64, 640, 641,\n", - " : 642, 643, 644, 645, 646, 647, 648, 649, 65, 650, 651, 652, 653,\n", - " : 654, 655, 656, 657, 658, 659, 66, 660, 661, 662, 663, 664, 665,\n", - " : 666, 667, 668, 669, 67, 670, 671, 672, 673, 674, 675, 676, 677,\n", - " : 678, 679, 68, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689,\n", - " : 69, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 7, 70, 700,\n", - " : 701, 702, 703, 704, 705, 706, 707, 708, 709, 71, 710, 711, 712,\n", - " : 713, 714, 715, 716, 717, 718, 719, 72, 720, 721, 722, 723, 724,\n", - " : 725, 726, 727, 728, 729, 73, 730, 731, 732, 733, 734, 735, 736,\n", - " : 737, 738, 739, 74, 740, 741, 742, 743, 744, 745, 746, 747, 748,\n", - " : 749, 75, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 76,\n", - " : 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 77, 770, 771,\n", - " : 772, 773, 774, 775, 776, 777, 778, 779, 78, 780, 781, 782, 783,\n", - " : 784, 785, 786, 787, 788, 789, 79, 790, 791, 792, 793, 794, 795,\n", - " : 796, 797, 798, 799, 8, 80, 800, 801, 802, 803, 804, 805, 806, 807,\n", - " : 808, 809, 81, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819,\n", - " : 82, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 83, 830,\n", - " : 831, 832, 833, 834, 835, 836, 837, 838, 839, 84, 840, 841, 842,\n", - " : 843, 844, 845, 846, 847, 848, 849, 85, 850, 851, 852, 853, 854,\n", - " : 855, 856, 857, 858, 859, 86, 860, 861, 862, 863, 864, 865, 866,\n", - " : 867, 868, 869, 87, 870, 871, 872, 873, 874, 875, 876, 877, 878,\n", - " : 879, 88, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 89,\n", - " : 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 9, 90, 900, 901,\n", - " : 902, 903, 904, 905, 906, 907, 908, 909, 91, 910, 911, 912, 913,\n", - " : 914, 915, 916, 917, 918, 919, 92, 920, 921, 922, 923, 924, 925,\n", - " : 926, 927, 928, 929, 93, 930, 931, 932, 933, 934, 935, 936, 937,\n", - " : 938, 939, 94, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949,\n", - " : 95, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 96, 960,\n", - " : 961, 962, 963, 964, 965, 966, 967, 968, 969, 97, 970, 971, 972,\n", - " : 973, 974, 975, 976, 977, 978, 979, 98, 980, 981, 982, 983, 984,\n", - " : 985, 986, 987, 988, 989, 99, 990, 991, 992, 993, 994, 995, 996,\n", - " : 997, 998, 999, foo" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root.info" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members1
No. arrays1
No. groups0
Arraysbaz
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 1\n", - "No. arrays : 1\n", - "No. groups : 0\n", - "Arrays : baz" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root['foo/bar'].info" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/store_benchmark.ipynb b/notebooks/store_benchmark.ipynb deleted file mode 100644 index 869e7df608..0000000000 --- a/notebooks/store_benchmark.ipynb +++ /dev/null @@ -1,1303 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are lies, damn lies and benchmarks..." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.2.0a2.dev22+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'6.2.5'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import bsddb3\n", - "bsddb3.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.93'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import lmdb\n", - "lmdb.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import dbm.gnu\n", - "import dbm.ndbm" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import shutil\n", - "bench_dir = '../data/bench'\n", - "\n", - "\n", - "def clean():\n", - " if os.path.isdir(bench_dir):\n", - " shutil.rmtree(bench_dir)\n", - " os.makedirs(bench_dir)\n", - "\n", - " \n", - "def setup(a, name='foo/bar'):\n", - " global fdict_z, hdict_z, lmdb_z, gdbm_z, ndbm_z, bdbm_btree_z, bdbm_hash_z, zip_z, dir_z\n", - " \n", - " clean()\n", - " fdict_root = zarr.group(store=dict())\n", - " hdict_root = zarr.group(store=zarr.DictStore())\n", - " lmdb_root = zarr.group(store=zarr.LMDBStore(os.path.join(bench_dir, 'lmdb')))\n", - " gdbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'gdbm'), open=dbm.gnu.open))\n", - " ndbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'ndbm'), open=dbm.ndbm.open))\n", - " bdbm_btree_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_btree'), open=bsddb3.btopen))\n", - " bdbm_hash_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_hash'), open=bsddb3.hashopen))\n", - " zip_root = zarr.group(store=zarr.ZipStore(os.path.join(bench_dir, 'zip'), mode='w'))\n", - " dir_root = zarr.group(store=zarr.DirectoryStore(os.path.join(bench_dir, 'dir')))\n", - "\n", - " fdict_z = fdict_root.empty_like(name, a)\n", - " hdict_z = hdict_root.empty_like(name, a)\n", - " lmdb_z = lmdb_root.empty_like(name, a)\n", - " gdbm_z = gdbm_root.empty_like(name, a)\n", - " ndbm_z = ndbm_root.empty_like(name, a)\n", - " bdbm_btree_z = bdbm_btree_root.empty_like(name, a)\n", - " bdbm_hash_z = bdbm_hash_root.empty_like(name, a)\n", - " zip_z = zip_root.empty_like(name, a)\n", - " dir_z = dir_root.empty_like(name, a)\n", - "\n", - " # check compression ratio\n", - " fdict_z[:] = a\n", - " return fdict_z.info\n", - " \n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Main benchmarks" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def save(a, z):\n", - " if isinstance(z.store, zarr.ZipStore):\n", - " # needed for zip benchmarks to avoid duplicate entries\n", - " z.store.clear()\n", - " z[:] = a\n", - " if hasattr(z.store, 'flush'):\n", - " z.store.flush()\n", - " \n", - " \n", - "def load(z, a):\n", - " z.get_basic_selection(out=a)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## arange" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.core.Array
Data typeint64
Shape(500000000,)
Chunk shape(488282,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes4000000000 (3.7G)
No. bytes stored59269657 (56.5M)
Storage ratio67.5
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (500000000,)\n", - "Chunk shape : (488282,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 4000000000 (3.7G)\n", - "No. bytes stored : 59269657 (56.5M)\n", - "Storage ratio : 67.5\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(500000000)\n", - "setup(a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### save" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "324 ms ± 60.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "302 ms ± 11.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "316 ms ± 12.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "938 ms ± 111 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "406 ms ± 8.93 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.43 s ± 156 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.24 s ± 260 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "519 ms ± 59.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "609 ms ± 48.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, dir_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### load" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "454 ms ± 56.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(fdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "428 ms ± 13.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(hdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "429 ms ± 19.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(lmdb_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "459 ms ± 10 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(gdbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "473 ms ± 5.71 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(ndbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "504 ms ± 8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(bdbm_btree_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "519 ms ± 9.59 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(bdbm_hash_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "575 ms ± 12.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(zip_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "494 ms ± 10.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(dir_z, a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## randint" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.core.Array
Data typeint64
Shape(500000000,)
Chunk shape(488282,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes4000000000 (3.7G)
No. bytes stored2020785466 (1.9G)
Storage ratio2.0
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (500000000,)\n", - "Chunk shape : (488282,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 4000000000 (3.7G)\n", - "No. bytes stored : 2020785466 (1.9G)\n", - "Storage ratio : 2.0\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.random.seed(42)\n", - "a = np.random.randint(0, 2**30, size=500000000)\n", - "setup(a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### save" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "670 ms ± 78.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "611 ms ± 6.11 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "846 ms ± 24 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.35 s ± 785 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4.62 s ± 1.09 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "7.84 s ± 1.66 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.49 s ± 808 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.68 s ± 441 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.55 s ± 1.24 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, dir_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### load" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "566 ms ± 72.8 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(fdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "521 ms ± 16.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(hdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "532 ms ± 16.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(lmdb_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.2 s ± 10.9 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(gdbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.18 s ± 13.2 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(ndbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.59 s ± 16.7 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(bdbm_btree_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.61 s ± 7.31 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(bdbm_hash_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2.33 s ± 19.8 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(zip_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "790 ms ± 56 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(dir_z, a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### dask" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "import dask.array as da" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [], - "source": [ - "def dask_op(source, sink, chunks=None):\n", - " if isinstance(sink.store, zarr.ZipStore):\n", - " sink.store.clear()\n", - " if chunks is None:\n", - " try:\n", - " chunks = sink.chunks\n", - " except AttributeError:\n", - " chunks = source.chunks\n", - " d = da.from_array(source, chunks=chunks, asarray=False, fancy=False, lock=False)\n", - " result = (d // 2) * 2\n", - " da.store(result, sink, lock=False)\n", - " if hasattr(sink.store, 'flush'):\n", - " sink.store.flush()\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Compare sources" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.6 s, sys: 1.8 s, total: 17.4 s\n", - "Wall time: 3.07 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.5 s, sys: 104 ms, total: 16.6 s\n", - "Wall time: 2.59 s\n" - ] - } - ], - "source": [ - "%time dask_op(hdict_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.1 s, sys: 524 ms, total: 15.6 s\n", - "Wall time: 3.02 s\n" - ] - } - ], - "source": [ - "%time dask_op(lmdb_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.5 s, sys: 712 ms, total: 17.2 s\n", - "Wall time: 3.13 s\n" - ] - } - ], - "source": [ - "%time dask_op(gdbm_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.3 s, sys: 604 ms, total: 16.9 s\n", - "Wall time: 3.22 s\n" - ] - } - ], - "source": [ - "%time dask_op(ndbm_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 19.6 s, sys: 1.24 s, total: 20.9 s\n", - "Wall time: 3.27 s\n" - ] - } - ], - "source": [ - "%time dask_op(bdbm_btree_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 20.3 s, sys: 1.08 s, total: 21.4 s\n", - "Wall time: 3.53 s\n" - ] - } - ], - "source": [ - "%time dask_op(bdbm_hash_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.7 s, sys: 700 ms, total: 16.4 s\n", - "Wall time: 3.12 s\n" - ] - } - ], - "source": [ - "%time dask_op(zip_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 17.4 s, sys: 1.08 s, total: 18.5 s\n", - "Wall time: 2.91 s\n" - ] - } - ], - "source": [ - "%time dask_op(dir_z, fdict_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Compare sinks" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.8 s, sys: 1.4 s, total: 17.2 s\n", - "Wall time: 3.04 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.2 s, sys: 1.6 s, total: 17.8 s\n", - "Wall time: 2.71 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.8 s, sys: 3.05 s, total: 19.8 s\n", - "Wall time: 8.01 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 17.9 s, sys: 3.01 s, total: 20.9 s\n", - "Wall time: 5.46 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.8 s, sys: 3.39 s, total: 17.2 s\n", - "Wall time: 7.87 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.9 s, sys: 3.27 s, total: 17.2 s\n", - "Wall time: 6.73 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.9 s, sys: 2.5 s, total: 16.4 s\n", - "Wall time: 3.8 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.7 s, sys: 3.72 s, total: 19.4 s\n", - "Wall time: 3.1 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, dir_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "lmdb_z.store.close()\n", - "gdbm_z.store.close()\n", - "ndbm_z.store.close()\n", - "bdbm_btree_z.store.close()\n", - "bdbm_hash_z.store.close()\n", - "zip_z.store.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/zip_benchmark.ipynb b/notebooks/zip_benchmark.ipynb deleted file mode 100644 index 6805552422..0000000000 --- a/notebooks/zip_benchmark.ipynb +++ /dev/null @@ -1,343 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.2.dev0+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3L/calldata/genotype, (7449486, 773, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 10.7G; nbytes_stored: 193.5M; ratio: 56.7; initialized: 11380/11380\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: ZipStore" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "store = zarr.ZipStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes.zip',\n", - " mode='r')\n", - "grp = zarr.Group(store)\n", - "z = grp['3L/calldata/genotype']\n", - "z" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 1832 function calls in 0.024 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 0.024 0.024 {built-in method builtins.exec}\n", - " 1 0.000 0.000 0.024 0.024 :1()\n", - " 1 0.000 0.000 0.024 0.024 core.py:292(__getitem__)\n", - " 20 0.000 0.000 0.023 0.001 core.py:539(_chunk_getitem)\n", - " 20 0.000 0.000 0.020 0.001 core.py:679(_decode_chunk)\n", - " 20 0.000 0.000 0.020 0.001 codecs.py:355(decode)\n", - " 20 0.020 0.001 0.020 0.001 {zarr.blosc.decompress}\n", - " 20 0.000 0.000 0.002 0.000 storage.py:766(__getitem__)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:1235(open)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:821(read)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:901(_read1)\n", - " 80 0.000 0.000 0.001 0.000 zipfile.py:660(read)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:854(_update_crc)\n", - " 40 0.000 0.000 0.000 0.000 {built-in method zlib.crc32}\n", - " 80 0.000 0.000 0.000 0.000 {method 'read' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:937(_read2)\n", - " 80 0.000 0.000 0.000 0.000 core.py:390()\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:953(close)\n", - " 20 0.000 0.000 0.000 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - " 20 0.000 0.000 0.000 0.000 util.py:106(is_total_slice)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:708(__init__)\n", - " 20 0.000 0.000 0.000 0.000 {method 'decode' of 'bytes' objects}\n", - " 20 0.000 0.000 0.000 0.000 core.py:676(_chunk_key)\n", - " 80 0.000 0.000 0.000 0.000 {method 'seek' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 {built-in method numpy.core.multiarray.frombuffer}\n", - " 80 0.000 0.000 0.000 0.000 core.py:398()\n", - " 20 0.000 0.000 0.000 0.000 {method 'join' of 'str' objects}\n", - " 20 0.000 0.000 0.000 0.000 core.py:386()\n", - " 20 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 40 0.000 0.000 0.000 0.000 util.py:121()\n", - " 231 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance}\n", - " 20 0.000 0.000 0.000 0.000 cp437.py:14(decode)\n", - " 80 0.000 0.000 0.000 0.000 {method 'tell' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:667(close)\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _struct.unpack}\n", - " 140 0.000 0.000 0.000 0.000 {built-in method builtins.max}\n", - " 20 0.000 0.000 0.000 0.000 {function ZipExtFile.close at 0x7f8cd5ca2048}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:1194(getinfo)\n", - " 140 0.000 0.000 0.000 0.000 {built-in method builtins.min}\n", - " 20 0.000 0.000 0.000 0.000 threading.py:1224(current_thread)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:654(__init__)\n", - " 1 0.000 0.000 0.000 0.000 util.py:195(get_chunk_range)\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _codecs.charmap_decode}\n", - " 1 0.000 0.000 0.000 0.000 util.py:166(normalize_array_selection)\n", - " 1 0.000 0.000 0.000 0.000 util.py:198()\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:1715(_fpclose)\n", - " 20 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", - " 63 0.000 0.000 0.000 0.000 {built-in method builtins.len}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method numpy.core.multiarray.empty}\n", - " 2 0.000 0.000 0.000 0.000 util.py:182()\n", - " 20 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _thread.get_ident}\n", - " 1 0.000 0.000 0.000 0.000 util.py:130(normalize_axis_selection)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:636(_get_decompressor)\n", - " 20 0.000 0.000 0.000 0.000 threading.py:1298(main_thread)\n", - " 4 0.000 0.000 0.000 0.000 core.py:373()\n", - " 3 0.000 0.000 0.000 0.000 util.py:187()\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "import cProfile\n", - "cProfile.run('z[:10]', sort='cumtime')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.11.0'" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import dask\n", - "import dask.array as da\n", - "dask.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d = da.from_array(z, chunks=z.chunks)\n", - "d" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3min 35s, sys: 4.36 s, total: 3min 40s\n", - "Wall time: 29.5 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[3, 0],\n", - " [1, 0],\n", - " [2, 0],\n", - " ..., \n", - " [2, 8],\n", - " [8, 8],\n", - " [0, 1]])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d.sum(axis=1).compute()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3L/calldata/genotype, (7449486, 773, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 10.7G; nbytes_stored: 193.5M; ratio: 56.7; initialized: 11380/11380\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: DirectoryStore" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# compare with same data via directory store\n", - "store_dir = zarr.DirectoryStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes')\n", - "grp_dir = zarr.Group(store_dir)\n", - "z_dir = grp_dir['3L/calldata/genotype']\n", - "z_dir" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d_dir = da.from_array(z_dir, chunks=z_dir.chunks)\n", - "d_dir" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3min 39s, sys: 4.91 s, total: 3min 44s\n", - "Wall time: 31.1 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[3, 0],\n", - " [1, 0],\n", - " [2, 0],\n", - " ..., \n", - " [2, 8],\n", - " [8, 8],\n", - " [0, 1]])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d_dir.sum(axis=1).compute()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/pyproject.toml b/pyproject.toml index 05db0860a8..6c18563a1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,12 @@ requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" +[tool.hatch.build.targets.sdist] +exclude = [ + "/.github", + "/bench", + "/docs", +] [project] name = "zarr" @@ -57,6 +63,7 @@ keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] # User extras remote = [ "fsspec>=2023.10.0", + "obstore>=0.5.1", ] gpu = [ "cupy-cuda12x", @@ -64,20 +71,26 @@ gpu = [ # Development extras test = [ "coverage", - "pytest", - "pytest-cov", - "s3fs", + # Pin possibly due to https://github.com/pytest-dev/pytest-cov/issues/693 + "pytest<8.4", "pytest-asyncio", + "pytest-cov", "pytest-accept", - "moto[s3,server]", - "requests", "rich", "mypy", "hypothesis", - "universal-pathlib", + "pytest-xdist", +] +remote_tests = [ + 'zarr[remote]', + "botocore", + "s3fs>=2023.10.0", + "moto[s3,server]", + "requests", ] optional = ["rich", "universal-pathlib"] docs = [ + # Doc building 'sphinx==8.1.3', 'sphinx-autobuild>=2021.3.14', 'sphinx-autoapi==3.4.0', @@ -87,69 +100,76 @@ docs = [ 'sphinx-reredirects', 'pydata-sphinx-theme', 'numpydoc', + # Changelog generation + 'towncrier', + # Optional dependencies to run examples 'numcodecs[msgpack]', 'rich', - 's3fs', + 's3fs>=2023.10.0', + 'astroid<4' ] [project.urls] "Bug Tracker" = "https://github.com/zarr-developers/zarr-python/issues" -Changelog = "https://zarr.readthedocs.io/en/stable/release.html" +Changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" Discussions = "https://github.com/zarr-developers/zarr-python/discussions" Documentation = "https://zarr.readthedocs.io/" Homepage = "https://github.com/zarr-developers/zarr-python" +[dependency-groups] +dev = [ + "ipykernel>=6.29.5", + "pip>=25.0.1", +] + [tool.coverage.report] exclude_lines = [ "pragma: no cover", + "if TYPE_CHECKING:", "pragma: ${PY_MAJOR_VERSION} no cover", '.*\.\.\.' # Ignore "..." lines ] [tool.coverage.run] omit = [ - "src/zarr/meta_v1.py", "bench/compress_normal.py", ] [tool.hatch] version.source = "vcs" -build.hooks.vcs.version-file = "src/zarr/_version.py" + +[tool.hatch.build] +hooks.vcs.version-file = "src/zarr/_version.py" [tool.hatch.envs.test] dependencies = [ "numpy~={matrix:numpy}", - "universal_pathlib", ] features = ["test"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] -version = ["minimal"] +numpy = ["1.25", "2.2"] +deps = ["minimal", "optional"] -[[tool.hatch.envs.test.matrix]] -python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] -features = ["optional"] - -[[tool.hatch.envs.test.matrix]] -python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] -features = ["gpu"] +[tool.hatch.envs.test.overrides] +matrix.deps.dependencies = [ + {value = "zarr[remote, remote_tests, test, optional]", if = ["optional"]} +] [tool.hatch.envs.test.scripts] -run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=src" -run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=src" +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" run = "run-coverage --no-cov" +run-pytest = "run" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +run-hypothesis = "run-coverage -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.doctest] -features = ["test", "optional"] +features = ["test", "optional", "remote", "remote_tests"] description = "Test environment for doctests" [tool.hatch.envs.doctest.scripts] @@ -166,15 +186,15 @@ features = ["test", "gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2.1"] +numpy = ["1.25", "2.2"] version = ["minimal"] [tool.hatch.envs.gputest.scripts] -run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=src" +run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.docs] @@ -190,17 +210,13 @@ dependencies = [ 'packaging @ git+https://github.com/pypa/packaging', 'numpy', # from scientific-python-nightly-wheels 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', - 'fsspec @ git+https://github.com/fsspec/filesystem_spec', 's3fs @ git+https://github.com/fsspec/s3fs', 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', 'typing_extensions @ git+https://github.com/python/typing_extensions', 'donfig @ git+https://github.com/pytroll/donfig', + 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', # test deps - 'hypothesis', - 'pytest', - 'pytest-cov', - 'pytest-asyncio', - 'moto[s3]', + 'zarr[test]', ] [tool.hatch.envs.upstream.env-vars] @@ -212,6 +228,9 @@ PIP_PRE = "1" run = "pytest --verbose" run-mypy = "mypy src" run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" list-env = "pip list" [tool.hatch.envs.min_deps] @@ -221,27 +240,28 @@ See Spec 0000 for details and drop schedule: https://scientific-python.org/specs """ python = "3.11" dependencies = [ + 'zarr[remote]', 'packaging==22.*', 'numpy==1.25.*', 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs - 'fsspec==2022.10.0', - 's3fs==2022.10.0', + 'fsspec==2023.10.0', + 's3fs==2023.10.0', 'universal_pathlib==0.0.22', 'typing_extensions==4.9.*', 'donfig==0.8.*', + 'obstore==0.5.*', # test deps - 'hypothesis', - 'pytest', - 'pytest-cov', - 'pytest-asyncio', - 'moto[s3]', + 'zarr[test]', + 'zarr[remote_tests]', ] [tool.hatch.envs.min_deps.scripts] run = "pytest --verbose" run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" - +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" [tool.ruff] line-length = 100 @@ -261,9 +281,9 @@ extend-exclude = [ "buck-out", "build", "dist", - "notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6 "venv", "docs", + "tests/test_regression/scripts/", # these are scripts that use a different version of python "src/zarr/v2/", "tests/v2/", ] @@ -272,8 +292,8 @@ extend-exclude = [ extend-select = [ "ANN", # flake8-annotations "B", # flake8-bugbear - "EXE", # flake8-executable "C4", # flake8-comprehensions + "EXE", # flake8-executable "FA", # flake8-future-annotations "FLY", # flynt "FURB", # refurb @@ -291,7 +311,7 @@ extend-select = [ "RUF", "SIM", # flake8-simplify "SLOT", # flake8-slots - "TCH", # flake8-type-checking + "TC", # flake8-type-checking "TRY", # tryceratops "UP", # pyupgrade "W", # pycodestyle warnings @@ -319,8 +339,7 @@ ignore = [ "Q003", "COM812", "COM819", - "ISC001", - "ISC002", + "TC006", ] [tool.ruff.lint.extend-per-file-ignores] @@ -331,32 +350,39 @@ python_version = "3.11" ignore_missing_imports = true namespace_packages = false - strict = true warn_unreachable = true - enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] [[tool.mypy.overrides]] module = [ - "zarr.v2.*", + "tests.package_with_entrypoint.*", + "zarr.testing.stateful", + "tests.test_codecs.test_transpose", + "tests.test_config", + "tests.test_store.test_zip", + "tests.test_store.test_local", + "tests.test_store.test_fsspec", + "tests.test_store.test_memory", + "tests.test_codecs.test_codecs", ] -ignore_errors = true +strict = false +# TODO: Move the next modules up to the strict = false section +# and fix the errors [[tool.mypy.overrides]] module = [ - "zarr.testing.stateful", # lots of hypothesis decorator errors - "tests.package_with_entrypoint.*", - "tests.test_codecs.test_codecs", - "tests.test_codecs.test_transpose", "tests.test_metadata.*", - "tests.test_store.*", - "tests.test_config", + "tests.test_store.test_core", + "tests.test_store.test_logging", + "tests.test_store.test_object", + "tests.test_store.test_stateful", + "tests.test_store.test_wrapper", "tests.test_group", "tests.test_indexing", "tests.test_properties", "tests.test_sync", - "tests.test_v2", + "tests.test_regression.scripts.*" ] ignore_errors = true @@ -375,15 +401,22 @@ addopts = [ "--durations=10", "-ra", "--strict-config", "--strict-markers", ] filterwarnings = [ - "error:::zarr.*", - "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", - "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", - "ignore:Creating a zarr.buffer.gpu.*:UserWarning", - "ignore:Duplicate name:UserWarning", # from ZipFile - "ignore:.*is currently not part in the Zarr format 3 specification.*:UserWarning", + "error", + # TODO: explicitly filter or catch the warnings below where we expect them to be emitted in the tests + "ignore:Consolidated metadata is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__.*:UserWarning", + "ignore:Automatic shard shape inference is experimental and may change without notice.*:UserWarning", + "ignore:The codec .* is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:The dtype .* is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:Use zarr.create_array instead.:DeprecationWarning", + "ignore:Duplicate name.*:UserWarning", + "ignore:The `compressor` argument is deprecated. Use `compressors` instead.:UserWarning", + "ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.:UserWarning", + "ignore:Unclosed client session None: + """ + Print version info for use in bug reports. + """ + import platform + from importlib.metadata import version + + def print_packages(packages: list[str]) -> None: + not_installed = [] + for package in packages: + try: + print(f"{package}: {version(package)}") + except ModuleNotFoundError: + not_installed.append(package) + if not_installed: + print("\n**Not Installed:**") + for package in not_installed: + print(package) + + required = [ + "packaging", + "numpy", + "numcodecs", + "typing_extensions", + "donfig", + ] + optional = [ + "botocore", + "cupy-cuda12x", + "fsspec", + "numcodecs", + "s3fs", + "gcsfs", + "universal-pathlib", + "rich", + "obstore", + ] + + print(f"platform: {platform.platform()}") + print(f"python: {platform.python_version()}") + print(f"zarr: {__version__}\n") + print("**Required dependencies:**") + print_packages(required) + print("\n**Optional dependencies:**") + print_packages(optional) + + __all__ = [ "Array", "AsyncArray", @@ -50,8 +100,10 @@ "create", "create_array", "create_group", + "create_hierarchy", "empty", "empty_like", + "from_array", "full", "full_like", "group", @@ -63,6 +115,7 @@ "open_consolidated", "open_group", "open_like", + "print_debug_info", "save", "save_array", "save_group", diff --git a/src/zarr/abc/buffer.py b/src/zarr/abc/buffer.py new file mode 100644 index 0000000000..3d5ac07157 --- /dev/null +++ b/src/zarr/abc/buffer.py @@ -0,0 +1,9 @@ +from zarr.core.buffer.core import ArrayLike, Buffer, BufferPrototype, NDArrayLike, NDBuffer + +__all__ = [ + "ArrayLike", + "Buffer", + "BufferPrototype", + "NDArrayLike", + "NDBuffer", +] diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index fabd042dbe..d9e3520d42 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import abstractmethod -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar from zarr.abc.metadata import Metadata from zarr.core.buffer import Buffer, NDBuffer @@ -12,11 +12,10 @@ from collections.abc import Awaitable, Callable, Iterable from typing import Self - import numpy as np - from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.indexing import SelectorTuple __all__ = [ @@ -93,7 +92,13 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: """ return self - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, + *, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: """Validates that the codec configuration is compatible with the array metadata. Raises errors when the codec configuration is not compatible. @@ -285,7 +290,9 @@ def supports_partial_decode(self) -> bool: ... def supports_partial_encode(self) -> bool: ... @abstractmethod - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, *, shape: ChunkCoords, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGrid + ) -> None: """Validates that all codec configurations are compatible with the array metadata. Raises errors when a codec configuration is not compatible. @@ -357,7 +364,7 @@ async def encode( @abstractmethod async def read( self, - batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -379,7 +386,7 @@ async def read( @abstractmethod async def write( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index bd0a7ad503..1fbdb3146c 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from asyncio import gather +from dataclasses import dataclass from itertools import starmap from typing import TYPE_CHECKING, Protocol, runtime_checkable @@ -19,7 +20,34 @@ __all__ = ["ByteGetter", "ByteSetter", "Store", "set_or_delete"] -ByteRangeRequest: TypeAlias = tuple[int | None, int | None] + +@dataclass +class RangeByteRequest: + """Request a specific byte range""" + + start: int + """The start of the byte range request (inclusive).""" + end: int + """The end of the byte range request (exclusive).""" + + +@dataclass +class OffsetByteRequest: + """Request all bytes starting from a given byte offset""" + + offset: int + """The byte offset for the offset range request.""" + + +@dataclass +class SuffixByteRequest: + """Request up to the last `n` bytes""" + + suffix: int + """The number of bytes from the suffix to request.""" + + +ByteRequest: TypeAlias = RangeByteRequest | OffsetByteRequest | SuffixByteRequest class Store(ABC): @@ -55,6 +83,27 @@ async def open(cls, *args: Any, **kwargs: Any) -> Self: await store._open() return store + def with_read_only(self, read_only: bool = False) -> Store: + """ + Return a new store with a new read_only setting. + + The new store points to the same location with the specified new read_only state. + The returned Store is not automatically opened, and this store is + not automatically closed. + + Parameters + ---------- + read_only + If True, the store will be created in read-only mode. Defaults to False. + + Returns + ------- + A new store of the same type with the new read only attribute. + """ + raise NotImplementedError( + f"with_read_only is not implemented for the {type(self)} store type." + ) + def __enter__(self) -> Self: """Enter a context manager that will close the store upon exiting.""" return self @@ -141,14 +190,20 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: """Retrieve the value associated with a given key. Parameters ---------- key : str - byte_range : tuple[int | None, int | None], optional + prototype : BufferPrototype + The prototype of the output buffer. Stores may support a default buffer prototype. + byte_range : ByteRequest, optional + ByteRequest may be one of the following. If not provided, all data associated with the key is retrieved. + - RangeByteRequest(int, int): Request a specific range of bytes in the form (start, end). The end is exclusive. If the given range is zero-length or starts after the end of the object, an error will be returned. Additionally, if the range ends after the end of the object, the entire remainder of the object will be returned. Otherwise, the exact requested range will be returned. + - OffsetByteRequest(int): Request all bytes starting from a given byte offset. This is equivalent to bytes={int}- as an HTTP header. + - SuffixByteRequest(int): Request the last int bytes. Note that here, int is the size of the request, not the byte offset. This is equivalent to bytes=-{int} as an HTTP header. Returns ------- @@ -160,12 +215,14 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: """Retrieve possibly partial values from given key_ranges. Parameters ---------- + prototype : BufferPrototype + The prototype of the output buffer. Stores may support a default buffer prototype. key_ranges : Iterable[tuple[str, tuple[int | None, int | None]]] Ordered set of key, range pairs, a key may occur multiple times with different ranges @@ -228,6 +285,18 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: """ await gather(*starmap(self.set, values)) + @property + def supports_consolidated_metadata(self) -> bool: + """ + Does the store support consolidated metadata?. + + If it doesn't an error will be raised on requests to consolidate the metadata. + Returning `False` can be useful for stores which implement their own + consolidation mechanism outside of the zarr-python implementation. + """ + + return True + @property @abstractmethod def supports_deletes(self) -> bool: @@ -338,7 +407,7 @@ def close(self) -> None: self._is_open = False async def _get_many( - self, requests: Iterable[tuple[str, BufferPrototype, ByteRangeRequest | None]] + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] ) -> AsyncGenerator[tuple[str, Buffer | None], None]: """ Retrieve a collection of objects from storage. In general this method does not guarantee @@ -416,17 +485,17 @@ async def getsize_prefix(self, prefix: str) -> int: @runtime_checkable class ByteGetter(Protocol): async def get( - self, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: ... @runtime_checkable class ByteSetter(Protocol): async def get( - self, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: ... - async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -> None: ... + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: ... async def delete(self) -> None: ... diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 2e98a43f94..3b53095636 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -9,34 +9,48 @@ import numpy.typing as npt from typing_extensions import deprecated -from zarr.core.array import Array, AsyncArray, create_array, get_array_metadata -from zarr.core.array_spec import ArrayConfig, ArrayConfigLike +from zarr.abc.store import Store +from zarr.core.array import ( + Array, + AsyncArray, + CompressorLike, + _get_default_chunk_encoding_v2, + create_array, + from_array, + get_array_metadata, +) +from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams from zarr.core.buffer import NDArrayLike from zarr.core.common import ( JSON, AccessModeLiteral, ChunkCoords, + DimensionNames, MemoryOrder, ZarrFormat, _default_zarr_format, _warn_order_kwarg, _warn_write_empty_chunks_kwarg, - parse_dtype, ) -from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata -from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata -from zarr.core.metadata.v2 import _default_compressor, _default_filters -from zarr.errors import NodeTypeValidationError -from zarr.storage import ( - StoreLike, - make_store_path, +from zarr.core.dtype import ZDTypeLike, get_data_type_from_native_dtype, parse_data_type +from zarr.core.group import ( + AsyncGroup, + ConsolidatedMetadata, + GroupMetadata, + create_hierarchy, ) +from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata +from zarr.errors import GroupNotFoundError, NodeTypeValidationError +from zarr.storage import StorePath +from zarr.storage._common import make_store_path if TYPE_CHECKING: from collections.abc import Iterable from zarr.abc.codec import Codec + from zarr.core.buffer import NDArrayLikeOrScalar from zarr.core.chunk_key_encodings import ChunkKeyEncoding + from zarr.storage import StoreLike # TODO: this type could use some more thought ArrayLike = AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array | npt.NDArray[Any] @@ -50,8 +64,10 @@ "copy_store", "create", "create_array", + "create_hierarchy", "empty", "empty_like", + "from_array", "full", "full_like", "group", @@ -74,7 +90,7 @@ _READ_MODES: tuple[AccessModeLiteral, ...] = ("r", "r+", "a") _CREATE_MODES: tuple[AccessModeLiteral, ...] = ("a", "w", "w-") -_OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("a", "r+", "w") +_OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("w",) def _infer_overwrite(mode: AccessModeLiteral) -> bool: @@ -160,7 +176,8 @@ async def consolidate_metadata( Consolidate the metadata of all nodes in a hierarchy. Upon completion, the metadata of the root node in the Zarr hierarchy will be - updated to include all the metadata of child nodes. + updated to include all the metadata of child nodes. For Stores that do + not support consolidated metadata, this operation raises a ``TypeError``. Parameters ---------- @@ -180,15 +197,25 @@ async def consolidate_metadata( ------- group: AsyncGroup The group, with the ``consolidated_metadata`` field set to include - the metadata of each child node. + the metadata of each child node. If the Store doesn't support + consolidated metadata, this function raises a `TypeError`. + See ``Store.supports_consolidated_metadata``. """ store_path = await make_store_path(store, path=path) + if not store_path.store.supports_consolidated_metadata: + store_name = type(store_path.store).__name__ + raise TypeError( + f"The Zarr Store in use ({store_name}) doesn't support consolidated metadata", + ) + group = await AsyncGroup.open(store_path, zarr_format=zarr_format, use_consolidated=False) group.store_path.store._check_writable() - members_metadata = {k: v.metadata async for k, v in group.members(max_depth=None)} - + members_metadata = { + k: v.metadata + async for k, v in group.members(max_depth=None, use_consolidated_for_children=False) + } # While consolidating, we want to be explicit about when child groups # are empty by inserting an empty dict for consolidated_metadata.metadata for k, v in members_metadata.items(): @@ -212,7 +239,6 @@ async def consolidate_metadata( group, metadata=metadata, ) - await group._save_metadata() return group @@ -235,7 +261,7 @@ async def load( path: str | None = None, zarr_format: ZarrFormat | None = None, zarr_version: ZarrFormat | None = None, -) -> NDArrayLike | dict[str, NDArrayLike]: +) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: """Load data from an array or group into memory. Parameters @@ -273,7 +299,7 @@ async def load( async def open( *, store: StoreLike | None = None, - mode: AccessModeLiteral = "a", + mode: AccessModeLiteral | None = None, zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, @@ -291,6 +317,7 @@ async def open( read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). + If the store is read-only, the default is 'r'; otherwise, it is 'a'. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional @@ -308,7 +335,11 @@ async def open( Return type depends on what exists in the given store. """ zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) - + if mode is None: + if isinstance(store, (Store, StorePath)) and store.read_only: + mode = "r" + else: + mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) # TODO: the mode check below seems wrong! @@ -316,7 +347,7 @@ async def open( try: metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we fix typing for array metadata dicts - _metadata_dict = cast(ArrayMetadataDict, metadata_dict) + _metadata_dict = cast("ArrayMetadataDict", metadata_dict) # for v2, the above would already have raised an exception if not an array zarr_format = _metadata_dict["zarr_format"] is_v3_array = zarr_format == 3 and _metadata_dict.get("node_type") == "array" @@ -425,11 +456,12 @@ async def save_array( shape = arr.shape chunks = getattr(arr, "chunks", None) # for array-likes with chunks attribute overwrite = kwargs.pop("overwrite", None) or _infer_overwrite(mode) + zarr_dtype = get_data_type_from_native_dtype(arr.dtype) new = await AsyncArray._create( store_path, zarr_format=zarr_format, shape=shape, - dtype=arr.dtype, + dtype=zarr_dtype, chunks=chunks, overwrite=overwrite, **kwargs, @@ -489,13 +521,12 @@ async def save_group( raise ValueError("at least one array must be provided") aws = [] for i, arr in enumerate(args): - _path = f"{path}/arr_{i}" if path is not None else f"arr_{i}" aws.append( save_array( store_path, arr, zarr_format=zarr_format, - path=_path, + path=f"arr_{i}", storage_options=storage_options, ) ) @@ -530,7 +561,7 @@ async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = async def array( - data: npt.ArrayLike, **kwargs: Any + data: npt.ArrayLike | Array, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Create an array filled with `data`. @@ -547,13 +578,16 @@ async def array( The new array. """ + if isinstance(data, Array): + return await from_array(data=data, **kwargs) + # ensure data is array-like if not hasattr(data, "shape") or not hasattr(data, "dtype"): data = np.asanyarray(data) # setup dtype kw_dtype = kwargs.get("dtype") - if kw_dtype is None: + if kw_dtype is None and hasattr(data, "dtype"): kwargs["dtype"] = data.dtype else: kwargs["dtype"] = kw_dtype @@ -801,7 +835,6 @@ async def open_group( warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) - if attributes is None: attributes = {} @@ -821,15 +854,15 @@ async def open_group( overwrite=overwrite, attributes=attributes, ) - raise FileNotFoundError(f"Unable to find group: {store_path}") + raise GroupNotFoundError(store, store_path.path) async def create( shape: ChunkCoords | int, *, # Note: this is a change from v2 chunks: ChunkCoords | int | None = None, # TODO: v2 allowed chunks=True - dtype: npt.DTypeLike | None = None, - compressor: dict[str, JSON] | None = None, # TODO: default and type change + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", fill_value: Any | None = 0, # TODO: need type order: MemoryOrder | None = None, store: str | StoreLike | None = None, @@ -857,9 +890,9 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, **kwargs: Any, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Create an array. @@ -975,21 +1008,13 @@ async def create( _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) or _default_zarr_format() ) - + zdtype = parse_data_type(dtype, zarr_format=zarr_format) if zarr_format == 2: - if chunks is None: - chunks = shape - dtype = parse_dtype(dtype, zarr_format) + default_filters, default_compressor = _get_default_chunk_encoding_v2(zdtype) if not filters: - filters = _default_filters(dtype) - if not compressor: - compressor = _default_compressor(dtype) - elif zarr_format == 3 and chunk_shape is None: # type: ignore[redundant-expr] - if chunks is not None: - chunk_shape = chunks - chunks = None - else: - chunk_shape = shape + filters = default_filters # type: ignore[assignment] + if compressor == "auto": + compressor = default_compressor if synchronizer is not None: warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2) @@ -1003,11 +1028,6 @@ async def create( warnings.warn("object_codec is not yet implemented", RuntimeWarning, stacklevel=2) if read_only is not None: warnings.warn("read_only is not yet implemented", RuntimeWarning, stacklevel=2) - if dimension_separator is not None and zarr_format == 3: - raise ValueError( - "dimension_separator is not supported for zarr format 3, use chunk_key_encoding instead" - ) - if order is not None: _warn_order_kwarg() if write_empty_chunks is not None: @@ -1021,7 +1041,7 @@ async def create( mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - config_dict: ArrayConfigLike = {} + config_dict: ArrayConfigParams = {} if write_empty_chunks is not None: if config is not None: @@ -1032,15 +1052,13 @@ async def create( ) warnings.warn(UserWarning(msg), stacklevel=1) config_dict["write_empty_chunks"] = write_empty_chunks - if order is not None: - if config is not None: - msg = ( - "Both order and config keyword arguments are set. " - "This is redundant. When both are set, order will be ignored and " - "config will be used." - ) - warnings.warn(UserWarning(msg), stacklevel=1) - config_dict["order"] = order + if order is not None and config is not None: + msg = ( + "Both order and config keyword arguments are set. " + "This is redundant. When both are set, order will be ignored and " + "config will be used." + ) + warnings.warn(UserWarning(msg), stacklevel=1) config_parsed = ArrayConfig.from_dict(config_dict) @@ -1048,12 +1066,13 @@ async def create( store_path, shape=shape, chunks=chunks, - dtype=dtype, + dtype=zdtype, compressor=compressor, fill_value=fill_value, overwrite=overwrite, filters=filters, dimension_separator=dimension_separator, + order=order, zarr_format=zarr_format, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, @@ -1068,7 +1087,8 @@ async def create( async def empty( shape: ChunkCoords, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty array. + """Create an empty array with the specified shape. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -1090,7 +1110,8 @@ async def empty( async def empty_like( a: ArrayLike, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty array like `a`. + """Create an empty array like `a`. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -1103,6 +1124,12 @@ async def empty_like( ------- Array The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. """ like_kwargs = _like_args(a, kwargs) return await empty(**like_kwargs) diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index f8bee9fcef..f2dc8757d6 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -7,13 +7,15 @@ import zarr.api.asynchronous as async_api import zarr.core.array from zarr._compat import _deprecate_positional_args -from zarr.core.array import Array, AsyncArray +from zarr.core.array import Array, AsyncArray, CompressorLike from zarr.core.group import Group from zarr.core.sync import sync +from zarr.core.sync_group import create_hierarchy if TYPE_CHECKING: from collections.abc import Iterable + import numpy as np import numpy.typing as npt from zarr.abc.codec import Codec @@ -24,17 +26,19 @@ SerializerLike, ShardsLike, ) - from zarr.core.array_spec import ArrayConfig, ArrayConfigLike - from zarr.core.buffer import NDArrayLike + from zarr.core.array_spec import ArrayConfigLike + from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike from zarr.core.common import ( JSON, AccessModeLiteral, ChunkCoords, + DimensionNames, MemoryOrder, ShapeLike, ZarrFormat, ) + from zarr.core.dtype import ZDTypeLike from zarr.storage import StoreLike __all__ = [ @@ -45,8 +49,10 @@ "copy_store", "create", "create_array", + "create_hierarchy", "empty", "empty_like", + "from_array", "full", "full_like", "group", @@ -76,7 +82,8 @@ def consolidate_metadata( Consolidate the metadata of all nodes in a hierarchy. Upon completion, the metadata of the root node in the Zarr hierarchy will be - updated to include all the metadata of child nodes. + updated to include all the metadata of child nodes. For Stores that do + not use consolidated metadata, this operation raises a `TypeError`. Parameters ---------- @@ -96,7 +103,10 @@ def consolidate_metadata( ------- group: Group The group, with the ``consolidated_metadata`` field set to include - the metadata of each child node. + the metadata of each child node. If the Store doesn't support + consolidated metadata, this function raises a `TypeError`. + See ``Store.supports_consolidated_metadata``. + """ return Group(sync(async_api.consolidate_metadata(store, path=path, zarr_format=zarr_format))) @@ -118,7 +128,7 @@ def load( path: str | None = None, zarr_format: ZarrFormat | None = None, zarr_version: ZarrFormat | None = None, -) -> NDArrayLike | dict[str, NDArrayLike]: +) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: """Load data from an array or group into memory. Parameters @@ -153,7 +163,7 @@ def load( def open( store: StoreLike | None = None, *, - mode: AccessModeLiteral = "a", + mode: AccessModeLiteral | None = None, zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, @@ -171,6 +181,7 @@ def open( read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). + If the store is read-only, the default is 'r'; otherwise, it is 'a'. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional @@ -356,7 +367,7 @@ def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> An # TODO: add type annotations for kwargs -def array(data: npt.ArrayLike, **kwargs: Any) -> Array: +def array(data: npt.ArrayLike | Array, **kwargs: Any) -> Array: """Create an array filled with `data`. Parameters @@ -593,9 +604,9 @@ def create( shape: ChunkCoords | int, *, # Note: this is a change from v2 chunks: ChunkCoords | int | bool | None = None, - dtype: npt.DTypeLike | None = None, - compressor: dict[str, JSON] | None = None, # TODO: default and type change - fill_value: Any | None = 0, # TODO: need type + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + fill_value: Any | None = None, # TODO: need type order: MemoryOrder | None = None, store: str | StoreLike | None = None, synchronizer: Any | None = None, @@ -622,9 +633,9 @@ def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, **kwargs: Any, ) -> Array: """Create an array. @@ -694,7 +705,7 @@ def create( storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. - config : ArrayConfig or ArrayConfigLike, optional + config : ArrayConfigLike, optional Runtime configuration of the array. If provided, will override the default values from `zarr.config.array`. @@ -744,8 +755,9 @@ def create_array( store: str | StoreLike, *, name: str | None = None, - shape: ShapeLike, - dtype: npt.DTypeLike, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunkCoords | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", @@ -755,11 +767,12 @@ def create_array( order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, + write_data: bool = True, ) -> Array: """Create an array. @@ -772,10 +785,14 @@ def create_array( name : str or None, optional The name of the array within the store. If ``name`` is ``None``, the array will be located at the root of the store. - shape : ChunkCoords - Shape of the array. - dtype : npt.DTypeLike - Data type of the array. + shape : ChunkCoords, optional + Shape of the array. Can be ``None`` if ``data`` is provided. + dtype : ZDTypeLike, optional + Data type of the array. Can be ``None`` if ``data`` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``, + or ``None``. chunks : ChunkCoords, optional Chunk shape of the array. If not specified, default are guessed based on the shape and dtype. @@ -847,8 +864,14 @@ def create_array( Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. - config : ArrayConfig or ArrayConfigLike, optional + If `True`, all existing paths in the store will be deleted. + config : ArrayConfigLike, optional Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the ``data`` parameter + then ``write_data`` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If ``write_data`` is ``False``, then the + array will be left empty. Returns ------- @@ -858,7 +881,7 @@ def create_array( Examples -------- >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') + >>> store = zarr.storage.MemoryStore() >>> arr = await zarr.create_array( >>> store=store, >>> shape=(100,100), @@ -874,6 +897,221 @@ def create_array( name=name, shape=shape, dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + write_data=write_data, + ) + ) + ) + + +def from_array( + store: str | StoreLike, + *, + data: Array | npt.ArrayLike, + write_data: bool = True, + name: str | None = None, + chunks: Literal["auto", "keep"] | ChunkCoords = "keep", + shards: ShardsLike | None | Literal["keep"] = "keep", + filters: FiltersLike | Literal["keep"] = "keep", + compressors: CompressorsLike | Literal["keep"] = "keep", + serializer: SerializerLike | Literal["keep"] = "keep", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, +) -> Array: + """Create an array from an existing array or array-like. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file for the new array. + data : Array | array-like + The array to copy. + write_data : bool, default True + Whether to copy the data from the input array to the new array. + If ``write_data`` is ``False``, the new array will be created with the same metadata as the + input array, but without any data. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + chunks : ChunkCoords or "auto" or "keep", optional + Chunk shape of the array. + Following values are supported: + + - "auto": Automatically determine the chunk shape based on the array's shape and dtype. + - "keep": Retain the chunk shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the chunk shape. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". + shards : ChunkCoords, optional + Shard shape of the array. + Following values are supported: + + - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. + - "keep": Retain the shard shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the shard shape. + - None: No sharding. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise None. + filters : Iterable[Codec] or "auto" or "keep", optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + + Following values are supported: + + - Iterable[Codec]: List of filters to apply to the array. + - "auto": Automatically determine the filters based on the array's dtype. + - "keep": Retain the filters of the data array if it is a zarr Array. + + If no ``filters`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + compressors : Iterable[Codec] or "auto" or "keep", optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + + Following values are supported: + + - Iterable[Codec]: List of compressors to apply to the array. + - "auto": Automatically determine the compressors based on the array's dtype. + - "keep": Retain the compressors of the input array if it is a zarr Array. + + If no ``compressors`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + + Following values are supported: + + - dict[str, JSON]: A dict representation of an ``ArrayBytesCodec``. + - ArrayBytesCodec: An instance of ``ArrayBytesCodec``. + - "auto": a default serializer will be used. These defaults can be changed by modifying the value of + ``array.v3_default_serializer`` in :mod:`zarr.core.config`. + - "keep": Retain the serializer of the input array if it is a zarr Array. + + fill_value : Any, optional + Fill value for the array. + If not specified, defaults to the fill value of the data array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If not specified, defaults to the memory order of the data array. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + If not specified, defaults to the zarr format of the data array. + attributes : dict, optional + Attributes for the array. + If not specified, defaults to the attributes of the data array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + If not specified and the data array has the same zarr format as the target array, + the chunk key encoding of the data array is used. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + If not specified, defaults to the dimension names of the data array. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + Array + The array. + + Examples + -------- + Create an array from an existing Array:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> store2 = zarr.storage.LocalStore('example.zarr') + >>> arr = zarr.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='int32', + >>> fill_value=0) + >>> arr2 = zarr.from_array(store2, data=arr) + + + Create an array from an existing NumPy array:: + + >>> import numpy as np + >>> arr3 = zarr.from_array( + zarr.storage.MemoryStore(), + >>> data=np.arange(10000, dtype='i4').reshape(100, 100), + >>> ) + + + Create an array from any array-like object:: + + >>> arr4 = zarr.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=[[1, 2], [3, 4]], + >>> ) + + >>> arr4[...] + array([[1, 2],[3, 4]]) + + Create an array from an existing Array without copying the data:: + + >>> arr5 = zarr.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=arr4, + >>> write_data=False, + >>> ) + + >>> arr5[...] + array([[0, 0],[0, 0]]) + """ + return Array( + sync( + zarr.core.array.from_array( + store, + data=data, + write_data=write_data, + name=name, chunks=chunks, shards=shards, filters=filters, @@ -895,7 +1133,8 @@ def create_array( # TODO: add type annotations for kwargs def empty(shape: ChunkCoords, **kwargs: Any) -> Array: - """Create an empty array. + """Create an empty array with the specified shape. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -921,7 +1160,8 @@ def empty(shape: ChunkCoords, **kwargs: Any) -> Array: # TODO: move ArrayLike to common module # TODO: add type annotations for kwargs def empty_like(a: ArrayLike, **kwargs: Any) -> Array: - """Create an empty array like another array. + """Create an empty array like another array. The contents will be filled with the + array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -934,6 +1174,12 @@ def empty_like(a: ArrayLike, **kwargs: Any) -> Array: ------- Array The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. """ return Array(sync(async_api.empty_like(a, **kwargs))) diff --git a/src/zarr/buffer/__init__.py b/src/zarr/buffer/__init__.py new file mode 100644 index 0000000000..db393f66c7 --- /dev/null +++ b/src/zarr/buffer/__init__.py @@ -0,0 +1,12 @@ +""" +Implementations of the Zarr Buffer interface. + +See Also +======== +zarr.abc.buffer: Abstract base class for the Zarr Buffer interface. +""" + +from zarr.buffer import cpu, gpu +from zarr.core.buffer import default_buffer_prototype + +__all__ = ["cpu", "default_buffer_prototype", "gpu"] diff --git a/src/zarr/buffer/cpu.py b/src/zarr/buffer/cpu.py new file mode 100644 index 0000000000..5307927c06 --- /dev/null +++ b/src/zarr/buffer/cpu.py @@ -0,0 +1,15 @@ +from zarr.core.buffer.cpu import ( + Buffer, + NDBuffer, + as_numpy_array_wrapper, + buffer_prototype, + numpy_buffer_prototype, +) + +__all__ = [ + "Buffer", + "NDBuffer", + "as_numpy_array_wrapper", + "buffer_prototype", + "numpy_buffer_prototype", +] diff --git a/src/zarr/buffer/gpu.py b/src/zarr/buffer/gpu.py new file mode 100644 index 0000000000..dbdc1b1357 --- /dev/null +++ b/src/zarr/buffer/gpu.py @@ -0,0 +1,7 @@ +from zarr.core.buffer.gpu import Buffer, NDBuffer, buffer_prototype + +__all__ = [ + "Buffer", + "NDBuffer", + "buffer_prototype", +] diff --git a/src/zarr/codecs/_v2.py b/src/zarr/codecs/_v2.py index 53edc1f4a1..08853f27f1 100644 --- a/src/zarr/codecs/_v2.py +++ b/src/zarr/codecs/_v2.py @@ -46,9 +46,9 @@ async def _decode_single( chunk = ensure_ndarray_like(chunk) # special case object dtype, because incorrect handling can lead to # segfaults and other bad things happening - if chunk_spec.dtype != object: + if chunk_spec.dtype.dtype_cls is not np.dtypes.ObjectDType: try: - chunk = chunk.view(chunk_spec.dtype) + chunk = chunk.view(chunk_spec.dtype.to_native_dtype()) except TypeError: # this will happen if the dtype of the chunk # does not match the dtype of the array spec i.g. if @@ -56,7 +56,7 @@ async def _decode_single( # is an object array. In this case, we need to convert the object # array to the correct dtype. - chunk = np.array(chunk).astype(chunk_spec.dtype) + chunk = np.array(chunk).astype(chunk_spec.dtype.to_native_dtype()) elif chunk.dtype != object: # If we end up here, someone must have hacked around with the filters. @@ -80,7 +80,7 @@ async def _encode_single( chunk = chunk_array.as_ndarray_like() # ensure contiguous and correct order - chunk = chunk.astype(chunk_spec.dtype, order=chunk_spec.order, copy=False) + chunk = chunk.astype(chunk_spec.dtype.to_native_dtype(), order=chunk_spec.order, copy=False) # apply filters if self.filters: diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index 54a23c9c57..1c5e52e9a4 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -8,10 +8,12 @@ import numcodecs from numcodecs.blosc import Blosc +from packaging.version import Version from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_enum, parse_named_configuration +from zarr.core.dtype.common import HasItemSize from zarr.registry import register_codec if TYPE_CHECKING: @@ -55,7 +57,7 @@ class BloscCname(Enum): zlib = "zlib" -# See https://zarr.readthedocs.io/en/stable/tutorial.html#configuring-blosc +# See https://zarr.readthedocs.io/en/stable/user-guide/performance.html#configuring-blosc numcodecs.blosc.use_threads = False @@ -136,14 +138,16 @@ def to_dict(self) -> dict[str, JSON]: } def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: - dtype = array_spec.dtype + item_size = 1 + if isinstance(array_spec.dtype, HasItemSize): + item_size = array_spec.dtype.item_size new_codec = self if new_codec.typesize is None: - new_codec = replace(new_codec, typesize=dtype.itemsize) + new_codec = replace(new_codec, typesize=item_size) if new_codec.shuffle is None: new_codec = replace( new_codec, - shuffle=(BloscShuffle.bitshuffle if dtype.itemsize == 1 else BloscShuffle.shuffle), + shuffle=(BloscShuffle.bitshuffle if item_size == 1 else BloscShuffle.shuffle), ) return new_codec @@ -163,6 +167,9 @@ def _blosc_codec(self) -> Blosc: "shuffle": map_shuffle_str_to_int[self.shuffle], "blocksize": self.blocksize, } + # See https://github.com/zarr-developers/numcodecs/pull/713 + if Version(numcodecs.__version__) >= Version("0.16.0"): + config_dict["typesize"] = self.typesize return Blosc.from_config(config_dict) async def _decode_single( diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 78c7b22fbc..d663a3b2cc 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -10,6 +10,7 @@ from zarr.abc.codec import ArrayBytesCodec from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer from zarr.core.common import JSON, parse_enum, parse_named_configuration +from zarr.core.dtype.common import HasEndianness from zarr.registry import register_codec if TYPE_CHECKING: @@ -56,7 +57,7 @@ def to_dict(self) -> dict[str, JSON]: return {"name": "bytes", "configuration": {"endian": self.endian.value}} def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: - if array_spec.dtype.itemsize == 0: + if not isinstance(array_spec.dtype, HasEndianness): if self.endian is not None: return replace(self, endian=None) elif self.endian is None: @@ -71,15 +72,12 @@ async def _decode_single( chunk_spec: ArraySpec, ) -> NDBuffer: assert isinstance(chunk_bytes, Buffer) - if chunk_spec.dtype.itemsize > 0: - if self.endian == Endian.little: - prefix = "<" - else: - prefix = ">" - dtype = np.dtype(f"{prefix}{chunk_spec.dtype.str[1:]}") + # TODO: remove endianness enum in favor of literal union + endian_str = self.endian.value if self.endian is not None else None + if isinstance(chunk_spec.dtype, HasEndianness): + dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] else: - dtype = np.dtype(f"|{chunk_spec.dtype.str[1:]}") - + dtype = chunk_spec.dtype.to_native_dtype() as_array_like = chunk_bytes.as_array_like() if isinstance(as_array_like, NDArrayLike): as_nd_array_like = as_array_like @@ -114,7 +112,7 @@ async def _encode_single( nd_array = chunk_array.as_ndarray_like() # Flatten the nd-array (only copy if needed) and reinterpret as bytes - nd_array = nd_array.ravel().view(dtype="b") + nd_array = nd_array.ravel().view(dtype="B") return chunk_spec.prototype.buffer.from_array_like(nd_array) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index 3a6624ad25..6da673ceac 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -40,7 +40,9 @@ async def _decode_single( inner_bytes = data[:-4] # Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved - computed_checksum = np.uint32(crc32c(cast(typing_extensions.Buffer, inner_bytes))).tobytes() + computed_checksum = np.uint32( + crc32c(cast("typing_extensions.Buffer", inner_bytes)) + ).tobytes() stored_checksum = bytes(crc32_bytes) if computed_checksum != stored_checksum: raise ValueError( @@ -55,9 +57,9 @@ async def _encode_single( ) -> Buffer | None: data = chunk_bytes.as_numpy_array() # Calculate the checksum and "cast" it to a numpy array - checksum = np.array([crc32c(cast(typing_extensions.Buffer, data))], dtype=np.uint32) + checksum = np.array([crc32c(cast("typing_extensions.Buffer", data))], dtype=np.uint32) # Append the checksum (as bytes) to the data - return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("b"))) + return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B"))) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length + 4 diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index a01145b3b2..cd8676b4d1 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -17,7 +17,13 @@ Codec, CodecPipeline, ) -from zarr.abc.store import ByteGetter, ByteRangeRequest, ByteSetter +from zarr.abc.store import ( + ByteGetter, + ByteRequest, + ByteSetter, + RangeByteRequest, + SuffixByteRequest, +) from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.core.array_spec import ArrayConfig, ArraySpec @@ -37,6 +43,7 @@ parse_shapelike, product, ) +from zarr.core.dtype.npy.int import UInt64 from zarr.core.indexing import ( BasicIndexer, SelectorTuple, @@ -52,6 +59,7 @@ from typing import Self from zarr.core.common import JSON + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType MAX_UINT_64 = 2**64 - 1 ShardMapping = Mapping[ChunkCoords, Buffer] @@ -77,12 +85,12 @@ class _ShardingByteGetter(ByteGetter): chunk_coords: ChunkCoords async def get( - self, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: assert byte_range is None, "byte_range is not supported within shards" - assert ( - prototype == default_buffer_prototype() - ), f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}" + assert prototype == default_buffer_prototype(), ( + f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}" + ) return self.shard_dict.get(self.chunk_coords) @@ -90,7 +98,7 @@ async def get( class _ShardingByteSetter(_ShardingByteGetter, ByteSetter): shard_dict: ShardMutableMapping - async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -> None: + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: assert byte_range is None, "byte_range is not supported within shards" self.shard_dict[self.chunk_coords] = value @@ -109,7 +117,7 @@ class _ShardIndex(NamedTuple): def chunks_per_shard(self) -> ChunkCoords: result = tuple(self.offsets_and_lengths.shape[0:-1]) # The cast is required until https://github.com/numpy/numpy/pull/27211 is merged - return cast(ChunkCoords, result) + return cast("ChunkCoords", result) def _localize_chunk(self, chunk_coords: ChunkCoords) -> ChunkCoords: return tuple( @@ -129,7 +137,7 @@ def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None: if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64): return None else: - return (int(chunk_start), int(chunk_len)) + return (int(chunk_start), int(chunk_start + chunk_len)) def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: slice | None) -> None: localized_chunk = self._localize_chunk(chunk_coords) @@ -203,7 +211,7 @@ def create_empty( def __getitem__(self, chunk_coords: ChunkCoords) -> Buffer: chunk_byte_slice = self.index.get_chunk_slice(chunk_coords) if chunk_byte_slice: - return self.buf[chunk_byte_slice[0] : (chunk_byte_slice[0] + chunk_byte_slice[1])] + return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]] raise KeyError def __len__(self) -> int: @@ -349,7 +357,11 @@ def __init__( object.__setattr__(self, "index_location", index_location_parsed) # Use instance-local lru_cache to avoid memory leaks - object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) + + # numpy void scalars are not hashable, which means an array spec with a fill value that is + # a numpy void scalar will break the lru_cache. This is commented for now but should be + # fixed. See https://github.com/zarr-developers/zarr-python/issues/3054 + # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) @@ -365,7 +377,7 @@ def __setstate__(self, state: dict[str, Any]) -> None: object.__setattr__(self, "index_location", parse_index_location(config["index_location"])) # Use instance-local lru_cache to avoid memory leaks - object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) + # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) @@ -396,7 +408,13 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: return replace(self, codecs=evolved_codecs) return self - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, + *, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: if len(self.chunk_shape) != len(shape): raise ValueError( "The shard's `chunk_shape` and array's `shape` need to have the same number of dimensions." @@ -433,7 +451,10 @@ async def _decode_single( # setup output array out = chunk_spec.prototype.nd_buffer.create( - shape=shard_shape, dtype=shard_spec.dtype, order=shard_spec.order, fill_value=0 + shape=shard_shape, + dtype=shard_spec.dtype.to_native_dtype(), + order=shard_spec.order, + fill_value=0, ) shard_dict = await _ShardReader.from_bytes(shard_bytes, self, chunks_per_shard) @@ -449,8 +470,9 @@ async def _decode_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], out, ) @@ -476,11 +498,14 @@ async def _decode_partial_single( # setup output array out = shard_spec.prototype.nd_buffer.create( - shape=indexer.shape, dtype=shard_spec.dtype, order=shard_spec.order, fill_value=0 + shape=indexer.shape, + dtype=shard_spec.dtype.to_native_dtype(), + order=shard_spec.order, + fill_value=0, ) indexed_chunks = list(indexer) - all_chunk_coords = {chunk_coords for chunk_coords, _, _ in indexed_chunks} + all_chunk_coords = {chunk_coords for chunk_coords, *_ in indexed_chunks} # reading bytes of all requested chunks shard_dict: ShardMapping = {} @@ -504,7 +529,8 @@ async def _decode_partial_single( chunk_byte_slice = shard_index.get_chunk_slice(chunk_coords) if chunk_byte_slice: chunk_bytes = await byte_getter.get( - prototype=chunk_spec.prototype, byte_range=chunk_byte_slice + prototype=chunk_spec.prototype, + byte_range=RangeByteRequest(chunk_byte_slice[0], chunk_byte_slice[1]), ) if chunk_bytes: shard_dict[chunk_coords] = chunk_bytes @@ -517,12 +543,17 @@ async def _decode_partial_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], out, ) - return out + + if hasattr(indexer, "sel_shape"): + return out.reshape(indexer.sel_shape) + else: + return out async def _encode_single( self, @@ -551,8 +582,9 @@ async def _encode_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], shard_array, ) @@ -594,8 +626,9 @@ async def _encode_partial_single( chunk_spec, chunk_selection, out_selection, + is_complete_shard, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], shard_array, ) @@ -663,12 +696,12 @@ def _shard_index_size(self, chunks_per_shard: ChunkCoords) -> int: def _get_index_chunk_spec(self, chunks_per_shard: ChunkCoords) -> ArraySpec: return ArraySpec( shape=chunks_per_shard + (2,), - dtype=np.dtype(" ArraySpec: @@ -696,11 +729,12 @@ async def _load_shard_index_maybe( shard_index_size = self._shard_index_size(chunks_per_shard) if self.index_location == ShardingCodecIndexLocation.start: index_bytes = await byte_getter.get( - prototype=numpy_buffer_prototype(), byte_range=(0, shard_index_size) + prototype=numpy_buffer_prototype(), + byte_range=RangeByteRequest(0, shard_index_size), ) else: index_bytes = await byte_getter.get( - prototype=numpy_buffer_prototype(), byte_range=(-shard_index_size, None) + prototype=numpy_buffer_prototype(), byte_range=SuffixByteRequest(shard_index_size) ) if index_bytes is not None: return await self._decode_shard_index(index_bytes, chunks_per_shard) diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index 1aa1eb40e2..be89690441 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -12,10 +12,11 @@ from zarr.registry import register_codec if TYPE_CHECKING: - from typing import Any, Self + from typing import Self from zarr.core.buffer import NDBuffer from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]: @@ -23,7 +24,7 @@ def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]: raise TypeError(f"Expected an iterable. Got {data} instead.") if not all(isinstance(a, int) for a in data): raise TypeError(f"Expected an iterable of integers. Got {data} instead.") - return tuple(cast(Iterable[int], data)) + return tuple(cast("Iterable[int]", data)) @dataclass(frozen=True) @@ -45,7 +46,12 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: def to_dict(self) -> dict[str, JSON]: return {"name": "transpose", "configuration": {"order": tuple(self.order)}} - def validate(self, shape: tuple[int, ...], dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, + shape: tuple[int, ...], + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: if len(self.order) != len(shape): raise ValueError( f"The `order` tuple needs have as many entries as there are dimensions in the array. Got {self.order}." diff --git a/src/zarr/codecs/vlen_utf8.py b/src/zarr/codecs/vlen_utf8.py index 0ef423793d..b7c0418b2e 100644 --- a/src/zarr/codecs/vlen_utf8.py +++ b/src/zarr/codecs/vlen_utf8.py @@ -10,7 +10,6 @@ from zarr.abc.codec import ArrayBytesCodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.common import JSON, parse_named_configuration -from zarr.core.strings import cast_to_string_dtype from zarr.registry import register_codec if TYPE_CHECKING: @@ -49,6 +48,7 @@ def to_dict(self) -> dict[str, JSON]: def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: return self + # TODO: expand the tests for this function async def _decode_single( self, chunk_bytes: Buffer, @@ -60,8 +60,7 @@ async def _decode_single( decoded = _vlen_utf8_codec.decode(raw_bytes) assert decoded.dtype == np.object_ decoded.shape = chunk_spec.shape - # coming out of the code, we know this is safe, so don't issue a warning - as_string_dtype = cast_to_string_dtype(decoded, safe=True) + as_string_dtype = decoded.astype(chunk_spec.dtype.to_native_dtype(), copy=False) return chunk_spec.prototype.nd_buffer.from_numpy_array(as_string_dtype) async def _encode_single( diff --git a/src/zarr/core/__init__.py b/src/zarr/core/__init__.py index cbacfe3422..03a108dbbf 100644 --- a/src/zarr/core/__init__.py +++ b/src/zarr/core/__init__.py @@ -1,3 +1,8 @@ +""" +The ``zarr.core`` module is considered private API and should not be imported +directly by 3rd-party code. +""" + from __future__ import annotations from zarr.core.buffer import Buffer, NDBuffer # noqa: F401 diff --git a/src/zarr/core/_info.py b/src/zarr/core/_info.py index 845552c8be..d57d17f934 100644 --- a/src/zarr/core/_info.py +++ b/src/zarr/core/_info.py @@ -1,13 +1,15 @@ +from __future__ import annotations + import dataclasses import textwrap -from typing import Any, Literal +from typing import TYPE_CHECKING, Literal -import numcodecs.abc -import numpy as np +if TYPE_CHECKING: + import numcodecs.abc -from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec -from zarr.core.common import ZarrFormat -from zarr.core.metadata.v3 import DataType + from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec + from zarr.core.common import ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType @dataclasses.dataclass(kw_only=True) @@ -67,7 +69,7 @@ def byte_info(size: int) -> str: return f"{size} ({human_readable_size(size)})" -@dataclasses.dataclass(kw_only=True) +@dataclasses.dataclass(kw_only=True, frozen=True, slots=True) class ArrayInfo: """ Visual summary for an Array. @@ -78,7 +80,8 @@ class ArrayInfo: _type: Literal["Array"] = "Array" _zarr_format: ZarrFormat - _data_type: np.dtype[Any] | DataType + _data_type: ZDType[TBaseDType, TBaseScalar] + _fill_value: object _shape: tuple[int, ...] _shard_shape: tuple[int, ...] | None = None _chunk_shape: tuple[int, ...] | None = None @@ -97,6 +100,7 @@ def __repr__(self) -> str: Type : {_type} Zarr format : {_zarr_format} Data type : {_data_type} + Fill value : {_fill_value} Shape : {_shape}""") if self._shard_shape is not None: diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 915158cb5a..cd6b33a28c 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4,7 +4,7 @@ import warnings from asyncio import gather from collections.abc import Iterable -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from itertools import starmap from logging import getLogger from typing import ( @@ -22,22 +22,25 @@ import numcodecs import numcodecs.abc import numpy as np -import numpy.typing as npt from typing_extensions import deprecated +import zarr from zarr._compat import _deprecate_positional_args from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.abc.store import Store, set_or_delete from zarr.codecs._v2 import V2Codec +from zarr.codecs.bytes import BytesCodec from zarr.core._info import ArrayInfo from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, parse_array_config from zarr.core.attributes import Attributes from zarr.core.buffer import ( BufferPrototype, NDArrayLike, + NDArrayLikeOrScalar, NDBuffer, default_buffer_prototype, ) +from zarr.core.buffer.cpu import buffer_prototype as cpu_buffer_prototype from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition, normalize_chunks from zarr.core.chunk_key_encodings import ( ChunkKeyEncoding, @@ -51,18 +54,25 @@ ZARRAY_JSON, ZATTRS_JSON, ChunkCoords, + DimensionNames, MemoryOrder, ShapeLike, ZarrFormat, _default_zarr_format, _warn_order_kwarg, concurrent_map, - parse_dtype, parse_order, parse_shapelike, product, ) +from zarr.core.config import categorize_data_type from zarr.core.config import config as zarr_config +from zarr.core.dtype import ( + ZDType, + ZDTypeLike, + parse_data_type, +) +from zarr.core.dtype.common import HasEndianness, HasItemSize from zarr.core.indexing import ( BasicIndexer, BasicSelection, @@ -98,12 +108,11 @@ T_ArrayMetadata, ) from zarr.core.metadata.v2 import ( - _default_compressor, - _default_filters, + CompressorLikev2, parse_compressor, parse_filters, ) -from zarr.core.metadata.v3 import DataType, parse_node_type_array +from zarr.core.metadata.v3 import parse_node_type_array from zarr.core.sync import sync from zarr.errors import MetadataValidationError from zarr.registry import ( @@ -112,16 +121,20 @@ _parse_bytes_bytes_codec, get_pipeline_class, ) -from zarr.storage import StoreLike, make_store_path -from zarr.storage._common import StorePath, ensure_no_existing_node +from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path +from zarr.storage._utils import _relativize_path if TYPE_CHECKING: from collections.abc import Iterator, Sequence from typing import Self + import numpy.typing as npt + from zarr.abc.codec import CodecPipeline from zarr.codecs.sharding import ShardingCodecIndexLocation + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar from zarr.core.group import AsyncGroup + from zarr.storage import StoreLike # Array and AsyncArray are defined in the base ``zarr`` namespace @@ -134,7 +147,8 @@ def parse_array_metadata(data: Any) -> ArrayMetadata: if isinstance(data, ArrayMetadata): return data elif isinstance(data, dict): - if data["zarr_format"] == 3: + zarr_format = data.get("zarr_format") + if zarr_format == 3: meta_out = ArrayV3Metadata.from_dict(data) if len(meta_out.storage_transformers) > 0: msg = ( @@ -143,9 +157,11 @@ def parse_array_metadata(data: Any) -> ArrayMetadata: ) raise ValueError(msg) return meta_out - elif data["zarr_format"] == 2: + elif zarr_format == 2: return ArrayV2Metadata.from_dict(data) - raise TypeError + else: + raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3") + raise TypeError # pragma: no cover def create_codec_pipeline(metadata: ArrayMetadata) -> CodecPipeline: @@ -154,8 +170,7 @@ def create_codec_pipeline(metadata: ArrayMetadata) -> CodecPipeline: elif isinstance(metadata, ArrayV2Metadata): v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor) return get_pipeline_class().from_codecs([v2_codec]) - else: - raise TypeError + raise TypeError # pragma: no cover async def get_array_metadata( @@ -163,19 +178,20 @@ async def get_array_metadata( ) -> dict[str, JSON]: if zarr_format == 2: zarray_bytes, zattrs_bytes = await gather( - (store_path / ZARRAY_JSON).get(), (store_path / ZATTRS_JSON).get() + (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), ) if zarray_bytes is None: raise FileNotFoundError(store_path) elif zarr_format == 3: - zarr_json_bytes = await (store_path / ZARR_JSON).get() + zarr_json_bytes = await (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype) if zarr_json_bytes is None: raise FileNotFoundError(store_path) elif zarr_format is None: zarr_json_bytes, zarray_bytes, zattrs_bytes = await gather( - (store_path / ZARR_JSON).get(), - (store_path / ZARRAY_JSON).get(), - (store_path / ZATTRS_JSON).get(), + (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), ) if zarr_json_bytes is not None and zarray_bytes is not None: # warn and favor v3 @@ -219,7 +235,7 @@ class AsyncArray(Generic[T_ArrayMetadata]): The metadata of the array. store_path : StorePath The path to the Zarr store. - config : ArrayConfig, optional + config : ArrayConfigLike, optional The runtime configuration of the array, by default None. Attributes @@ -244,7 +260,7 @@ def __init__( self: AsyncArray[ArrayV2Metadata], metadata: ArrayV2Metadata | ArrayV2MetadataDict, store_path: StorePath, - config: ArrayConfig | None = None, + config: ArrayConfigLike | None = None, ) -> None: ... @overload @@ -252,33 +268,21 @@ def __init__( self: AsyncArray[ArrayV3Metadata], metadata: ArrayV3Metadata | ArrayV3MetadataDict, store_path: StorePath, - config: ArrayConfig | None = None, + config: ArrayConfigLike | None = None, ) -> None: ... def __init__( self, metadata: ArrayMetadata | ArrayMetadataDict, store_path: StorePath, - config: ArrayConfig | None = None, + config: ArrayConfigLike | None = None, ) -> None: - if isinstance(metadata, dict): - zarr_format = metadata["zarr_format"] - # TODO: remove this when we extensively type the dict representation of metadata - _metadata = cast(dict[str, JSON], metadata) - if zarr_format == 2: - metadata = ArrayV2Metadata.from_dict(_metadata) - elif zarr_format == 3: - metadata = ArrayV3Metadata.from_dict(_metadata) - else: - raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3") - metadata_parsed = parse_array_metadata(metadata) - - config = ArrayConfig.from_dict({}) if config is None else config + config_parsed = parse_array_config(config) object.__setattr__(self, "metadata", metadata_parsed) object.__setattr__(self, "store_path", store_path) - object.__setattr__(self, "_config", config) + object.__setattr__(self, "_config", config_parsed) object.__setattr__(self, "codec_pipeline", create_codec_pipeline(metadata=metadata_parsed)) # this overload defines the function signature when zarr_format is 2 @@ -290,7 +294,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: Literal[2], fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -298,11 +302,11 @@ async def create( dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLikev2 | Literal["auto"] = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV2Metadata]: ... # this overload defines the function signature when zarr_format is 3 @@ -314,7 +318,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: Literal[3], fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -327,11 +331,11 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV3Metadata]: ... @overload @@ -342,7 +346,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: Literal[3] = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -355,11 +359,11 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV3Metadata]: ... @overload @@ -370,7 +374,7 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -383,17 +387,17 @@ async def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: ... @classmethod @@ -405,30 +409,30 @@ async def create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, # v3 only chunk_shape: ShapeLike | None = None, chunk_key_encoding: ( - ChunkKeyEncoding + ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Method to create a new asynchronous array instance. @@ -441,7 +445,7 @@ async def create( The store where the array will be created. shape : ShapeLike The shape of the array. - dtype : npt.DTypeLike + dtype : ZDTypeLike The data type of the array. zarr_format : ZarrFormat, optional The Zarr format version (default is 3). @@ -453,7 +457,7 @@ async def create( The shape of the array's chunks Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. If not specified, default are guessed based on the shape and dtype. - chunk_key_encoding : ChunkKeyEncoding, optional + chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. Default is ``("default", "/")``. @@ -470,7 +474,7 @@ async def create( These defaults can be changed by modifying the value of ``array.v3_default_filters``, ``array.v3_default_serializer`` and ``array.v3_default_compressors`` in :mod:`zarr.core.config`. - dimension_names : Iterable[str], optional + dimension_names : Iterable[str | None], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. chunks : ShapeLike, optional @@ -506,7 +510,7 @@ async def create( Whether to raise an error if the store already exists (default is False). data : npt.ArrayLike, optional The data to be inserted into the array (default is None). - config : ArrayConfig or ArrayConfigLike, optional + config : ArrayConfigLike, optional Runtime configuration for the array. Returns @@ -546,47 +550,50 @@ async def _create( *, # v2 and v3 shape: ShapeLike, - dtype: npt.DTypeLike, + dtype: ZDTypeLike | ZDType[TBaseDType, TBaseScalar], zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, # v3 only chunk_shape: ShapeLike | None = None, chunk_key_encoding: ( - ChunkKeyEncoding + ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Method to create a new asynchronous array instance. See :func:`AsyncArray.create` for more details. Deprecated in favor of :func:`zarr.api.asynchronous.create_array`. """ + + dtype_parsed = parse_data_type(dtype, zarr_format=zarr_format) store_path = await make_store_path(store) - dtype_parsed = parse_dtype(dtype, zarr_format) shape = parse_shapelike(shape) if chunks is not None and chunk_shape is not None: raise ValueError("Only one of chunk_shape or chunks can be provided.") - + item_size = 1 + if isinstance(dtype_parsed, HasItemSize): + item_size = dtype_parsed.item_size if chunks: - _chunks = normalize_chunks(chunks, shape, dtype_parsed.itemsize) + _chunks = normalize_chunks(chunks, shape, item_size) else: - _chunks = normalize_chunks(chunk_shape, shape, dtype_parsed.itemsize) + _chunks = normalize_chunks(chunk_shape, shape, item_size) config_parsed = parse_array_config(config) result: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] @@ -599,13 +606,14 @@ async def _create( raise ValueError( "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead." ) - if compressor is not None: + if compressor != "auto": raise ValueError( "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead." ) if order is not None: _warn_order_kwarg() + config_parsed = replace(config_parsed, order=order) result = await cls._create_v3( store_path, @@ -660,24 +668,72 @@ async def _create( return result + @staticmethod + def _create_metadata_v3( + shape: ShapeLike, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_shape: ChunkCoords, + fill_value: Any | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + attributes: dict[str, JSON] | None = None, + ) -> ArrayV3Metadata: + """ + Create an instance of ArrayV3Metadata. + """ + filters: tuple[ArrayArrayCodec, ...] + compressors: tuple[BytesBytesCodec, ...] + + shape = parse_shapelike(shape) + if codecs is None: + filters, serializer, compressors = _get_default_chunk_encoding_v3(dtype) + codecs_parsed = (*filters, serializer, *compressors) + else: + codecs_parsed = tuple(codecs) + + chunk_key_encoding_parsed: ChunkKeyEncodingLike + if chunk_key_encoding is None: + chunk_key_encoding_parsed = {"name": "default", "separator": "/"} + else: + chunk_key_encoding_parsed = chunk_key_encoding + + if fill_value is None: + # v3 spec will not allow a null fill value + fill_value_parsed = dtype.default_scalar() + else: + fill_value_parsed = fill_value + + chunk_grid_parsed = RegularChunkGrid(chunk_shape=chunk_shape) + return ArrayV3Metadata( + shape=shape, + data_type=dtype, + chunk_grid=chunk_grid_parsed, + chunk_key_encoding=chunk_key_encoding_parsed, + fill_value=fill_value_parsed, + codecs=codecs_parsed, # type: ignore[arg-type] + dimension_names=tuple(dimension_names) if dimension_names else None, + attributes=attributes or {}, + ) + @classmethod async def _create_v3( cls, store_path: StorePath, *, shape: ShapeLike, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], chunk_shape: ChunkCoords, config: ArrayConfig, fill_value: Any | None = None, chunk_key_encoding: ( - ChunkKeyEncoding + ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, attributes: dict[str, JSON] | None = None, overwrite: bool = False, ) -> AsyncArray[ArrayV3Metadata]: @@ -689,13 +745,6 @@ async def _create_v3( else: await ensure_no_existing_node(store_path, zarr_format=3) - shape = parse_shapelike(shape) - codecs = list(codecs) if codecs is not None else _get_default_codecs(np.dtype(dtype)) - - if chunk_key_encoding is None: - chunk_key_encoding = ("default", "/") - assert chunk_key_encoding is not None - if isinstance(chunk_key_encoding, tuple): chunk_key_encoding = ( V2ChunkKeyEncoding(separator=chunk_key_encoding[1]) @@ -703,43 +752,63 @@ async def _create_v3( else DefaultChunkKeyEncoding(separator=chunk_key_encoding[1]) ) - if dtype.kind in "UTS": - warn( - f"The dtype `{dtype}` is currently not part in the Zarr format 3 specification. It " - "may not be supported by other zarr implementations and may change in the future.", - category=UserWarning, - stacklevel=2, - ) - - metadata = ArrayV3Metadata( + metadata = cls._create_metadata_v3( shape=shape, - data_type=dtype, - chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), - chunk_key_encoding=chunk_key_encoding, + dtype=dtype, + chunk_shape=chunk_shape, fill_value=fill_value, + chunk_key_encoding=chunk_key_encoding, codecs=codecs, - dimension_names=tuple(dimension_names) if dimension_names else None, - attributes=attributes or {}, + dimension_names=dimension_names, + attributes=attributes, ) array = cls(metadata=metadata, store_path=store_path, config=config) await array._save_metadata(metadata, ensure_parents=True) return array + @staticmethod + def _create_metadata_v2( + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunks: ChunkCoords, + order: MemoryOrder, + dimension_separator: Literal[".", "/"] | None = None, + fill_value: float | None = None, + filters: Iterable[dict[str, JSON] | numcodecs.abc.Codec] | None = None, + compressor: CompressorLikev2 = None, + attributes: dict[str, JSON] | None = None, + ) -> ArrayV2Metadata: + if dimension_separator is None: + dimension_separator = "." + if fill_value is None: + fill_value = dtype.default_scalar() # type: ignore[assignment] + return ArrayV2Metadata( + shape=shape, + dtype=dtype, + chunks=chunks, + order=order, + dimension_separator=dimension_separator, + fill_value=fill_value, + compressor=compressor, + filters=filters, + attributes=attributes, + ) + @classmethod async def _create_v2( cls, store_path: StorePath, *, shape: ChunkCoords, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], chunks: ChunkCoords, order: MemoryOrder, config: ArrayConfig, dimension_separator: Literal[".", "/"] | None = None, fill_value: float | None = None, filters: Iterable[dict[str, JSON] | numcodecs.abc.Codec] | None = None, - compressor: dict[str, JSON] | numcodecs.abc.Codec | None = None, + compressor: CompressorLike = "auto", attributes: dict[str, JSON] | None = None, overwrite: bool = False, ) -> AsyncArray[ArrayV2Metadata]: @@ -751,30 +820,29 @@ async def _create_v2( else: await ensure_no_existing_node(store_path, zarr_format=2) - if dimension_separator is None: - dimension_separator = "." - - dtype = parse_dtype(dtype, zarr_format=2) - - # inject VLenUTF8 for str dtype if not already present - if np.issubdtype(dtype, np.str_): - filters = filters or [] - from numcodecs.vlen import VLenUTF8 - - if not any(isinstance(x, VLenUTF8) or x["id"] == "vlen-utf8" for x in filters): - filters = list(filters) + [VLenUTF8()] + compressor_parsed: CompressorLikev2 + if compressor == "auto": + _, compressor_parsed = _get_default_chunk_encoding_v2(dtype) + elif isinstance(compressor, BytesBytesCodec): + raise ValueError( + "Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. " + "Use a numcodecs codec directly instead." + ) + else: + compressor_parsed = compressor - metadata = ArrayV2Metadata( + metadata = cls._create_metadata_v2( shape=shape, - dtype=np.dtype(dtype), + dtype=dtype, chunks=chunks, order=order, dimension_separator=dimension_separator, fill_value=fill_value, - compressor=compressor, filters=filters, + compressor=compressor_parsed, attributes=attributes, ) + array = cls(metadata=metadata, store_path=store_path, config=config) await array._save_metadata(metadata, ensure_parents=True) return array @@ -835,14 +903,14 @@ async def open( Examples -------- >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') + >>> store = zarr.storage.MemoryStore() >>> async_arr = await AsyncArray.open(store) # doctest: +ELLIPSIS """ store_path = await make_store_path(store) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints - _metadata_dict = cast(ArrayV3MetadataDict, metadata_dict) + _metadata_dict = cast("ArrayV3MetadataDict", metadata_dict) return cls(store_path=store_path, metadata=_metadata_dict) @property @@ -970,7 +1038,17 @@ def compressors(self) -> tuple[numcodecs.abc.Codec, ...] | tuple[BytesBytesCodec ) @property - def dtype(self) -> np.dtype[Any]: + def _zdtype(self) -> ZDType[TBaseDType, TBaseScalar]: + """ + The zarr-specific representation of the array data type + """ + if self.metadata.zarr_format == 2: + return self.metadata.dtype + else: + return self.metadata.data_type + + @property + def dtype(self) -> TBaseDType: """Returns the data type of the array. Returns @@ -978,7 +1056,7 @@ def dtype(self) -> np.dtype[Any]: np.dtype Data type of the array """ - return self.metadata.dtype + return self._zdtype.to_native_dtype() @property def order(self) -> MemoryOrder: @@ -989,7 +1067,10 @@ def order(self) -> MemoryOrder: bool Memory order of the array """ - return self._config.order + if self.metadata.zarr_format == 2: + return self.metadata.order + else: + return self._config.order @property def attrs(self) -> dict[str, JSON]: @@ -1203,7 +1284,7 @@ async def _get_selection( prototype: BufferPrototype, out: NDBuffer | None = None, fields: Fields | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: # check fields are sensible out_dtype = check_fields(fields, self.dtype) @@ -1221,26 +1302,32 @@ async def _get_selection( out_buffer = prototype.nd_buffer.create( shape=indexer.shape, dtype=out_dtype, - order=self._config.order, + order=self.order, fill_value=self.metadata.fill_value, ) if product(indexer.shape) > 0: + # need to use the order from the metadata for v2 + _config = self._config + if self.metadata.zarr_format == 2: + _config = replace(_config, order=self.order) + # reading chunks and decoding them await self.codec_pipeline.read( [ ( self.store_path / self.metadata.encode_chunk_key(chunk_coords), - self.metadata.get_chunk_spec( - chunk_coords, self._config, prototype=prototype - ), + self.metadata.get_chunk_spec(chunk_coords, _config, prototype=prototype), chunk_selection, out_selection, + is_complete_chunk, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer ], out_buffer, drop_axes=indexer.drop_axes, ) + if isinstance(indexer, BasicIndexer) and indexer.shape == (): + return out_buffer.as_scalar() return out_buffer.as_ndarray_like() async def getitem( @@ -1248,7 +1335,7 @@ async def getitem( selection: BasicSelection, *, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """ Asynchronous function that retrieves a subset of the array's data based on the provided selection. @@ -1261,13 +1348,13 @@ async def getitem( Returns ------- - NDArrayLike + NDArrayLikeOrScalar The retrieved subset of the array's data. Examples -------- >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') + >>> store = zarr.storage.MemoryStore() >>> async_arr = await zarr.api.asynchronous.create_array( ... store=store, ... shape=(100,100), @@ -1292,7 +1379,7 @@ async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = F """ Asynchronously save the array metadata. """ - to_save = metadata.to_buffer_dict(default_buffer_prototype()) + to_save = metadata.to_buffer_dict(cpu_buffer_prototype) awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] if ensure_parents: @@ -1304,7 +1391,7 @@ async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = F [ (parent.store_path / key).set_if_not_exists(value) for key, value in parent.metadata.to_buffer_dict( - default_buffer_prototype() + cpu_buffer_prototype ).items() ] ) @@ -1329,36 +1416,43 @@ async def _set_selection( if isinstance(array_like, np._typing._SupportsArrayFunc): # TODO: need to handle array types that don't support __array_function__ # like PyTorch and JAX - array_like_ = cast(np._typing._SupportsArrayFunc, array_like) - value = np.asanyarray(value, dtype=self.metadata.dtype, like=array_like_) + array_like_ = cast("np._typing._SupportsArrayFunc", array_like) + value = np.asanyarray(value, dtype=self.dtype, like=array_like_) else: if not hasattr(value, "shape"): - value = np.asarray(value, self.metadata.dtype) + value = np.asarray(value, self.dtype) # assert ( # value.shape == indexer.shape # ), f"shape of value doesn't match indexer shape. Expected {indexer.shape}, got {value.shape}" - if not hasattr(value, "dtype") or value.dtype.name != self.metadata.dtype.name: + if not hasattr(value, "dtype") or value.dtype.name != self.dtype.name: if hasattr(value, "astype"): # Handle things that are already NDArrayLike more efficiently - value = value.astype(dtype=self.metadata.dtype, order="A") + value = value.astype(dtype=self.dtype, order="A") else: - value = np.array(value, dtype=self.metadata.dtype, order="A") - value = cast(NDArrayLike, value) + value = np.array(value, dtype=self.dtype, order="A") + value = cast("NDArrayLike", value) + # We accept any ndarray like object from the user and convert it # to a NDBuffer (or subclass). From this point onwards, we only pass # Buffer and NDBuffer between components. value_buffer = prototype.nd_buffer.from_ndarray_like(value) + # need to use the order from the metadata for v2 + _config = self._config + if self.metadata.zarr_format == 2: + _config = replace(_config, order=self.metadata.order) + # merging with existing data and encoding chunks await self.codec_pipeline.write( [ ( self.store_path / self.metadata.encode_chunk_key(chunk_coords), - self.metadata.get_chunk_spec(chunk_coords, self._config, prototype), + self.metadata.get_chunk_spec(chunk_coords, _config, prototype), chunk_selection, out_selection, + is_complete_chunk, ) - for chunk_coords, chunk_selection, out_selection in indexer + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer ], value_buffer, drop_axes=indexer.drop_axes, @@ -1546,8 +1640,6 @@ async def update_attributes(self, new_attributes: dict[str, JSON]) -> Self: - The updated attributes will be merged with existing attributes, and any conflicts will be overwritten by the new values. """ - # metadata.attributes is "frozen" so we simply clear and update the dict - self.metadata.attributes.clear() self.metadata.attributes.update(new_attributes) # Write new metadata @@ -1619,15 +1711,10 @@ async def info_complete(self) -> Any: def _info( self, count_chunks_initialized: int | None = None, count_bytes_stored: int | None = None ) -> Any: - _data_type: np.dtype[Any] | DataType - if isinstance(self.metadata, ArrayV2Metadata): - _data_type = self.metadata.dtype - else: - _data_type = self.metadata.data_type - return ArrayInfo( _zarr_format=self.metadata.zarr_format, - _data_type=_data_type, + _data_type=self._zdtype, + _fill_value=self.metadata.fill_value, _shape=self.shape, _order=self.order, _shard_shape=self.shards, @@ -1646,7 +1733,9 @@ def _info( # TODO: Array can be a frozen data class again once property setters (e.g. shape) are removed @dataclass(frozen=False) class Array: - """Instantiate an array from an initialized store.""" + """ + A Zarr array. + """ _async_array: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] @@ -1659,7 +1748,7 @@ def create( *, # v2 and v3 shape: ChunkCoords, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -1672,16 +1761,16 @@ def create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ChunkCoords | None = None, dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> Array: """Creates a new Array instance from an initialized store. @@ -1694,13 +1783,13 @@ def create( The array store that has already been initialized. shape : ChunkCoords The shape of the array. - dtype : npt.DTypeLike + dtype : ZDTypeLike The data type of the array. chunk_shape : ChunkCoords, optional The shape of the Array's chunks. Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. If not specified, default are guessed based on the shape and dtype. - chunk_key_encoding : ChunkKeyEncoding, optional + chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. Default is ``("default", "/")``. @@ -1717,7 +1806,7 @@ def create( These defaults can be changed by modifying the value of ``array.v3_default_filters``, ``array.v3_default_serializer`` and ``array.v3_default_compressors`` in :mod:`zarr.core.config`. - dimension_names : Iterable[str], optional + dimension_names : Iterable[str | None], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. chunks : ChunkCoords, optional @@ -1788,7 +1877,7 @@ def _create( *, # v2 and v3 shape: ChunkCoords, - dtype: npt.DTypeLike, + dtype: ZDTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -1801,16 +1890,16 @@ def _create( | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, - dimension_names: Iterable[str] | None = None, + dimension_names: DimensionNames = None, # v2 only chunks: ChunkCoords | None = None, dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: CompressorLike = "auto", # runtime overwrite: bool = False, - config: ArrayConfig | ArrayConfigLike | None = None, + config: ArrayConfigLike | None = None, ) -> Array: """Creates a new Array instance from an initialized store. See :func:`Array.create` for more details. @@ -1995,10 +2084,11 @@ def path(self) -> str: @property def name(self) -> str: + """Array name following h5py convention.""" return self._async_array.name @property - def basename(self) -> str | None: + def basename(self) -> str: """Final component of name.""" return self._async_array.basename @@ -2206,14 +2296,15 @@ def __array__( msg = "`copy=False` is not supported. This method always creates a copy." raise ValueError(msg) - arr_np = self[...] + arr = self[...] + arr_np: NDArrayLike = np.array(arr, dtype=dtype) if dtype is not None: arr_np = arr_np.astype(dtype) return arr_np - def __getitem__(self, selection: Selection) -> NDArrayLike: + def __getitem__(self, selection: Selection) -> NDArrayLikeOrScalar: """Retrieve data for an item or region of the array. Parameters @@ -2224,8 +2315,8 @@ def __getitem__(self, selection: Selection) -> NDArrayLike: Returns ------- - NDArrayLike - An array-like containing the data for the requested region. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested region. Examples -------- @@ -2358,11 +2449,11 @@ def __getitem__(self, selection: Selection) -> NDArrayLike: """ fields, pure_selection = pop_fields(selection) if is_pure_fancy_indexing(pure_selection, self.ndim): - return self.vindex[cast(CoordinateSelection | MaskSelection, selection)] + return self.vindex[cast("CoordinateSelection | MaskSelection", selection)] elif is_pure_orthogonal_indexing(pure_selection, self.ndim): return self.get_orthogonal_selection(pure_selection, fields=fields) else: - return self.get_basic_selection(cast(BasicSelection, pure_selection), fields=fields) + return self.get_basic_selection(cast("BasicSelection", pure_selection), fields=fields) def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None: """Modify data for an item or region of the array. @@ -2457,11 +2548,11 @@ def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None: """ fields, pure_selection = pop_fields(selection) if is_pure_fancy_indexing(pure_selection, self.ndim): - self.vindex[cast(CoordinateSelection | MaskSelection, selection)] = value + self.vindex[cast("CoordinateSelection | MaskSelection", selection)] = value elif is_pure_orthogonal_indexing(pure_selection, self.ndim): self.set_orthogonal_selection(pure_selection, value, fields=fields) else: - self.set_basic_selection(cast(BasicSelection, pure_selection), value, fields=fields) + self.set_basic_selection(cast("BasicSelection", pure_selection), value, fields=fields) @_deprecate_positional_args def get_basic_selection( @@ -2471,7 +2562,7 @@ def get_basic_selection( out: NDBuffer | None = None, prototype: BufferPrototype | None = None, fields: Fields | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve data for an item or region of the array. Parameters @@ -2489,8 +2580,8 @@ def get_basic_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested region. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested region. Examples -------- @@ -2691,7 +2782,7 @@ def get_orthogonal_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve data by making a selection for each dimension of the array. For example, if an array has 2 dimensions, allows selecting specific rows and/or columns. The selection for each dimension can be either an integer (indexing a @@ -2713,8 +2804,8 @@ def get_orthogonal_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested selection. Examples -------- @@ -2927,7 +3018,7 @@ def get_mask_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing a Boolean array of the same shape as the array against which the selection is being made, where True values indicate a selected item. @@ -2947,8 +3038,8 @@ def get_mask_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested selection. Examples -------- @@ -3089,7 +3180,7 @@ def get_coordinate_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing the indices (coordinates) for each selected item. @@ -3107,8 +3198,8 @@ def get_coordinate_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested coordinate selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested coordinate selection. Examples -------- @@ -3277,7 +3368,7 @@ def get_block_selection( out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, - ) -> NDArrayLike: + ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing the indices (coordinates) for each selected item. @@ -3295,8 +3386,8 @@ def get_block_selection( Returns ------- - NDArrayLike - An array-like containing the data for the requested block selection. + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested block selection. Examples -------- @@ -3579,7 +3670,7 @@ def update_attributes(self, new_attributes: dict[str, JSON]) -> Array: # TODO: remove this cast when type inference improves new_array = sync(self._async_array.update_attributes(new_attributes)) # TODO: remove this cast when type inference improves - _new_array = cast(AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], new_array) + _new_array = cast("AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]", new_array) return type(self)(_new_array) def __repr__(self) -> str: @@ -3663,7 +3754,12 @@ async def chunks_initialized( store_contents = [ x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path) ] - return tuple(chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents) + store_contents_relative = [ + _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents + ] + return tuple( + chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents_relative + ) def _build_parents( @@ -3697,13 +3793,6 @@ def _build_parents( return parents -def _get_default_codecs( - np_dtype: np.dtype[Any], -) -> tuple[Codec, ...]: - filters, serializer, compressors = _get_default_chunk_encoding_v3(np_dtype) - return filters + (serializer,) + compressors - - FiltersLike: TypeAlias = ( Iterable[dict[str, JSON] | ArrayArrayCodec | numcodecs.abc.Codec] | ArrayArrayCodec @@ -3712,7 +3801,11 @@ def _get_default_codecs( | Literal["auto"] | None ) -CompressorLike: TypeAlias = dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec | None +# Union of acceptable types for users to pass in for both v2 and v3 compressors +CompressorLike: TypeAlias = ( + dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec | Literal["auto"] | None +) + CompressorsLike: TypeAlias = ( Iterable[dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec] | dict[str, JSON] @@ -3732,39 +3825,297 @@ class ShardsConfigParam(TypedDict): ShardsLike: TypeAlias = ChunkCoords | ShardsConfigParam | Literal["auto"] -async def create_array( +async def from_array( store: str | StoreLike, *, + data: Array | npt.ArrayLike, + write_data: bool = True, name: str | None = None, - shape: ShapeLike, - dtype: npt.DTypeLike, - chunks: ChunkCoords | Literal["auto"] = "auto", - shards: ShardsLike | None = None, - filters: FiltersLike = "auto", - compressors: CompressorsLike = "auto", - serializer: SerializerLike = "auto", + chunks: Literal["auto", "keep"] | ChunkCoords = "keep", + shards: ShardsLike | None | Literal["keep"] = "keep", + filters: FiltersLike | Literal["keep"] = "keep", + compressors: CompressorsLike | Literal["keep"] = "keep", + serializer: SerializerLike | Literal["keep"] = "keep", fill_value: Any | None = None, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an array. + """Create an array from an existing array or array-like. Parameters ---------- store : str or Store - Store or path to directory in file system or name of zip file. + Store or path to directory in file system or name of zip file for the new array. + data : Array | array-like + The array to copy. + write_data : bool, default True + Whether to copy the data from the input array to the new array. + If ``write_data`` is ``False``, the new array will be created with the same metadata as the + input array, but without any data. name : str or None, optional The name of the array within the store. If ``name`` is ``None``, the array will be located at the root of the store. + chunks : ChunkCoords or "auto" or "keep", optional + Chunk shape of the array. + Following values are supported: + + - "auto": Automatically determine the chunk shape based on the array's shape and dtype. + - "keep": Retain the chunk shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the chunk shape. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". + shards : ChunkCoords, optional + Shard shape of the array. + Following values are supported: + + - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. + - "keep": Retain the shard shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the shard shape. + - None: No sharding. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise None. + filters : Iterable[Codec] or "auto" or "keep", optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + + Following values are supported: + + - Iterable[Codec]: List of filters to apply to the array. + - "auto": Automatically determine the filters based on the array's dtype. + - "keep": Retain the filters of the data array if it is a zarr Array. + + If no ``filters`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + compressors : Iterable[Codec] or "auto" or "keep", optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + + Following values are supported: + + - Iterable[Codec]: List of compressors to apply to the array. + - "auto": Automatically determine the compressors based on the array's dtype. + - "keep": Retain the compressors of the input array if it is a zarr Array. + + If no ``compressors`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + + Following values are supported: + + - dict[str, JSON]: A dict representation of an ``ArrayBytesCodec``. + - ArrayBytesCodec: An instance of ``ArrayBytesCodec``. + - "auto": a default serializer will be used. These defaults can be changed by modifying the value of + ``array.v3_default_serializer`` in :mod:`zarr.core.config`. + - "keep": Retain the serializer of the input array if it is a zarr Array. + + fill_value : Any, optional + Fill value for the array. + If not specified, defaults to the fill value of the data array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If not specified, defaults to the memory order of the data array. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + If not specified, defaults to the zarr format of the data array. + attributes : dict, optional + Attributes for the array. + If not specified, defaults to the attributes of the data array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + If not specified and the data array has the same zarr format as the target array, + the chunk key encoding of the data array is used. + dimension_names : Iterable[str | None], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + If not specified, defaults to the dimension names of the data array. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + AsyncArray + The array. + + Examples + -------- + Create an array from an existing Array:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> store2 = zarr.storage.LocalStore('example.zarr') + >>> arr = zarr.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='int32', + >>> fill_value=0) + >>> arr2 = await zarr.api.asynchronous.from_array(store2, data=arr) + + + Create an array from an existing NumPy array:: + + >>> arr3 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=np.arange(10000, dtype='i4').reshape(100, 100), + >>> ) + + + Create an array from any array-like object:: + + >>> arr4 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=[[1, 2], [3, 4]], + >>> ) + + >>> await arr4.getitem(...) + array([[1, 2],[3, 4]]) + + Create an array from an existing Array without copying the data:: + + >>> arr5 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=Array(arr4), + >>> write_data=False, + >>> ) + + >>> await arr5.getitem(...) + array([[0, 0],[0, 0]]) + """ + mode: Literal["a"] = "a" + config_parsed = parse_array_config(config) + store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + + ( + chunks, + shards, + filters, + compressors, + serializer, + fill_value, + order, + zarr_format, + chunk_key_encoding, + dimension_names, + ) = _parse_keep_array_attr( + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + ) + if not hasattr(data, "dtype") or not hasattr(data, "shape"): + data = np.array(data) + + result = await init_array( + store_path=store_path, + shape=data.shape, + dtype=data.dtype, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + overwrite=overwrite, + config=config_parsed, + ) + + if write_data: + if isinstance(data, Array): + + async def _copy_array_region(chunk_coords: ChunkCoords | slice, _data: Array) -> None: + arr = await _data._async_array.getitem(chunk_coords) + await result.setitem(chunk_coords, arr) + + # Stream data from the source array to the new array + await concurrent_map( + [(region, data) for region in result._iter_chunk_regions()], + _copy_array_region, + zarr.core.config.config.get("async.concurrency"), + ) + else: + + async def _copy_arraylike_region(chunk_coords: slice, _data: NDArrayLike) -> None: + await result.setitem(chunk_coords, _data[chunk_coords]) + + # Stream data from the source array to the new array + await concurrent_map( + [(region, data) for region in result._iter_chunk_regions()], + _copy_arraylike_region, + zarr.core.config.config.get("async.concurrency"), + ) + return result + + +async def init_array( + *, + store_path: StorePath, + shape: ShapeLike, + dtype: ZDTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + overwrite: bool = False, + config: ArrayConfigLike | None, +) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: + """Create and persist an array metadata document. + + Parameters + ---------- + store_path : StorePath + StorePath instance. The path attribute is the name of the array to initialize. shape : ChunkCoords Shape of the array. - dtype : npt.DTypeLike + dtype : ZDTypeLike Data type of the array. chunks : ChunkCoords, optional Chunk shape of the array. @@ -3825,37 +4176,22 @@ async def create_array( The zarr format to use when saving. attributes : dict, optional Attributes for the array. - chunk_key_encoding : ChunkKeyEncoding, optional + chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. - config : ArrayConfig or ArrayConfigLike, optional - Runtime configuration for the array. + config : ArrayConfigLike or None, optional + Configuration for this array. Returns ------- AsyncArray - The array. - - Examples - -------- - >>> import zarr - >>> store = zarr.storage.MemoryStore(mode='w') - >>> async_arr = await zarr.api.asynchronous.create_array( - >>> store=store, - >>> shape=(100,100), - >>> chunks=(10,10), - >>> dtype='i4', - >>> fill_value=0) - + The AsyncArray. """ if zarr_format is None: @@ -3863,20 +4199,32 @@ async def create_array( from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation - mode: Literal["a"] = "a" - dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) - config_parsed = parse_array_config(config) + zdtype = parse_data_type(dtype, zarr_format=zarr_format) shape_parsed = parse_shapelike(shape) chunk_key_encoding_parsed = _parse_chunk_key_encoding( chunk_key_encoding, zarr_format=zarr_format ) - store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + + if overwrite: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + + item_size = 1 + if isinstance(zdtype, HasItemSize): + item_size = zdtype.item_size + shard_shape_parsed, chunk_shape_parsed = _auto_partition( - array_shape=shape_parsed, shard_shape=shards, chunk_shape=chunks, dtype=dtype_parsed + array_shape=shape_parsed, + shard_shape=shards, + chunk_shape=chunks, + item_size=item_size, ) chunks_out: tuple[int, ...] - result: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] - + meta: ArrayV2Metadata | ArrayV3Metadata if zarr_format == 2: if shard_shape_parsed is not None: msg = ( @@ -3889,9 +4237,8 @@ async def create_array( raise ValueError("Zarr format 2 arrays do not support `serializer`.") filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( - compressor=compressors, filters=filters, dtype=np.dtype(dtype) + compressor=compressors, filters=filters, dtype=zdtype ) - if dimension_names is not None: raise ValueError("Zarr format 2 arrays do not support dimension names.") if order is None: @@ -3899,10 +4246,9 @@ async def create_array( else: order_parsed = order - result = await AsyncArray._create_v2( - store_path=store_path, + meta = AsyncArray._create_metadata_v2( shape=shape_parsed, - dtype=dtype_parsed, + dtype=zdtype, chunks=chunk_shape_parsed, dimension_separator=chunk_key_encoding_parsed.separator, fill_value=fill_value, @@ -3910,17 +4256,15 @@ async def create_array( filters=filters_parsed, compressor=compressor_parsed, attributes=attributes, - overwrite=overwrite, - config=config_parsed, ) else: array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( compressors=compressors, filters=filters, serializer=serializer, - dtype=dtype_parsed, + dtype=zdtype, ) - sub_codecs = cast(tuple[Codec, ...], (*array_array, array_bytes, *bytes_bytes)) + sub_codecs = cast("tuple[Codec, ...]", (*array_array, array_bytes, *bytes_bytes)) codecs_out: tuple[Codec, ...] if shard_shape_parsed is not None: index_location = None @@ -3933,7 +4277,7 @@ async def create_array( ) sharding_codec.validate( shape=chunk_shape_parsed, - dtype=dtype_parsed, + dtype=zdtype, chunk_grid=RegularChunkGrid(chunk_shape=shard_shape_parsed), ) codecs_out = (sharding_codec,) @@ -3942,25 +4286,298 @@ async def create_array( chunks_out = chunk_shape_parsed codecs_out = sub_codecs - result = await AsyncArray._create_v3( - store_path=store_path, + if config is None: + config = {} + if order is not None and isinstance(config, dict): + config["order"] = config.get("order", order) + + meta = AsyncArray._create_metadata_v3( shape=shape_parsed, - dtype=dtype_parsed, + dtype=zdtype, fill_value=fill_value, - attributes=attributes, chunk_shape=chunks_out, chunk_key_encoding=chunk_key_encoding_parsed, codecs=codecs_out, dimension_names=dimension_names, + attributes=attributes, + ) + + arr = AsyncArray(metadata=meta, store_path=store_path, config=config) + await arr._save_metadata(meta, ensure_parents=True) + return arr + + +async def create_array( + store: str | StoreLike, + *, + name: str | None = None, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + shape : ChunkCoords, optional + Shape of the array. Can be ``None`` if ``data`` is provided. + dtype : ZDTypeLike | None + Data type of the array. Can be ``None`` if ``data`` is provided. + data : Array-like data to use for initializing the array. If this parameter is provided, the + ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``, + or ``None``. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the ``data`` parameter + then ``write_data`` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If ``write_data`` is ``False``, then the + array will be left empty. + + Returns + ------- + AsyncArray + The array. + + Examples + -------- + >>> import zarr + >>> store = zarr.storage.MemoryStore(mode='w') + >>> async_arr = await zarr.api.asynchronous.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='i4', + >>> fill_value=0) + + """ + data_parsed, shape_parsed, dtype_parsed = _parse_data_params( + data=data, shape=shape, dtype=dtype + ) + if data_parsed is not None: + return await from_array( + store, + data=data_parsed, + write_data=write_data, + name=name, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, overwrite=overwrite, - config=config_parsed, + config=config, ) + else: + mode: Literal["a"] = "a" - return result + store_path = await make_store_path( + store, path=name, mode=mode, storage_options=storage_options + ) + return await init_array( + store_path=store_path, + shape=shape_parsed, + dtype=dtype_parsed, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + overwrite=overwrite, + config=config, + ) + + +def _parse_keep_array_attr( + data: Array | npt.ArrayLike, + chunks: Literal["auto", "keep"] | ChunkCoords, + shards: ShardsLike | None | Literal["keep"], + filters: FiltersLike | Literal["keep"], + compressors: CompressorsLike | Literal["keep"], + serializer: SerializerLike | Literal["keep"], + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: ZarrFormat | None, + chunk_key_encoding: ChunkKeyEncodingLike | None, + dimension_names: DimensionNames, +) -> tuple[ + ChunkCoords | Literal["auto"], + ShardsLike | None, + FiltersLike, + CompressorsLike, + SerializerLike, + Any | None, + MemoryOrder | None, + ZarrFormat, + ChunkKeyEncodingLike | None, + DimensionNames, +]: + if isinstance(data, Array): + if chunks == "keep": + chunks = data.chunks + if shards == "keep": + shards = data.shards + if zarr_format is None: + zarr_format = data.metadata.zarr_format + if filters == "keep": + if zarr_format == data.metadata.zarr_format: + filters = data.filters or None + else: + filters = "auto" + if compressors == "keep": + if zarr_format == data.metadata.zarr_format: + compressors = data.compressors or None + else: + compressors = "auto" + if serializer == "keep": + if zarr_format == 3 and data.metadata.zarr_format == 3: + serializer = cast("SerializerLike", data.serializer) + else: + serializer = "auto" + if fill_value is None: + fill_value = data.fill_value + if order is None: + order = data.order + if chunk_key_encoding is None and zarr_format == data.metadata.zarr_format: + if isinstance(data.metadata, ArrayV2Metadata): + chunk_key_encoding = {"name": "v2", "separator": data.metadata.dimension_separator} + elif isinstance(data.metadata, ArrayV3Metadata): + chunk_key_encoding = data.metadata.chunk_key_encoding + if dimension_names is None and data.metadata.zarr_format == 3: + dimension_names = data.metadata.dimension_names + else: + if chunks == "keep": + chunks = "auto" + if shards == "keep": + shards = None + if zarr_format is None: + zarr_format = 3 + if filters == "keep": + filters = "auto" + if compressors == "keep": + compressors = "auto" + if serializer == "keep": + serializer = "auto" + return ( + chunks, + shards, + filters, + compressors, + serializer, + fill_value, + order, + zarr_format, + chunk_key_encoding, + dimension_names, + ) def _parse_chunk_key_encoding( - data: ChunkKeyEncoding | ChunkKeyEncodingLike | None, zarr_format: ZarrFormat + data: ChunkKeyEncodingLike | None, zarr_format: ZarrFormat ) -> ChunkKeyEncoding: """ Take an implicit specification of a chunk key encoding and parse it into a ChunkKeyEncoding object. @@ -3984,62 +4601,50 @@ def _parse_chunk_key_encoding( def _get_default_chunk_encoding_v3( - np_dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: """ Get the default ArrayArrayCodecs, ArrayBytesCodec, and BytesBytesCodec for a given dtype. """ - dtype = DataType.from_numpy(np_dtype) - if dtype == DataType.string: - dtype_key = "string" - elif dtype == DataType.bytes: - dtype_key = "bytes" - else: - dtype_key = "numeric" - default_filters = zarr_config.get("array.v3_default_filters").get(dtype_key) - default_serializer = zarr_config.get("array.v3_default_serializer").get(dtype_key) - default_compressors = zarr_config.get("array.v3_default_compressors").get(dtype_key) + dtype_category = categorize_data_type(dtype) - filters = tuple(_parse_array_array_codec(codec_dict) for codec_dict in default_filters) - serializer = _parse_array_bytes_codec(default_serializer) - compressors = tuple(_parse_bytes_bytes_codec(codec_dict) for codec_dict in default_compressors) + filters = zarr_config.get("array.v3_default_filters").get(dtype_category) + compressors = zarr_config.get("array.v3_default_compressors").get(dtype_category) + serializer = zarr_config.get("array.v3_default_serializer").get(dtype_category) - return filters, serializer, compressors + return ( + tuple(_parse_array_array_codec(f) for f in filters), + _parse_array_bytes_codec(serializer), + tuple(_parse_bytes_bytes_codec(c) for c in compressors), + ) def _get_default_chunk_encoding_v2( - np_dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[numcodecs.abc.Codec, ...] | None, numcodecs.abc.Codec | None]: """ Get the default chunk encoding for Zarr format 2 arrays, given a dtype """ + dtype_category = categorize_data_type(dtype) + filters = zarr_config.get("array.v2_default_filters").get(dtype_category) + compressor = zarr_config.get("array.v2_default_compressor").get(dtype_category) + if filters is not None: + filters = tuple(numcodecs.get_codec(f) for f in filters) - compressor_dict = _default_compressor(np_dtype) - filter_dicts = _default_filters(np_dtype) - - compressor = None - if compressor_dict is not None: - compressor = numcodecs.get_codec(compressor_dict) - - filters = None - if filter_dicts is not None: - filters = tuple(numcodecs.get_codec(f) for f in filter_dicts) - - return filters, compressor + return filters, numcodecs.get_codec(compressor) def _parse_chunk_encoding_v2( *, compressor: CompressorsLike, filters: FiltersLike, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[numcodecs.abc.Codec, ...] | None, numcodecs.abc.Codec | None]: """ Generate chunk encoding classes for Zarr format 2 arrays with optional defaults. """ default_filters, default_compressor = _get_default_chunk_encoding_v2(dtype) - _filters: tuple[numcodecs.abc.Codec, ...] | None _compressor: numcodecs.abc.Codec | None @@ -4078,7 +4683,7 @@ def _parse_chunk_encoding_v3( compressors: CompressorsLike, filters: FiltersLike, serializer: SerializerLike, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: """ Generate chunk encoding classes for v3 arrays with optional defaults. @@ -4096,12 +4701,15 @@ def _parse_chunk_encoding_v3( if isinstance(filters, dict | Codec): maybe_array_array = (filters,) else: - maybe_array_array = cast(Iterable[Codec | dict[str, JSON]], filters) + maybe_array_array = cast("Iterable[Codec | dict[str, JSON]]", filters) out_array_array = tuple(_parse_array_array_codec(c) for c in maybe_array_array) if serializer == "auto": out_array_bytes = default_array_bytes else: + # TODO: ensure that the serializer is compatible with the ndarray produced by the + # array-array codecs. For example, if a sequence of array-array codecs produces an + # array with a single-byte data type, then the serializer should not specify endiannesss. out_array_bytes = _parse_array_bytes_codec(serializer) if compressors is None: @@ -4113,23 +4721,86 @@ def _parse_chunk_encoding_v3( if isinstance(compressors, dict | Codec): maybe_bytes_bytes = (compressors,) else: - maybe_bytes_bytes = cast(Iterable[Codec | dict[str, JSON]], compressors) + maybe_bytes_bytes = cast("Iterable[Codec | dict[str, JSON]]", compressors) out_bytes_bytes = tuple(_parse_bytes_bytes_codec(c) for c in maybe_bytes_bytes) + # specialize codecs as needed given the dtype + + # TODO: refactor so that the config only contains the name of the codec, and we use the dtype + # to create the codec instance, instead of storing a dict representation of a full codec. + + # TODO: ensure that the serializer is compatible with the ndarray produced by the + # array-array codecs. For example, if a sequence of array-array codecs produces an + # array with a single-byte data type, then the serializer should not specify endiannesss. + if isinstance(out_array_bytes, BytesCodec) and not isinstance(dtype, HasEndianness): + # The default endianness in the bytescodec might not be None, so we need to replace it + out_array_bytes = replace(out_array_bytes, endian=None) return out_array_array, out_array_bytes, out_bytes_bytes def _parse_deprecated_compressor( - compressor: CompressorLike | None, compressors: CompressorsLike + compressor: CompressorLike | None, compressors: CompressorsLike, zarr_format: int = 3 ) -> CompressorsLike | None: - if compressor: + if compressor != "auto": if compressors != "auto": raise ValueError("Cannot specify both `compressor` and `compressors`.") - warn( - "The `compressor` argument is deprecated. Use `compressors` instead.", - category=UserWarning, - stacklevel=2, - ) - compressors = (compressor,) + if zarr_format == 3: + warn( + "The `compressor` argument is deprecated. Use `compressors` instead.", + category=UserWarning, + stacklevel=2, + ) + if compressor is None: + # "no compression" + compressors = () + else: + compressors = (compressor,) + elif zarr_format == 2 and compressor == compressors == "auto": + compressors = ({"id": "blosc"},) return compressors + + +def _parse_data_params( + *, + data: np.ndarray[Any, np.dtype[Any]] | None, + shape: ShapeLike | None, + dtype: ZDTypeLike | None, +) -> tuple[np.ndarray[Any, np.dtype[Any]] | None, ShapeLike, ZDTypeLike]: + """ + Ensure an array-like ``data`` parameter is consistent with the ``dtype`` and ``shape`` + parameters. + """ + if data is None: + if shape is None: + msg = ( + "The data parameter was set to None, but shape was not specified. " + "Either provide a value for data, or specify shape." + ) + raise ValueError(msg) + shape_out = shape + if dtype is None: + msg = ( + "The data parameter was set to None, but dtype was not specified." + "Either provide an array-like value for data, or specify dtype." + ) + raise ValueError(msg) + dtype_out = dtype + else: + if shape is not None: + msg = ( + "The data parameter was used, but the shape parameter was also " + "used. This is an error. Either use the data parameter, or the shape parameter, " + "but not both." + ) + raise ValueError(msg) + shape_out = data.shape + if dtype is not None: + msg = ( + "The data parameter was used, but the dtype parameter was also " + "used. This is an error. Either use the data parameter, or the dtype parameter, " + "but not both." + ) + raise ValueError(msg) + dtype_out = data.dtype + return data, shape_out, dtype_out diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py index b1a6a3cad0..279bf6edf0 100644 --- a/src/zarr/core/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -3,8 +3,6 @@ from dataclasses import dataclass, fields from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast -import numpy as np - from zarr.core.common import ( MemoryOrder, parse_bool, @@ -19,9 +17,10 @@ from zarr.core.buffer import BufferPrototype from zarr.core.common import ChunkCoords + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType -class ArrayConfigLike(TypedDict): +class ArrayConfigParams(TypedDict): """ A TypedDict model of the attributes of an ArrayConfig class, but with no required fields. This allows for partial construction of an ArrayConfig, with the assumption that the unset @@ -56,15 +55,15 @@ def __init__(self, order: MemoryOrder, write_empty_chunks: bool) -> None: object.__setattr__(self, "write_empty_chunks", write_empty_chunks_parsed) @classmethod - def from_dict(cls, data: ArrayConfigLike) -> Self: + def from_dict(cls, data: ArrayConfigParams) -> Self: """ Create an ArrayConfig from a dict. The keys of that dict are a subset of the attributes of the ArrayConfig class. Any keys missing from that dict will be set to the the values in the ``array`` namespace of ``zarr.config``. """ - kwargs_out: ArrayConfigLike = {} + kwargs_out: ArrayConfigParams = {} for f in fields(ArrayConfig): - field_name = cast(Literal["order", "write_empty_chunks"], f.name) + field_name = cast("Literal['order', 'write_empty_chunks']", f.name) if field_name not in data: kwargs_out[field_name] = zarr_config.get(f"array.{field_name}") else: @@ -72,7 +71,10 @@ def from_dict(cls, data: ArrayConfigLike) -> Self: return cls(**kwargs_out) -def parse_array_config(data: ArrayConfig | ArrayConfigLike | None) -> ArrayConfig: +ArrayConfigLike = ArrayConfig | ArrayConfigParams + + +def parse_array_config(data: ArrayConfigLike | None) -> ArrayConfig: """ Convert various types of data to an ArrayConfig. """ @@ -87,7 +89,7 @@ def parse_array_config(data: ArrayConfig | ArrayConfigLike | None) -> ArrayConfi @dataclass(frozen=True) class ArraySpec: shape: ChunkCoords - dtype: np.dtype[Any] + dtype: ZDType[TBaseDType, TBaseScalar] fill_value: Any config: ArrayConfig prototype: BufferPrototype @@ -95,17 +97,16 @@ class ArraySpec: def __init__( self, shape: ChunkCoords, - dtype: np.dtype[Any], + dtype: ZDType[TBaseDType, TBaseScalar], fill_value: Any, config: ArrayConfig, prototype: BufferPrototype, ) -> None: shape_parsed = parse_shapelike(shape) - dtype_parsed = np.dtype(dtype) fill_value_parsed = parse_fill_value(fill_value) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "dtype", dtype_parsed) + object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "fill_value", fill_value_parsed) object.__setattr__(self, "config", config) object.__setattr__(self, "prototype", prototype) diff --git a/src/zarr/core/attributes.py b/src/zarr/core/attributes.py index 7f9864d1b5..e699c4f66d 100644 --- a/src/zarr/core/attributes.py +++ b/src/zarr/core/attributes.py @@ -28,7 +28,7 @@ def __setitem__(self, key: str, value: JSON) -> None: def __delitem__(self, key: str) -> None: new_attrs = dict(self._obj.metadata.attributes) del new_attrs[key] - self._obj = self._obj.update_attributes(new_attrs) + self.put(new_attrs) def __iter__(self) -> Iterator[str]: return iter(self._obj.metadata.attributes) @@ -50,6 +50,7 @@ def put(self, d: dict[str, JSON]) -> None: >>> attrs {'a': 3, 'c': 4} """ + self._obj.metadata.attributes.clear() self._obj = self._obj.update_attributes(d) def asdict(self) -> dict[str, JSON]: diff --git a/src/zarr/core/buffer/__init__.py b/src/zarr/core/buffer/__init__.py index ccb41e291c..ebec61a372 100644 --- a/src/zarr/core/buffer/__init__.py +++ b/src/zarr/core/buffer/__init__.py @@ -3,6 +3,7 @@ Buffer, BufferPrototype, NDArrayLike, + NDArrayLikeOrScalar, NDBuffer, default_buffer_prototype, ) @@ -13,6 +14,7 @@ "Buffer", "BufferPrototype", "NDArrayLike", + "NDArrayLikeOrScalar", "NDBuffer", "default_buffer_prototype", "numpy_buffer_prototype", diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 85a7351fc7..0e24c5b326 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -105,6 +105,10 @@ def __eq__(self, other: object) -> Self: # type: ignore[explicit-override, over """ +ScalarType = int | float | complex | bytes | str | bool | np.generic +NDArrayLikeOrScalar = ScalarType | NDArrayLike + + def check_item_key_is_1d_contiguous(key: Any) -> None: """Raises error if `key` isn't a 1d contiguous slice""" if not isinstance(key, slice): @@ -139,7 +143,7 @@ class Buffer(ABC): def __init__(self, array_like: ArrayLike) -> None: if array_like.ndim != 1: raise ValueError("array_like: only 1-dim allowed") - if array_like.dtype != np.dtype("b"): + if array_like.dtype != np.dtype("B"): raise ValueError("array_like: only byte dtype allowed") self._data = array_like @@ -155,7 +159,7 @@ def create_zero_length(cls) -> Self: if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( - cast(ArrayLike, None) + cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod @@ -203,7 +207,7 @@ def from_buffer(cls, buffer: Buffer) -> Self: if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( - cast(ArrayLike, None) + cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod @@ -223,7 +227,7 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self: if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( - cast(ArrayLike, None) + cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker def as_array_like(self) -> ArrayLike: @@ -251,6 +255,19 @@ def as_numpy_array(self) -> npt.NDArray[Any]: """ ... + def as_buffer_like(self) -> BytesLike: + """Returns the buffer as an object that implements the Python buffer protocol. + + Notes + ----- + Might have to copy data, since the implementation uses `.as_numpy_array()`. + + Returns + ------- + An object that implements the Python buffer protocol + """ + return memoryview(self.as_numpy_array()) # type: ignore[arg-type] + def to_bytes(self) -> bytes: """Returns the buffer as `bytes` (host memory). @@ -302,7 +319,7 @@ class NDBuffer: Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer - is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. @@ -354,7 +371,7 @@ def create( "Cannot call abstract method on the abstract class 'NDBuffer'" ) return cls( - cast(NDArrayLike, None) + cast("NDArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod @@ -391,7 +408,7 @@ def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: "Cannot call abstract method on the abstract class 'NDBuffer'" ) return cls( - cast(NDArrayLike, None) + cast("NDArrayLike", None) ) # This line will never be reached, but it satisfies the type checker def as_ndarray_like(self) -> NDArrayLike: @@ -419,6 +436,12 @@ def as_numpy_array(self) -> npt.NDArray[Any]: """ ... + def as_scalar(self) -> ScalarType: + """Returns the buffer as a scalar value""" + if self._data.size != 1: + raise ValueError("Buffer does not contain a single scalar value") + return cast("ScalarType", self.as_numpy_array()[()]) + @property def dtype(self) -> np.dtype[Any]: return self._data.dtype @@ -470,7 +493,11 @@ def all_equal(self, other: Any, equal_nan: bool = True) -> bool: # every single time we have to write data? _data, other = np.broadcast_arrays(self._data, other) return np.array_equal( - self._data, other, equal_nan=equal_nan if self._data.dtype.kind not in "USTO" else False + self._data, + other, + equal_nan=equal_nan + if self._data.dtype.kind not in ("U", "S", "T", "O", "V") + else False, ) def fill(self, value: Any) -> None: diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 5019075496..3140d75111 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -49,7 +49,7 @@ def __init__(self, array_like: ArrayLike) -> None: @classmethod def create_zero_length(cls) -> Self: - return cls(np.array([], dtype="b")) + return cls(np.array([], dtype="B")) @classmethod def from_buffer(cls, buffer: core.Buffer) -> Self: @@ -92,7 +92,7 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self: ------- New buffer representing `bytes_like` """ - return cls.from_array_like(np.frombuffer(bytes_like, dtype="b")) + return cls.from_array_like(np.frombuffer(bytes_like, dtype="B")) def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). @@ -111,7 +111,7 @@ def __add__(self, other: core.Buffer) -> Self: """Concatenate two buffers""" other_array = other.as_array_like() - assert other_array.dtype == np.dtype("b") + assert other_array.dtype == np.dtype("B") return self.__class__( np.concatenate((np.asanyarray(self._data), np.asanyarray(other_array))) ) @@ -131,7 +131,7 @@ class NDBuffer(core.NDBuffer): Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer - is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. @@ -154,10 +154,11 @@ def create( order: Literal["C", "F"] = "C", fill_value: Any | None = None, ) -> Self: - ret = cls(np.empty(shape=tuple(shape), dtype=dtype, order=order)) - if fill_value is not None: - ret.fill(fill_value) - return ret + # np.zeros is much faster than np.full, and therefore using it when possible is better. + if fill_value is None or (isinstance(fill_value, int) and fill_value == 0): + return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order)) + else: + return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order)) @classmethod def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: @@ -223,5 +224,10 @@ def numpy_buffer_prototype() -> core.BufferPrototype: return core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) -register_buffer(Buffer) -register_ndbuffer(NDBuffer) +register_buffer(Buffer, qualname="zarr.buffer.cpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.buffer.cpu.NDBuffer") + + +# backwards compatibility +register_buffer(Buffer, qualname="zarr.core.buffer.cpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.cpu.NDBuffer") diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index 6941c8897e..7ea6d53fe3 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -13,6 +13,10 @@ from zarr.core.buffer import core from zarr.core.buffer.core import ArrayLike, BufferPrototype, NDArrayLike +from zarr.registry import ( + register_buffer, + register_ndbuffer, +) if TYPE_CHECKING: from collections.abc import Iterable @@ -55,7 +59,7 @@ def __init__(self, array_like: ArrayLike) -> None: if array_like.ndim != 1: raise ValueError("array_like: only 1-dim allowed") - if array_like.dtype != np.dtype("b"): + if array_like.dtype != np.dtype("B"): raise ValueError("array_like: only byte dtype allowed") if not hasattr(array_like, "__cuda_array_interface__"): @@ -80,7 +84,7 @@ def create_zero_length(cls) -> Self: ------- New empty 0-length buffer """ - return cls(cp.array([], dtype="b")) + return cls(cp.array([], dtype="B")) @classmethod def from_buffer(cls, buffer: core.Buffer) -> Self: @@ -96,14 +100,14 @@ def from_buffer(cls, buffer: core.Buffer) -> Self: @classmethod def from_bytes(cls, bytes_like: BytesLike) -> Self: - return cls.from_array_like(cp.frombuffer(bytes_like, dtype="b")) + return cls.from_array_like(cp.frombuffer(bytes_like, dtype="B")) def as_numpy_array(self) -> npt.NDArray[Any]: - return cast(npt.NDArray[Any], cp.asnumpy(self._data)) + return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) def __add__(self, other: core.Buffer) -> Self: other_array = other.as_array_like() - assert other_array.dtype == np.dtype("b") + assert other_array.dtype == np.dtype("B") gpu_other = Buffer(other_array) gpu_other_array = gpu_other.as_array_like() return self.__class__( @@ -125,7 +129,7 @@ class NDBuffer(core.NDBuffer): Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer - is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. @@ -200,7 +204,7 @@ def as_numpy_array(self) -> npt.NDArray[Any]: ------- NumPy array of this buffer (might be a data copy) """ - return cast(npt.NDArray[Any], cp.asnumpy(self._data)) + return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) def __getitem__(self, key: Any) -> Self: return self.__class__(self._data.__getitem__(key)) @@ -215,3 +219,10 @@ def __setitem__(self, key: Any, value: Any) -> None: buffer_prototype = BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) + +register_buffer(Buffer, qualname="zarr.buffer.gpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.buffer.gpu.NDBuffer") + +# backwards compatibility +register_buffer(Buffer, qualname="zarr.core.buffer.gpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.gpu.NDBuffer") diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index d3e40c26ed..4bf03c89de 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -64,6 +64,9 @@ def _guess_chunks( if isinstance(shape, int): shape = (shape,) + if typesize == 0: + return shape + ndims = len(shape) # require chunks to have non-zero length for all dimensions chunks = np.maximum(np.array(shape, dtype="=f8"), 1) @@ -204,7 +207,7 @@ def _auto_partition( array_shape: tuple[int, ...], chunk_shape: tuple[int, ...] | Literal["auto"], shard_shape: ShardsLike | None, - dtype: np.dtype[Any], + item_size: int, ) -> tuple[tuple[int, ...] | None, tuple[int, ...]]: """ Automatically determine the shard shape and chunk shape for an array, given the shape and dtype of the array. @@ -214,7 +217,6 @@ def _auto_partition( of the array; if the `chunk_shape` is also "auto", then the chunks will be set heuristically as well, given the dtype and shard shape. Otherwise, the chunks will be returned as-is. """ - item_size = dtype.itemsize if shard_shape is None: _shards_out: None | tuple[int, ...] = None if chunk_shape == "auto": diff --git a/src/zarr/core/chunk_key_encodings.py b/src/zarr/core/chunk_key_encodings.py index 95ce9108f3..91dfc90365 100644 --- a/src/zarr/core/chunk_key_encodings.py +++ b/src/zarr/core/chunk_key_encodings.py @@ -2,7 +2,10 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Literal, TypedDict, cast +from typing import TYPE_CHECKING, Literal, TypeAlias, TypedDict, cast + +if TYPE_CHECKING: + from typing import NotRequired from zarr.abc.metadata import Metadata from zarr.core.common import ( @@ -17,12 +20,12 @@ def parse_separator(data: JSON) -> SeparatorLiteral: if data not in (".", "/"): raise ValueError(f"Expected an '.' or '/' separator. Got {data} instead.") - return cast(SeparatorLiteral, data) + return cast("SeparatorLiteral", data) -class ChunkKeyEncodingLike(TypedDict): +class ChunkKeyEncodingParams(TypedDict): name: Literal["v2", "default"] - separator: SeparatorLiteral + separator: NotRequired[SeparatorLiteral] @dataclass(frozen=True) @@ -36,9 +39,7 @@ def __init__(self, *, separator: SeparatorLiteral) -> None: object.__setattr__(self, "separator", separator_parsed) @classmethod - def from_dict( - cls, data: dict[str, JSON] | ChunkKeyEncoding | ChunkKeyEncodingLike - ) -> ChunkKeyEncoding: + def from_dict(cls, data: dict[str, JSON] | ChunkKeyEncodingLike) -> ChunkKeyEncoding: if isinstance(data, ChunkKeyEncoding): return data @@ -46,6 +47,9 @@ def from_dict( if "name" in data and "separator" in data: data = {"name": data["name"], "configuration": {"separator": data["separator"]}} + # TODO: remove this cast when we are statically typing the JSON metadata completely. + data = cast("dict[str, JSON]", data) + # configuration is optional for chunk key encodings name_parsed, config_parsed = parse_named_configuration(data, require_configuration=False) if name_parsed == "default": @@ -73,6 +77,9 @@ def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: pass +ChunkKeyEncodingLike: TypeAlias = ChunkKeyEncodingParams | ChunkKeyEncoding + + @dataclass(frozen=True) class DefaultChunkKeyEncoding(ChunkKeyEncoding): name: Literal["default"] = "default" diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index 583ca01c5e..23c27e40c6 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -16,20 +16,18 @@ ) from zarr.core.common import ChunkCoords, concurrent_map from zarr.core.config import config -from zarr.core.indexing import SelectorTuple, is_scalar, is_total_slice -from zarr.core.metadata.v2 import _default_fill_value +from zarr.core.indexing import SelectorTuple, is_scalar from zarr.registry import register_pipeline if TYPE_CHECKING: from collections.abc import Iterable, Iterator from typing import Self - import numpy as np - from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType T = TypeVar("T") U = TypeVar("U") @@ -56,6 +54,19 @@ def resolve_batched(codec: Codec, chunk_specs: Iterable[ArraySpec]) -> Iterable[ return [codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] +def fill_value_or_default(chunk_spec: ArraySpec) -> Any: + fill_value = chunk_spec.fill_value + if fill_value is None: + # Zarr V2 allowed `fill_value` to be null in the metadata. + # Zarr V3 requires it to be set. This has already been + # validated when decoding the metadata, but we support reading + # Zarr V2 data and need to support the case where fill_value + # is None. + return chunk_spec.dtype.default_scalar() + else: + return fill_value + + @dataclass(frozen=True) class BatchedCodecPipeline(CodecPipeline): """Default codec pipeline. @@ -121,7 +132,9 @@ def __iter__(self) -> Iterator[Codec]: yield self.array_bytes_codec yield from self.bytes_bytes_codecs - def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: + def validate( + self, *, shape: ChunkCoords, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGrid + ) -> None: for codec in self: codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid) @@ -230,7 +243,7 @@ async def encode_partial_batch( async def read_batch( self, - batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -238,44 +251,31 @@ async def read_batch( chunk_array_batch = await self.decode_partial_batch( [ (byte_getter, chunk_selection, chunk_spec) - for byte_getter, chunk_spec, chunk_selection, _ in batch_info + for byte_getter, chunk_spec, chunk_selection, *_ in batch_info ] ) - for chunk_array, (_, chunk_spec, _, out_selection) in zip( + for chunk_array, (_, chunk_spec, _, out_selection, _) in zip( chunk_array_batch, batch_info, strict=False ): if chunk_array is not None: out[out_selection] = chunk_array else: - fill_value = chunk_spec.fill_value - - if fill_value is None: - # Zarr V2 allowed `fill_value` to be null in the metadata. - # Zarr V3 requires it to be set. This has already been - # validated when decoding the metadata, but we support reading - # Zarr V2 data and need to support the case where fill_value - # is None. - fill_value = _default_fill_value(dtype=chunk_spec.dtype) - - out[out_selection] = fill_value + out[out_selection] = fill_value_or_default(chunk_spec) else: chunk_bytes_batch = await concurrent_map( - [ - (byte_getter, array_spec.prototype) - for byte_getter, array_spec, _, _ in batch_info - ], + [(byte_getter, array_spec.prototype) for byte_getter, array_spec, *_ in batch_info], lambda byte_getter, prototype: byte_getter.get(prototype), config.get("async.concurrency"), ) chunk_array_batch = await self.decode_batch( [ (chunk_bytes, chunk_spec) - for chunk_bytes, (_, chunk_spec, _, _) in zip( + for chunk_bytes, (_, chunk_spec, *_) in zip( chunk_bytes_batch, batch_info, strict=False ) ], ) - for chunk_array, (_, chunk_spec, chunk_selection, out_selection) in zip( + for chunk_array, (_, chunk_spec, chunk_selection, out_selection, _) in zip( chunk_array_batch, batch_info, strict=False ): if chunk_array is not None: @@ -284,10 +284,7 @@ async def read_batch( tmp = tmp.squeeze(axis=drop_axes) out[out_selection] = tmp else: - fill_value = chunk_spec.fill_value - if fill_value is None: - fill_value = _default_fill_value(dtype=chunk_spec.dtype) - out[out_selection] = fill_value + out[out_selection] = fill_value_or_default(chunk_spec) def _merge_chunk_array( self, @@ -296,20 +293,12 @@ def _merge_chunk_array( out_selection: SelectorTuple, chunk_spec: ArraySpec, chunk_selection: SelectorTuple, + is_complete_chunk: bool, drop_axes: tuple[int, ...], ) -> NDBuffer: - if is_total_slice(chunk_selection, chunk_spec.shape) and value.shape == chunk_spec.shape: - return value - if existing_chunk_array is None: - chunk_array = chunk_spec.prototype.nd_buffer.create( - shape=chunk_spec.shape, - dtype=chunk_spec.dtype, - order=chunk_spec.order, - fill_value=chunk_spec.fill_value, - ) - else: - chunk_array = existing_chunk_array.copy() # make a writable copy - if chunk_selection == () or is_scalar(value.as_ndarray_like(), chunk_spec.dtype): + if chunk_selection == () or is_scalar( + value.as_ndarray_like(), chunk_spec.dtype.to_native_dtype() + ): chunk_value = value else: chunk_value = value[out_selection] @@ -322,12 +311,26 @@ def _merge_chunk_array( for idx in range(chunk_spec.ndim) ) chunk_value = chunk_value[item] + if is_complete_chunk and chunk_value.shape == chunk_spec.shape: + # TODO: For the last chunk, we could have is_complete_chunk=True + # that is smaller than the chunk_spec.shape but this throws + # an error in the _decode_single + return chunk_value + if existing_chunk_array is None: + chunk_array = chunk_spec.prototype.nd_buffer.create( + shape=chunk_spec.shape, + dtype=chunk_spec.dtype.to_native_dtype(), + order=chunk_spec.order, + fill_value=fill_value_or_default(chunk_spec), + ) + else: + chunk_array = existing_chunk_array.copy() # make a writable copy chunk_array[chunk_selection] = chunk_value return chunk_array async def write_batch( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -337,14 +340,14 @@ async def write_batch( await self.encode_partial_batch( [ (byte_setter, value, chunk_selection, chunk_spec) - for byte_setter, chunk_spec, chunk_selection, out_selection in batch_info + for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info ], ) else: await self.encode_partial_batch( [ (byte_setter, value[out_selection], chunk_selection, chunk_spec) - for byte_setter, chunk_spec, chunk_selection, out_selection in batch_info + for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info ], ) @@ -361,10 +364,10 @@ async def _read_key( chunk_bytes_batch = await concurrent_map( [ ( - None if is_total_slice(chunk_selection, chunk_spec.shape) else byte_setter, + None if is_complete_chunk else byte_setter, chunk_spec.prototype, ) - for byte_setter, chunk_spec, chunk_selection, _ in batch_info + for byte_setter, chunk_spec, chunk_selection, _, is_complete_chunk in batch_info ], _read_key, config.get("async.concurrency"), @@ -372,7 +375,7 @@ async def _read_key( chunk_array_decoded = await self.decode_batch( [ (chunk_bytes, chunk_spec) - for chunk_bytes, (_, chunk_spec, _, _) in zip( + for chunk_bytes, (_, chunk_spec, *_) in zip( chunk_bytes_batch, batch_info, strict=False ) ], @@ -380,21 +383,31 @@ async def _read_key( chunk_array_merged = [ self._merge_chunk_array( - chunk_array, value, out_selection, chunk_spec, chunk_selection, drop_axes - ) - for chunk_array, (_, chunk_spec, chunk_selection, out_selection) in zip( - chunk_array_decoded, batch_info, strict=False + chunk_array, + value, + out_selection, + chunk_spec, + chunk_selection, + is_complete_chunk, + drop_axes, ) + for chunk_array, ( + _, + chunk_spec, + chunk_selection, + out_selection, + is_complete_chunk, + ) in zip(chunk_array_decoded, batch_info, strict=False) ] chunk_array_batch: list[NDBuffer | None] = [] - for chunk_array, (_, chunk_spec, _, _) in zip( + for chunk_array, (_, chunk_spec, *_) in zip( chunk_array_merged, batch_info, strict=False ): if chunk_array is None: chunk_array_batch.append(None) # type: ignore[unreachable] else: if not chunk_spec.config.write_empty_chunks and chunk_array.all_equal( - chunk_spec.fill_value + fill_value_or_default(chunk_spec) ): chunk_array_batch.append(None) else: @@ -403,7 +416,7 @@ async def _read_key( chunk_bytes_batch = await self.encode_batch( [ (chunk_array, chunk_spec) - for chunk_array, (_, chunk_spec, _, _) in zip( + for chunk_array, (_, chunk_spec, *_) in zip( chunk_array_batch, batch_info, strict=False ) ], @@ -418,7 +431,7 @@ async def _write_key(byte_setter: ByteSetter, chunk_bytes: Buffer | None) -> Non await concurrent_map( [ (byte_setter, chunk_bytes) - for chunk_bytes, (byte_setter, _, _, _) in zip( + for chunk_bytes, (byte_setter, *_) in zip( chunk_bytes_batch, batch_info, strict=False ) ], @@ -446,7 +459,7 @@ async def encode( async def read( self, - batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -461,7 +474,7 @@ async def read( async def write( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 7205b8c206..2ba5914ea5 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -4,22 +4,21 @@ import functools import operator import warnings -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from enum import Enum from itertools import starmap from typing import ( TYPE_CHECKING, Any, + Generic, Literal, + TypedDict, TypeVar, cast, overload, ) -import numpy as np - from zarr.core.config import config as zarr_config -from zarr.core.strings import _STRING_DTYPE if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator @@ -31,16 +30,24 @@ ZATTRS_JSON = ".zattrs" ZMETADATA_V2_JSON = ".zmetadata" -ByteRangeRequest = tuple[int | None, int | None] BytesLike = bytes | bytearray | memoryview ShapeLike = tuple[int, ...] | int ChunkCoords = tuple[int, ...] ChunkCoordsLike = Iterable[int] ZarrFormat = Literal[2, 3] NodeType = Literal["array", "group"] -JSON = str | int | float | Mapping[str, "JSON"] | tuple["JSON", ...] | None +JSON = str | int | float | Mapping[str, "JSON"] | Sequence["JSON"] | None MemoryOrder = Literal["C", "F"] AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] +DimensionNames = Iterable[str | None] | None + +TName = TypeVar("TName", bound=str) +TConfig = TypeVar("TConfig", bound=Mapping[str, object]) + + +class NamedConfig(TypedDict, Generic[TName, TConfig]): + name: TName + configuration: TConfig def product(tup: ChunkCoords) -> int: @@ -158,7 +165,7 @@ def parse_fill_value(data: Any) -> Any: def parse_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): - return cast(Literal["C", "F"], data) + return cast("Literal['C', 'F']", data) raise ValueError(f"Expected one of ('C', 'F'), got {data} instead.") @@ -168,16 +175,6 @@ def parse_bool(data: Any) -> bool: raise ValueError(f"Expected bool, got {data} instead.") -def parse_dtype(dtype: Any, zarr_format: ZarrFormat) -> np.dtype[Any]: - if dtype is str or dtype == "str": - if zarr_format == 2: - # special case as object - return np.dtype("object") - else: - return _STRING_DTYPE - return np.dtype(dtype) - - def _warn_write_empty_chunks_kwarg() -> None: # TODO: link to docs page on array configuration in this message msg = ( @@ -202,4 +199,4 @@ def _warn_order_kwarg() -> None: def _default_zarr_format() -> ZarrFormat: """Return the default zarr_version""" - return cast(ZarrFormat, int(zarr_config.get("default_zarr_format", 3))) + return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 7920d220a4..05d048ef74 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -29,15 +29,28 @@ from __future__ import annotations -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast from donfig import Config as DConfig +if TYPE_CHECKING: + from donfig.config_obj import ConfigSet + + from zarr.core.dtype.wrapper import ZDType + class BadConfigError(ValueError): _msg = "bad Config: %r" +# These values are used for rough categorization of data types +# we use this for choosing a default encoding scheme based on the data type. Specifically, +# these categories are keys in a configuration dictionary. +# it is not a part of the ZDType class because these categories are more of an implementation detail +# of our config system rather than a useful attribute of any particular data type. +DTypeCategory = Literal["variable-length-string", "default"] + + class Config(DConfig): # type: ignore[misc] """The Config will collect configuration from config files and environment variables @@ -56,6 +69,14 @@ def reset(self) -> None: self.clear() self.refresh() + def enable_gpu(self) -> ConfigSet: + """ + Configure Zarr to use GPUs where possible. + """ + return self.set( + {"buffer": "zarr.buffer.gpu.Buffer", "ndbuffer": "zarr.buffer.gpu.NDBuffer"} + ) + # The default configuration for zarr config = Config( @@ -67,30 +88,24 @@ def reset(self) -> None: "order": "C", "write_empty_chunks": False, "v2_default_compressor": { - "numeric": {"id": "zstd", "level": 0, "checksum": False}, - "string": {"id": "zstd", "level": 0, "checksum": False}, - "bytes": {"id": "zstd", "level": 0, "checksum": False}, + "default": {"id": "zstd", "level": 0, "checksum": False}, + "variable-length-string": {"id": "zstd", "level": 0, "checksum": False}, }, "v2_default_filters": { - "numeric": None, - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], + "default": None, + "variable-length-string": [{"id": "vlen-utf8"}], }, - "v3_default_filters": {"numeric": [], "string": [], "bytes": []}, + "v3_default_filters": {"default": [], "variable-length-string": []}, "v3_default_serializer": { - "numeric": {"name": "bytes", "configuration": {"endian": "little"}}, - "string": {"name": "vlen-utf8"}, - "bytes": {"name": "vlen-bytes"}, + "default": {"name": "bytes", "configuration": {"endian": "little"}}, + "variable-length-string": {"name": "vlen-utf8"}, }, "v3_default_compressors": { - "numeric": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], - "string": [ + "default": [ {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, ], - "bytes": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, + "variable-length-string": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}} ], }, }, @@ -113,8 +128,8 @@ def reset(self) -> None: "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", }, - "buffer": "zarr.core.buffer.cpu.Buffer", - "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", } ], ) @@ -122,6 +137,20 @@ def reset(self) -> None: def parse_indexing_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): - return cast(Literal["C", "F"], data) + return cast("Literal['C', 'F']", data) msg = f"Expected one of ('C', 'F'), got {data} instead." raise ValueError(msg) + + +def categorize_data_type(dtype: ZDType[Any, Any]) -> DTypeCategory: + """ + Classify a ZDType. The return value is a string which belongs to the type ``DTypeCategory``. + + This is used by the config system to determine how to encode arrays with the associated data type + when the user has not specified a particular serialization scheme. + """ + from zarr.core.dtype import VariableLengthUTF8 + + if isinstance(dtype, VariableLengthUTF8): + return "variable-length-string" + return "default" diff --git a/src/zarr/core/dtype/__init__.py b/src/zarr/core/dtype/__init__.py new file mode 100644 index 0000000000..735690d4bc --- /dev/null +++ b/src/zarr/core/dtype/__init__.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, TypeAlias + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeJSON, +) +from zarr.core.dtype.npy.bool import Bool +from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes +from zarr.core.dtype.npy.complex import Complex64, Complex128 +from zarr.core.dtype.npy.float import Float16, Float32, Float64 +from zarr.core.dtype.npy.int import Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 +from zarr.core.dtype.npy.structured import ( + Structured, +) +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 + +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + +from collections.abc import Mapping + +import numpy as np +import numpy.typing as npt + +from zarr.core.common import JSON +from zarr.core.dtype.npy.string import ( + FixedLengthUTF32, + VariableLengthUTF8, +) +from zarr.core.dtype.registry import DataTypeRegistry +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +__all__ = [ + "Bool", + "Complex64", + "Complex128", + "DataTypeRegistry", + "DataTypeValidationError", + "DateTime64", + "FixedLengthUTF32", + "Float16", + "Float32", + "Float64", + "Int8", + "Int16", + "Int32", + "Int64", + "NullTerminatedBytes", + "RawBytes", + "Structured", + "TBaseDType", + "TBaseScalar", + "TimeDelta64", + "TimeDelta64", + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "VariableLengthUTF8", + "ZDType", + "data_type_registry", + "parse_data_type", +] + +data_type_registry = DataTypeRegistry() + +IntegerDType = Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 +INTEGER_DTYPE: Final = Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 + +FloatDType = Float16 | Float32 | Float64 +FLOAT_DTYPE: Final = Float16, Float32, Float64 + +ComplexFloatDType = Complex64 | Complex128 +COMPLEX_FLOAT_DTYPE: Final = Complex64, Complex128 + +StringDType = FixedLengthUTF32 | VariableLengthUTF8 +STRING_DTYPE: Final = FixedLengthUTF32, VariableLengthUTF8 + +TimeDType = DateTime64 | TimeDelta64 +TIME_DTYPE: Final = DateTime64, TimeDelta64 + +BytesDType = RawBytes | NullTerminatedBytes | VariableLengthBytes +BYTES_DTYPE: Final = RawBytes, NullTerminatedBytes, VariableLengthBytes + +AnyDType = ( + Bool + | IntegerDType + | FloatDType + | ComplexFloatDType + | StringDType + | BytesDType + | Structured + | TimeDType + | VariableLengthBytes +) +# mypy has trouble inferring the type of variablelengthstring dtype, because its class definition +# depends on the installed numpy version. That's why the type: ignore statement is needed here. +ANY_DTYPE: Final = ( + Bool, + *INTEGER_DTYPE, + *FLOAT_DTYPE, + *COMPLEX_FLOAT_DTYPE, + *STRING_DTYPE, + *BYTES_DTYPE, + Structured, + *TIME_DTYPE, + VariableLengthBytes, +) + +# This type models inputs that can be coerced to a ZDType +ZDTypeLike: TypeAlias = npt.DTypeLike | ZDType[TBaseDType, TBaseScalar] | Mapping[str, JSON] | str + +for dtype in ANY_DTYPE: + # mypy does not know that all the elements of ANY_DTYPE are subclasses of ZDType + data_type_registry.register(dtype._zarr_v3_name, dtype) # type: ignore[arg-type] + + +# TODO: find a better name for this function +def get_data_type_from_native_dtype(dtype: npt.DTypeLike) -> ZDType[TBaseDType, TBaseScalar]: + """ + Get a data type wrapper (an instance of ``ZDType``) from a native data type, e.g. a numpy dtype. + """ + if not isinstance(dtype, np.dtype): + na_dtype: np.dtype[np.generic] + if isinstance(dtype, list): + # this is a valid _VoidDTypeLike check + na_dtype = np.dtype([tuple(d) for d in dtype]) + else: + na_dtype = np.dtype(dtype) + else: + na_dtype = dtype + return data_type_registry.match_dtype(dtype=na_dtype) + + +def get_data_type_from_json( + dtype_spec: DTypeJSON, *, zarr_format: ZarrFormat +) -> ZDType[TBaseDType, TBaseScalar]: + """ + Given a JSON representation of a data type and a Zarr format version, + attempt to create a ZDType instance from the registered ZDType classes. + """ + return data_type_registry.match_json(dtype_spec, zarr_format=zarr_format) + + +def parse_data_type( + dtype_spec: ZDTypeLike, + *, + zarr_format: ZarrFormat, +) -> ZDType[TBaseDType, TBaseScalar]: + """ + Interpret the input as a ZDType instance. + """ + if isinstance(dtype_spec, ZDType): + return dtype_spec + # dict and zarr_format 3 means that we have a JSON object representation of the dtype + if zarr_format == 3 and isinstance(dtype_spec, Mapping): + return get_data_type_from_json(dtype_spec, zarr_format=3) + # otherwise, we have either a numpy dtype string, or a zarr v3 dtype string, and in either case + # we can create a numpy dtype from it, and do the dtype inference from that + return get_data_type_from_native_dtype(dtype_spec) # type: ignore[arg-type] diff --git a/src/zarr/core/dtype/common.py b/src/zarr/core/dtype/common.py new file mode 100644 index 0000000000..6f61b6775e --- /dev/null +++ b/src/zarr/core/dtype/common.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import warnings +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import ( + ClassVar, + Final, + Generic, + Literal, + TypedDict, + TypeGuard, + TypeVar, +) + +from zarr.core.common import NamedConfig + +EndiannessStr = Literal["little", "big"] +ENDIANNESS_STR: Final = "little", "big" + +SpecialFloatStrings = Literal["NaN", "Infinity", "-Infinity"] +SPECIAL_FLOAT_STRINGS: Final = ("NaN", "Infinity", "-Infinity") + +JSONFloatV2 = float | SpecialFloatStrings +JSONFloatV3 = float | SpecialFloatStrings | str + +ObjectCodecID = Literal["vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2"] +# These are the ids of the known object codecs for zarr v2. +OBJECT_CODEC_IDS: Final = ("vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2") + +# This is a wider type than our standard JSON type because we need +# to work with typeddict objects which are assignable to Mapping[str, object] +DTypeJSON = str | int | float | Sequence["DTypeJSON"] | None | Mapping[str, object] + +# The DTypeJSON_V2 type exists because ZDType.from_json takes a single argument, which must contain +# all the information necessary to decode the data type. Zarr v2 supports multiple distinct +# data types that all used the "|O" data type identifier. These data types can only be +# discriminated on the basis of their "object codec", i.e. a special data type specific +# compressor or filter. So to figure out what data type a zarr v2 array has, we need the +# data type identifier from metadata, as well as an object codec id if the data type identifier +# is "|O". +# So we will pack the name of the dtype alongside the name of the object codec id, if applicable, +# in a single dict, and pass that to the data type inference logic. +# These type variables have a very wide bound because the individual zdtype +# classes can perform a very specific type check. + +# This is the JSON representation of a structured dtype in zarr v2 +StructuredName_V2 = Sequence["str | StructuredName_V2"] + +# This models the type of the name a dtype might have in zarr v2 array metadata +DTypeName_V2 = StructuredName_V2 | str + +TDTypeNameV2_co = TypeVar("TDTypeNameV2_co", bound=DTypeName_V2, covariant=True) +TObjectCodecID_co = TypeVar("TObjectCodecID_co", bound=None | str, covariant=True) + + +class DTypeConfig_V2(TypedDict, Generic[TDTypeNameV2_co, TObjectCodecID_co]): + name: TDTypeNameV2_co + object_codec_id: TObjectCodecID_co + + +DTypeSpec_V2 = DTypeConfig_V2[DTypeName_V2, None | str] + + +def check_structured_dtype_v2_inner(data: object) -> TypeGuard[StructuredName_V2]: + """ + A type guard for the inner elements of a structured dtype. This is a recursive check because + the type is itself recursive. + + This check ensures that all the elements are 2-element sequences beginning with a string + and ending with either another string or another 2-element sequence beginning with a string and + ending with another instance of that type. + """ + if isinstance(data, (str, Mapping)): + return False + if not isinstance(data, Sequence): + return False + if len(data) != 2: + return False + if not (isinstance(data[0], str)): + return False + if isinstance(data[-1], str): + return True + elif isinstance(data[-1], Sequence): + return check_structured_dtype_v2_inner(data[-1]) + return False + + +def check_structured_dtype_name_v2(data: Sequence[object]) -> TypeGuard[StructuredName_V2]: + return all(check_structured_dtype_v2_inner(d) for d in data) + + +def check_dtype_name_v2(data: object) -> TypeGuard[DTypeName_V2]: + """ + Type guard for narrowing the type of a python object to an valid zarr v2 dtype name. + """ + if isinstance(data, str): + return True + elif isinstance(data, Sequence): + return check_structured_dtype_name_v2(data) + return False + + +def check_dtype_spec_v2(data: object) -> TypeGuard[DTypeSpec_V2]: + """ + Type guard for narrowing a python object to an instance of DTypeSpec_V2 + """ + if not isinstance(data, Mapping): + return False + if set(data.keys()) != {"name", "object_codec_id"}: + return False + if not check_dtype_name_v2(data["name"]): + return False + return isinstance(data["object_codec_id"], str | None) + + +# By comparison, The JSON representation of a dtype in zarr v3 is much simpler. +# It's either a string, or a structured dict +DTypeSpec_V3 = str | NamedConfig[str, Mapping[str, object]] + + +def check_dtype_spec_v3(data: object) -> TypeGuard[DTypeSpec_V3]: + """ + Type guard for narrowing the type of a python object to an instance of + DTypeSpec_V3, i.e either a string or a dict with a "name" field that's a string and a + "configuration" field that's a mapping with string keys. + """ + if isinstance(data, str) or ( # noqa: SIM103 + isinstance(data, Mapping) + and set(data.keys()) == {"name", "configuration"} + and isinstance(data["configuration"], Mapping) + and all(isinstance(k, str) for k in data["configuration"]) + ): + return True + return False + + +def unpack_dtype_json(data: DTypeSpec_V2 | DTypeSpec_V3) -> DTypeJSON: + """ + Return the array metadata form of the dtype JSON representation. For the Zarr V3 form of dtype + metadata, this is a no-op. For the Zarr V2 form of dtype metadata, this unpacks the dtype name. + """ + if isinstance(data, Mapping) and set(data.keys()) == {"name", "object_codec_id"}: + return data["name"] + return data + + +class DataTypeValidationError(ValueError): ... + + +class ScalarTypeValidationError(ValueError): ... + + +@dataclass(frozen=True) +class HasLength: + """ + A mix-in class for data types with a length attribute, such as fixed-size collections + of unicode strings, or bytes. + """ + + length: int + + +@dataclass(frozen=True) +class HasEndianness: + """ + A mix-in class for data types with an endianness attribute + """ + + endianness: EndiannessStr = "little" + + +@dataclass(frozen=True) +class HasItemSize: + """ + A mix-in class for data types with an item size attribute. + This mix-in bears a property ``item_size``, which denotes the size of each element of the data + type, in bytes. + """ + + @property + def item_size(self) -> int: + raise NotImplementedError + + +@dataclass(frozen=True) +class HasObjectCodec: + """ + A mix-in class for data types that require an object codec id. + This class bears the property ``object_codec_id``, which is the string name of an object + codec that is required to encode and decode the data type. + + In zarr-python 2.x certain data types like variable-length strings or variable-length arrays + used the catch-all numpy "object" data type for their in-memory representation. But these data + types cannot be stored as numpy object data types, because the object data type does not define + a fixed memory layout. So these data types required a special codec, called an "object codec", + that effectively defined a compact representation for the data type, which was used to encode + and decode the data type. + + Zarr-python 2.x would not allow the creation of arrays with the "object" data type if an object + codec was not specified, and thus the name of the object codec is effectively part of the data + type model. + """ + + object_codec_id: ClassVar[str] + + +class UnstableSpecificationWarning(FutureWarning): ... + + +def v3_unstable_dtype_warning(dtype: object) -> None: + """ + Emit this warning when a data type does not have a stable zarr v3 spec + """ + msg = ( + f"The data type ({dtype}) does not have a Zarr V3 specification. " + "That means that the representation of array saved with this data type may change without " + "warning in a future version of Zarr Python. " + "Arrays stored with this data type may be unreadable by other Zarr libraries. " + "Use this data type at your own risk! " + "Check https://github.com/zarr-developers/zarr-extensions/tree/main/data-types for the " + "status of data type specifications for Zarr V3." + ) + warnings.warn(msg, category=UnstableSpecificationWarning, stacklevel=2) diff --git a/src/zarr/core/dtype/npy/__init__.py b/src/zarr/core/dtype/npy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/zarr/core/dtype/npy/bool.py b/src/zarr/core/dtype/npy/bool.py new file mode 100644 index 0000000000..d8d52468bf --- /dev/null +++ b/src/zarr/core/dtype/npy/bool.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True, kw_only=True, slots=True) +class Bool(ZDType[np.dtypes.BoolDType, np.bool_], HasItemSize): + """ + Wrapper for numpy boolean dtype. + + Attributes + ---------- + name : str + The name of the dtype. + dtype_cls : ClassVar[type[np.dtypes.BoolDType]] + The numpy dtype class. + """ + + _zarr_v3_name: ClassVar[Literal["bool"]] = "bool" + _zarr_v2_name: ClassVar[Literal["|b1"]] = "|b1" + dtype_cls = np.dtypes.BoolDType + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Bool from a np.dtype('bool') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.BoolDType: + """ + Create a NumPy boolean dtype instance from this ZDType + """ + return self.dtype_cls() + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|b1"], None]]: + """ + Check that the input is a valid JSON representation of a Bool. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == cls._zarr_v2_name + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["bool"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_name!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|b1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["bool"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|b1"], None] | Literal["bool"]: + if zarr_format == 2: + return {"name": self._zarr_v2_name, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> bool: + # Anything can become a bool + return True + + def cast_scalar(self, data: object) -> np.bool_: + if self._check_scalar(data): + return np.bool_(data) + msg = f"Cannot convert object with type {type(data)} to a numpy boolean." + raise TypeError(msg) + + def default_scalar(self) -> np.bool_: + """ + Get the default value for the boolean dtype. + + Returns + ------- + np.bool_ + The default value. + """ + return np.False_ + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> bool: + """ + Convert a scalar to a python bool. + + Parameters + ---------- + data : object + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + bool + The JSON-serializable format. + """ + return bool(data) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bool_: + """ + Read a JSON-serializable value as a numpy boolean scalar. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + np.bool_ + The numpy boolean scalar. + """ + if self._check_scalar(data): + return np.bool_(data) + raise TypeError(f"Invalid type: {data}. Expected a boolean.") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 diff --git a/src/zarr/core/dtype/npy/bytes.py b/src/zarr/core/dtype/npy/bytes.py new file mode 100644 index 0000000000..e363c75053 --- /dev/null +++ b/src/zarr/core/dtype/npy/bytes.py @@ -0,0 +1,369 @@ +from __future__ import annotations + +import base64 +import re +from dataclasses import dataclass +from typing import Any, ClassVar, Literal, Self, TypedDict, TypeGuard, cast, overload + +import numpy as np + +from zarr.core.common import JSON, NamedConfig, ZarrFormat +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasItemSize, + HasLength, + HasObjectCodec, + check_dtype_spec_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import check_json_str +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +BytesLike = np.bytes_ | str | bytes | int + + +class FixedLengthBytesConfig(TypedDict): + length_bytes: int + + +NullTerminatedBytesJSONV3 = NamedConfig[Literal["null_terminated_bytes"], FixedLengthBytesConfig] +RawBytesJSONV3 = NamedConfig[Literal["raw_bytes"], FixedLengthBytesConfig] + + +@dataclass(frozen=True, kw_only=True) +class NullTerminatedBytes(ZDType[np.dtypes.BytesDType[int], np.bytes_], HasLength, HasItemSize): + dtype_cls = np.dtypes.BytesDType + _zarr_v3_name: ClassVar[Literal["null_terminated_bytes"]] = "null_terminated_bytes" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(length=dtype.itemsize) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.BytesDType[int]: + return self.dtype_cls(self.length) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid representation of a numpy S dtype. We expect + something like ``{"name": "|S10", "object_codec_id": None}`` + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^\|S\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[NullTerminatedBytesJSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and "length_bytes" in data["configuration"] + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls(length=int(name[2:])) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|S1', '|S2', etc" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"]) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> NullTerminatedBytesJSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | NullTerminatedBytesJSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return { + "name": self._zarr_v3_name, + "configuration": {"length_bytes": self.length}, + } + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: + # this is generous for backwards compatibility + return isinstance(data, BytesLike) + + def _cast_scalar_unchecked(self, data: BytesLike) -> np.bytes_: + # We explicitly truncate the result because of the following numpy behavior: + # >>> x = np.dtype('S3').type('hello world') + # >>> x + # np.bytes_(b'hello world') + # >>> x.dtype + # dtype('S11') + + if isinstance(data, int): + return self.to_native_dtype().type(str(data)[: self.length]) + else: + return self.to_native_dtype().type(data[: self.length]) + + def cast_scalar(self, data: object) -> np.bytes_: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy bytes scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.bytes_: + return np.bytes_(b"") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + as_bytes = self.cast_scalar(data) + return base64.standard_b64encode(as_bytes).decode("ascii") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bytes_: + if check_json_str(data): + return self.to_native_dtype().type(base64.standard_b64decode(data.encode("ascii"))) + raise TypeError( + f"Invalid type: {data}. Expected a base64-encoded string." + ) # pragma: no cover + + @property + def item_size(self) -> int: + return self.length + + +@dataclass(frozen=True, kw_only=True) +class RawBytes(ZDType[np.dtypes.VoidDType[int], np.void], HasLength, HasItemSize): + # np.dtypes.VoidDType is specified in an odd way in numpy + # it cannot be used to create instances of the dtype + # so we have to tell mypy to ignore this here + dtype_cls = np.dtypes.VoidDType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["raw_bytes"]] = "raw_bytes" + + @classmethod + def _check_native_dtype( + cls: type[Self], dtype: TBaseDType + ) -> TypeGuard[np.dtypes.VoidDType[Any]]: + """ + Numpy void dtype comes in two forms: + * If the ``fields`` attribute is ``None``, then the dtype represents N raw bytes. + * If the ``fields`` attribute is not ``None``, then the dtype represents a structured dtype, + + In this check we ensure that ``fields`` is ``None``. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return cls.dtype_cls is type(dtype) and dtype.fields is None # type: ignore[has-type] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(length=dtype.itemsize) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + + def to_native_dtype(self) -> np.dtypes.VoidDType[int]: + # Numpy does not allow creating a void type + # by invoking np.dtypes.VoidDType directly + return cast("np.dtypes.VoidDType[int]", np.dtype(f"V{self.length}")) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid representation of a numpy S dtype. We expect + something like ``{"name": "|V10", "object_codec_id": None}`` + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^\|V\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[RawBytesJSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"length_bytes"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls(length=int(name[2:])) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|V1', '|V2', etc" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"]) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> RawBytesJSONV3: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | RawBytesJSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return {"name": self._zarr_v3_name, "configuration": {"length_bytes": self.length}} + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> bool: + return isinstance(data, np.bytes_ | str | bytes | np.void) + + def _cast_scalar_unchecked(self, data: object) -> np.void: + native_dtype = self.to_native_dtype() + # Without the second argument, numpy will return a void scalar for dtype V1. + # The second argument ensures that, if native_dtype is something like V10, + # the result will actually be a V10 scalar. + return native_dtype.type(data, native_dtype) + + def cast_scalar(self, data: object) -> np.void: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy void scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.void: + return self.to_native_dtype().type(("\x00" * self.length).encode("ascii")) + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return base64.standard_b64encode(self.cast_scalar(data).tobytes()).decode("ascii") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: + if check_json_str(data): + return self.to_native_dtype().type(base64.standard_b64decode(data)) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + @property + def item_size(self) -> int: + return self.length + + +@dataclass(frozen=True, kw_only=True) +class VariableLengthBytes(ZDType[np.dtypes.ObjectDType, bytes], HasObjectCodec): + dtype_cls = np.dtypes.ObjectDType + _zarr_v3_name: ClassVar[Literal["variable_length_bytes"]] = "variable_length_bytes" + object_codec_id: ClassVar[Literal["vlen-bytes"]] = "vlen-bytes" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.ObjectDType: + return self.dtype_cls() + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]]]: + """ + Check that the input is a valid JSON representation of a numpy O dtype, and that the + object codec id is appropriate for variable-length UTF-8 strings. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == "|O" + and data["object_codec_id"] == cls.object_codec_id + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_bytes"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O' and an object_codec_id of {cls.object_codec_id}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json( + self, zarr_format: Literal[2] + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_bytes"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]] | Literal["variable_length_bytes"]: + if zarr_format == 2: + return {"name": "|O", "object_codec_id": self.object_codec_id} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def default_scalar(self) -> bytes: + return b"" + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return base64.standard_b64encode(data).decode("ascii") # type: ignore[arg-type] + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> bytes: + if check_json_str(data): + return base64.standard_b64decode(data.encode("ascii")) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: + return isinstance(data, BytesLike) + + def _cast_scalar_unchecked(self, data: BytesLike) -> bytes: + if isinstance(data, str): + return bytes(data, encoding="utf-8") + return bytes(data) + + def cast_scalar(self, data: object) -> bytes: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to bytes." + raise TypeError(msg) diff --git a/src/zarr/core/dtype/npy/common.py b/src/zarr/core/dtype/npy/common.py new file mode 100644 index 0000000000..264561f25c --- /dev/null +++ b/src/zarr/core/dtype/npy/common.py @@ -0,0 +1,503 @@ +from __future__ import annotations + +import base64 +import struct +import sys +from collections.abc import Sequence +from typing import ( + TYPE_CHECKING, + Any, + Final, + Literal, + SupportsComplex, + SupportsFloat, + SupportsIndex, + SupportsInt, + TypeGuard, + TypeVar, +) + +import numpy as np + +from zarr.core.dtype.common import ( + ENDIANNESS_STR, + SPECIAL_FLOAT_STRINGS, + EndiannessStr, + JSONFloatV2, + JSONFloatV3, +) + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +IntLike = SupportsInt | SupportsIndex | bytes | str +FloatLike = SupportsIndex | SupportsFloat | bytes | str +ComplexLike = SupportsFloat | SupportsIndex | SupportsComplex | bytes | str | None +DateTimeUnit = Literal[ + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" +] +DATETIME_UNIT: Final = ( + "Y", + "M", + "W", + "D", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", + "generic", +) + +NumpyEndiannessStr = Literal[">", "<", "="] +NUMPY_ENDIANNESS_STR: Final = ">", "<", "=" + +TFloatDType_co = TypeVar( + "TFloatDType_co", + bound=np.dtypes.Float16DType | np.dtypes.Float32DType | np.dtypes.Float64DType, + covariant=True, +) +TFloatScalar_co = TypeVar( + "TFloatScalar_co", bound=np.float16 | np.float32 | np.float64, covariant=True +) + +TComplexDType_co = TypeVar( + "TComplexDType_co", bound=np.dtypes.Complex64DType | np.dtypes.Complex128DType, covariant=True +) +TComplexScalar_co = TypeVar("TComplexScalar_co", bound=np.complex64 | np.complex128, covariant=True) + + +def endianness_from_numpy_str(endianness: NumpyEndiannessStr) -> EndiannessStr: + """ + Convert a numpy endianness string literal to a human-readable literal value. + + Parameters + ---------- + endianness : Literal[">", "<", "="] + The numpy string representation of the endianness. + + Returns + ------- + Endianness + The human-readable representation of the endianness. + + Raises + ------ + ValueError + If the endianness is invalid. + """ + match endianness: + case "=": + # Use the local system endianness + return sys.byteorder + case "<": + return "little" + case ">": + return "big" + raise ValueError(f"Invalid endianness: {endianness!r}. Expected one of {NUMPY_ENDIANNESS_STR}") + + +def endianness_to_numpy_str(endianness: EndiannessStr) -> NumpyEndiannessStr: + """ + Convert an endianness literal to its numpy string representation. + + Parameters + ---------- + endianness : Endianness + The endianness to convert. + + Returns + ------- + Literal[">", "<"] + The numpy string representation of the endianness. + + Raises + ------ + ValueError + If the endianness is invalid. + """ + match endianness: + case "little": + return "<" + case "big": + return ">" + raise ValueError( + f"Invalid endianness: {endianness!r}. Expected one of {ENDIANNESS_STR} or None" + ) + + +def get_endianness_from_numpy_dtype(dtype: np.dtype[np.generic]) -> EndiannessStr: + """ + Gets the endianness from a numpy dtype that has an endianness. This function will + raise a ValueError if the numpy data type does not have a concrete endianness. + """ + endianness = dtype.byteorder + if dtype.byteorder in NUMPY_ENDIANNESS_STR: + return endianness_from_numpy_str(endianness) # type: ignore [arg-type] + raise ValueError(f"The dtype {dtype} has an unsupported endianness: {endianness}") + + +def float_from_json_v2(data: JSONFloatV2) -> float: + """ + Convert a JSON float to a float (Zarr v2). + + Parameters + ---------- + data : JSONFloat + The JSON float to convert. + + Returns + ------- + float + The float value. + """ + match data: + case "NaN": + return float("nan") + case "Infinity": + return float("inf") + case "-Infinity": + return float("-inf") + case _: + return float(data) + + +def float_from_json_v3(data: JSONFloatV3) -> float: + """ + Convert a JSON float to a float (v3). + + Parameters + ---------- + data : JSONFloat + The JSON float to convert. + + Returns + ------- + float + The float value. + + Notes + ----- + Zarr V3 allows floats to be stored as hex strings. To quote the spec: + "...for float32, "NaN" is equivalent to "0x7fc00000". + This representation is the only way to specify a NaN value other than the specific NaN value + denoted by "NaN"." + """ + + if isinstance(data, str): + if data in SPECIAL_FLOAT_STRINGS: + return float_from_json_v2(data) # type: ignore[arg-type] + if not data.startswith("0x"): + msg = ( + f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" + " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." + ) + raise ValueError(msg) + if len(data[2:]) == 4: + dtype_code = ">e" + elif len(data[2:]) == 8: + dtype_code = ">f" + elif len(data[2:]) == 16: + dtype_code = ">d" + else: + msg = ( + f"Invalid hexadecimal float value: {data!r}. " + "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" + ) + raise ValueError(msg) + return float(struct.unpack(dtype_code, bytes.fromhex(data[2:]))[0]) + return float_from_json_v2(data) + + +def bytes_from_json(data: str, *, zarr_format: ZarrFormat) -> bytes: + """ + Convert a JSON string to bytes + + Parameters + ---------- + data : str + The JSON string to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + bytes + The bytes. + """ + if zarr_format == 2: + return base64.b64decode(data.encode("ascii")) + # TODO: differentiate these as needed. This is a spec question. + if zarr_format == 3: + return base64.b64decode(data.encode("ascii")) + raise ValueError(f"Invalid zarr format: {zarr_format}. Expected 2 or 3.") # pragma: no cover + + +def bytes_to_json(data: bytes, zarr_format: ZarrFormat) -> str: + """ + Convert bytes to JSON. + + Parameters + ---------- + data : bytes + The bytes to store. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The bytes encoded as ascii using the base64 alphabet. + """ + # TODO: decide if we are going to make this implementation zarr format-specific + return base64.b64encode(data).decode("ascii") + + +def float_to_json_v2(data: float | np.floating[Any]) -> JSONFloatV2: + """ + Convert a float to JSON (v2). + + Parameters + ---------- + data : float or np.floating + The float value to convert. + + Returns + ------- + JSONFloat + The JSON representation of the float. + """ + if np.isnan(data): + return "NaN" + elif np.isinf(data): + return "Infinity" if data > 0 else "-Infinity" + return float(data) + + +def float_to_json_v3(data: float | np.floating[Any]) -> JSONFloatV3: + """ + Convert a float to JSON (v3). + + Parameters + ---------- + data : float or np.floating + The float value to convert. + + Returns + ------- + JSONFloat + The JSON representation of the float. + """ + # v3 can in principle handle distinct NaN values, but numpy does not represent these explicitly + # so we just reuse the v2 routine here + return float_to_json_v2(data) + + +def complex_float_to_json_v3( + data: complex | np.complexfloating[Any, Any], +) -> tuple[JSONFloatV3, JSONFloatV3]: + """ + Convert a complex number to JSON as defined by the Zarr V3 spec. + + Parameters + ---------- + data : complex or np.complexfloating + The complex value to convert. + + Returns + ------- + tuple[JSONFloat, JSONFloat] + The JSON representation of the complex number. + """ + return float_to_json_v3(data.real), float_to_json_v3(data.imag) + + +def complex_float_to_json_v2( + data: complex | np.complexfloating[Any, Any], +) -> tuple[JSONFloatV2, JSONFloatV2]: + """ + Convert a complex number to JSON as defined by the Zarr V2 spec. + + Parameters + ---------- + data : complex | np.complexfloating + The complex value to convert. + + Returns + ------- + tuple[JSONFloat, JSONFloat] + The JSON representation of the complex number. + """ + return float_to_json_v2(data.real), float_to_json_v2(data.imag) + + +def complex_float_from_json_v2(data: tuple[JSONFloatV2, JSONFloatV2]) -> complex: + """ + Convert a JSON complex float to a complex number (v2). + + Parameters + ---------- + data : tuple[JSONFloat, JSONFloat] + The JSON complex float to convert. + + Returns + ------- + np.complexfloating + The complex number. + """ + return complex(float_from_json_v2(data[0]), float_from_json_v2(data[1])) + + +def complex_float_from_json_v3(data: tuple[JSONFloatV3, JSONFloatV3]) -> complex: + """ + Convert a JSON complex float to a complex number (v3). + + Parameters + ---------- + data : tuple[JSONFloat, JSONFloat] + The JSON complex float to convert. + + Returns + ------- + np.complexfloating + The complex number. + """ + return complex(float_from_json_v3(data[0]), float_from_json_v3(data[1])) + + +def check_json_float_v2(data: JSON) -> TypeGuard[JSONFloatV2]: + """ + Check if a JSON value represents a float (v2). + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a float, False otherwise. + """ + if data == "NaN" or data == "Infinity" or data == "-Infinity": + return True + return isinstance(data, float | int) + + +def check_json_float_v3(data: JSON) -> TypeGuard[JSONFloatV3]: + """ + Check if a JSON value represents a float (v3). + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a float, False otherwise. + """ + return check_json_float_v2(data) or (isinstance(data, str) and data.startswith("0x")) + + +def check_json_complex_float_v2(data: JSON) -> TypeGuard[tuple[JSONFloatV2, JSONFloatV2]]: + """ + Check if a JSON value represents a complex float, as per the behavior of zarr-python 2.x + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a complex float, False otherwise. + """ + return ( + not isinstance(data, str) + and isinstance(data, Sequence) + and len(data) == 2 + and check_json_float_v2(data[0]) + and check_json_float_v2(data[1]) + ) + + +def check_json_complex_float_v3(data: JSON) -> TypeGuard[tuple[JSONFloatV3, JSONFloatV3]]: + """ + Check if a JSON value represents a complex float, as per the zarr v3 spec + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a complex float, False otherwise. + """ + return ( + not isinstance(data, str) + and isinstance(data, Sequence) + and len(data) == 2 + and check_json_float_v3(data[0]) + and check_json_float_v3(data[1]) + ) + + +def check_json_int(data: JSON) -> TypeGuard[int]: + """ + Check if a JSON value is an integer. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is an integer, False otherwise. + """ + return bool(isinstance(data, int)) + + +def check_json_str(data: JSON) -> TypeGuard[str]: + """ + Check if a JSON value is a string. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a string, False otherwise. + """ + return bool(isinstance(data, str)) + + +def check_json_bool(data: JSON) -> TypeGuard[bool]: + """ + Check if a JSON value is a boolean. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a boolean, False otherwise. + """ + return isinstance(data, bool) diff --git a/src/zarr/core/dtype/npy/complex.py b/src/zarr/core/dtype/npy/complex.py new file mode 100644 index 0000000000..38e506f1bc --- /dev/null +++ b/src/zarr/core/dtype/npy/complex.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + TypeGuard, + overload, +) + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + ComplexLike, + TComplexDType_co, + TComplexScalar_co, + check_json_complex_float_v2, + check_json_complex_float_v3, + complex_float_from_json_v2, + complex_float_from_json_v3, + complex_float_to_json_v2, + complex_float_to_json_v3, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True) +class BaseComplex(ZDType[TComplexDType_co, TComplexScalar_co], HasEndianness, HasItemSize): + # This attribute holds the possible zarr v2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> TComplexDType_co: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> str: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[ComplexLike]: + return isinstance(data, ComplexLike) + + def _cast_scalar_unchecked(self, data: ComplexLike) -> TComplexScalar_co: + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TComplexScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy float scalar." + raise TypeError(msg) + + def default_scalar(self) -> TComplexScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TComplexScalar_co: + """ + Read a JSON-serializable value as a numpy float. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy float. + """ + if zarr_format == 2: + if check_json_complex_float_v2(data): + return self._cast_scalar_unchecked(complex_float_from_json_v2(data)) + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + elif zarr_format == 3: + if check_json_complex_float_v3(data): + return self._cast_scalar_unchecked(complex_float_from_json_v3(data)) + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: + """ + Convert an object to a JSON-serializable float. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + JSON + The JSON-serializable form of the complex number, which is a list of two floats, + each of which is encoding according to a zarr-format-specific encoding. + """ + if zarr_format == 2: + return complex_float_to_json_v2(self.cast_scalar(data)) + elif zarr_format == 3: + return complex_float_to_json_v3(self.cast_scalar(data)) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True) +class Complex64(BaseComplex[np.dtypes.Complex64DType, np.complex64]): + dtype_cls = np.dtypes.Complex64DType + _zarr_v3_name: ClassVar[Literal["complex64"]] = "complex64" + _zarr_v2_names: ClassVar[tuple[str, ...]] = (">c8", " int: + return 8 + + +@dataclass(frozen=True, kw_only=True) +class Complex128(BaseComplex[np.dtypes.Complex128DType, np.complex128], HasEndianness): + dtype_cls = np.dtypes.Complex128DType + _zarr_v3_name: ClassVar[Literal["complex128"]] = "complex128" + _zarr_v2_names: ClassVar[tuple[str, ...]] = (">c16", " int: + return 16 diff --git a/src/zarr/core/dtype/npy/float.py b/src/zarr/core/dtype/npy/float.py new file mode 100644 index 0000000000..7b7243993f --- /dev/null +++ b/src/zarr/core/dtype/npy/float.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + ScalarTypeValidationError, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + FloatLike, + TFloatDType_co, + TFloatScalar_co, + check_json_float_v2, + check_json_float_v3, + endianness_to_numpy_str, + float_from_json_v2, + float_from_json_v3, + float_to_json_v2, + float_to_json_v3, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True) +class BaseFloat(ZDType[TFloatDType_co, TFloatScalar_co], HasEndianness, HasItemSize): + # This attribute holds the possible zarr v2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> TFloatDType_co: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> str: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[FloatLike]: + return isinstance(data, FloatLike) + + def _cast_scalar_unchecked(self, data: FloatLike) -> TFloatScalar_co: + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TFloatScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy float scalar." + raise ScalarTypeValidationError(msg) + + def default_scalar(self) -> TFloatScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TFloatScalar_co: + """ + Read a JSON-serializable value as a numpy float. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy float. + """ + if zarr_format == 2: + if check_json_float_v2(data): + return self._cast_scalar_unchecked(float_from_json_v2(data)) + else: + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + elif zarr_format == 3: + if check_json_float_v3(data): + return self._cast_scalar_unchecked(float_from_json_v3(data)) + else: + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> float | str: + """ + Convert an object to a JSON-serializable float. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + JSON + The JSON-serializable form of the float, which is potentially a number or a string. + See the zarr specifications for details on the JSON encoding for floats. + """ + if zarr_format == 2: + return float_to_json_v2(self.cast_scalar(data)) + elif zarr_format == 3: + return float_to_json_v3(self.cast_scalar(data)) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True) +class Float16(BaseFloat[np.dtypes.Float16DType, np.float16]): + dtype_cls = np.dtypes.Float16DType + _zarr_v3_name = "float16" + _zarr_v2_names: ClassVar[tuple[Literal[">f2"], Literal["f2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class Float32(BaseFloat[np.dtypes.Float32DType, np.float32]): + dtype_cls = np.dtypes.Float32DType + _zarr_v3_name = "float32" + _zarr_v2_names: ClassVar[tuple[Literal[">f4"], Literal["f4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class Float64(BaseFloat[np.dtypes.Float64DType, np.float64]): + dtype_cls = np.dtypes.Float64DType + _zarr_v3_name = "float64" + _zarr_v2_names: ClassVar[tuple[Literal[">f8"], Literal["f8", " int: + return 8 diff --git a/src/zarr/core/dtype/npy/int.py b/src/zarr/core/dtype/npy/int.py new file mode 100644 index 0000000000..79d3ce2d47 --- /dev/null +++ b/src/zarr/core/dtype/npy/int.py @@ -0,0 +1,686 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + SupportsIndex, + SupportsInt, + TypeGuard, + TypeVar, + overload, +) + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + check_json_int, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +_NumpyIntDType = ( + np.dtypes.Int8DType + | np.dtypes.Int16DType + | np.dtypes.Int32DType + | np.dtypes.Int64DType + | np.dtypes.UInt8DType + | np.dtypes.UInt16DType + | np.dtypes.UInt32DType + | np.dtypes.UInt64DType +) +_NumpyIntScalar = ( + np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64 +) +TIntDType_co = TypeVar("TIntDType_co", bound=_NumpyIntDType, covariant=True) +TIntScalar_co = TypeVar("TIntScalar_co", bound=_NumpyIntScalar, covariant=True) +IntLike = SupportsInt | SupportsIndex | bytes | str + + +@dataclass(frozen=True) +class BaseInt(ZDType[TIntDType_co, TIntScalar_co], HasItemSize): + # This attribute holds the possible zarr V2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def _check_json_v2(cls, data: object) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: object) -> TypeGuard[str]: + """ + Check that a JSON value is consistent with the zarr v3 spec for this data type. + """ + return data == cls._zarr_v3_name + + def _check_scalar(self, data: object) -> TypeGuard[IntLike]: + """ + Check that a python object is IntLike + """ + return isinstance(data, IntLike) + + def _cast_scalar_unchecked(self, data: IntLike) -> TIntScalar_co: + """ + Create an integer without any type checking of the input. + """ + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TIntScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy integer." + raise TypeError(msg) + + def default_scalar(self) -> TIntScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TIntScalar_co: + """ + Read a JSON-serializable value as a numpy int scalar. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy scalar. + """ + if check_json_int(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected an integer.") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: + """ + Convert an object to JSON-serializable scalar. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + int + The JSON-serializable form of the scalar. + """ + return int(self.cast_scalar(data)) + + +@dataclass(frozen=True, kw_only=True) +class Int8(BaseInt[np.dtypes.Int8DType, np.int8]): + dtype_cls = np.dtypes.Int8DType + _zarr_v3_name: ClassVar[Literal["int8"]] = "int8" + _zarr_v2_names: ClassVar[tuple[Literal["|i1"]]] = ("|i1",) + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Int8 from a np.dtype('int8') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.Int8DType: + return self.dtype_cls() + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|i1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["int8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|i1"], None] | Literal["int8"]: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self._zarr_v2_names[0], "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 + + +@dataclass(frozen=True, kw_only=True) +class UInt8(BaseInt[np.dtypes.UInt8DType, np.uint8]): + dtype_cls = np.dtypes.UInt8DType + _zarr_v3_name: ClassVar[Literal["uint8"]] = "uint8" + _zarr_v2_names: ClassVar[tuple[Literal["|u1"]]] = ("|u1",) + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Bool from a np.dtype('uint8') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.UInt8DType: + return self.dtype_cls() + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|u1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["uint8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|u1"], None] | Literal["uint8"]: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self._zarr_v2_names[0], "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 + + +@dataclass(frozen=True, kw_only=True) +class Int16(BaseInt[np.dtypes.Int16DType, np.int16], HasEndianness): + dtype_cls = np.dtypes.Int16DType + _zarr_v3_name: ClassVar[Literal["int16"]] = "int16" + _zarr_v2_names: ClassVar[tuple[Literal[">i2"], Literal["i2", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int16DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names!r}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i2", " Literal["int16"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class UInt16(BaseInt[np.dtypes.UInt16DType, np.uint16], HasEndianness): + dtype_cls = np.dtypes.UInt16DType + _zarr_v3_name: ClassVar[Literal["uint16"]] = "uint16" + _zarr_v2_names: ClassVar[tuple[Literal[">u2"], Literal["u2", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.UInt16DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u2", " Literal["uint16"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class Int32(BaseInt[np.dtypes.Int32DType, np.int32], HasEndianness): + dtype_cls = np.dtypes.Int32DType + _zarr_v3_name: ClassVar[Literal["int32"]] = "int32" + _zarr_v2_names: ClassVar[tuple[Literal[">i4"], Literal["i4", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int32DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i4", " Literal["int32"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class UInt32(BaseInt[np.dtypes.UInt32DType, np.uint32], HasEndianness): + dtype_cls = np.dtypes.UInt32DType + _zarr_v3_name: ClassVar[Literal["uint32"]] = "uint32" + _zarr_v2_names: ClassVar[tuple[Literal[">u4"], Literal["u4", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.UInt32DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u4", " Literal["uint32"]: ... + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class Int64(BaseInt[np.dtypes.Int64DType, np.int64], HasEndianness): + dtype_cls = np.dtypes.Int64DType + _zarr_v3_name: ClassVar[Literal["int64"]] = "int64" + _zarr_v2_names: ClassVar[tuple[Literal[">i8"], Literal["i8", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int64DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i8", " Literal["int64"]: ... + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i8", " int: + return 8 + + +@dataclass(frozen=True, kw_only=True) +class UInt64(BaseInt[np.dtypes.UInt64DType, np.uint64], HasEndianness): + dtype_cls = np.dtypes.UInt64DType + _zarr_v3_name: ClassVar[Literal["uint64"]] = "uint64" + _zarr_v2_names: ClassVar[tuple[Literal[">u8"], Literal["u8", " np.dtypes.UInt64DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u8", " Literal["uint64"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u8", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + @property + def item_size(self) -> int: + return 8 diff --git a/src/zarr/core/dtype/npy/string.py b/src/zarr/core/dtype/npy/string.py new file mode 100644 index 0000000000..4a1114617a --- /dev/null +++ b/src/zarr/core/dtype/npy/string.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Protocol, + Self, + TypedDict, + TypeGuard, + overload, + runtime_checkable, +) + +import numpy as np + +from zarr.core.common import NamedConfig +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + HasLength, + HasObjectCodec, + check_dtype_spec_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import ( + check_json_str, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TDType_co, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType + +_NUMPY_SUPPORTS_VLEN_STRING = hasattr(np.dtypes, "StringDType") + + +@runtime_checkable +class SupportsStr(Protocol): + def __str__(self) -> str: ... + + +class LengthBytesConfig(TypedDict): + length_bytes: int + + +# TODO: Fix this terrible name +FixedLengthUTF32JSONV3 = NamedConfig[Literal["fixed_length_utf32"], LengthBytesConfig] + + +@dataclass(frozen=True, kw_only=True) +class FixedLengthUTF32( + ZDType[np.dtypes.StrDType[int], np.str_], HasEndianness, HasLength, HasItemSize +): + dtype_cls = np.dtypes.StrDType + _zarr_v3_name: ClassVar[Literal["fixed_length_utf32"]] = "fixed_length_utf32" + code_point_bytes: ClassVar[int] = 4 # utf32 is 4 bytes per code point + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + endianness = get_endianness_from_numpy_dtype(dtype) + return cls( + length=dtype.itemsize // (cls.code_point_bytes), + endianness=endianness, + ) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.StrDType[int]: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls(self.length).newbyteorder(byte_order) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of a numpy U dtype. + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^[><]U\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[FixedLengthUTF32JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and "configuration" in data + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"length_bytes"} + and isinstance(data["configuration"]["length_bytes"], int) + ) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> FixedLengthUTF32JSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | FixedLengthUTF32JSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return { + "name": self._zarr_v3_name, + "configuration": {"length_bytes": self.length * self.code_point_bytes}, + } + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Construct the numpy dtype instead of string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + raise DataTypeValidationError( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string representation of a numpy U dtype." + ) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"] // cls.code_point_bytes) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + def default_scalar(self) -> np.str_: + return np.str_("") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return str(data) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.str_: + if check_json_str(data): + return self.to_native_dtype().type(data) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[str | np.str_ | bytes | int]: + # this is generous for backwards compatibility + return isinstance(data, str | np.str_ | bytes | int) + + def cast_scalar(self, data: object) -> np.str_: + if self._check_scalar(data): + # We explicitly truncate before casting because of the following numpy behavior: + # >>> x = np.dtype('U3').type('hello world') + # >>> x + # np.str_('hello world') + # >>> x.dtype + # dtype('U11') + + if isinstance(data, int): + return self.to_native_dtype().type(str(data)[: self.length]) + else: + return self.to_native_dtype().type(data[: self.length]) + raise TypeError( + f"Cannot convert object with type {type(data)} to a numpy unicode string scalar." + ) + + @property + def item_size(self) -> int: + return self.length * self.code_point_bytes + + +def check_vlen_string_json_scalar(data: object) -> TypeGuard[int | str | float]: + """ + This function checks the type of JSON-encoded variable length strings. It is generous for + backwards compatibility, as zarr-python v2 would use ints for variable length strings + fill values + """ + return isinstance(data, int | str | float) + + +# VariableLengthUTF8 is defined in two places, conditioned on the version of numpy. +# If numpy 2 is installed, then VariableLengthUTF8 is defined with the numpy variable length +# string dtype as the native dtype. Otherwise, VariableLengthUTF8 is defined with the numpy object +# dtype as the native dtype. +class UTF8Base(ZDType[TDType_co, str], HasObjectCodec): + """ + A base class for the variable length UTF-8 string data type. This class should not be used + as data type, but as a base class for other variable length string data types. + """ + + _zarr_v3_name: ClassVar[Literal["variable_length_utf8"]] = "variable_length_utf8" + object_codec_id: ClassVar[Literal["vlen-utf8"]] = "vlen-utf8" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]]: + """ + Check that the input is a valid JSON representation of a numpy O dtype, and that the + object codec id is appropriate for variable-length UTF-8 strings. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == "|O" + and data["object_codec_id"] == cls.object_codec_id + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_utf8"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O'" + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json( + self, zarr_format: Literal[2] + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_utf8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]] | Literal["variable_length_utf8"]: + if zarr_format == 2: + return {"name": "|O", "object_codec_id": self.object_codec_id} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def default_scalar(self) -> str: + return "" + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected a string.") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> str: + if not check_vlen_string_json_scalar(data): + raise TypeError(f"Invalid type: {data}. Expected a string or number.") + return str(data) + + def _check_scalar(self, data: object) -> TypeGuard[SupportsStr]: + return isinstance(data, SupportsStr) + + def _cast_scalar_unchecked(self, data: SupportsStr) -> str: + return str(data) + + def cast_scalar(self, data: object) -> str: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Cannot convert object with type {type(data)} to a python string.") + + +if _NUMPY_SUPPORTS_VLEN_STRING: + + @dataclass(frozen=True, kw_only=True) + class VariableLengthUTF8(UTF8Base[np.dtypes.StringDType]): # type: ignore[type-var] + dtype_cls = np.dtypes.StringDType + + def to_native_dtype(self) -> np.dtypes.StringDType: + return self.dtype_cls() + +else: + # Numpy pre-2 does not have a variable length string dtype, so we use the Object dtype instead. + @dataclass(frozen=True, kw_only=True) + class VariableLengthUTF8(UTF8Base[np.dtypes.ObjectDType]): # type: ignore[no-redef] + dtype_cls = np.dtypes.ObjectDType + + def to_native_dtype(self) -> np.dtypes.ObjectDType: + return self.dtype_cls() diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py new file mode 100644 index 0000000000..07e3000826 --- /dev/null +++ b/src/zarr/core/dtype/npy/structured.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Self, TypeGuard, cast, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + DTypeSpec_V3, + HasItemSize, + StructuredName_V2, + check_dtype_spec_v2, + check_structured_dtype_name_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import ( + bytes_from_json, + bytes_to_json, + check_json_str, +) +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr.core.common import JSON, NamedConfig, ZarrFormat + +StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int + + +@dataclass(frozen=True, kw_only=True) +class Structured(ZDType[np.dtypes.VoidDType[int], np.void], HasItemSize): + dtype_cls = np.dtypes.VoidDType # type: ignore[assignment] + _zarr_v3_name = "structured" + fields: tuple[tuple[str, ZDType[TBaseDType, TBaseScalar]], ...] + + @classmethod + def _check_native_dtype(cls, dtype: TBaseDType) -> TypeGuard[np.dtypes.VoidDType[int]]: + """ + Check that this dtype is a numpy structured dtype + + Parameters + ---------- + dtype : np.dtypes.DTypeLike + The dtype to check. + + Returns + ------- + TypeGuard[np.dtypes.VoidDType] + True if the dtype matches, False otherwise. + """ + return isinstance(dtype, cls.dtype_cls) and dtype.fields is not None # type: ignore[has-type] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + from zarr.core.dtype import get_data_type_from_native_dtype + + fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] + if cls._check_native_dtype(dtype): + # fields of a structured numpy dtype are either 2-tuples or 3-tuples. we only + # care about the first element in either case. + for key, (dtype_instance, *_) in dtype.fields.items(): # type: ignore[union-attr] + dtype_wrapped = get_data_type_from_native_dtype(dtype_instance) + fields.append((key, dtype_wrapped)) + + return cls(fields=tuple(fields)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + + def to_native_dtype(self) -> np.dtypes.VoidDType[int]: + return cast( + "np.dtypes.VoidDType[int]", + np.dtype([(key, dtype.to_native_dtype()) for (key, dtype) in self.fields]), + ) + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[StructuredName_V2, None]]: + return ( + check_dtype_spec_v2(data) + and not isinstance(data["name"], str) + and check_structured_dtype_name_v2(data["name"]) + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3( + cls, data: DTypeJSON + ) -> TypeGuard[NamedConfig[Literal["structured"], dict[str, Sequence[tuple[str, DTypeJSON]]]]]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"fields"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + # avoid circular import + from zarr.core.dtype import get_data_type_from_json + + if cls._check_json_v2(data): + # structured dtypes are constructed directly from a list of lists + # note that we do not handle the object codec here! this will prevent structured + # dtypes from containing object dtypes. + return cls( + fields=tuple( # type: ignore[misc] + ( # type: ignore[misc] + f_name, + get_data_type_from_json( + {"name": f_dtype, "object_codec_id": None}, zarr_format=2 + ), + ) + for f_name, f_dtype in data["name"] + ) + ) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON array of arrays" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + # avoid circular import + from zarr.core.dtype import get_data_type_from_json + + if cls._check_json_v3(data): + config = data["configuration"] + meta_fields = config["fields"] + return cls( + fields=tuple( + (f_name, get_data_type_from_json(f_dtype, zarr_format=3)) + for f_name, f_dtype in meta_fields + ) + ) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON object with the key {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[StructuredName_V2, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> DTypeSpec_V3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[StructuredName_V2, None] | DTypeSpec_V3: + if zarr_format == 2: + fields = [ + [f_name, f_dtype.to_json(zarr_format=zarr_format)["name"]] + for f_name, f_dtype in self.fields + ] + return {"name": fields, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + fields = [ + [f_name, f_dtype.to_json(zarr_format=zarr_format)] # type: ignore[list-item] + for f_name, f_dtype in self.fields + ] + base_dict = { + "name": self._zarr_v3_name, + "configuration": {"fields": fields}, + } + return cast("DTypeSpec_V3", base_dict) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[StructuredScalarLike]: + # TODO: implement something more precise here! + return isinstance(data, (bytes, list, tuple, int, np.void)) + + def _cast_scalar_unchecked(self, data: StructuredScalarLike) -> np.void: + na_dtype = self.to_native_dtype() + if isinstance(data, bytes): + res = np.frombuffer(data, dtype=na_dtype)[0] + elif isinstance(data, list | tuple): + res = np.array([tuple(data)], dtype=na_dtype)[0] + else: + res = np.array([data], dtype=na_dtype)[0] + return cast("np.void", res) + + def cast_scalar(self, data: object) -> np.void: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy structured scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.void: + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: + if check_json_str(data): + as_bytes = bytes_from_json(data, zarr_format=zarr_format) + dtype = self.to_native_dtype() + return cast("np.void", np.array([as_bytes]).view(dtype)[0]) + raise TypeError(f"Invalid type: {data}. Expected a string.") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return bytes_to_json(self.cast_scalar(data).tobytes(), zarr_format) + + @property + def item_size(self) -> int: + # Lets have numpy do the arithmetic here + return self.to_native_dtype().itemsize diff --git a/src/zarr/core/dtype/npy/time.py b/src/zarr/core/dtype/npy/time.py new file mode 100644 index 0000000000..1f9080475c --- /dev/null +++ b/src/zarr/core/dtype/npy/time.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + TypedDict, + TypeGuard, + TypeVar, + cast, + get_args, + overload, +) + +import numpy as np + +from zarr.core.common import NamedConfig +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + DATETIME_UNIT, + DateTimeUnit, + check_json_int, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +_DTypeName = Literal["datetime64", "timedelta64"] +TimeDeltaLike = str | int | bytes | np.timedelta64 | timedelta | None +DateTimeLike = str | int | bytes | np.datetime64 | datetime | None + + +def datetime_from_int(data: int, *, unit: DateTimeUnit, scale_factor: int) -> np.datetime64: + """ + Convert an integer to a datetime64. + + Parameters + ---------- + data : int + The integer to convert. + unit : DateTimeUnit + The unit of the datetime64. + scale_factor : int + The scale factor of the datetime64. + + Returns + ------- + np.datetime64 + The datetime64 value. + """ + dtype_name = f"datetime64[{scale_factor}{unit}]" + return cast("np.datetime64", np.int64(data).view(dtype_name)) + + +def datetimelike_to_int(data: np.datetime64 | np.timedelta64) -> int: + """ + Convert a datetime64 or a timedelta64 to an integer. + + Parameters + ---------- + data : np.datetime64 | np.timedelta64 + The value to convert. + + Returns + ------- + int + An integer representation of the scalar. + """ + return data.view(np.int64).item() + + +def check_json_time(data: JSON) -> TypeGuard[Literal["NaT"] | int]: + """ + Type guard to check if the input JSON data is the literal string "NaT" + or an integer. + """ + return check_json_int(data) or data == "NaT" + + +BaseTimeDType_co = TypeVar( + "BaseTimeDType_co", + bound=np.dtypes.TimeDelta64DType | np.dtypes.DateTime64DType, + covariant=True, +) +BaseTimeScalar_co = TypeVar( + "BaseTimeScalar_co", bound=np.timedelta64 | np.datetime64, covariant=True +) + + +class TimeConfig(TypedDict): + unit: DateTimeUnit + scale_factor: int + + +DateTime64JSONV3 = NamedConfig[Literal["numpy.datetime64"], TimeConfig] +TimeDelta64JSONV3 = NamedConfig[Literal["numpy.timedelta64"], TimeConfig] + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TimeDTypeBase(ZDType[BaseTimeDType_co, BaseTimeScalar_co], HasEndianness, HasItemSize): + _zarr_v2_names: ClassVar[tuple[str, ...]] + # this attribute exists so that we can programmatically create a numpy dtype instance + # because the particular numpy dtype we are wrapping does not allow direct construction via + # cls.dtype_cls() + _numpy_name: ClassVar[_DTypeName] + scale_factor: int + unit: DateTimeUnit + + def __post_init__(self) -> None: + if self.scale_factor < 1: + raise ValueError(f"scale_factor must be > 0, got {self.scale_factor}.") + if self.scale_factor >= 2**31: + raise ValueError(f"scale_factor must be < 2147483648, got {self.scale_factor}.") + if self.unit not in get_args(DateTimeUnit): + raise ValueError(f"unit must be one of {get_args(DateTimeUnit)}, got {self.unit!r}.") + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + unit, scale_factor = np.datetime_data(dtype.name) + unit = cast("DateTimeUnit", unit) + return cls( + unit=unit, + scale_factor=scale_factor, + endianness=get_endianness_from_numpy_dtype(dtype), + ) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> BaseTimeDType_co: + # Numpy does not allow creating datetime64 or timedelta64 via + # np.dtypes.{dtype_name}() + # so we use np.dtype with a formatted string. + dtype_string = f"{self._numpy_name}[{self.scale_factor}{self.unit}]" + return np.dtype(dtype_string).newbyteorder(endianness_to_numpy_str(self.endianness)) # type: ignore[return-value] + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> DateTime64JSONV3 | TimeDelta64JSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | DateTime64JSONV3 | TimeDelta64JSONV3: + if zarr_format == 2: + name = self.to_native_dtype().str + return {"name": name, "object_codec_id": None} + elif zarr_format == 3: + return cast( + "DateTime64JSONV3 | TimeDelta64JSONV3", + { + "name": self._zarr_v3_name, + "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, + }, + ) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: + return datetimelike_to_int(data) # type: ignore[arg-type] + + @property + def item_size(self) -> int: + return 8 + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TimeDelta64(TimeDTypeBase[np.dtypes.TimeDelta64DType, np.timedelta64], HasEndianness): + """ + A wrapper for the ``TimeDelta64`` data type defined in numpy. + Scalars of this type can be created by performing arithmetic with ``DateTime64`` scalars. + Like ``DateTime64``, ``TimeDelta64`` is parametrized by a unit, but unlike ``DateTime64``, the + unit for ``TimeDelta64`` is optional. + """ + + # mypy infers the type of np.dtypes.TimeDelta64DType to be + # "Callable[[Literal['Y', 'M', 'W', 'D'] | Literal['h', 'm', 's', 'ms', 'us', 'ns', 'ps', 'fs', 'as']], Never]" + dtype_cls = np.dtypes.TimeDelta64DType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["numpy.timedelta64"]] = "numpy.timedelta64" + _zarr_v2_names = (">m8", " TypeGuard[DTypeConfig_V2[str, None]]: + if not check_dtype_spec_v2(data): + return False + name = data["name"] + # match m[M], etc + # consider making this a standalone function + if not isinstance(name, str): + return False + if not name.startswith(cls._zarr_v2_names): + return False + if len(name) == 3: + # no unit, and + # we already checked that this string is either m8 + return True + else: + return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"unit", "scale_factor"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " + f"representation of an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + unit = data["configuration"]["unit"] + scale_factor = data["configuration"]["scale_factor"] + return cls(unit=unit, scale_factor=scale_factor) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " + f"with a 'name' key with the value 'numpy.timedelta64', " + "and a 'configuration' key with a value of a dict with a 'unit' key and a " + "'scale_factor' key" + ) + raise DataTypeValidationError(msg) + + def _check_scalar(self, data: object) -> TypeGuard[TimeDeltaLike]: + if data is None: + return True + return isinstance(data, str | int | bytes | np.timedelta64 | timedelta) + + def _cast_scalar_unchecked(self, data: TimeDeltaLike) -> np.timedelta64: + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + + def cast_scalar(self, data: object) -> np.timedelta64: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy timedelta64 scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.timedelta64: + return np.timedelta64("NaT") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.timedelta64: + if check_json_time(data): + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DateTime64(TimeDTypeBase[np.dtypes.DateTime64DType, np.datetime64], HasEndianness): + dtype_cls = np.dtypes.DateTime64DType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["numpy.datetime64"]] = "numpy.datetime64" + _zarr_v2_names = (">M8", " TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that JSON input is a string representation of a NumPy datetime64 data type, like "M8[10s]". This function can be used as a type guard to narrow the type of unknown JSON + input. + """ + if not check_dtype_spec_v2(data): + return False + name = data["name"] + if not isinstance(name, str): + return False + if not name.startswith(cls._zarr_v2_names): + return False + if len(name) == 3: + # no unit, and + # we already checked that this string is either M8 + return True + else: + return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"unit", "scale_factor"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " + f"representation of an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + unit = data["configuration"]["unit"] + scale_factor = data["configuration"]["scale_factor"] + return cls(unit=unit, scale_factor=scale_factor) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " + f"with a 'name' key with the value 'numpy.datetime64', " + "and a 'configuration' key with a value of a dict with a 'unit' key and a " + "'scale_factor' key" + ) + raise DataTypeValidationError(msg) + + def _check_scalar(self, data: object) -> TypeGuard[DateTimeLike]: + if data is None: + return True + return isinstance(data, str | int | bytes | np.datetime64 | datetime) + + def _cast_scalar_unchecked(self, data: DateTimeLike) -> np.datetime64: + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + + def cast_scalar(self, data: object) -> np.datetime64: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy datetime scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.datetime64: + return np.datetime64("NaT") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.datetime64: + if check_json_time(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover diff --git a/src/zarr/core/dtype/registry.py b/src/zarr/core/dtype/registry.py new file mode 100644 index 0000000000..1d2a97a90a --- /dev/null +++ b/src/zarr/core/dtype/registry.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import contextlib +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Self + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeJSON, +) + +if TYPE_CHECKING: + from importlib.metadata import EntryPoint + + from zarr.core.common import ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +# This class is different from the other registry classes, which inherit from +# dict. IMO it's simpler to just do a dataclass. But long-term we should +# have just 1 registry class in use. +@dataclass(frozen=True, kw_only=True) +class DataTypeRegistry: + contents: dict[str, type[ZDType[TBaseDType, TBaseScalar]]] = field( + default_factory=dict, init=False + ) + + lazy_load_list: list[EntryPoint] = field(default_factory=list, init=False) + + def lazy_load(self) -> None: + for e in self.lazy_load_list: + self.register(e.load()._zarr_v3_name, e.load()) + + self.lazy_load_list.clear() + + def register(self: Self, key: str, cls: type[ZDType[TBaseDType, TBaseScalar]]) -> None: + # don't register the same dtype twice + if key not in self.contents or self.contents[key] != cls: + self.contents[key] = cls + + def unregister(self, key: str) -> None: + """Unregister a data type by its key.""" + if key in self.contents: + del self.contents[key] + else: + raise KeyError(f"Data type '{key}' not found in registry.") + + def get(self, key: str) -> type[ZDType[TBaseDType, TBaseScalar]]: + return self.contents[key] + + def match_dtype(self, dtype: TBaseDType) -> ZDType[TBaseDType, TBaseScalar]: + if dtype == np.dtype("O"): + msg = ( + f"Zarr data type resolution from {dtype} failed. " + 'Attempted to resolve a zarr data type from a numpy "Object" data type, which is ' + 'ambiguous, as multiple zarr data types can be represented by the numpy "Object" ' + "data type. " + "In this case you should construct your array by providing a specific Zarr data " + 'type. For a list of Zarr data types that are compatible with the numpy "Object"' + "data type, see https://github.com/zarr-developers/zarr-python/issues/3117" + ) + raise ValueError(msg) + matched: list[ZDType[TBaseDType, TBaseScalar]] = [] + for val in self.contents.values(): + with contextlib.suppress(DataTypeValidationError): + matched.append(val.from_native_dtype(dtype)) + if len(matched) == 1: + return matched[0] + elif len(matched) > 1: + msg = ( + f"Zarr data type resolution from {dtype} failed. " + f"Multiple data type wrappers found that match dtype '{dtype}': {matched}. " + "You should unregister one of these data types, or avoid Zarr data type inference " + "entirely by providing a specific Zarr data type when creating your array." + "For more information, see https://github.com/zarr-developers/zarr-python/issues/3117" + ) + raise ValueError(msg) + raise ValueError(f"No Zarr data type found that matches dtype '{dtype!r}'") + + def match_json( + self, data: DTypeJSON, *, zarr_format: ZarrFormat + ) -> ZDType[TBaseDType, TBaseScalar]: + for val in self.contents.values(): + try: + return val.from_json(data, zarr_format=zarr_format) + except DataTypeValidationError: + pass + raise ValueError(f"No Zarr data type found that matches {data!r}") diff --git a/src/zarr/core/dtype/wrapper.py b/src/zarr/core/dtype/wrapper.py new file mode 100644 index 0000000000..7be97fa4b4 --- /dev/null +++ b/src/zarr/core/dtype/wrapper.py @@ -0,0 +1,297 @@ +""" +Wrapper for native array data types. + +The ``ZDType`` class is an abstract base class for wrapping native array data types, e.g. NumPy dtypes. +``ZDType`` provides a common interface for working with data types in a way that is independent of the +underlying data type system. + +The wrapper class encapsulates a native data type. Instances of the class can be created from a +native data type instance, and a native data type instance can be created from an instance of the +wrapper class. + +The wrapper class is responsible for: +- Serializing and deserializing a native data type to Zarr V2 or Zarr V3 metadata. + This ensures that the data type can be properly stored and retrieved from array metadata. +- Serializing and deserializing scalar values to Zarr V2 or Zarr V3 metadata. This is important for + storing a fill value for an array in a manner that is valid for the data type. + +You can add support for a new data type in Zarr by subclassing ``ZDType`` wrapper class and adapt its methods +to support your native data type. The wrapper class must be added to a data type registry +(defined elsewhere) before array creation routines or array reading routines can use your new data +type. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Generic, + Literal, + Self, + TypeGuard, + TypeVar, + overload, +) + +import numpy as np + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2, DTypeSpec_V3 + +# This the upper bound for the scalar types we support. It's numpy scalars + str, +# because the new variable-length string dtype in numpy does not have a corresponding scalar type +TBaseScalar = np.generic | str | bytes +# This is the bound for the dtypes that we support. If we support non-numpy dtypes, +# then this bound will need to be widened. +TBaseDType = np.dtype[np.generic] + +# These two type parameters are covariant because we want +# x : ZDType[BaseDType, BaseScalar] = ZDType[SubDType, SubScalar] +# to type check +TScalar_co = TypeVar("TScalar_co", bound=TBaseScalar, covariant=True) +TDType_co = TypeVar("TDType_co", bound=TBaseDType, covariant=True) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ZDType(ABC, Generic[TDType_co, TScalar_co]): + """ + Abstract base class for wrapping native array data types, e.g. numpy dtypes + + Attributes + ---------- + dtype_cls : ClassVar[type[TDType]] + The wrapped dtype class. This is a class variable. + _zarr_v3_name : ClassVar[str] + The name given to the data type by a Zarr v3 data type specification. This is a + class variable, and it should generally be unique across different data types. + """ + + # this class will create a native data type + # mypy currently disallows class variables to contain type parameters + # but it seems OK for us to use it here: + # https://github.com/python/typing/discussions/1424#discussioncomment-7989934 + dtype_cls: ClassVar[type[TDType_co]] # type: ignore[misc] + _zarr_v3_name: ClassVar[str] + + @classmethod + def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[TDType_co]: + """ + Check that a native data type matches the dtype_cls class attribute. Used as a type guard. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return type(dtype) is cls.dtype_cls + + @classmethod + @abstractmethod + def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self: + """ + Create a ZDType instance from a native data type. The default implementation first performs + a type check via ``cls._check_native_dtype``. If that type check succeeds, the ZDType class + instance is created. + + This method is used when taking a user-provided native data type, like a NumPy data type, + and creating the corresponding ZDType instance from them. + + Parameters + ---------- + dtype : TDType + The native data type object to wrap. + + Returns + ------- + Self + The ZDType that wraps the native data type. + + Raises + ------ + TypeError + If the native data type is not consistent with the wrapped data type. + """ + ... + + @abstractmethod + def to_native_dtype(self: Self) -> TDType_co: + """ + Return an instance of the wrapped data type. This operation inverts ``from_native_dtype``. + + Returns + ------- + TDType + The native data type wrapped by this ZDType. + """ + ... + + @classmethod + @abstractmethod + def _from_json_v2(cls: type[Self], data: DTypeJSON) -> Self: ... + + @classmethod + @abstractmethod + def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: ... + + @classmethod + def from_json(cls: type[Self], data: DTypeJSON, *, zarr_format: ZarrFormat) -> Self: + """ + Create an instance of this ZDType from JSON data. + + Parameters + ---------- + data : DTypeJSON + The JSON representation of the data type. The type annotation includes + Mapping[str, object] to accommodate typed dictionaries. + + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + Self + The wrapped data type. + """ + if zarr_format == 2: + return cls._from_json_v2(data) + if zarr_format == 3: + return cls._from_json_v3(data) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @overload + def to_json(self, zarr_format: Literal[2]) -> DTypeSpec_V2: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> DTypeSpec_V3: ... + + @abstractmethod + def to_json(self, zarr_format: ZarrFormat) -> DTypeSpec_V2 | DTypeSpec_V3: + """ + Serialize this ZDType to JSON. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + DTypeJSON_V2 | DTypeJSON_V3 + The JSON-serializable representation of the wrapped data type + """ + ... + + @abstractmethod + def _check_scalar(self, data: object) -> bool: + """ + Check that an python object is a valid scalar value for the wrapped data type. + + Parameters + ---------- + data : object + A value to check. + + Returns + ------- + Bool + True if the object is valid, False otherwise. + """ + ... + + @abstractmethod + def cast_scalar(self, data: object) -> TScalar_co: + """ + Cast a python object to the wrapped scalar type. + The type of the provided scalar is first checked for compatibility. + If it's incompatible with the associated scalar type, a ``TypeError`` will be raised. + + Parameters + ---------- + data : object + The python object to cast. + + Returns + ------- + TScalar + The cast value. + """ + + @abstractmethod + def default_scalar(self) -> TScalar_co: + """ + Get the default scalar value for the wrapped data type. This is a method, rather than an + attribute, because the default value for some data types depends on parameters that are + not known until a concrete data type is wrapped. For example, data types parametrized by a + length like fixed-length strings or bytes will generate scalars consistent with that length. + + Returns + ------- + TScalar + The default value for this data type. + """ + ... + + @abstractmethod + def from_json_scalar(self: Self, data: JSON, *, zarr_format: ZarrFormat) -> TScalar_co: + """ + Read a JSON-serializable value as a scalar. + + Parameters + ---------- + data : JSON + A JSON representation of a scalar value. + zarr_format : ZarrFormat + The zarr format version. This is specified because the JSON serialization of scalars + differs between Zarr V2 and Zarr V3. + + Returns + ------- + TScalar + The deserialized scalar value. + """ + ... + + @abstractmethod + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: + """ + Serialize a python object to the JSON representation of a scalar. The value will first be + cast to the scalar type associated with this ZDType, then serialized to JSON. + + Parameters + ---------- + data : object + The value to convert. + zarr_format : ZarrFormat + The zarr format version. This is specified because the JSON serialization of scalars + differs between Zarr V2 and Zarr V3. + + Returns + ------- + JSON + The JSON-serialized scalar. + """ + ... + + +def scalar_failed_type_check_msg( + cls_instance: ZDType[TBaseDType, TBaseScalar], bad_scalar: object +) -> str: + """ + Generate an error message reporting that a particular value failed a type check when attempting + to cast that value to a scalar. + """ + return ( + f"The value {bad_scalar!r} failed a type check. " + f"It cannot be safely cast to a scalar compatible with {cls_instance}. " + f"Consult the documentation for {cls_instance} to determine the possible values that can " + "be cast to scalars of the wrapped data type." + ) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index ebdc63364e..4c8ced21f4 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -7,6 +7,7 @@ import warnings from collections import defaultdict from dataclasses import asdict, dataclass, field, fields, replace +from itertools import accumulate from typing import TYPE_CHECKING, Literal, TypeVar, assert_never, cast, overload import numpy as np @@ -40,6 +41,7 @@ ZGROUP_JSON, ZMETADATA_V2_JSON, ChunkCoords, + DimensionNames, NodeType, ShapeLike, ZarrFormat, @@ -47,19 +49,27 @@ ) from zarr.core.config import config from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata -from zarr.core.metadata.v3 import V3JsonEncoder from zarr.core.sync import SyncMixin, sync -from zarr.errors import MetadataValidationError -from zarr.storage import StoreLike, StorePath, make_store_path -from zarr.storage._common import ensure_no_existing_node +from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError +from zarr.storage import StoreLike, StorePath +from zarr.storage._common import ensure_no_existing_node, make_store_path +from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Generator, Iterable, Iterator + from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Coroutine, + Generator, + Iterable, + Iterator, + Mapping, + ) from typing import Any from zarr.core.array_spec import ArrayConfig, ArrayConfigLike from zarr.core.buffer import Buffer, BufferPrototype - from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike + from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike from zarr.core.common import MemoryOrder logger = logging.getLogger("zarr.group") @@ -70,7 +80,7 @@ def parse_zarr_format(data: Any) -> ZarrFormat: """Parse the zarr_format field from metadata.""" if data in (2, 3): - return cast(ZarrFormat, data) + return cast("ZarrFormat", data) msg = f"Invalid zarr_format. Expected one of 2 or 3. Got {data}." raise ValueError(msg) @@ -78,7 +88,7 @@ def parse_zarr_format(data: Any) -> ZarrFormat: def parse_node_type(data: Any) -> NodeType: """Parse the node_type field from metadata.""" if data in ("array", "group"): - return cast(Literal["array", "group"], data) + return cast("Literal['array', 'group']", data) raise MetadataValidationError("node_type", "array or group", data) @@ -325,7 +335,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: if self.zarr_format == 3: return { ZARR_JSON: prototype.buffer.from_bytes( - json.dumps(self.to_dict(), cls=V3JsonEncoder).encode() + json.dumps(self.to_dict(), indent=json_indent, allow_nan=False).encode() ) } else: @@ -334,7 +344,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: json.dumps({"zarr_format": self.zarr_format}, indent=json_indent).encode() ), ZATTRS_JSON: prototype.buffer.from_bytes( - json.dumps(self.attributes, indent=json_indent).encode() + json.dumps(self.attributes, indent=json_indent, allow_nan=False).encode() ), } if self.consolidated_metadata: @@ -345,7 +355,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] assert isinstance(consolidated_metadata, dict) for k, v in consolidated_metadata.items(): - attrs = v.pop("attributes", None) + attrs = v.pop("attributes", {}) d[f"{k}/{ZATTRS_JSON}"] = attrs if "shape" in v: # it's an array @@ -362,8 +372,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: items[ZMETADATA_V2_JSON] = prototype.buffer.from_bytes( json.dumps( - {"metadata": d, "zarr_consolidated_format": 1}, - cls=V3JsonEncoder, + {"metadata": d, "zarr_consolidated_format": 1}, allow_nan=False ).encode() ) @@ -407,6 +416,15 @@ def to_dict(self) -> dict[str, Any]: return result +@dataclass(frozen=True) +class ImplicitGroupMarker(GroupMetadata): + """ + Marker for an implicit group. Instances of this class are only used in the context of group + creation as a placeholder to represent groups that should only be created if they do not + already exist in storage + """ + + @dataclass(frozen=True) class AsyncGroup: """ @@ -416,6 +434,9 @@ class AsyncGroup: metadata: GroupMetadata store_path: StorePath + # TODO: make this correct and work + # TODO: ensure that this can be bound properly to subclass of AsyncGroup + @classmethod async def from_store( cls, @@ -460,10 +481,11 @@ async def open( By default, consolidated metadata is used if it's present in the store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file - for Zarr format 2). + for Zarr format 2) and the Store supports it. - To explicitly require consolidated metadata, set ``use_consolidated=True``, - which will raise an exception if consolidated metadata is not found. + To explicitly require consolidated metadata, set ``use_consolidated=True``. + In this case, if the Store doesn't support consolidation or consolidated metadata is + not found, a ``ValueError`` exception is raised. To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, which will fall back to using the regular, non consolidated metadata. @@ -473,6 +495,16 @@ async def open( to load consolidated metadata from a non-default key. """ store_path = await make_store_path(store) + if not store_path.store.supports_consolidated_metadata: + # Fail if consolidated metadata was requested but the Store doesn't support it + if use_consolidated: + store_name = type(store_path.store).__name__ + raise ValueError( + f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." + ) + + # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation + use_consolidated = False consolidated_key = ZMETADATA_V2_JSON @@ -573,8 +605,8 @@ def _from_bytes_v2( v2_consolidated_metadata = json.loads(consolidated_metadata_bytes.to_bytes()) v2_consolidated_metadata = v2_consolidated_metadata["metadata"] # We already read zattrs and zgroup. Should we ignore these? - v2_consolidated_metadata.pop(".zattrs") - v2_consolidated_metadata.pop(".zgroup") + v2_consolidated_metadata.pop(".zattrs", None) + v2_consolidated_metadata.pop(".zgroup", None) consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) @@ -590,6 +622,7 @@ def _from_bytes_v2( consolidated_metadata[path].update(v) else: raise ValueError(f"Invalid file type '{kind}' at path '{path}") + group_metadata["consolidated_metadata"] = { "metadata": dict(consolidated_metadata), "kind": "inline", @@ -666,55 +699,12 @@ async def getitem( # Consolidated metadata lets us avoid some I/O operations so try that first. if self.metadata.consolidated_metadata is not None: return self._getitem_consolidated(store_path, key, prefix=self.name) - - # Note: - # in zarr-python v2, we first check if `key` references an Array, else if `key` references - # a group,using standalone `contains_array` and `contains_group` functions. These functions - # are reusable, but for v3 they would perform redundant I/O operations. - # Not clear how much of that strategy we want to keep here. - elif self.metadata.zarr_format == 3: - zarr_json_bytes = await (store_path / ZARR_JSON).get() - if zarr_json_bytes is None: - raise KeyError(key) - else: - zarr_json = json.loads(zarr_json_bytes.to_bytes()) - if zarr_json["node_type"] == "group": - return type(self).from_dict(store_path, zarr_json) - elif zarr_json["node_type"] == "array": - return AsyncArray.from_dict(store_path, zarr_json) - else: - raise ValueError(f"unexpected node_type: {zarr_json['node_type']}") - elif self.metadata.zarr_format == 2: - # Q: how do we like optimistically fetching .zgroup, .zarray, and .zattrs? - # This guarantees that we will always make at least one extra request to the store - zgroup_bytes, zarray_bytes, zattrs_bytes = await asyncio.gather( - (store_path / ZGROUP_JSON).get(), - (store_path / ZARRAY_JSON).get(), - (store_path / ZATTRS_JSON).get(), + try: + return await get_node( + store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format ) - - if zgroup_bytes is None and zarray_bytes is None: - raise KeyError(key) - - # unpack the zarray, if this is None then we must be opening a group - zarray = json.loads(zarray_bytes.to_bytes()) if zarray_bytes else None - # unpack the zattrs, this can be None if no attrs were written - zattrs = json.loads(zattrs_bytes.to_bytes()) if zattrs_bytes is not None else {} - - if zarray is not None: - # TODO: update this once the V2 array support is part of the primary array class - zarr_json = {**zarray, "attributes": zattrs} - return AsyncArray.from_dict(store_path, zarr_json) - else: - zgroup = ( - json.loads(zgroup_bytes.to_bytes()) - if zgroup_bytes is not None - else {"zarr_format": self.metadata.zarr_format} - ) - zarr_json = {**zgroup, "attributes": zattrs} - return type(self).from_dict(store_path, zarr_json) - else: - raise ValueError(f"unexpected zarr_format: {self.metadata.zarr_format}") + except FileNotFoundError as e: + raise KeyError(key) from e def _getitem_consolidated( self, store_path: StorePath, key: str, prefix: str @@ -1015,13 +1005,13 @@ async def create_array( shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", - compressor: CompressorLike = None, + compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = 0, order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, @@ -1118,8 +1108,9 @@ async def create_array( AsyncArray """ - - compressors = _parse_deprecated_compressor(compressor, compressors) + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) return await create_array( store=self.store_path, name=name, @@ -1275,8 +1266,6 @@ async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: ------- self : AsyncGroup """ - # metadata.attributes is "frozen" so we simply clear and update the dict - self.metadata.attributes.clear() self.metadata.attributes.update(new_attributes) # Write new metadata @@ -1319,6 +1308,8 @@ async def nmembers( async def members( self, max_depth: int | None = 0, + *, + use_consolidated_for_children: bool = True, ) -> AsyncGenerator[ tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], None, @@ -1336,6 +1327,11 @@ async def members( default, (``max_depth=0``) only immediate children are included. Set ``max_depth=None`` to include all nodes, and some positive integer to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. Returns ------- @@ -1346,18 +1342,52 @@ async def members( """ if max_depth is not None and max_depth < 0: raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") - async for item in self._members(max_depth=max_depth, current_depth=0): + async for item in self._members( + max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children + ): yield item - async def _members( - self, max_depth: int | None, current_depth: int - ) -> AsyncGenerator[ + def _members_consolidated( + self, max_depth: int | None, prefix: str = "" + ) -> Generator[ tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], None, ]: + consolidated_metadata = self.metadata.consolidated_metadata + + do_recursion = max_depth is None or max_depth > 0 + + # we kind of just want the top-level keys. + if consolidated_metadata is not None: + for key in consolidated_metadata.metadata: + obj = self._getitem_consolidated( + self.store_path, key, prefix=self.name + ) # Metadata -> Group/Array + key = f"{prefix}/{key}".lstrip("/") + yield key, obj + + if do_recursion and isinstance(obj, AsyncGroup): + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + yield from obj._members_consolidated(new_depth, prefix=key) + + async def _members( + self, max_depth: int | None, *, use_consolidated_for_children: bool = True + ) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None + ]: + skip_keys: tuple[str, ...] + if self.metadata.zarr_format == 2: + skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") + elif self.metadata.zarr_format == 3: + skip_keys = ("zarr.json",) + else: + raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") + if self.metadata.consolidated_metadata is not None: - # we should be able to do members without any additional I/O - members = self._members_consolidated(max_depth, current_depth) + members = self._members_consolidated(max_depth=max_depth) for member in members: yield member return @@ -1371,66 +1401,94 @@ async def _members( ) raise ValueError(msg) - # would be nice to make these special keys accessible programmatically, - # and scoped to specific zarr versions - # especially true for `.zmetadata` which is configurable - _skip_keys = ("zarr.json", ".zgroup", ".zattrs", ".zmetadata") - - # hmm lots of I/O and logic interleaved here. - # We *could* have an async gen over self.metadata.consolidated_metadata.metadata.keys() - # and plug in here. `getitem` will skip I/O. - # Kinda a shame to have all the asyncio task overhead though, when it isn't needed. - - async for key in self.store_path.store.list_dir(self.store_path.path): - if key in _skip_keys: - continue - try: - obj = await self.getitem(key) - yield (key, obj) - - if ( - ((max_depth is None) or (current_depth < max_depth)) - and hasattr(obj.metadata, "node_type") - and obj.metadata.node_type == "group" - ): - # the assert is just for mypy to know that `obj.metadata.node_type` - # implies an AsyncGroup, not an AsyncArray - assert isinstance(obj, AsyncGroup) - async for child_key, val in obj._members( - max_depth=max_depth, current_depth=current_depth + 1 - ): - yield f"{key}/{child_key}", val - except KeyError: - # keyerror is raised when `key` names an object (in the object storage sense), - # as opposed to a prefix, in the store under the prefix associated with this group - # in which case `key` cannot be the name of a sub-array or sub-group. - warnings.warn( - f"Object at {key} is not recognized as a component of a Zarr hierarchy.", - UserWarning, - stacklevel=1, - ) - - def _members_consolidated( - self, max_depth: int | None, current_depth: int, prefix: str = "" - ) -> Generator[ - tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], - None, + # enforce a concurrency limit by passing a semaphore to all the recursive functions + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + async for member in _iter_members_deep( + self, + max_depth=max_depth, + skip_keys=skip_keys, + semaphore=semaphore, + use_consolidated_for_children=use_consolidated_for_children, + ): + yield member + + async def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] ]: - consolidated_metadata = self.metadata.consolidated_metadata + """ + Create a hierarchy of arrays or groups rooted at this group. - # we kind of just want the top-level keys. - if consolidated_metadata is not None: - for key in consolidated_metadata.metadata: - obj = self._getitem_consolidated( - self.store_path, key, prefix=self.name - ) # Metadata -> Group/Array - key = f"{prefix}/{key}".lstrip("/") - yield key, obj + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. - if ((max_depth is None) or (current_depth < max_depth)) and isinstance( - obj, AsyncGroup - ): - yield from obj._members_consolidated(max_depth, current_depth + 1, prefix=key) + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncArray | AsyncGroup]. + """ + # check that all the nodes have the same zarr_format as Self + prefix = self.path + nodes_parsed = {} + for key, value in nodes.items(): + if value.zarr_format != self.metadata.zarr_format: + msg = ( + "The zarr_format of the nodes must be the same as the parent group. " + f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" + f" has zarr_format {self.metadata.zarr_format}." + ) + raise ValueError(msg) + if normalize_path(key) == "": + msg = ( + "The input defines a root node, but a root node already exists, namely this Group instance." + "It is an error to use this method to create a root node. " + "Remove the root node from the input dict, or use a function like " + "create_rooted_hierarchy to create a rooted hierarchy." + ) + raise ValueError(msg) + else: + nodes_parsed[_join_paths([prefix, key])] = value + + async for key, node in create_hierarchy( + store=self.store, + nodes=nodes_parsed, + overwrite=overwrite, + ): + if prefix == "": + out_key = key + else: + out_key = key.removeprefix(prefix + "/") + yield out_key, node async def keys(self) -> AsyncGenerator[str, None]: """Iterate over member names.""" @@ -1523,7 +1581,8 @@ async def tree(self, expand: bool | None = None, level: int | None = None) -> An async def empty( self, *, name: str, shape: ChunkCoords, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty array in this Group. + """Create an empty array with the specified shape in this Group. The contents will + be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -1540,7 +1599,6 @@ async def empty( retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - return await async_api.empty(shape=shape, store=self.store_path, path=name, **kwargs) async def zeros( @@ -1617,7 +1675,8 @@ async def full( async def empty_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: - """Create an empty sub-array like `data`. + """Create an empty sub-array like `data`. The contents will be filled with + the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -1710,6 +1769,10 @@ async def move(self, source: str, dest: str) -> None: @dataclass(frozen=True) class Group(SyncMixin): + """ + A Zarr group. + """ + _async_group: AsyncGroup @classmethod @@ -2045,15 +2108,99 @@ def nmembers(self, max_depth: int | None = 0) -> int: return self._sync(self._async_group.nmembers(max_depth=max_depth)) - def members(self, max_depth: int | None = 0) -> tuple[tuple[str, Array | Group], ...]: + def members( + self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True + ) -> tuple[tuple[str, Array | Group], ...]: """ - Return the sub-arrays and sub-groups of this group as a tuple of (name, array | group) - pairs + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. """ _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) + def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> Iterator[tuple[str, Group | Array]]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------- + tuple[str, Array | Group]. + + Examples + -------- + >>> import zarr + >>> from zarr.core.group import GroupMetadata + >>> root = zarr.create_group(store={}) + >>> for key, val in root.create_hierarchy({'a/b/c': GroupMetadata()}): + ... print(key, val) + ... + + + + """ + for key, node in self._sync_iter( + self._async_group.create_hierarchy(nodes, overwrite=overwrite) + ): + yield (key, _parse_async_node(node)) + def keys(self) -> Generator[str, None]: """Return an iterator over group member names. @@ -2270,13 +2417,13 @@ def create_array( shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", - compressor: CompressorLike = None, + compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = 0, order: MemoryOrder | None = "C", attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, @@ -2372,7 +2519,9 @@ def create_array( ------- AsyncArray """ - compressors = _parse_deprecated_compressor(compressor, compressors) + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) return Array( self._sync( self._async_group.create_array( @@ -2465,7 +2614,8 @@ def require_array(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array: @_deprecate_positional_args def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: - """Create an empty array in this Group. + """Create an empty array with the specified shape in this Group. The contents will be filled with + the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -2554,7 +2704,8 @@ def full( @_deprecate_positional_args def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array: - """Create an empty sub-array like `data`. + """Create an empty sub-array like `data`. The contents will be filled + with the array's fill value or zeros if no fill value is provided. Parameters ---------- @@ -2569,6 +2720,12 @@ def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> ------- Array The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. """ return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) @@ -2659,8 +2816,8 @@ def array( fill_value: Any | None = 0, order: MemoryOrder | None = "C", attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, - dimension_names: Iterable[str] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfig | ArrayConfigLike | None = None, @@ -2755,6 +2912,8 @@ def array( Whether to overwrite an array with the same name in the store, if one exists. config : ArrayConfig or ArrayConfigLike, optional Runtime configuration for the array. + data : array_like + The data to fill the array with. Returns ------- @@ -2763,7 +2922,7 @@ def array( compressors = _parse_deprecated_compressor(compressor, compressors) return Array( self._sync( - self._async_group.create_array( + self._async_group.create_dataset( name=name, shape=shape, dtype=dtype, @@ -2780,6 +2939,773 @@ def array( overwrite=overwrite, storage_options=storage_options, config=config, + data=data, ) ) ) + + +async def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] +]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncGroup | AsyncArray] + This function yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> from zarr.api.asynchronous import create_hierarchy + >>> from zarr.storage import MemoryStore + >>> from zarr.core.group import GroupMetadata + >>> import asyncio + >>> store = MemoryStore() + >>> nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + >>> async def run(): + ... print(dict([x async for x in create_hierarchy(store=store, nodes=nodes)])) + >>> asyncio.run(run()) + # {'a': , '': } + """ + # normalize the keys to be valid paths + nodes_normed_keys = _normalize_path_keys(nodes) + + # ensure that all nodes have the same zarr_format, and add implicit groups as needed + nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) + redundant_implicit_groups = [] + + # empty hierarchies should be a no-op + if len(nodes_parsed) > 0: + # figure out which zarr format we are using + zarr_format = next(iter(nodes_parsed.values())).zarr_format + + # check which implicit groups will require materialization + implicit_group_keys = tuple( + filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) + ) + # read potential group metadata for each implicit group + maybe_extant_group_coros = ( + _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys + ) + maybe_extant_groups = await asyncio.gather( + *maybe_extant_group_coros, return_exceptions=True + ) + + for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): + if isinstance(value, BaseException): + if isinstance(value, FileNotFoundError): + # this is fine -- there was no group there, so we will create one + pass + else: + raise value + else: + # a loop exists already at ``key``, so we can avoid creating anything there + redundant_implicit_groups.append(key) + + if overwrite: + # we will remove any nodes that collide with arrays and non-implicit groups defined in + # nodes + + # track the keys of nodes we need to delete + to_delete_keys = [] + to_delete_keys.extend( + [k for k, v in nodes_parsed.items() if k not in implicit_group_keys] + ) + await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) + else: + # This type is long. + coros: ( + Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] + | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] + ) + if zarr_format == 2: + coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) + elif zarr_format == 3: + coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) + else: # pragma: no cover + raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover + + extant_node_query = dict( + zip( + nodes_parsed.keys(), + await asyncio.gather(*coros, return_exceptions=True), + strict=False, + ) + ) + # iterate over the existing arrays / groups and figure out which of them conflict + # with the arrays / groups we want to create + for key, extant_node in extant_node_query.items(): + proposed_node = nodes_parsed[key] + if isinstance(extant_node, BaseException): + if isinstance(extant_node, FileNotFoundError): + # ignore FileNotFoundError, because they represent nodes we can safely create + pass + else: + # Any other exception is a real error + raise extant_node + else: + # this is a node that already exists, but a node with the same key was specified + # in nodes_parsed. + if isinstance(extant_node, GroupMetadata): + # a group already exists where we want to create a group + if isinstance(proposed_node, ImplicitGroupMarker): + # we have proposed an implicit group, which is OK -- we will just skip + # creating this particular metadata document + redundant_implicit_groups.append(key) + else: + # we have proposed an explicit group, which is an error, given that a + # group already exists. + raise ContainsGroupError(store, key) + elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): + # we are trying to overwrite an existing array. this is an error. + raise ContainsArrayError(store, key) + + nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} + + for k, v in nodes_parsed.items(): + if k not in redundant_implicit_groups: + if isinstance(v, ImplicitGroupMarker): + nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) + else: + nodes_explicit[k] = v + + async for key, node in create_nodes(store=store, nodes=nodes_explicit): + yield key, node + + +async def create_nodes( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] +]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + AsyncGroup | AsyncArray + The created nodes in the order they are created. + """ + + # Note: the only way to alter this value is via the config. If that's undesirable for some reason, + # then we should consider adding a keyword argument this this function + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + create_tasks: list[Coroutine[None, None, str]] = [] + + for key, value in nodes.items(): + # make the key absolute + create_tasks.extend(_persist_metadata(store, key, value, semaphore=semaphore)) + + created_object_keys = [] + + for coro in asyncio.as_completed(create_tasks): + created_key = await coro + # we need this to track which metadata documents were written so that we can yield a + # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. + created_object_keys.append(created_key) + + # get the node name from the object key + if len(created_key.split("/")) == 1: + # this is the root node + meta_out = nodes[""] + node_name = "" + else: + # turn "foo/" into "foo" + node_name = created_key[: created_key.rfind("/")] + meta_out = nodes[node_name] + if meta_out.zarr_format == 3: + yield node_name, _build_node(store=store, path=node_name, metadata=meta_out) + else: + # For zarr v2 + # we only want to yield when both the metadata and attributes are created + # so we track which keys have been created, and wait for both the meta key and + # the attrs key to be created before yielding back the AsyncArray / AsyncGroup + + attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys + + if isinstance(meta_out, GroupMetadata): + meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys + else: + meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys + + if meta_done and attrs_done: + yield node_name, _build_node(store=store, path=node_name, metadata=meta_out) + + continue + + +def _get_roots( + data: Iterable[str], +) -> tuple[str, ...]: + """ + Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of + path segments. + """ + if "" in data: + return ("",) + keys_split = sorted((key.split("/") for key in data), key=len) + groups: defaultdict[int, list[str]] = defaultdict(list) + for key_split in keys_split: + groups[len(key_split)].append("/".join(key_split)) + return tuple(groups[min(groups.keys())]) + + +def _parse_hierarchy_dict( + *, + data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: + """ + Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into + a dict of str: node pairs that models a valid, complete Zarr hierarchy. + + If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, + then return a dict with the exact same data as the input. + + Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make + the hierarchy complete. + + For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two + groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function + to that input will result in a return value of + {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups + were added. + + The input is also checked for the following conditions; an error is raised if any are violated: + + - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). + - All arrays and groups must have the same ``zarr_format`` value. + + This function ensures that the input is transformed into a specification of a complete and valid + Zarr hierarchy. + """ + + # ensure that all nodes have the same zarr format + data_purified = _ensure_consistent_zarr_format(data) + + # ensure that keys are normalized to zarr paths + data_normed_keys = _normalize_path_keys(data_purified) + + # insert an implicit root group if a root was not specified + # but not if an empty dict was provided, because any empty hierarchy has no nodes + if len(data_normed_keys) > 0 and "" not in data_normed_keys: + z_format = next(iter(data_normed_keys.values())).zarr_format + data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} + + out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} + + for k, v in data_normed_keys.items(): + key_split = k.split("/") + + # get every parent path + *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) + + for subpath in subpaths: + # If a component is not already in the output dict, add ImplicitGroupMetadata + if subpath not in out: + out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) + else: + if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): + msg = ( + f"The node at {subpath} contains other nodes, but it is not a Zarr group. " + "This is invalid. Only Zarr groups can contain other nodes." + ) + raise ValueError(msg) + return out + + +def _ensure_consistent_zarr_format( + data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: + """ + Ensure that all values of the input dict have the same zarr format. If any do not, + then a value error is raised. + """ + observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} + + for k, v in data.items(): + observed_zarr_formats[v.zarr_format].append(k) + + if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " + f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." + "Ensure that all nodes have the same Zarr format." + ) + raise ValueError(msg) + + return cast( + "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", + data, + ) + + +async def _getitem_semaphore( + node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None +) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup: + """ + Wrap Group.getitem with an optional semaphore. + + If the semaphore parameter is an + asyncio.Semaphore instance, then the getitem operation is performed inside an async context + manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked + without a context manager. + """ + if semaphore is not None: + async with semaphore: + return await node.getitem(key) + else: + return await node.getitem(key) + + +async def _iter_members( + node: AsyncGroup, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None, +) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None +]: + """ + Iterate over the arrays and groups contained in a group. + + Parameters + ---------- + node : AsyncGroup + The group to traverse. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + + Yields + ------ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup] + """ + + # retrieve keys from storage + keys = [key async for key in node.store.list_dir(node.path)] + keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) + + node_tasks = tuple( + asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) + for key in keys_filtered + ) + + for fetched_node_coro in asyncio.as_completed(node_tasks): + try: + fetched_node = await fetched_node_coro + except KeyError as e: + # keyerror is raised when `key` names an object (in the object storage sense), + # as opposed to a prefix, in the store under the prefix associated with this group + # in which case `key` cannot be the name of a sub-array or sub-group. + warnings.warn( + f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", + UserWarning, + stacklevel=1, + ) + continue + match fetched_node: + case AsyncArray() | AsyncGroup(): + yield fetched_node.basename, fetched_node + case _: + raise ValueError(f"Unexpected type: {type(fetched_node)}") + + +async def _iter_members_deep( + group: AsyncGroup, + *, + max_depth: int | None, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None = None, + use_consolidated_for_children: bool = True, +) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None +]: + """ + Iterate over the arrays and groups contained in a group, and optionally the + arrays and groups contained in those groups. + + Parameters + ---------- + group : AsyncGroup + The group to traverse. + max_depth : int | None + The maximum depth of recursion. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Yields + ------ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup] + """ + + to_recurse = {} + do_recursion = max_depth is None or max_depth > 0 + + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): + is_group = isinstance(node, AsyncGroup) + if ( + is_group + and not use_consolidated_for_children + and node.metadata.consolidated_metadata is not None # type: ignore [union-attr] + ): + node = cast("AsyncGroup", node) + # We've decided not to trust consolidated metadata at this point, because we're + # reconsolidating the metadata, for example. + node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) + yield name, node + if is_group and do_recursion: + node = cast("AsyncGroup", node) + to_recurse[name] = _iter_members_deep( + node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore + ) + + for prefix, subgroup_iter in to_recurse.items(): + async for name, node in subgroup_iter: + key = f"{prefix}/{name}".lstrip("/") + yield key, node + + +async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / zarr.json. If no such document is found, raise a + FileNotFoundError. + """ + zarr_json_bytes = await store.get( + _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() + ) + if zarr_json_bytes is None: + raise FileNotFoundError(path) + else: + zarr_json = json.loads(zarr_json_bytes.to_bytes()) + return _build_metadata_v3(zarr_json) + + +async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / (.zgroup | .zarray). If no such document is found, + raise a FileNotFoundError. + """ + # TODO: consider first fetching array metadata, and only fetching group metadata when we don't + # find an array + zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( + store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), + ) + + if zattrs_bytes is None: + zattrs = {} + else: + zattrs = json.loads(zattrs_bytes.to_bytes()) + + # TODO: decide how to handle finding both array and group metadata. The spec does not seem to + # consider this situation. A practical approach would be to ignore that combination, and only + # return the array metadata. + if zarray_bytes is not None: + zmeta = json.loads(zarray_bytes.to_bytes()) + else: + if zgroup_bytes is None: + # neither .zarray or .zgroup were found results in KeyError + raise FileNotFoundError(path) + else: + zmeta = json.loads(zgroup_bytes.to_bytes()) + + return _build_metadata_v2(zmeta, zattrs) + + +async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v2(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v3(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata( + store: Store, path: str, *, zarr_format: ZarrFormat +) -> GroupMetadata: + if zarr_format == 2: + return await _read_group_metadata_v2(store=store, path=path) + return await _read_group_metadata_v3(store=store, path=path) + + +def _build_metadata_v3(zarr_json: dict[str, JSON]) -> ArrayV3Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. + """ + if "node_type" not in zarr_json: + raise MetadataValidationError("node_type", "array or group", "nothing (the key is missing)") + match zarr_json: + case {"node_type": "array"}: + return ArrayV3Metadata.from_dict(zarr_json) + case {"node_type": "group"}: + return GroupMetadata.from_dict(zarr_json) + case _: # pragma: no cover + raise ValueError( + "invalid value for `node_type` key in metadata document" + ) # pragma: no cover + + +def _build_metadata_v2( + zarr_json: dict[str, JSON], attrs_json: dict[str, JSON] +) -> ArrayV2Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. + """ + match zarr_json: + case {"shape": _}: + return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}) + case _: # pragma: no cover + return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}) + + +@overload +def _build_node( + *, store: Store, path: str, metadata: ArrayV2Metadata +) -> AsyncArray[ArrayV2Metadata]: ... + + +@overload +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata +) -> AsyncArray[ArrayV3Metadata]: ... + + +@overload +def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... + + +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Take a metadata object and return a node (AsyncArray or AsyncGroup). + """ + store_path = StorePath(store=store, path=path) + match metadata: + case ArrayV2Metadata() | ArrayV3Metadata(): + return AsyncArray(metadata, store_path=store_path) + case GroupMetadata(): + return AsyncGroup(metadata, store_path=store_path) + case _: # pragma: no cover + raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover + + +async def _get_node_v2(store: Store, path: str) -> AsyncArray[ArrayV2Metadata] | AsyncGroup: + """ + Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v2(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def _get_node_v3(store: Store, path: str) -> AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v3(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def get_node( + store: Store, path: str, zarr_format: ZarrFormat +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Get an AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + + match zarr_format: + case 2: + return await _get_node_v2(store=store, path=path) + case 3: + return await _get_node_v3(store=store, path=path) + case _: # pragma: no cover + raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover + + +async def _set_return_key( + *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None +) -> str: + """ + Write a value to storage at the given key. The key is returned. + Useful when saving values via routines that return results in execution order, + like asyncio.as_completed, because in this case we need to know which key was saved in order + to yield the right object to the caller. + + Parameters + ---------- + store : Store + The store to save the value to. + key : str + The key to save the value to. + value : Buffer + The value to save. + semaphore : asyncio.Semaphore | None + An optional semaphore to use to limit the number of concurrent writes. + """ + + if semaphore is not None: + async with semaphore: + await store.set(key, value) + else: + await store.set(key, value) + return key + + +def _persist_metadata( + store: Store, + path: str, + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata, + semaphore: asyncio.Semaphore | None = None, +) -> tuple[Coroutine[None, None, str], ...]: + """ + Prepare to save a metadata document to storage, returning a tuple of coroutines that must be awaited. + """ + + to_save = metadata.to_buffer_dict(default_buffer_prototype()) + return tuple( + _set_return_key(store=store, key=_join_paths([path, key]), value=value, semaphore=semaphore) + for key, value in to_save.items() + ) + + +async def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """ + Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. + This function ensures that its input contains a specification of a root node, + calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. + """ + roots = _get_roots(nodes.keys()) + if len(roots) != 1: + msg = ( + "The input does not specify a root node. " + "This function can only create hierarchies that contain a root node, which is " + "defined as a group that is ancestral to all the other arrays and " + "groups in the hierarchy, or a single array." + ) + raise ValueError(msg) + else: + root_key = roots[0] + + nodes_created = [ + x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) + ] + return dict(nodes_created)[root_key] diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index ca227be094..c11889f7f4 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from zarr.core.array import Array - from zarr.core.buffer import NDArrayLike + from zarr.core.buffer import NDArrayLikeOrScalar from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import ChunkCoords @@ -289,9 +289,9 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or def get_chunk_shape(chunk_grid: ChunkGrid) -> ChunkCoords: from zarr.core.chunk_grids import RegularChunkGrid - assert isinstance( - chunk_grid, RegularChunkGrid - ), "Only regular chunk grid is supported, currently." + assert isinstance(chunk_grid, RegularChunkGrid), ( + "Only regular chunk grid is supported, currently." + ) return chunk_grid.chunk_shape @@ -321,12 +321,12 @@ class ChunkDimProjection(NamedTuple): Selection of items from chunk array. dim_out_sel Selection of items in target (output) array. - """ dim_chunk_ix: int dim_chunk_sel: Selector dim_out_sel: Selector | None + is_complete_chunk: bool @dataclass(frozen=True) @@ -346,7 +346,8 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: dim_offset = dim_chunk_ix * self.dim_chunk_len dim_chunk_sel = self.dim_sel - dim_offset dim_out_sel = None - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + is_complete_chunk = self.dim_chunk_len == 1 + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) @dataclass(frozen=True) @@ -420,7 +421,10 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: dim_out_sel = slice(dim_out_offset, dim_out_offset + dim_chunk_nitems) - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + is_complete_chunk = ( + dim_chunk_sel_start == 0 and (self.stop >= dim_limit) and self.step in [1, None] + ) + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) def check_selection_length(selection: SelectionNormalized, shape: ChunkCoords) -> None: @@ -462,7 +466,7 @@ def replace_ellipsis(selection: Any, shape: ChunkCoords) -> SelectionNormalized: # check selection not too long check_selection_length(selection, shape) - return cast(SelectionNormalized, selection) + return cast("SelectionNormalized", selection) def replace_lists(selection: SelectionNormalized) -> SelectionNormalized: @@ -477,7 +481,7 @@ def replace_lists(selection: SelectionNormalized) -> SelectionNormalized: def ensure_tuple(v: Any) -> SelectionNormalized: if not isinstance(v, tuple): v = (v,) - return cast(SelectionNormalized, v) + return cast("SelectionNormalized", v) class ChunkProjection(NamedTuple): @@ -493,12 +497,14 @@ class ChunkProjection(NamedTuple): Selection of items from chunk array. out_selection Selection of items in target (output) array. - + is_complete_chunk: + True if a complete chunk is indexed """ chunk_coords: ChunkCoords chunk_selection: tuple[Selector, ...] | npt.NDArray[np.intp] out_selection: tuple[Selector, ...] | npt.NDArray[np.intp] | slice + is_complete_chunk: bool def is_slice(s: Any) -> TypeGuard[slice]: @@ -574,8 +580,8 @@ def __iter__(self) -> Iterator[ChunkProjection]: out_selection = tuple( p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) @@ -643,8 +649,9 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] stop = self.chunk_nitems_cumsum[dim_chunk_ix] dim_out_sel = slice(start, stop) + is_complete_chunk = False # TODO - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) class Order(Enum): @@ -783,8 +790,8 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: # find region in chunk dim_offset = dim_chunk_ix * self.dim_chunk_len dim_chunk_sel = self.dim_sel[start:stop] - dim_offset - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) + is_complete_chunk = False # TODO + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) def slice_to_range(s: slice, length: int) -> range: @@ -811,7 +818,7 @@ def ix_(selection: Any, shape: ChunkCoords) -> npt.NDArray[np.intp]: # now get numpy to convert to a coordinate selection selection = np.ix_(*selection) - return cast(npt.NDArray[np.intp], selection) + return cast("npt.NDArray[np.intp]", selection) def oindex(a: npt.NDArray[Any], selection: Selection) -> npt.NDArray[Any]: @@ -921,7 +928,8 @@ def __iter__(self) -> Iterator[ChunkProjection]: if not is_basic_selection(out_selection): out_selection = ix_(out_selection, self.shape) - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) @@ -929,7 +937,7 @@ class OIndex: array: Array # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool - def __getitem__(self, selection: OrthogonalSelection | Array) -> NDArrayLike: + def __getitem__(self, selection: OrthogonalSelection | Array) -> NDArrayLikeOrScalar: from zarr.core.array import Array # if input is a Zarr array, we materialize it now. @@ -940,7 +948,7 @@ def __getitem__(self, selection: OrthogonalSelection | Array) -> NDArrayLike: new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.get_orthogonal_selection( - cast(OrthogonalSelection, new_selection), fields=fields + cast("OrthogonalSelection", new_selection), fields=fields ) def __setitem__(self, selection: OrthogonalSelection, value: npt.ArrayLike) -> None: @@ -948,7 +956,7 @@ def __setitem__(self, selection: OrthogonalSelection, value: npt.ArrayLike) -> N new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.set_orthogonal_selection( - cast(OrthogonalSelection, new_selection), value, fields=fields + cast("OrthogonalSelection", new_selection), value, fields=fields ) @@ -1030,26 +1038,26 @@ def __iter__(self) -> Iterator[ChunkProjection]: out_selection = tuple( p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) class BlockIndex: array: Array - def __getitem__(self, selection: BasicSelection) -> NDArrayLike: + def __getitem__(self, selection: BasicSelection) -> NDArrayLikeOrScalar: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) - return self.array.get_block_selection(cast(BasicSelection, new_selection), fields=fields) + return self.array.get_block_selection(cast("BasicSelection", new_selection), fields=fields) def __setitem__(self, selection: BasicSelection, value: npt.ArrayLike) -> None: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.set_block_selection( - cast(BasicSelection, new_selection), value, fields=fields + cast("BasicSelection", new_selection), value, fields=fields ) @@ -1097,12 +1105,12 @@ def __init__( nchunks = reduce(operator.mul, cdata_shape, 1) # some initial normalization - selection_normalized = cast(CoordinateSelectionNormalized, ensure_tuple(selection)) + selection_normalized = cast("CoordinateSelectionNormalized", ensure_tuple(selection)) selection_normalized = tuple( np.asarray([i]) if is_integer(i) else i for i in selection_normalized ) selection_normalized = cast( - CoordinateSelectionNormalized, replace_lists(selection_normalized) + "CoordinateSelectionNormalized", replace_lists(selection_normalized) ) # validation @@ -1198,15 +1206,16 @@ def __iter__(self) -> Iterator[ChunkProjection]: for (dim_sel, dim_chunk_offset) in zip(self.selection, chunk_offsets, strict=True) ) - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) + is_complete_chunk = False # TODO + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) class MaskIndexer(CoordinateIndexer): def __init__(self, selection: MaskSelection, shape: ChunkCoords, chunk_grid: ChunkGrid) -> None: # some initial normalization - selection_normalized = cast(tuple[MaskSelection], ensure_tuple(selection)) - selection_normalized = cast(tuple[MaskSelection], replace_lists(selection_normalized)) + selection_normalized = cast("tuple[MaskSelection]", ensure_tuple(selection)) + selection_normalized = cast("tuple[MaskSelection]", replace_lists(selection_normalized)) # validation if not is_mask_selection(selection_normalized, shape): @@ -1227,7 +1236,9 @@ class VIndex: array: Array # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool - def __getitem__(self, selection: CoordinateSelection | MaskSelection | Array) -> NDArrayLike: + def __getitem__( + self, selection: CoordinateSelection | MaskSelection | Array + ) -> NDArrayLikeOrScalar: from zarr.core.array import Array # if input is a Zarr array, we materialize it now. @@ -1300,14 +1311,14 @@ def pop_fields(selection: SelectionWithFields) -> tuple[Fields | None, Selection elif not isinstance(selection, tuple): # single selection item, no fields # leave selection as-is - return None, cast(Selection, selection) + return None, cast("Selection", selection) else: # multiple items, split fields from selection items fields: Fields = [f for f in selection if isinstance(f, str)] fields = fields[0] if len(fields) == 1 else fields selection_tuple = tuple(s for s in selection if not isinstance(s, str)) selection = cast( - Selection, selection_tuple[0] if len(selection_tuple) == 1 else selection_tuple + "Selection", selection_tuple[0] if len(selection_tuple) == 1 else selection_tuple ) return fields, selection @@ -1361,29 +1372,6 @@ def c_order_iter(chunks_per_shard: ChunkCoords) -> Iterator[ChunkCoords]: return itertools.product(*(range(x) for x in chunks_per_shard)) -def is_total_slice(item: Selection, shape: ChunkCoords) -> bool: - """Determine whether `item` specifies a complete slice of array with the - given `shape`. Used to optimize __setitem__ operations on the Chunk - class.""" - - # N.B., assume shape is normalized - if item == slice(None): - return True - if isinstance(item, slice): - item = (item,) - if isinstance(item, tuple): - return all( - isinstance(dim_sel, slice) - and ( - (dim_sel == slice(None)) - or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) - ) - for dim_sel, dim_len in zip(item, shape, strict=False) - ) - else: - raise TypeError(f"expected slice or tuple of slices, found {item!r}") - - def get_indexer( selection: SelectionWithFields, shape: ChunkCoords, chunk_grid: ChunkGrid ) -> Indexer: @@ -1392,12 +1380,12 @@ def get_indexer( new_selection = ensure_tuple(selection) new_selection = replace_lists(new_selection) if is_coordinate_selection(new_selection, shape): - return CoordinateIndexer(cast(CoordinateSelection, selection), shape, chunk_grid) + return CoordinateIndexer(cast("CoordinateSelection", selection), shape, chunk_grid) elif is_mask_selection(new_selection, shape): - return MaskIndexer(cast(MaskSelection, selection), shape, chunk_grid) + return MaskIndexer(cast("MaskSelection", selection), shape, chunk_grid) else: raise VindexInvalidSelectionError(new_selection) elif is_pure_orthogonal_indexing(pure_selection, len(shape)): - return OrthogonalIndexer(cast(OrthogonalSelection, selection), shape, chunk_grid) + return OrthogonalIndexer(cast("OrthogonalSelection", selection), shape, chunk_grid) else: - return BasicIndexer(cast(BasicSelection, selection), shape, chunk_grid) + return BasicIndexer(cast("BasicSelection", selection), shape, chunk_grid) diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index b95433068a..3ac75e0418 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -1,22 +1,31 @@ from __future__ import annotations -import base64 -from collections.abc import Iterable -from enum import Enum +import warnings +from collections.abc import Iterable, Sequence from functools import cached_property -from typing import TYPE_CHECKING, TypedDict, cast +from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict, cast import numcodecs.abc from zarr.abc.metadata import Metadata +from zarr.core.chunk_grids import RegularChunkGrid +from zarr.core.dtype import get_data_type_from_json +from zarr.core.dtype.common import OBJECT_CODEC_IDS, DTypeSpec_V2 if TYPE_CHECKING: - from typing import Any, Literal, Self + from typing import Literal, Self import numpy.typing as npt from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.common import ChunkCoords + from zarr.core.dtype.wrapper import ( + TBaseDType, + TBaseScalar, + TDType_co, + TScalar_co, + ZDType, + ) import json from dataclasses import dataclass, field, fields, replace @@ -25,9 +34,14 @@ import numpy as np from zarr.core.array_spec import ArrayConfig, ArraySpec -from zarr.core.chunk_grids import RegularChunkGrid from zarr.core.chunk_key_encodings import parse_separator -from zarr.core.common import JSON, ZARRAY_JSON, ZATTRS_JSON, MemoryOrder, parse_shapelike +from zarr.core.common import ( + JSON, + ZARRAY_JSON, + ZATTRS_JSON, + MemoryOrder, + parse_shapelike, +) from zarr.core.config import config, parse_indexing_order from zarr.core.metadata.common import parse_attributes @@ -41,16 +55,20 @@ class ArrayV2MetadataDict(TypedDict): attributes: dict[str, JSON] +# Union of acceptable types for v2 compressors +CompressorLikev2: TypeAlias = dict[str, JSON] | numcodecs.abc.Codec | None + + @dataclass(frozen=True, kw_only=True) class ArrayV2Metadata(Metadata): shape: ChunkCoords chunks: ChunkCoords - dtype: np.dtype[Any] - fill_value: int | float | str | bytes | None = 0 + dtype: ZDType[TBaseDType, TBaseScalar] + fill_value: int | float | str | bytes | None = None order: MemoryOrder = "C" filters: tuple[numcodecs.abc.Codec, ...] | None = None dimension_separator: Literal[".", "/"] = "." - compressor: numcodecs.abc.Codec | None = None + compressor: CompressorLikev2 attributes: dict[str, JSON] = field(default_factory=dict) zarr_format: Literal[2] = field(init=False, default=2) @@ -58,12 +76,12 @@ def __init__( self, *, shape: ChunkCoords, - dtype: npt.DTypeLike, + dtype: ZDType[TDType_co, TScalar_co], chunks: ChunkCoords, fill_value: Any, order: MemoryOrder, dimension_separator: Literal[".", "/"] = ".", - compressor: numcodecs.abc.Codec | dict[str, JSON] | None = None, + compressor: CompressorLikev2 = None, filters: Iterable[numcodecs.abc.Codec | dict[str, JSON]] | None = None, attributes: dict[str, JSON] | None = None, ) -> None: @@ -71,18 +89,20 @@ def __init__( Metadata for a Zarr format 2 array. """ shape_parsed = parse_shapelike(shape) - dtype_parsed = parse_dtype(dtype) chunks_parsed = parse_shapelike(chunks) - compressor_parsed = parse_compressor(compressor) order_parsed = parse_indexing_order(order) dimension_separator_parsed = parse_separator(dimension_separator) filters_parsed = parse_filters(filters) - fill_value_parsed = parse_fill_value(fill_value, dtype=dtype_parsed) + fill_value_parsed: TBaseScalar | None + if fill_value is not None: + fill_value_parsed = dtype.cast_scalar(fill_value) + else: + fill_value_parsed = fill_value attributes_parsed = parse_attributes(attributes) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "dtype", dtype_parsed) + object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "chunks", chunks_parsed) object.__setattr__(self, "compressor", compressor_parsed) object.__setattr__(self, "order", order_parsed) @@ -107,87 +127,104 @@ def shards(self) -> ChunkCoords | None: return None def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - def _json_convert( - o: Any, - ) -> Any: - if isinstance(o, np.dtype): - if o.fields is None: - return o.str - else: - return o.descr - if isinstance(o, numcodecs.abc.Codec): - return o.get_config() - if np.isscalar(o): - out: Any - if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): - # https://github.com/zarr-developers/zarr-python/issues/2119 - # `.item()` on a datetime type might or might not return an - # integer, depending on the value. - # Explicitly cast to an int first, and then grab .item() - out = o.view("i8").item() - else: - # convert numpy scalar to python type, and pass - # python types through - out = getattr(o, "item", lambda: o)() - if isinstance(out, complex): - # python complex types are not JSON serializable, so we use the - # serialization defined in the zarr v3 spec - return [out.real, out.imag] - return out - if isinstance(o, Enum): - return o.name - raise TypeError - zarray_dict = self.to_dict() zattrs_dict = zarray_dict.pop("attributes", {}) json_indent = config.get("json_indent") return { ZARRAY_JSON: prototype.buffer.from_bytes( - json.dumps(zarray_dict, default=_json_convert, indent=json_indent).encode() + json.dumps(zarray_dict, indent=json_indent, allow_nan=False).encode() ), ZATTRS_JSON: prototype.buffer.from_bytes( - json.dumps(zattrs_dict, indent=json_indent).encode() + json.dumps(zattrs_dict, indent=json_indent, allow_nan=False).encode() ), } @classmethod def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: - # make a copy to protect the original from modification + # Make a copy to protect the original from modification. _data = data.copy() - # check that the zarr_format attribute is correct + # Check that the zarr_format attribute is correct. _ = parse_zarr_format(_data.pop("zarr_format")) - dtype = parse_dtype(_data["dtype"]) - if dtype.kind in "SV": - fill_value_encoded = _data.get("fill_value") - if fill_value_encoded is not None: - fill_value = base64.standard_b64decode(fill_value_encoded) - _data["fill_value"] = fill_value + # To resolve a numpy object dtype array, we need to search for an object codec, + # which could be in filters or as a compressor. + # we will reference a hard-coded collection of object codec ids for this search. + + _filters, _compressor = (data.get("filters"), data.get("compressor")) + if _filters is not None: + _filters = cast("tuple[dict[str, JSON], ...]", _filters) + object_codec_id = get_object_codec_id(tuple(_filters) + (_compressor,)) + else: + object_codec_id = get_object_codec_id((_compressor,)) + # we add a layer of indirection here around the dtype attribute of the array metadata + # because we also need to know the object codec id, if any, to resolve the data type + dtype_spec: DTypeSpec_V2 = { + "name": data["dtype"], + "object_codec_id": object_codec_id, + } + dtype = get_data_type_from_json(dtype_spec, zarr_format=2) + + _data["dtype"] = dtype + fill_value_encoded = _data.get("fill_value") + if fill_value_encoded is not None: + fill_value = dtype.from_json_scalar(fill_value_encoded, zarr_format=2) + _data["fill_value"] = fill_value # zarr v2 allowed arbitrary keys here. # We don't want the ArrayV2Metadata constructor to fail just because someone put an # extra key in the metadata. expected = {x.name for x in fields(cls)} - # https://github.com/zarr-developers/zarr-python/issues/2269 - # handle the renames expected |= {"dtype", "chunks"} + # check if `filters` is an empty sequence; if so use None instead and raise a warning + filters = _data.get("filters") + if ( + isinstance(filters, Sequence) + and not isinstance(filters, (str, bytes)) + and len(filters) == 0 + ): + msg = ( + "Found an empty list of filters in the array metadata document. " + "This is contrary to the Zarr V2 specification, and will cause an error in the future. " + "Use None (or Null in a JSON document) instead of an empty list of filters." + ) + warnings.warn(msg, UserWarning, stacklevel=1) + _data["filters"] = None + _data = {k: v for k, v in _data.items() if k in expected} return cls(**_data) def to_dict(self) -> dict[str, JSON]: zarray_dict = super().to_dict() + if isinstance(zarray_dict["compressor"], numcodecs.abc.Codec): + codec_config = zarray_dict["compressor"].get_config() + # Hotfix for https://github.com/zarr-developers/zarr-python/issues/2647 + if codec_config["id"] == "zstd" and not codec_config.get("checksum", False): + codec_config.pop("checksum") + zarray_dict["compressor"] = codec_config + + if zarray_dict["filters"] is not None: + raw_filters = zarray_dict["filters"] + # TODO: remove this when we can stratically type the output JSON data structure + # entirely + if not isinstance(raw_filters, list | tuple): + raise TypeError("Invalid type for filters. Expected a list or tuple.") + new_filters = [] + for f in raw_filters: + if isinstance(f, numcodecs.abc.Codec): + new_filters.append(f.get_config()) + else: + new_filters.append(f) + zarray_dict["filters"] = new_filters - if self.dtype.kind in "SV" and self.fill_value is not None: - # There's a relationship between self.dtype and self.fill_value - # that mypy isn't aware of. The fact that we have S or V dtype here - # means we should have a bytes-type fill_value. - fill_value = base64.standard_b64encode(cast(bytes, self.fill_value)).decode("ascii") + # serialize the fill value after dtype-specific JSON encoding + if self.fill_value is not None: + fill_value = self.dtype.to_json_scalar(self.fill_value, zarr_format=2) zarray_dict["fill_value"] = fill_value - _ = zarray_dict.pop("dtype") - zarray_dict["dtype"] = self.dtype.str + # pull the "name" attribute out of the dtype spec returned by self.dtype.to_json + zarray_dict["dtype"] = self.dtype.to_json(zarr_format=2)["name"] return zarray_dict @@ -214,6 +251,8 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]: + if isinstance(data, list): # this is a valid _VoidDTypeLike check + data = [tuple(d) for d in data] return np.dtype(data) @@ -240,7 +279,11 @@ def parse_filters(data: object) -> tuple[numcodecs.abc.Codec, ...] | None: else: msg = f"Invalid filter at index {idx}. Expected a numcodecs.abc.Codec or a dict representation of numcodecs.abc.Codec. Got {type(val)} instead." raise TypeError(msg) - return tuple(out) + if len(out) == 0: + # Per the v2 spec, an empty tuple is not allowed -- use None to express "no filters" + return None + else: + return tuple(out) # take a single codec instance and wrap it in a tuple if isinstance(data, numcodecs.abc.Codec): return (data,) @@ -270,109 +313,19 @@ def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: return data -def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: +def get_object_codec_id(maybe_object_codecs: Sequence[JSON]) -> str | None: """ - Parse a potential fill value into a value that is compatible with the provided dtype. - - Parameters - ---------- - fill_value : Any - A potential fill value. - dtype : np.dtype[Any] - A numpy dtype. - - Returns - ------- - An instance of `dtype`, or `None`, or any python object (in the case of an object dtype) - """ - - if fill_value is None or dtype.hasobject: - # no fill value - pass - elif not isinstance(fill_value, np.void) and fill_value == 0: - # this should be compatible across numpy versions for any array type, including - # structured arrays - fill_value = np.zeros((), dtype=dtype)[()] - - elif dtype.kind == "U": - # special case unicode because of encoding issues on Windows if passed through numpy - # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713 - - if not isinstance(fill_value, str): - raise ValueError( - f"fill_value {fill_value!r} is not valid for dtype {dtype}; must be a unicode string" - ) - else: - try: - if isinstance(fill_value, bytes) and dtype.kind == "V": - # special case for numpy 1.14 compatibility - fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()] - else: - fill_value = np.array(fill_value, dtype=dtype)[()] - - except Exception as e: - msg = f"Fill_value {fill_value} is not valid for dtype {dtype}." - raise ValueError(msg) from e - - return fill_value - - -def _default_fill_value(dtype: np.dtype[Any]) -> Any: - """ - Get the default fill value for a type. - - Notes - ----- - This differs from :func:`parse_fill_value`, which parses a fill value - stored in the Array metadata into an in-memory value. This only gives - the default fill value for some type. - - This is useful for reading Zarr format 2 arrays, which allow the fill - value to be unspecified. - """ - if dtype.kind == "S": - return b"" - elif dtype.kind in "UO": - return "" - else: - return dtype.type(0) - - -def _default_compressor( - dtype: np.dtype[Any], -) -> dict[str, JSON] | None: - """Get the default filters and compressor for a dtype. - - https://numpy.org/doc/2.1/reference/generated/numpy.dtype.kind.html - """ - default_compressor = config.get("array.v2_default_compressor") - if dtype.kind in "biufcmM": - dtype_key = "numeric" - elif dtype.kind in "U": - dtype_key = "string" - elif dtype.kind in "OSV": - dtype_key = "bytes" - else: - raise ValueError(f"Unsupported dtype kind {dtype.kind}") - - return cast(dict[str, JSON] | None, default_compressor.get(dtype_key, None)) - - -def _default_filters( - dtype: np.dtype[Any], -) -> list[dict[str, JSON]] | None: - """Get the default filters and compressor for a dtype. - - https://numpy.org/doc/2.1/reference/generated/numpy.dtype.kind.html + Inspect a sequence of codecs / filters for an "object codec", i.e. a codec + that can serialize object arrays to contiguous bytes. Zarr python + maintains a hard-coded set of object codec ids. If any element from the input + has an id that matches one of the hard-coded object codec ids, that id + is returned immediately. """ - default_filters = config.get("array.v2_default_filters") - if dtype.kind in "biufcmM": - dtype_key = "numeric" - elif dtype.kind in "U": - dtype_key = "string" - elif dtype.kind in "OSV": - dtype_key = "bytes" - else: - raise ValueError(f"Unsupported dtype kind {dtype.kind}") - - return cast(list[dict[str, JSON]] | None, default_filters.get(dtype_key, None)) + object_codec_id = None + for maybe_object_codec in maybe_object_codecs: + if ( + isinstance(maybe_object_codec, dict) + and maybe_object_codec.get("id") in OBJECT_CODEC_IDS + ): + return cast("str", maybe_object_codec["id"]) + return object_codec_id diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 13a275a6a1..84872d3dbd 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -1,10 +1,11 @@ from __future__ import annotations -import warnings -from typing import TYPE_CHECKING, TypedDict, overload +from typing import TYPE_CHECKING, TypedDict from zarr.abc.metadata import Metadata from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype import VariableLengthUTF8, ZDType, get_data_type_from_json +from zarr.core.dtype.common import check_dtype_spec_v3 if TYPE_CHECKING: from typing import Self @@ -12,44 +13,31 @@ from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import JSON, ChunkCoords + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar + import json -from collections.abc import Iterable, Sequence +from collections.abc import Iterable from dataclasses import dataclass, field, replace -from enum import Enum -from typing import Any, Literal, cast - -import numcodecs.abc -import numpy as np -import numpy.typing as npt +from typing import Any, Literal from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid -from zarr.core.chunk_key_encodings import ChunkKeyEncoding +from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike from zarr.core.common import ( JSON, ZARR_JSON, ChunkCoords, + DimensionNames, parse_named_configuration, parse_shapelike, ) from zarr.core.config import config from zarr.core.metadata.common import parse_attributes -from zarr.core.strings import _NUMPY_SUPPORTS_VLEN_STRING -from zarr.core.strings import _STRING_DTYPE as STRING_NP_DTYPE from zarr.errors import MetadataValidationError, NodeTypeValidationError from zarr.registry import get_codec_class -DEFAULT_DTYPE = "float64" - -# Keep in sync with _replace_special_floats -SPECIAL_FLOATS_ENCODED = { - "Infinity": np.inf, - "-Infinity": -np.inf, - "NaN": np.nan, -} - def parse_zarr_format(data: object) -> Literal[3]: if data == 3: @@ -92,7 +80,7 @@ def validate_array_bytes_codec(codecs: tuple[Codec, ...]) -> ArrayBytesCodec: return abcs[0] -def validate_codecs(codecs: tuple[Codec, ...], dtype: DataType) -> None: +def validate_codecs(codecs: tuple[Codec, ...], dtype: ZDType[TBaseDType, TBaseScalar]) -> None: """Check that the codecs are valid for the given dtype""" from zarr.codecs.sharding import ShardingCodec @@ -105,14 +93,11 @@ def validate_codecs(codecs: tuple[Codec, ...], dtype: DataType) -> None: # we need to have special codecs if we are decoding vlen strings or bytestrings # TODO: use codec ID instead of class name codec_class_name = abc.__class__.__name__ - if dtype == DataType.string and not codec_class_name == "VLenUTF8Codec": + # TODO: Fix typing here + if isinstance(dtype, VariableLengthUTF8) and not codec_class_name == "VLenUTF8Codec": # type: ignore[unreachable] raise ValueError( f"For string dtype, ArrayBytesCodec must be `VLenUTF8Codec`, got `{codec_class_name}`." ) - if dtype == DataType.bytes and not codec_class_name == "VLenBytesCodec": - raise ValueError( - f"For bytes dtype, ArrayBytesCodec must be `VLenBytesCodec`, got `{codec_class_name}`." - ) def parse_dimension_names(data: object) -> tuple[str | None, ...] | None: @@ -142,66 +127,6 @@ def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]: ) -class V3JsonEncoder(json.JSONEncoder): - def __init__(self, *args: Any, **kwargs: Any) -> None: - self.indent = kwargs.pop("indent", config.get("json_indent")) - super().__init__(*args, **kwargs) - - def default(self, o: object) -> Any: - if isinstance(o, np.dtype): - return str(o) - if np.isscalar(o): - out: Any - if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): - # https://github.com/zarr-developers/zarr-python/issues/2119 - # `.item()` on a datetime type might or might not return an - # integer, depending on the value. - # Explicitly cast to an int first, and then grab .item() - out = o.view("i8").item() - else: - # convert numpy scalar to python type, and pass - # python types through - out = getattr(o, "item", lambda: o)() - if isinstance(out, complex): - # python complex types are not JSON serializable, so we use the - # serialization defined in the zarr v3 spec - return _replace_special_floats([out.real, out.imag]) - elif np.isnan(out): - return "NaN" - elif np.isinf(out): - return "Infinity" if out > 0 else "-Infinity" - return out - elif isinstance(o, Enum): - return o.name - # this serializes numcodecs compressors - # todo: implement to_dict for codecs - elif isinstance(o, numcodecs.abc.Codec): - config: dict[str, Any] = o.get_config() - return config - else: - return super().default(o) - - -def _replace_special_floats(obj: object) -> Any: - """Helper function to replace NaN/Inf/-Inf values with special strings - - Note: this cannot be done in the V3JsonEncoder because Python's `json.dumps` optimistically - converts NaN/Inf values to special types outside of the encoding step. - """ - if isinstance(obj, float): - if np.isnan(obj): - return "NaN" - elif np.isinf(obj): - return "Infinity" if obj > 0 else "-Infinity" - elif isinstance(obj, dict): - # Recursively replace in dictionaries - return {k: _replace_special_floats(v) for k, v in obj.items()} - elif isinstance(obj, list): - # Recursively replace in lists - return [_replace_special_floats(item) for item in obj] - return obj - - class ArrayV3MetadataDict(TypedDict): """ A typed dictionary model for zarr v3 metadata. @@ -214,13 +139,13 @@ class ArrayV3MetadataDict(TypedDict): @dataclass(frozen=True, kw_only=True) class ArrayV3Metadata(Metadata): shape: ChunkCoords - data_type: DataType + data_type: ZDType[TBaseDType, TBaseScalar] chunk_grid: ChunkGrid chunk_key_encoding: ChunkKeyEncoding fill_value: Any codecs: tuple[Codec, ...] attributes: dict[str, Any] = field(default_factory=dict) - dimension_names: tuple[str, ...] | None = None + dimension_names: tuple[str | None, ...] | None = None zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["array"] = field(default="array", init=False) storage_transformers: tuple[dict[str, JSON], ...] @@ -229,45 +154,41 @@ def __init__( self, *, shape: Iterable[int], - data_type: npt.DTypeLike | DataType, + data_type: ZDType[TBaseDType, TBaseScalar], chunk_grid: dict[str, JSON] | ChunkGrid, - chunk_key_encoding: dict[str, JSON] | ChunkKeyEncoding, - fill_value: Any, + chunk_key_encoding: ChunkKeyEncodingLike, + fill_value: object, codecs: Iterable[Codec | dict[str, JSON]], attributes: dict[str, JSON] | None, - dimension_names: Iterable[str] | None, + dimension_names: DimensionNames, storage_transformers: Iterable[dict[str, JSON]] | None = None, ) -> None: """ Because the class is a frozen dataclass, we set attributes using object.__setattr__ """ + shape_parsed = parse_shapelike(shape) - data_type_parsed = DataType.parse(data_type) chunk_grid_parsed = ChunkGrid.from_dict(chunk_grid) chunk_key_encoding_parsed = ChunkKeyEncoding.from_dict(chunk_key_encoding) dimension_names_parsed = parse_dimension_names(dimension_names) - if fill_value is None: - fill_value = default_fill_value(data_type_parsed) - # we pass a string here rather than an enum to make mypy happy - fill_value_parsed = parse_fill_value( - fill_value, dtype=cast(ALL_DTYPES, data_type_parsed.value) - ) + # Note: relying on a type method is numpy-specific + fill_value_parsed = data_type.cast_scalar(fill_value) attributes_parsed = parse_attributes(attributes) codecs_parsed_partial = parse_codecs(codecs) storage_transformers_parsed = parse_storage_transformers(storage_transformers) array_spec = ArraySpec( shape=shape_parsed, - dtype=data_type_parsed.to_numpy(), + dtype=data_type, fill_value=fill_value_parsed, config=ArrayConfig.from_dict({}), # TODO: config is not needed here. prototype=default_buffer_prototype(), # TODO: prototype is not needed here. ) codecs_parsed = tuple(c.evolve_from_array_spec(array_spec) for c in codecs_parsed_partial) - validate_codecs(codecs_parsed_partial, data_type_parsed) + validate_codecs(codecs_parsed_partial, data_type) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "data_type", data_type_parsed) + object.__setattr__(self, "data_type", data_type) object.__setattr__(self, "chunk_grid", chunk_grid_parsed) object.__setattr__(self, "chunk_key_encoding", chunk_key_encoding_parsed) object.__setattr__(self, "codecs", codecs_parsed) @@ -292,19 +213,16 @@ def _validate_metadata(self) -> None: if self.fill_value is None: raise ValueError("`fill_value` is required.") for codec in self.codecs: - codec.validate( - shape=self.shape, dtype=self.data_type.to_numpy(), chunk_grid=self.chunk_grid - ) - - @property - def dtype(self) -> np.dtype[Any]: - """Interpret Zarr dtype as NumPy dtype""" - return self.data_type.to_numpy() + codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid) @property def ndim(self) -> int: return len(self.shape) + @property + def dtype(self) -> ZDType[TBaseDType, TBaseScalar]: + return self.data_type + @property def chunks(self) -> ChunkCoords: if isinstance(self.chunk_grid, RegularChunkGrid): @@ -351,9 +269,9 @@ def inner_codecs(self) -> tuple[Codec, ...]: def get_chunk_spec( self, _chunk_coords: ChunkCoords, array_config: ArrayConfig, prototype: BufferPrototype ) -> ArraySpec: - assert isinstance( - self.chunk_grid, RegularChunkGrid - ), "Currently, only regular chunk grid is supported" + assert isinstance(self.chunk_grid, RegularChunkGrid), ( + "Currently, only regular chunk grid is supported" + ) return ArraySpec( shape=self.chunk_grid.chunk_shape, dtype=self.dtype, @@ -366,8 +284,13 @@ def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: return self.chunk_key_encoding.encode_chunk_key(chunk_coords) def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - d = _replace_special_floats(self.to_dict()) - return {ZARR_JSON: prototype.buffer.from_bytes(json.dumps(d, cls=V3JsonEncoder).encode())} + json_indent = config.get("json_indent") + d = self.to_dict() + return { + ZARR_JSON: prototype.buffer.from_bytes( + json.dumps(d, allow_nan=False, indent=json_indent).encode() + ) + } @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: @@ -379,18 +302,31 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: # check that the node_type attribute is correct _ = parse_node_type_array(_data.pop("node_type")) - # check that the data_type attribute is valid - data_type = DataType.parse(_data.pop("data_type")) + data_type_json = _data.pop("data_type") + if not check_dtype_spec_v3(data_type_json): + raise ValueError(f"Invalid data_type: {data_type_json!r}") + data_type = get_data_type_from_json(data_type_json, zarr_format=3) + + # check that the fill value is consistent with the data type + try: + fill = _data.pop("fill_value") + fill_value_parsed = data_type.from_json_scalar(fill, zarr_format=3) + except ValueError as e: + raise TypeError(f"Invalid fill_value: {fill!r}") from e # dimension_names key is optional, normalize missing to `None` _data["dimension_names"] = _data.pop("dimension_names", None) + # attributes key is optional, normalize missing to `None` _data["attributes"] = _data.pop("attributes", None) - return cls(**_data, data_type=data_type) # type: ignore[arg-type] + + return cls(**_data, fill_value=fill_value_parsed, data_type=data_type) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: out_dict = super().to_dict() - + out_dict["fill_value"] = self.data_type.to_json_scalar( + self.fill_value, zarr_format=self.zarr_format + ) if not isinstance(out_dict, dict): raise TypeError(f"Expected dict. Got {type(out_dict)}.") @@ -398,6 +334,15 @@ def to_dict(self) -> dict[str, JSON]: # the metadata document if out_dict["dimension_names"] is None: out_dict.pop("dimension_names") + + # TODO: replace the `to_dict` / `from_dict` on the `Metadata`` class with + # to_json, from_json, and have ZDType inherit from `Metadata` + # until then, we have this hack here, which relies on the fact that to_dict will pass through + # any non-`Metadata` fields as-is. + dtype_meta = out_dict["data_type"] + if isinstance(dtype_meta, ZDType): + out_dict["data_type"] = dtype_meta.to_json(zarr_format=3) # type: ignore[unreachable] + return out_dict def update_shape(self, shape: ChunkCoords) -> Self: @@ -405,299 +350,3 @@ def update_shape(self, shape: ChunkCoords) -> Self: def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) - - -# enum Literals can't be used in typing, so we have to restate all of the V3 dtypes as types -# https://github.com/python/typing/issues/781 - -BOOL_DTYPE = Literal["bool"] -BOOL = np.bool_ -INTEGER_DTYPE = Literal["int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64"] -INTEGER = np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64 -FLOAT_DTYPE = Literal["float16", "float32", "float64"] -FLOAT = np.float16 | np.float32 | np.float64 -COMPLEX_DTYPE = Literal["complex64", "complex128"] -COMPLEX = np.complex64 | np.complex128 -STRING_DTYPE = Literal["string"] -STRING = np.str_ -BYTES_DTYPE = Literal["bytes"] -BYTES = np.bytes_ - -ALL_DTYPES = BOOL_DTYPE | INTEGER_DTYPE | FLOAT_DTYPE | COMPLEX_DTYPE | STRING_DTYPE | BYTES_DTYPE - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: BOOL_DTYPE, -) -> BOOL: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: INTEGER_DTYPE, -) -> INTEGER: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: FLOAT_DTYPE, -) -> FLOAT: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: COMPLEX_DTYPE, -) -> COMPLEX: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: STRING_DTYPE, -) -> STRING: ... - - -@overload -def parse_fill_value( - fill_value: complex | str | bytes | np.generic | Sequence[Any] | bool, - dtype: BYTES_DTYPE, -) -> BYTES: ... - - -def parse_fill_value( - fill_value: Any, - dtype: ALL_DTYPES, -) -> Any: - """ - Parse `fill_value`, a potential fill value, into an instance of `dtype`, a data type. - If `fill_value` is `None`, then this function will return the result of casting the value 0 - to the provided data type. Otherwise, `fill_value` will be cast to the provided data type. - - Note that some numpy dtypes use very permissive casting rules. For example, - `np.bool_({'not remotely a bool'})` returns `True`. Thus this function should not be used for - validating that the provided fill value is a valid instance of the data type. - - Parameters - ---------- - fill_value : Any - A potential fill value. - dtype : str - A valid Zarr format 3 DataType. - - Returns - ------- - A scalar instance of `dtype` - """ - data_type = DataType(dtype) - if fill_value is None: - raise ValueError("Fill value cannot be None") - if data_type == DataType.string: - return np.str_(fill_value) - if data_type == DataType.bytes: - return np.bytes_(fill_value) - - # the rest are numeric types - np_dtype = cast(np.dtype[Any], data_type.to_numpy()) - - if isinstance(fill_value, Sequence) and not isinstance(fill_value, str): - if data_type in (DataType.complex64, DataType.complex128): - if len(fill_value) == 2: - decoded_fill_value = tuple( - SPECIAL_FLOATS_ENCODED.get(value, value) for value in fill_value - ) - # complex datatypes serialize to JSON arrays with two elements - return np_dtype.type(complex(*decoded_fill_value)) - else: - msg = ( - f"Got an invalid fill value for complex data type {data_type.value}." - f"Expected a sequence with 2 elements, but {fill_value!r} has " - f"length {len(fill_value)}." - ) - raise ValueError(msg) - msg = f"Cannot parse non-string sequence {fill_value!r} as a scalar with type {data_type.value}." - raise TypeError(msg) - - # Cast the fill_value to the given dtype - try: - # This warning filter can be removed after Zarr supports numpy>=2.0 - # The warning is saying that the future behavior of out of bounds casting will be to raise - # an OverflowError. In the meantime, we allow overflow and catch cases where - # fill_value != casted_value below. - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) - casted_value = np.dtype(np_dtype).type(fill_value) - except (ValueError, OverflowError, TypeError) as e: - raise ValueError(f"fill value {fill_value!r} is not valid for dtype {data_type}") from e - # Check if the value is still representable by the dtype - if (fill_value == "NaN" and np.isnan(casted_value)) or ( - fill_value in ["Infinity", "-Infinity"] and not np.isfinite(casted_value) - ): - pass - elif np_dtype.kind == "f": - # float comparison is not exact, especially when dtype str | bytes | np.generic: - if dtype == DataType.string: - return "" - elif dtype == DataType.bytes: - return b"" - else: - np_dtype = dtype.to_numpy() - np_dtype = cast(np.dtype[Any], np_dtype) - return np_dtype.type(0) # type: ignore[misc] - - -# For type checking -_bool = bool - - -class DataType(Enum): - bool = "bool" - int8 = "int8" - int16 = "int16" - int32 = "int32" - int64 = "int64" - uint8 = "uint8" - uint16 = "uint16" - uint32 = "uint32" - uint64 = "uint64" - float16 = "float16" - float32 = "float32" - float64 = "float64" - complex64 = "complex64" - complex128 = "complex128" - string = "string" - bytes = "bytes" - - @property - def byte_count(self) -> int | None: - data_type_byte_counts = { - DataType.bool: 1, - DataType.int8: 1, - DataType.int16: 2, - DataType.int32: 4, - DataType.int64: 8, - DataType.uint8: 1, - DataType.uint16: 2, - DataType.uint32: 4, - DataType.uint64: 8, - DataType.float16: 2, - DataType.float32: 4, - DataType.float64: 8, - DataType.complex64: 8, - DataType.complex128: 16, - } - try: - return data_type_byte_counts[self] - except KeyError: - # string and bytes have variable length - return None - - @property - def has_endianness(self) -> _bool: - return self.byte_count is not None and self.byte_count != 1 - - def to_numpy_shortname(self) -> str: - data_type_to_numpy = { - DataType.bool: "bool", - DataType.int8: "i1", - DataType.int16: "i2", - DataType.int32: "i4", - DataType.int64: "i8", - DataType.uint8: "u1", - DataType.uint16: "u2", - DataType.uint32: "u4", - DataType.uint64: "u8", - DataType.float16: "f2", - DataType.float32: "f4", - DataType.float64: "f8", - DataType.complex64: "c8", - DataType.complex128: "c16", - } - return data_type_to_numpy[self] - - def to_numpy(self) -> np.dtypes.StringDType | np.dtypes.ObjectDType | np.dtype[Any]: - # note: it is not possible to round trip DataType <-> np.dtype - # due to the fact that DataType.string and DataType.bytes both - # generally return np.dtype("O") from this function, even though - # they can originate as fixed-length types (e.g. " DataType: - if dtype.kind in "UT": - return DataType.string - elif dtype.kind == "S": - return DataType.bytes - elif not _NUMPY_SUPPORTS_VLEN_STRING and dtype.kind == "O": - # numpy < 2.0 does not support vlen string dtype - # so we fall back on object array of strings - return DataType.string - dtype_to_data_type = { - "|b1": "bool", - "bool": "bool", - "|i1": "int8", - " DataType: - if dtype is None: - return DataType[DEFAULT_DTYPE] - if isinstance(dtype, DataType): - return dtype - try: - return DataType(dtype) - except ValueError: - pass - try: - dtype = np.dtype(dtype) - except (ValueError, TypeError) as e: - raise ValueError(f"Invalid Zarr format 3 data_type: {dtype}") from e - # check that this is a valid v3 data_type - try: - data_type = DataType.from_numpy(dtype) - except KeyError as e: - raise ValueError(f"Invalid Zarr format 3 data_type: {dtype}") from e - return data_type diff --git a/src/zarr/core/strings.py b/src/zarr/core/strings.py deleted file mode 100644 index ffca0c3b0c..0000000000 --- a/src/zarr/core/strings.py +++ /dev/null @@ -1,86 +0,0 @@ -"""This module contains utilities for working with string arrays across -different versions of Numpy. -""" - -from typing import Any, Union, cast -from warnings import warn - -import numpy as np - -# _STRING_DTYPE is the in-memory datatype that will be used for V3 string arrays -# when reading data back from Zarr. -# Any valid string-like datatype should be fine for *setting* data. - -_STRING_DTYPE: Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"] -_NUMPY_SUPPORTS_VLEN_STRING: bool - - -def cast_array( - data: np.ndarray[Any, np.dtype[Any]], -) -> np.ndarray[Any, Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"]]: - raise NotImplementedError - - -try: - # this new vlen string dtype was added in NumPy 2.0 - _STRING_DTYPE = np.dtypes.StringDType() - _NUMPY_SUPPORTS_VLEN_STRING = True - - def cast_array( - data: np.ndarray[Any, np.dtype[Any]], - ) -> np.ndarray[Any, np.dtypes.StringDType | np.dtypes.ObjectDType]: - out = data.astype(_STRING_DTYPE, copy=False) - return cast(np.ndarray[Any, np.dtypes.StringDType], out) - -except AttributeError: - # if not available, we fall back on an object array of strings, as in Zarr < 3 - _STRING_DTYPE = np.dtypes.ObjectDType() - _NUMPY_SUPPORTS_VLEN_STRING = False - - def cast_array( - data: np.ndarray[Any, np.dtype[Any]], - ) -> np.ndarray[Any, Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"]]: - out = data.astype(_STRING_DTYPE, copy=False) - return cast(np.ndarray[Any, np.dtypes.ObjectDType], out) - - -def cast_to_string_dtype( - data: np.ndarray[Any, np.dtype[Any]], safe: bool = False -) -> np.ndarray[Any, Union["np.dtypes.StringDType", "np.dtypes.ObjectDType"]]: - """Take any data and attempt to cast to to our preferred string dtype. - - data : np.ndarray - The data to cast - - safe : bool - If True, do not issue a warning if the data is cast from object to string dtype. - - """ - if np.issubdtype(data.dtype, np.str_): - # legacy fixed-width string type (e.g. "= 2.", - stacklevel=2, - ) - return cast_array(data) - raise ValueError(f"Cannot cast dtype {data.dtype} to string dtype") diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index f7d4529478..ffb04e764d 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -3,6 +3,7 @@ import asyncio import atexit import logging +import os import threading from concurrent.futures import ThreadPoolExecutor, wait from typing import TYPE_CHECKING, TypeVar @@ -12,7 +13,7 @@ from zarr.core.config import config if TYPE_CHECKING: - from collections.abc import AsyncIterator, Coroutine + from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from typing import Any logger = logging.getLogger(__name__) @@ -54,9 +55,7 @@ def _get_executor() -> ThreadPoolExecutor: global _executor if not _executor: max_workers = config.get("threading.max_workers", None) - print(max_workers) - # if max_workers is not None and max_workers > 0: - # raise ValueError(max_workers) + logger.debug("Creating Zarr ThreadPoolExecutor with max_workers=%s", max_workers) _executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="zarr_pool") _get_loop().set_default_executor(_executor) return _executor @@ -91,6 +90,26 @@ def cleanup_resources() -> None: atexit.register(cleanup_resources) +def reset_resources_after_fork() -> None: + """ + Ensure that global resources are reset after a fork. Without this function, + forked processes will retain invalid references to the parent process's resources. + """ + global loop, iothread, _executor + # These lines are excluded from coverage because this function only runs in a child process, + # which is not observed by the test coverage instrumentation. Despite the apparent lack of + # test coverage, this function should be adequately tested by any test that uses Zarr IO with + # multiprocessing. + loop[0] = None # pragma: no cover + iothread[0] = None # pragma: no cover + _executor = None # pragma: no cover + + +# this is only available on certain operating systems +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=reset_resources_after_fork) + + async def _runner(coro: Coroutine[Any, Any, T]) -> T | BaseException: """ Await a coroutine and return the result of running it. If awaiting the coroutine raises an @@ -118,6 +137,9 @@ def sync( # NB: if the loop is not running *yet*, it is OK to submit work # and we will wait for it loop = _get_loop() + if _executor is None and config.get("threading.max_workers", None) is not None: + # trigger executor creation and attach to loop + _ = _get_executor() if not isinstance(loop, asyncio.AbstractEventLoop): raise TypeError(f"loop cannot be of type {type(loop)}") if loop.is_closed(): @@ -153,6 +175,7 @@ def _get_loop() -> asyncio.AbstractEventLoop: # repeat the check just in case the loop got filled between the # previous two calls from another thread if loop[0] is None: + logger.debug("Creating Zarr event loop") new_loop = asyncio.new_event_loop() loop[0] = new_loop iothread[0] = threading.Thread(target=new_loop.run_forever, name="zarr_io") @@ -192,3 +215,17 @@ async def iter_to_list() -> list[T]: return [item async for item in async_iterator] return self._sync(iter_to_list()) + + +async def _with_semaphore( + func: Callable[[], Awaitable[T]], semaphore: asyncio.Semaphore | None = None +) -> T: + """ + Await the result of invoking the no-argument-callable ``func`` within the context manager + provided by a Semaphore, if one is provided. Otherwise, just await the result of invoking + ``func``. + """ + if semaphore is None: + return await func() + async with semaphore: + return await func() diff --git a/src/zarr/core/sync_group.py b/src/zarr/core/sync_group.py new file mode 100644 index 0000000000..39d8a17992 --- /dev/null +++ b/src/zarr/core/sync_group.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr.core.group import Group, GroupMetadata, _parse_async_node +from zarr.core.group import create_hierarchy as create_hierarchy_async +from zarr.core.group import create_nodes as create_nodes_async +from zarr.core.group import create_rooted_hierarchy as create_rooted_hierarchy_async +from zarr.core.group import get_node as get_node_async +from zarr.core.sync import _collect_aiterator, sync + +if TYPE_CHECKING: + from collections.abc import Iterator + + from zarr.abc.store import Store + from zarr.core.array import Array + from zarr.core.common import ZarrFormat + from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata + + +def create_nodes( + *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] +) -> Iterator[tuple[str, Group | Array]]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + Group | Array + The created nodes. + """ + coro = create_nodes_async(store=store, nodes=nodes) + + for key, value in sync(_collect_aiterator(coro)): + yield key, _parse_async_node(value) + + +def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> Iterator[tuple[str, Group | Array]]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, Group | Array] + This function yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> from zarr import create_hierarchy + >>> from zarr.storage import MemoryStore + >>> from zarr.core.group import GroupMetadata + + >>> store = MemoryStore() + >>> nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + >>> nodes_created = dict(create_hierarchy(store=store, nodes=nodes)) + >>> print(nodes) + # {'a': GroupMetadata(attributes={'name': 'leaf'}, zarr_format=3, consolidated_metadata=None, node_type='group')} + """ + coro = create_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite) + + for key, value in sync(_collect_aiterator(coro)): + yield key, _parse_async_node(value) + + +def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> Group | Array: + """ + Create a Zarr hierarchy with a root, and return the root node, which could be a ``Group`` + or ``Array`` instance. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + overwrite : bool + Whether to overwrite existing nodes. Default is ``False``. + + Returns + ------- + Group | Array + """ + async_node = sync(create_rooted_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite)) + return _parse_async_node(async_node) + + +def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> Array | Group: + """ + Get an Array or Group from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + Array | Group + """ + + return _parse_async_node(sync(get_node_async(store=store, path=path, zarr_format=zarr_format))) diff --git a/src/zarr/dtype.py b/src/zarr/dtype.py new file mode 100644 index 0000000000..6e3789543b --- /dev/null +++ b/src/zarr/dtype.py @@ -0,0 +1,3 @@ +from zarr.core.dtype import ZDType, data_type_registry + +__all__ = ["ZDType", "data_type_registry"] diff --git a/src/zarr/errors.py b/src/zarr/errors.py index 441cdab9a3..4f972a6703 100644 --- a/src/zarr/errors.py +++ b/src/zarr/errors.py @@ -5,6 +5,7 @@ "ContainsArrayAndGroupError", "ContainsArrayError", "ContainsGroupError", + "GroupNotFoundError", "MetadataValidationError", "NodeTypeValidationError", ] @@ -21,6 +22,14 @@ def __init__(self, *args: Any) -> None: super().__init__(self._msg.format(*args)) +class GroupNotFoundError(BaseZarrError, FileNotFoundError): + """ + Raised when a group isn't found at a certain path. + """ + + _msg = "No group found in store {!r} at path {!r}" + + class ContainsGroupError(BaseZarrError): """Raised when a group already exists at a certain path.""" diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 704db3f704..eb345b24b1 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from zarr.core.config import BadConfigError, config +from zarr.core.dtype import data_type_registry if TYPE_CHECKING: from importlib.metadata import EntryPoint @@ -43,10 +44,13 @@ def __init__(self) -> None: def lazy_load(self) -> None: for e in self.lazy_load_list: self.register(e.load()) + self.lazy_load_list.clear() - def register(self, cls: type[T]) -> None: - self[fully_qualified_name(cls)] = cls + def register(self, cls: type[T], qualname: str | None = None) -> None: + if qualname is None: + qualname = fully_qualified_name(cls) + self[qualname] = cls __codec_registries: dict[str, Registry[Codec]] = defaultdict(Registry) @@ -58,12 +62,14 @@ def register(self, cls: type[T]) -> None: The registry module is responsible for managing implementations of codecs, pipelines, buffers and ndbuffers and collecting them from entrypoints. The implementation used is determined by the config. + +The registry module is also responsible for managing dtypes. """ def _collect_entrypoints() -> list[Registry[Any]]: """ - Collects codecs, pipelines, buffers and ndbuffers from entrypoints. + Collects codecs, pipelines, dtypes, buffers and ndbuffers from entrypoints. Entry points can either be single items or groups of items. Allowed syntax for entry_points.txt is e.g. @@ -86,6 +92,10 @@ def _collect_entrypoints() -> list[Registry[Any]]: __buffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="buffer")) __ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr.ndbuffer")) __ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="ndbuffer")) + + data_type_registry.lazy_load_list.extend(entry_points.select(group="zarr.data_type")) + data_type_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="data_type")) + __pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) __pipeline_registry.lazy_load_list.extend( entry_points.select(group="zarr", name="codec_pipeline") @@ -123,12 +133,12 @@ def register_pipeline(pipe_cls: type[CodecPipeline]) -> None: __pipeline_registry.register(pipe_cls) -def register_ndbuffer(cls: type[NDBuffer]) -> None: - __ndbuffer_registry.register(cls) +def register_ndbuffer(cls: type[NDBuffer], qualname: str | None = None) -> None: + __ndbuffer_registry.register(cls, qualname) -def register_buffer(cls: type[Buffer]) -> None: - __buffer_registry.register(cls) +def register_buffer(cls: type[Buffer], qualname: str | None = None) -> None: + __buffer_registry.register(cls, qualname) def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]: @@ -148,7 +158,8 @@ def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]: if len(codec_classes) == 1: return next(iter(codec_classes.values())) warnings.warn( - f"Codec '{key}' not configured in config. Selecting any implementation.", stacklevel=2 + f"Codec '{key}' not configured in config. Selecting any implementation.", + stacklevel=2, ) return list(codec_classes.values())[-1] selected_codec_cls = codec_classes[config_entry] diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index c092ade03e..6721139375 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -3,11 +3,12 @@ from types import ModuleType from typing import Any -from zarr.storage._common import StoreLike, StorePath, make_store_path +from zarr.storage._common import StoreLike, StorePath from zarr.storage._fsspec import FsspecStore from zarr.storage._local import LocalStore from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, MemoryStore +from zarr.storage._obstore import ObjectStore from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -17,11 +18,11 @@ "LocalStore", "LoggingStore", "MemoryStore", + "ObjectStore", "StoreLike", "StorePath", "WrapperStore", "ZipStore", - "make_store_path", ] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 523e470671..f264728cf2 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -1,10 +1,11 @@ from __future__ import annotations +import importlib.util import json from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZGROUP_JSON, AccessModeLiteral, ZarrFormat from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError @@ -12,6 +13,12 @@ from zarr.storage._memory import MemoryStore from zarr.storage._utils import normalize_path +_has_fsspec = importlib.util.find_spec("fsspec") +if _has_fsspec: + from fsspec.mapping import FSMap +else: + FSMap = None + if TYPE_CHECKING: from zarr.core.buffer import BufferPrototype @@ -41,16 +48,14 @@ class StorePath: def __init__(self, store: Store, path: str = "") -> None: self.store = store - self.path = path + self.path = normalize_path(path) @property def read_only(self) -> bool: return self.store.read_only @classmethod - async def open( - cls, store: Store, path: str, mode: AccessModeLiteral | None = None - ) -> StorePath: + async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = None) -> Self: """ Open StorePath based on the provided mode. @@ -67,6 +72,9 @@ async def open( ------ FileExistsError If the mode is 'w-' and the store path already exists. + ValueError + If the mode is not "r" and the store is read-only, or + if the mode is "r" and the store is not read-only. """ await store._ensure_open() @@ -78,6 +86,8 @@ async def open( if store.read_only and mode != "r": raise ValueError(f"Store is read-only but mode is '{mode}'") + if not store.read_only and mode == "r": + raise ValueError(f"Store is not read-only but mode is '{mode}'") match mode: case "w-": @@ -102,7 +112,7 @@ async def open( async def get( self, prototype: BufferPrototype | None = None, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: """ Read bytes from the store. @@ -111,7 +121,7 @@ async def get( ---------- prototype : BufferPrototype, optional The buffer prototype to use when reading the bytes. - byte_range : ByteRangeRequest, optional + byte_range : ByteRequest, optional The range of bytes to read. Returns @@ -123,7 +133,7 @@ async def get( prototype = default_buffer_prototype() return await self.store.get(self.path, prototype=prototype, byte_range=byte_range) - async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -> None: + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: """ Write bytes to the store. @@ -131,7 +141,7 @@ async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) - ---------- value : Buffer The buffer to write. - byte_range : ByteRangeRequest, optional + byte_range : ByteRequest, optional The range of bytes to write. If None, the entire buffer is written. Raises @@ -224,7 +234,7 @@ def __eq__(self, other: object) -> bool: return False -StoreLike = Store | StorePath | Path | str | dict[str, Buffer] +StoreLike: TypeAlias = Store | StorePath | FSMap | Path | str | dict[str, Buffer] async def make_store_path( @@ -311,9 +321,18 @@ async def make_store_path( # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. store = await MemoryStore.open(store_dict=store_like, read_only=_read_only) + elif _has_fsspec and isinstance(store_like, FSMap): + if path: + raise ValueError( + "'path' was provided but is not used for FSMap store_like objects. Specify the path when creating the FSMap instance instead." + ) + if storage_options: + raise ValueError( + "'storage_options was provided but is not used for FSMap store_like objects. Specify the storage options when creating the FSMap instance instead." + ) + store = FsspecStore.from_mapper(store_like, read_only=_read_only) else: - msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] - raise TypeError(msg) + raise TypeError(f"Unsupported type for store_like: '{type(store_like).__name__}'") result = await StorePath.open(store, path=path_normalized, mode=mode) diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 89d80320dd..4f6929456e 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -1,17 +1,30 @@ from __future__ import annotations +import json import warnings +from contextlib import suppress from typing import TYPE_CHECKING, Any -from zarr.abc.store import ByteRangeRequest, Store +from packaging.version import parse as parse_version + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.buffer import Buffer from zarr.storage._common import _dereference_path if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable + from fsspec import AbstractFileSystem from fsspec.asyn import AsyncFileSystem + from fsspec.mapping import FSMap - from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.buffer import BufferPrototype from zarr.core.common import BytesLike @@ -22,9 +35,45 @@ ) +def _make_async(fs: AbstractFileSystem) -> AsyncFileSystem: + """Convert a sync FSSpec filesystem to an async FFSpec filesystem + + If the filesystem class supports async operations, a new async instance is created + from the existing instance. + + If the filesystem class does not support async operations, the existing instance + is wrapped with AsyncFileSystemWrapper. + """ + import fsspec + + fsspec_version = parse_version(fsspec.__version__) + if fs.async_impl and fs.asynchronous: + # Already an async instance of an async filesystem, nothing to do + return fs + if fs.async_impl: + # Convert sync instance of an async fs to an async instance + fs_dict = json.loads(fs.to_json()) + fs_dict["asynchronous"] = True + return fsspec.AbstractFileSystem.from_json(json.dumps(fs_dict)) + + # Wrap sync filesystems with the async wrapper + if type(fs) is fsspec.implementations.local.LocalFileSystem and not fs.auto_mkdir: + raise ValueError( + f"LocalFilesystem {fs} was created with auto_mkdir=False but Zarr requires the filesystem to automatically create directories" + ) + if fsspec_version < parse_version("2024.12.0"): + raise ImportError( + f"The filesystem '{fs}' is synchronous, and the required " + "AsyncFileSystemWrapper is not available. Upgrade fsspec to version " + "2024.12.0 or later to enable this functionality." + ) + + return fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper(fs, asynchronous=True) + + class FsspecStore(Store): """ - A remote Store based on FSSpec + Store for remote data based on FSSpec. Parameters ---------- @@ -73,6 +122,7 @@ class FsspecStore(Store): fs: AsyncFileSystem allowed_exceptions: tuple[type[Exception], ...] + path: str def __init__( self, @@ -129,6 +179,38 @@ def from_upath( allowed_exceptions=allowed_exceptions, ) + @classmethod + def from_mapper( + cls, + fs_map: FSMap, + read_only: bool = False, + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> FsspecStore: + """ + Create a FsspecStore from a FSMap object. + + Parameters + ---------- + fs_map : FSMap + Fsspec mutable mapping object. + read_only : bool + Whether the store is read-only, defaults to False. + allowed_exceptions : tuple, optional + The exceptions that are allowed to be raised when accessing the + store. Defaults to ALLOWED_EXCEPTIONS. + + Returns + ------- + FsspecStore + """ + fs = _make_async(fs_map.fs) + return cls( + fs=fs, + path=fs_map.root, + read_only=read_only, + allowed_exceptions=allowed_exceptions, + ) + @classmethod def from_url( cls, @@ -166,6 +248,8 @@ def from_url( opts = {"asynchronous": True, **opts} fs, path = url_to_fs(url, **opts) + if not fs.async_impl: + fs = _make_async(fs) # fsspec is not consistent about removing the scheme from the path, so check and strip it here # https://github.com/fsspec/filesystem_spec/issues/1722 @@ -175,6 +259,15 @@ def from_url( return cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) + def with_read_only(self, read_only: bool = False) -> FsspecStore: + # docstring inherited + return type(self)( + fs=self.fs, + path=self.path, + allowed_exceptions=self.allowed_exceptions, + read_only=read_only, + ) + async def clear(self) -> None: # docstring inherited try: @@ -199,7 +292,7 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if not self._is_open: @@ -207,23 +300,26 @@ async def get( path = _dereference_path(self.path, key) try: - if byte_range: - # fsspec uses start/end, not start/length - start, length = byte_range - if start is not None and length is not None: - end = start + length - elif length is not None: - end = length - else: - end = None - value = prototype.buffer.from_bytes( - await ( - self.fs._cat_file(path, start=byte_range[0], end=end) - if byte_range - else self.fs._cat_file(path) + if byte_range is None: + value = prototype.buffer.from_bytes(await self.fs._cat_file(path)) + elif isinstance(byte_range, RangeByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file( + path, + start=byte_range.start, + end=byte_range.end, + ) ) - ) - + elif isinstance(byte_range, OffsetByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file(path, start=byte_range.offset, end=None) + ) + elif isinstance(byte_range, SuffixByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file(path, start=-byte_range.suffix, end=None) + ) + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") except self.allowed_exceptions: return None except OSError as e: @@ -244,6 +340,10 @@ async def set( if not self._is_open: await self._open() self._check_writable() + if not isinstance(value, Buffer): + raise TypeError( + f"FsspecStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) path = _dereference_path(self.path, key) # write data if byte_range: @@ -261,6 +361,19 @@ async def delete(self, key: str) -> None: except self.allowed_exceptions: pass + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + if not self.supports_deletes: + raise NotImplementedError( + "This method is only available for stores that support deletes." + ) + self._check_writable() + + path_to_delete = _dereference_path(self.path, prefix) + + with suppress(*self.allowed_exceptions): + await self.fs._rm(path_to_delete, recursive=True) + async def exists(self, key: str) -> bool: # docstring inherited path = _dereference_path(self.path, key) @@ -270,25 +383,35 @@ async def exists(self, key: str) -> bool: async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited if key_ranges: - paths, starts, stops = zip( - *( - ( - _dereference_path(self.path, k[0]), - k[1][0], - ((k[1][0] or 0) + k[1][1]) if k[1][1] is not None else None, - ) - for k in key_ranges - ), - strict=False, - ) + # _cat_ranges expects a list of paths, start, and end ranges, so we need to reformat each ByteRequest. + key_ranges = list(key_ranges) + paths: list[str] = [] + starts: list[int | None] = [] + stops: list[int | None] = [] + for key, byte_range in key_ranges: + paths.append(_dereference_path(self.path, key)) + if byte_range is None: + starts.append(None) + stops.append(None) + elif isinstance(byte_range, RangeByteRequest): + starts.append(byte_range.start) + stops.append(byte_range.end) + elif isinstance(byte_range, OffsetByteRequest): + starts.append(byte_range.offset) + stops.append(None) + elif isinstance(byte_range, SuffixByteRequest): + starts.append(-byte_range.suffix) + stops.append(None) + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") else: return [] # TODO: expectations for exceptions or missing keys? - res = await self.fs._cat_ranges(list(paths), starts, stops, on_error="return") + res = await self.fs._cat_ranges(paths, starts, stops, on_error="return") # the following is an s3-specific condition we probably don't want to leak res = [b"" if (isinstance(r, OSError) and "not satisfiable" in str(r)) else r for r in res] for r in res: @@ -306,7 +429,7 @@ async def set_partial_values( async def list(self) -> AsyncIterator[str]: # docstring inherited allfiles = await self.fs._find(self.path, detail=False, withdirs=False) - for onefile in (a.replace(self.path + "/", "") for a in allfiles): + for onefile in (a.removeprefix(self.path + "/") for a in allfiles): yield onefile async def list_dir(self, prefix: str) -> AsyncIterator[str]: diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index f4226792cb..43e585415d 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -7,7 +7,13 @@ from pathlib import Path from typing import TYPE_CHECKING -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) from zarr.core.buffer import Buffer from zarr.core.buffer.core import default_buffer_prototype from zarr.core.common import concurrent_map @@ -18,29 +24,20 @@ from zarr.core.buffer import BufferPrototype -def _get( - path: Path, prototype: BufferPrototype, byte_range: tuple[int | None, int | None] | None -) -> Buffer: - if byte_range is not None: - if byte_range[0] is None: - start = 0 - else: - start = byte_range[0] - - end = (start + byte_range[1]) if byte_range[1] is not None else None - else: +def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None) -> Buffer: + if byte_range is None: return prototype.buffer.from_bytes(path.read_bytes()) with path.open("rb") as f: size = f.seek(0, io.SEEK_END) - if start is not None: - if start >= 0: - f.seek(start) - else: - f.seek(max(0, size + start)) - if end is not None: - if end < 0: - end = size + end - return prototype.buffer.from_bytes(f.read(end - f.tell())) + if isinstance(byte_range, RangeByteRequest): + f.seek(byte_range.start) + return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) + elif isinstance(byte_range, OffsetByteRequest): + f.seek(byte_range.offset) + elif isinstance(byte_range, SuffixByteRequest): + f.seek(max(0, size - byte_range.suffix)) + else: + raise TypeError(f"Unexpected byte_range, got {byte_range}.") return prototype.buffer.from_bytes(f.read()) @@ -54,21 +51,23 @@ def _put( if start is not None: with path.open("r+b") as f: f.seek(start) - f.write(value.as_numpy_array().tobytes()) + # write takes any object supporting the buffer protocol + f.write(value.as_buffer_like()) return None else: - view = memoryview(value.as_numpy_array().tobytes()) + view = value.as_buffer_like() if exclusive: mode = "xb" else: mode = "wb" with path.open(mode=mode) as f: + # write takes any object supporting the buffer protocol return f.write(view) class LocalStore(Store): """ - Local file system store. + Store for the local file system. Parameters ---------- @@ -99,10 +98,17 @@ def __init__(self, root: Path | str, *, read_only: bool = False) -> None: root = Path(root) if not isinstance(root, Path): raise TypeError( - f'"root" must be a string or Path instance. Got an object with type {type(root)} instead.' + f"'root' must be a string or Path instance. Got an instance of {type(root)} instead." ) self.root = root + def with_read_only(self, read_only: bool = False) -> LocalStore: + # docstring inherited + return type(self)( + root=self.root, + read_only=read_only, + ) + async def _open(self) -> None: if not self.read_only: self.root.mkdir(parents=True, exist_ok=True) @@ -127,7 +133,7 @@ async def get( self, key: str, prototype: BufferPrototype | None = None, - byte_range: tuple[int | None, int | None] | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if prototype is None: @@ -145,7 +151,7 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited args = [] @@ -172,7 +178,9 @@ async def _set(self, key: str, value: Buffer, exclusive: bool = False) -> None: self._check_writable() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError("LocalStore.set(): `value` must a Buffer instance") + raise TypeError( + f"LocalStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) path = self.root / key await asyncio.to_thread(_put, path, value, start=None, exclusive=exclusive) @@ -209,6 +217,19 @@ async def delete(self, key: str) -> None: else: await asyncio.to_thread(path.unlink, True) # Q: we may want to raise if path is missing + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + self._check_writable() + path = self.root / prefix + if path.is_dir(): + shutil.rmtree(path) + elif path.is_file(): + raise ValueError(f"delete_dir was passed a {prefix=!r} that is a file") + else: + # Non-existent directory + # This path is tested by test_group:test_create_creates_parents for one + pass + async def exists(self, key: str) -> bool: # docstring inherited path = self.root / key @@ -239,5 +260,17 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: except (FileNotFoundError, NotADirectoryError): pass + async def move(self, dest_root: Path | str) -> None: + """ + Move the store to another path. The old root directory is deleted. + """ + if isinstance(dest_root, str): + dest_root = Path(dest_root) + os.makedirs(dest_root.parent, exist_ok=True) + if os.path.exists(dest_root): + raise FileExistsError(f"Destination root {dest_root} already exists.") + shutil.move(self.root, dest_root) + self.root = dest_root + async def getsize(self, key: str) -> int: return os.path.getsize(self.root / key) diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py index 450913e9d3..a2164a418f 100644 --- a/src/zarr/storage/_logging.py +++ b/src/zarr/storage/_logging.py @@ -2,26 +2,29 @@ import inspect import logging +import sys import time from collections import defaultdict from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self, TypeVar from zarr.abc.store import Store from zarr.storage._wrapper import WrapperStore if TYPE_CHECKING: - from collections.abc import AsyncIterator, Generator, Iterable + from collections.abc import AsyncGenerator, Generator, Iterable - from zarr.abc.store import ByteRangeRequest + from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, BufferPrototype counter: defaultdict[str, int] +T_Store = TypeVar("T_Store", bound=Store) -class LoggingStore(WrapperStore[Store]): + +class LoggingStore(WrapperStore[T_Store]): """ - Store wrapper that logs all calls to the wrapped store. + Store that logs all calls to another wrapped store. Parameters ---------- @@ -42,7 +45,7 @@ class LoggingStore(WrapperStore[Store]): def __init__( self, - store: Store, + store: T_Store, log_level: str = "DEBUG", log_handler: logging.Handler | None = None, ) -> None: @@ -67,7 +70,7 @@ def _configure_logger( def _default_handler(self) -> logging.Handler: """Define a default log handler""" - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stdout) handler.setLevel(self.log_level) handler.setFormatter( logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -85,7 +88,7 @@ def log(self, hint: Any = "") -> Generator[None, None, None]: op = f"{type(self._store).__name__}.{method}" if hint: op = f"{op}({hint})" - self.logger.info("Calling %s", op) + self.logger.info(" Calling %s", op) start_time = time.time() try: self.counter[method] += 1 @@ -94,6 +97,14 @@ def log(self, hint: Any = "") -> Generator[None, None, None]: end_time = time.time() self.logger.info("Finished %s [%.2f s]", op, end_time - start_time) + @classmethod + async def open(cls: type[Self], store_cls: type[T_Store], *args: Any, **kwargs: Any) -> Self: + log_level = kwargs.pop("log_level", "DEBUG") + log_handler = kwargs.pop("log_handler", None) + store = store_cls(*args, **kwargs) + await store._open() + return cls(store=store, log_level=log_level, log_handler=log_handler) + @property def supports_writes(self) -> bool: with self.log(): @@ -126,8 +137,7 @@ def _is_open(self) -> bool: @_is_open.setter def _is_open(self, value: bool) -> None: - with self.log(value): - self._store._is_open = value + raise NotImplementedError("LoggingStore must be opened via the `_open` method") async def _open(self) -> None: with self.log(): @@ -151,17 +161,17 @@ def __str__(self) -> str: return f"logging-{self._store}" def __repr__(self) -> str: - return f"LoggingStore({repr(self._store)!r})" + return f"LoggingStore({self._store.__class__.__name__}, '{self._store}')" def __eq__(self, other: object) -> bool: with self.log(other): - return self._store == other + return type(self) is type(other) and self._store.__eq__(other._store) # type: ignore[attr-defined] async def get( self, key: str, prototype: BufferPrototype, - byte_range: tuple[int | None, int | None] | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited with self.log(key): @@ -170,7 +180,7 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited keys = ",".join([k[0] for k in key_ranges]) @@ -205,19 +215,19 @@ async def set_partial_values( with self.log(keys): return await self._store.set_partial_values(key_start_values=key_start_values) - async def list(self) -> AsyncIterator[str]: + async def list(self) -> AsyncGenerator[str, None]: # docstring inherited with self.log(): async for key in self._store.list(): yield key - async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited with self.log(prefix): async for key in self._store.list_prefix(prefix=prefix): yield key - async def list_dir(self, prefix: str) -> AsyncIterator[str]: + async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited with self.log(prefix): async for key in self._store.list_dir(prefix=prefix): diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index 1f8dd75768..0dc6f13236 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -3,10 +3,10 @@ from logging import getLogger from typing import TYPE_CHECKING, Self -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, gpu from zarr.core.common import concurrent_map -from zarr.storage._utils import _normalize_interval_index +from zarr.storage._utils import _normalize_byte_range_index if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable, MutableMapping @@ -19,7 +19,7 @@ class MemoryStore(Store): """ - In-memory store. + Store for local memory. Parameters ---------- @@ -54,6 +54,13 @@ def __init__( store_dict = {} self._store_dict = store_dict + def with_read_only(self, read_only: bool = False) -> MemoryStore: + # docstring inherited + return type(self)( + store_dict=self._store_dict, + read_only=read_only, + ) + async def clear(self) -> None: # docstring inherited self._store_dict.clear() @@ -75,7 +82,7 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: tuple[int | None, int | None] | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if not self._is_open: @@ -83,20 +90,20 @@ async def get( assert isinstance(key, str) try: value = self._store_dict[key] - start, length = _normalize_interval_index(value, byte_range) - return prototype.buffer.from_buffer(value[start : start + length]) + start, stop = _normalize_byte_range_index(value, byte_range) + return prototype.buffer.from_buffer(value[start:stop]) except KeyError: return None async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited # All the key-ranges arguments goes with the same prototype - async def _get(key: str, byte_range: ByteRangeRequest) -> Buffer | None: + async def _get(key: str, byte_range: ByteRequest | None) -> Buffer | None: return await self.get(key, prototype=prototype, byte_range=byte_range) return await concurrent_map(key_ranges, _get, limit=None) @@ -111,7 +118,9 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None await self._ensure_open() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError(f"Expected Buffer. Got {type(value)}.") + raise TypeError( + f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) if byte_range is not None: buf = self._store_dict[key] @@ -171,8 +180,10 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: class GpuMemoryStore(MemoryStore): - """A GPU only memory store that stores every chunk in GPU memory irrespective - of the original location. + """ + Store for GPU memory. + + Stores every chunk in GPU memory irrespective of the original location. The dictionary of buffers to initialize this memory store with *must* be GPU Buffers. @@ -231,8 +242,9 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None self._check_writable() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError(f"Expected Buffer. Got {type(value)}.") - + raise TypeError( + f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) # Convert to gpu.Buffer gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value) await super().set(key, gpu_value, byte_range=byte_range) diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py new file mode 100644 index 0000000000..047ed07fbb --- /dev/null +++ b/src/zarr/storage/_obstore.py @@ -0,0 +1,495 @@ +from __future__ import annotations + +import asyncio +import contextlib +import pickle +from collections import defaultdict +from typing import TYPE_CHECKING, TypedDict + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.config import config + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Coroutine, Iterable + from typing import Any + + from obstore import ListResult, ListStream, ObjectMeta, OffsetRange, SuffixRange + from obstore.store import ObjectStore as _UpstreamObjectStore + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.common import BytesLike + +__all__ = ["ObjectStore"] + +_ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, +) + + +class ObjectStore(Store): + """ + Store that uses obstore for fast read/write from AWS, GCP, Azure. + + Parameters + ---------- + store : obstore.store.ObjectStore + An obstore store instance that is set up with the proper credentials. + read_only : bool + Whether to open the store in read-only mode. + + Warnings + -------- + ObjectStore is experimental and subject to API changes without notice. Please + raise an issue with any comments/concerns about the store. + """ + + store: _UpstreamObjectStore + """The underlying obstore instance.""" + + def __eq__(self, value: object) -> bool: + if not isinstance(value, ObjectStore): + return False + + if not self.read_only == value.read_only: + return False + + return self.store == value.store + + def __init__(self, store: _UpstreamObjectStore, *, read_only: bool = False) -> None: + if not store.__class__.__module__.startswith("obstore"): + raise TypeError(f"expected ObjectStore class, got {store!r}") + super().__init__(read_only=read_only) + self.store = store + + def with_read_only(self, read_only: bool = False) -> ObjectStore: + # docstring inherited + return type(self)( + store=self.store, + read_only=read_only, + ) + + def __str__(self) -> str: + return f"object_store://{self.store}" + + def __repr__(self) -> str: + return f"{type(self).__name__}({self})" + + def __getstate__(self) -> dict[Any, Any]: + state = self.__dict__.copy() + state["store"] = pickle.dumps(self.store) + return state + + def __setstate__(self, state: dict[Any, Any]) -> None: + state["store"] = pickle.loads(state["store"]) + self.__dict__.update(state) + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: + # docstring inherited + import obstore as obs + + try: + if byte_range is None: + resp = await obs.get_async(self.store, key) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + elif isinstance(byte_range, RangeByteRequest): + bytes = await obs.get_range_async( + self.store, key, start=byte_range.start, end=byte_range.end + ) + return prototype.buffer.from_bytes(bytes) # type: ignore[arg-type] + elif isinstance(byte_range, OffsetByteRequest): + resp = await obs.get_async( + self.store, key, options={"range": {"offset": byte_range.offset}} + ) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + elif isinstance(byte_range, SuffixByteRequest): + # some object stores (Azure) don't support suffix requests. In this + # case, our workaround is to first get the length of the object and then + # manually request the byte range at the end. + try: + resp = await obs.get_async( + self.store, key, options={"range": {"suffix": byte_range.suffix}} + ) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + except obs.exceptions.NotSupportedError: + head_resp = await obs.head_async(self.store, key) + file_size = head_resp["size"] + suffix_len = byte_range.suffix + buffer = await obs.get_range_async( + self.store, + key, + start=file_size - suffix_len, + length=suffix_len, + ) + return prototype.buffer.from_bytes(buffer) # type: ignore[arg-type] + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}") + except _ALLOWED_EXCEPTIONS: + return None + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + return await _get_partial_values(self.store, prototype=prototype, key_ranges=key_ranges) + + async def exists(self, key: str) -> bool: + # docstring inherited + import obstore as obs + + try: + await obs.head_async(self.store, key) + except FileNotFoundError: + return False + else: + return True + + @property + def supports_writes(self) -> bool: + # docstring inherited + return True + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + + buf = value.as_buffer_like() + await obs.put_async(self.store, key, buf) + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + buf = value.as_buffer_like() + with contextlib.suppress(obs.exceptions.AlreadyExistsError): + await obs.put_async(self.store, key, buf, mode="create") + + @property + def supports_deletes(self) -> bool: + # docstring inherited + return True + + async def delete(self, key: str) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + await obs.delete_async(self.store, key) + + @property + def supports_partial_writes(self) -> bool: + # docstring inherited + return False + + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, BytesLike]] + ) -> None: + # docstring inherited + raise NotImplementedError + + @property + def supports_listing(self) -> bool: + # docstring inherited + return True + + def list(self) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + objects: ListStream[list[ObjectMeta]] = obs.list(self.store) + return _transform_list(objects) + + def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + objects: ListStream[list[ObjectMeta]] = obs.list(self.store, prefix=prefix) + return _transform_list(objects) + + def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + coroutine = obs.list_with_delimiter_async(self.store, prefix=prefix) + return _transform_list_dir(coroutine, prefix) + + +async def _transform_list( + list_stream: ListStream[list[ObjectMeta]], +) -> AsyncGenerator[str, None]: + """ + Transform the result of list into an async generator of paths. + """ + async for batch in list_stream: + for item in batch: + yield item["path"] + + +async def _transform_list_dir( + list_result_coroutine: Coroutine[Any, Any, ListResult[list[ObjectMeta]]], prefix: str +) -> AsyncGenerator[str, None]: + """ + Transform the result of list_with_delimiter into an async generator of paths. + """ + list_result = await list_result_coroutine + + # We assume that the underlying object-store implementation correctly handles the + # prefix, so we don't double-check that the returned results actually start with the + # given prefix. + prefixes = [obj.lstrip(prefix).lstrip("/") for obj in list_result["common_prefixes"]] + objects = [obj["path"].removeprefix(prefix).lstrip("/") for obj in list_result["objects"]] + for item in prefixes + objects: + yield item + + +class _BoundedRequest(TypedDict): + """Range request with a known start and end byte. + + These requests can be multiplexed natively on the Rust side with + `obstore.get_ranges_async`. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + start: int + """Start byte offset.""" + + end: int + """End byte offset.""" + + +class _OtherRequest(TypedDict): + """Offset or suffix range requests. + + These requests cannot be concurrent on the Rust side, and each need their own call + to `obstore.get_async`, passing in the `range` parameter. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + path: str + """The path to request from.""" + + range: OffsetRange | None + # Note: suffix requests are handled separately because some object stores (Azure) + # don't support them + """The range request type.""" + + +class _SuffixRequest(TypedDict): + """Offset or suffix range requests. + + These requests cannot be concurrent on the Rust side, and each need their own call + to `obstore.get_async`, passing in the `range` parameter. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + path: str + """The path to request from.""" + + range: SuffixRange + """The suffix range.""" + + +class _Response(TypedDict): + """A response buffer associated with the original index that it should be restored to.""" + + original_request_index: int + """The positional index in the original key_ranges input""" + + buffer: Buffer + """The buffer returned from obstore's range request.""" + + +async def _make_bounded_requests( + store: _UpstreamObjectStore, + path: str, + requests: list[_BoundedRequest], + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make all bounded requests for a specific file. + + `obstore.get_ranges_async` allows for making concurrent requests for multiple ranges + within a single file, and will e.g. merge concurrent requests. This only uses one + single Python coroutine. + """ + import obstore as obs + + starts = [r["start"] for r in requests] + ends = [r["end"] for r in requests] + async with semaphore: + responses = await obs.get_ranges_async(store, path=path, starts=starts, ends=ends) + + buffer_responses: list[_Response] = [] + for request, response in zip(requests, responses, strict=True): + buffer_responses.append( + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(response), # type: ignore[arg-type] + } + ) + + return buffer_responses + + +async def _make_other_request( + store: _UpstreamObjectStore, + request: _OtherRequest, + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make offset or full-file requests. + + We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all + futures can be gathered together. + """ + import obstore as obs + + async with semaphore: + if request["range"] is None: + resp = await obs.get_async(store, request["path"]) + else: + resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) + buffer = await resp.bytes_async() + + return [ + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] + } + ] + + +async def _make_suffix_request( + store: _UpstreamObjectStore, + request: _SuffixRequest, + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make suffix requests. + + This is separated out from `_make_other_request` because some object stores (Azure) + don't support suffix requests. In this case, our workaround is to first get the + length of the object and then manually request the byte range at the end. + + We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all + futures can be gathered together. + """ + import obstore as obs + + async with semaphore: + try: + resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) + buffer = await resp.bytes_async() + except obs.exceptions.NotSupportedError: + head_resp = await obs.head_async(store, request["path"]) + file_size = head_resp["size"] + suffix_len = request["range"]["suffix"] + buffer = await obs.get_range_async( + store, + request["path"], + start=file_size - suffix_len, + length=suffix_len, + ) + + return [ + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] + } + ] + + +async def _get_partial_values( + store: _UpstreamObjectStore, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], +) -> list[Buffer | None]: + """Make multiple range requests. + + ObjectStore has a `get_ranges` method that will additionally merge nearby ranges, + but it's _per_ file. So we need to split these key_ranges into **per-file** key + ranges, and then reassemble the results in the original order. + + We separate into different requests: + + - One call to `obstore.get_ranges_async` **per target file** + - One call to `obstore.get_async` for each other request. + """ + key_ranges = list(key_ranges) + per_file_bounded_requests: dict[str, list[_BoundedRequest]] = defaultdict(list) + other_requests: list[_OtherRequest] = [] + suffix_requests: list[_SuffixRequest] = [] + + for idx, (path, byte_range) in enumerate(key_ranges): + if byte_range is None: + other_requests.append( + { + "original_request_index": idx, + "path": path, + "range": None, + } + ) + elif isinstance(byte_range, RangeByteRequest): + per_file_bounded_requests[path].append( + {"original_request_index": idx, "start": byte_range.start, "end": byte_range.end} + ) + elif isinstance(byte_range, OffsetByteRequest): + other_requests.append( + { + "original_request_index": idx, + "path": path, + "range": {"offset": byte_range.offset}, + } + ) + elif isinstance(byte_range, SuffixByteRequest): + suffix_requests.append( + { + "original_request_index": idx, + "path": path, + "range": {"suffix": byte_range.suffix}, + } + ) + else: + raise ValueError(f"Unsupported range input: {byte_range}") + + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + + futs: list[Coroutine[Any, Any, list[_Response]]] = [] + for path, bounded_ranges in per_file_bounded_requests.items(): + futs.append( + _make_bounded_requests(store, path, bounded_ranges, prototype, semaphore=semaphore) + ) + + for request in other_requests: + futs.append(_make_other_request(store, request, prototype, semaphore=semaphore)) # noqa: PERF401 + + for suffix_request in suffix_requests: + futs.append(_make_suffix_request(store, suffix_request, prototype, semaphore=semaphore)) # noqa: PERF401 + + buffers: list[Buffer | None] = [None] * len(key_ranges) + + for responses in await asyncio.gather(*futs): + for resp in responses: + buffers[resp["original_request_index"]] = resp["buffer"] + + return buffers diff --git a/src/zarr/storage/_utils.py b/src/zarr/storage/_utils.py index 7ba82b00fd..145790278c 100644 --- a/src/zarr/storage/_utils.py +++ b/src/zarr/storage/_utils.py @@ -2,9 +2,14 @@ import re from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar + +from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + + from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer @@ -44,25 +49,115 @@ def normalize_path(path: str | bytes | Path | None) -> str: return result -def _normalize_interval_index( - data: Buffer, interval: tuple[int | None, int | None] | None -) -> tuple[int, int]: +def _normalize_byte_range_index(data: Buffer, byte_range: ByteRequest | None) -> tuple[int, int]: """ - Convert an implicit interval into an explicit start and length + Convert an ByteRequest into an explicit start and stop """ - if interval is None: + if byte_range is None: start = 0 - length = len(data) + stop = len(data) + 1 + elif isinstance(byte_range, RangeByteRequest): + start = byte_range.start + stop = byte_range.end + elif isinstance(byte_range, OffsetByteRequest): + start = byte_range.offset + stop = len(data) + 1 + elif isinstance(byte_range, SuffixByteRequest): + start = len(data) - byte_range.suffix + stop = len(data) + 1 + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") + return (start, stop) + + +def _join_paths(paths: Iterable[str]) -> str: + """ + Filter out instances of '' and join the remaining strings with '/'. + + Parameters + ---------- + paths : Iterable[str] + + Returns + ------- + str + + Examples + -------- + >>> _join_paths(["", "a", "b"]) + 'a/b' + >>> _join_paths(["a", "b", "c"]) + 'a/b/c' + """ + return "/".join(filter(lambda v: v != "", paths)) + + +def _relativize_path(*, path: str, prefix: str) -> str: + """ + Make a "/"-delimited path relative to some prefix. If the prefix is '', then the path is + returned as-is. Otherwise, the prefix is removed from the path as well as the separator + string "/". + + If ``prefix`` is not the empty string and ``path`` does not start with ``prefix`` + followed by a "/" character, then an error is raised. + + This function assumes that the prefix does not end with "/". + + Parameters + ---------- + path : str + The path to make relative to the prefix. + prefix : str + The prefix to make the path relative to. + + Returns + ------- + str + + Examples + -------- + >>> _relativize_path(path="", prefix="a/b") + 'a/b' + >>> _relativize_path(path="a/b", prefix="a/b/c") + 'c' + """ + if prefix == "": + return path else: - maybe_start, maybe_len = interval - if maybe_start is None: - start = 0 - else: - start = maybe_start - - if maybe_len is None: - length = len(data) - start - else: - length = maybe_len - - return (start, length) + _prefix = prefix + "/" + if not path.startswith(_prefix): + raise ValueError(f"The first component of {path} does not start with {prefix}.") + return path.removeprefix(f"{prefix}/") + + +def _normalize_paths(paths: Iterable[str]) -> tuple[str, ...]: + """ + Normalize the input paths according to the normalization scheme used for zarr node paths. + If any two paths normalize to the same value, raise a ValueError. + """ + path_map: dict[str, str] = {} + for path in paths: + parsed = normalize_path(path) + if parsed in path_map: + msg = ( + f"After normalization, the value '{path}' collides with '{path_map[parsed]}'. " + f"Both '{path}' and '{path_map[parsed]}' normalize to the same value: '{parsed}'. " + f"You should use either '{path}' or '{path_map[parsed]}', but not both." + ) + raise ValueError(msg) + path_map[parsed] = path + return tuple(path_map.keys()) + + +T = TypeVar("T") + + +def _normalize_path_keys(data: Mapping[str, T]) -> dict[str, T]: + """ + Normalize the keys of the input dict according to the normalization scheme used for zarr node + paths. If any two keys in the input normalize to the same value, raise a ValueError. + Returns a dict where the keys are the elements of the input and the values are the + normalized form of each key. + """ + parsed_keys = _normalize_paths(data.keys()) + return dict(zip(parsed_keys, data.values(), strict=True)) diff --git a/src/zarr/storage/_wrapper.py b/src/zarr/storage/_wrapper.py index c160100084..f21d378191 100644 --- a/src/zarr/storage/_wrapper.py +++ b/src/zarr/storage/_wrapper.py @@ -7,7 +7,7 @@ from types import TracebackType from typing import Any, Self - from zarr.abc.store import ByteRangeRequest + from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.common import BytesLike @@ -18,7 +18,8 @@ class WrapperStore(Store, Generic[T_Store]): """ - A store class that wraps an existing ``Store`` instance. + Store that wraps an existing Store. + By default all of the store methods are delegated to the wrapped store instance, which is accessible via the ``._store`` attribute of this class. @@ -56,6 +57,14 @@ async def _ensure_open(self) -> None: async def is_empty(self, prefix: str) -> bool: return await self._store.is_empty(prefix) + @property + def _is_open(self) -> bool: + return self._store._is_open + + @_is_open.setter + def _is_open(self, value: bool) -> None: + raise NotImplementedError("WrapperStore must be opened via the `_open` method") + async def clear(self) -> None: return await self._store.clear() @@ -67,17 +76,23 @@ def _check_writable(self) -> None: return self._store._check_writable() def __eq__(self, value: object) -> bool: - return type(self) is type(value) and self._store.__eq__(value) + return type(self) is type(value) and self._store.__eq__(value._store) # type: ignore[attr-defined] + + def __str__(self) -> str: + return f"wrapping-{self._store}" + + def __repr__(self) -> str: + return f"WrapperStore({self._store.__class__.__name__}, '{self._store}')" async def get( - self, key: str, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: return await self._store.get(key, prototype, byte_range) async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: return await self._store.get_partial_values(prototype, key_ranges) @@ -133,7 +148,7 @@ def close(self) -> None: self._store.close() async def _get_many( - self, requests: Iterable[tuple[str, BufferPrototype, ByteRangeRequest | None]] + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] ) -> AsyncGenerator[tuple[str, Buffer | None], None]: async for req in self._store._get_many(requests): yield req diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index a186b3cf59..5d147deded 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -1,13 +1,20 @@ from __future__ import annotations import os +import shutil import threading import time import zipfile from pathlib import Path from typing import TYPE_CHECKING, Any, Literal -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) from zarr.core.buffer import Buffer, BufferPrototype if TYPE_CHECKING: @@ -18,7 +25,7 @@ class ZipStore(Store): """ - Storage class using a ZIP file. + Store using a ZIP file. Parameters ---------- @@ -101,11 +108,15 @@ def _sync_open(self) -> None: async def _open(self) -> None: self._sync_open() - def __getstate__(self) -> tuple[Path, ZipStoreAccessModeLiteral, int, bool]: - return self.path, self._zmode, self.compression, self.allowZip64 + def __getstate__(self) -> dict[str, Any]: + # We need a copy to not modify the state of the original store + state = self.__dict__.copy() + for attr in ["_zf", "_lock"]: + state.pop(attr, None) + return state - def __setstate__(self, state: Any) -> None: - self.path, self._zmode, self.compression, self.allowZip64 = state + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__ = state self._is_open = False self._sync_open() @@ -138,23 +149,26 @@ def _get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: + if not self._is_open: + self._sync_open() # docstring inherited try: with self._zf.open(key) as f: # will raise KeyError if byte_range is None: return prototype.buffer.from_bytes(f.read()) - start, length = byte_range - if start: - if start < 0: - start = f.seek(start, os.SEEK_END) + start - else: - start = f.seek(start, os.SEEK_SET) - if length: - return prototype.buffer.from_bytes(f.read(length)) + elif isinstance(byte_range, RangeByteRequest): + f.seek(byte_range.start) + return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) + size = f.seek(0, os.SEEK_END) + if isinstance(byte_range, OffsetByteRequest): + f.seek(byte_range.offset) + elif isinstance(byte_range, SuffixByteRequest): + f.seek(max(0, size - byte_range.suffix)) else: - return prototype.buffer.from_bytes(f.read()) + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + return prototype.buffer.from_bytes(f.read()) except KeyError: return None @@ -162,7 +176,7 @@ async def get( self, key: str, prototype: BufferPrototype, - byte_range: ByteRangeRequest | None = None, + byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited assert isinstance(key, str) @@ -173,7 +187,7 @@ async def get( async def get_partial_values( self, prototype: BufferPrototype, - key_ranges: Iterable[tuple[str, ByteRangeRequest]], + key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited out = [] @@ -183,6 +197,8 @@ async def get_partial_values( return out def _set(self, key: str, value: Buffer) -> None: + if not self._is_open: + self._sync_open() # generally, this should be called inside a lock keyinfo = zipfile.ZipInfo(filename=key, date_time=time.localtime(time.time())[:6]) keyinfo.compress_type = self.compression @@ -196,9 +212,13 @@ def _set(self, key: str, value: Buffer) -> None: async def set(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() + if not self._is_open: + self._sync_open() assert isinstance(key, str) if not isinstance(value, Buffer): - raise TypeError("ZipStore.set(): `value` must a Buffer instance") + raise TypeError( + f"ZipStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) with self._lock: self._set(key, value) @@ -264,8 +284,20 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: yield key else: for key in keys: - if key.startswith(prefix + "/") and key != prefix: + if key.startswith(prefix + "/") and key.strip("/") != prefix: k = key.removeprefix(prefix + "/").split("/")[0] if k not in seen: seen.add(k) yield k + + async def move(self, path: Path | str) -> None: + """ + Move the store to another path. + """ + if isinstance(path, str): + path = Path(path) + self.close() + os.makedirs(path.parent, exist_ok=True) + shutil.move(self.path, path) + self.path = path + await self._open() diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py index cc0f220807..f83d942549 100644 --- a/src/zarr/testing/stateful.py +++ b/src/zarr/testing/stateful.py @@ -17,10 +17,18 @@ import zarr from zarr import Array from zarr.abc.store import Store +from zarr.codecs.bytes import BytesCodec from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype from zarr.core.sync import SyncMixin from zarr.storage import LocalStore, MemoryStore -from zarr.testing.strategies import key_ranges, node_names, np_array_and_chunks, numpy_arrays +from zarr.testing.strategies import ( + basic_indices, + chunk_paths, + key_ranges, + node_names, + np_array_and_chunks, + numpy_arrays, +) from zarr.testing.strategies import keys as zarr_keys MAX_BINARY_SIZE = 100 @@ -68,6 +76,9 @@ def can_add(self, path: str) -> bool: # -------------------- store operations ----------------------- @rule(name=node_names, data=st.data()) def add_group(self, name: str, data: DataObject) -> None: + # Handle possible case-insensitive file systems (e.g. MacOS) + if isinstance(self.store, LocalStore): + name = name.lower() if self.all_groups: parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Group parent") else: @@ -90,6 +101,9 @@ def add_array( name: str, array_and_chunks: tuple[np.ndarray[Any, Any], tuple[int, ...]], ) -> None: + # Handle possible case-insensitive file systems (e.g. MacOS) + if isinstance(self.store, LocalStore): + name = name.lower() array, chunks = array_and_chunks fill_value = data.draw(npst.from_dtype(array.dtype)) if self.all_groups: @@ -102,9 +116,131 @@ def add_array( assume(self.can_add(path)) note(f"Adding array: path='{path}' shape={array.shape} chunks={chunks}") for store in [self.store, self.model]: - zarr.array(array, chunks=chunks, path=path, store=store, fill_value=fill_value) + zarr.array( + array, + chunks=chunks, + path=path, + store=store, + fill_value=fill_value, + # Chose bytes codec to avoid wasting time compressing the data being written + codecs=[BytesCodec()], + ) self.all_arrays.add(path) + @rule() + def clear(self) -> None: + note("clearing") + import zarr + + self._sync(self.store.clear()) + self._sync(self.model.clear()) + + assert self._sync(self.store.is_empty("/")) + assert self._sync(self.model.is_empty("/")) + + self.all_groups.clear() + self.all_arrays.clear() + + zarr.group(store=self.store) + zarr.group(store=self.model) + + # TODO: MemoryStore is broken? + # assert not self._sync(self.store.is_empty("/")) + # assert not self._sync(self.model.is_empty("/")) + + def draw_directory(self, data: DataObject) -> str: + group_st = st.sampled_from(sorted(self.all_groups)) if self.all_groups else st.nothing() + array_st = st.sampled_from(sorted(self.all_arrays)) if self.all_arrays else st.nothing() + array_or_group = data.draw(st.one_of(group_st, array_st)) + if data.draw(st.booleans()) and array_or_group in self.all_arrays: + arr = zarr.open_array(path=array_or_group, store=self.model) + path = data.draw( + st.one_of( + st.sampled_from([array_or_group]), + chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape).map( + lambda x: f"{array_or_group}/c/" + ), + ) + ) + else: + path = array_or_group + return path + + @precondition(lambda self: bool(self.all_groups)) + @rule(data=st.data()) + def check_list_dir(self, data: DataObject) -> None: + path = self.draw_directory(data) + note(f"list_dir for {path=!r}") + # Consider .list_dir("path/to/array") for an array with a single chunk. + # The MemoryStore model will return `"c", "zarr.json"` only if the chunk exists + # If that chunk was deleted, then `"c"` is not returned. + # LocalStore will not have this behaviour :/ + # There are similar consistency issues with delete_dir("/path/to/array/c/0/0") + assume(not isinstance(self.store, LocalStore)) + model_ls = sorted(self._sync_iter(self.model.list_dir(path))) + store_ls = sorted(self._sync_iter(self.store.list_dir(path))) + assert model_ls == store_ls, (model_ls, store_ls) + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def delete_chunk(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + arr = zarr.open_array(path=array, store=self.model) + chunk_path = data.draw(chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape, subset=False)) + path = f"{array}/c/{chunk_path}" + note(f"deleting chunk {path=!r}") + self._sync(self.model.delete(path)) + self._sync(self.store.delete(path)) + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def overwrite_array_basic_indexing(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + model_array = zarr.open_array(path=array, store=self.model) + store_array = zarr.open_array(path=array, store=self.store) + slicer = data.draw(basic_indices(shape=model_array.shape)) + note(f"overwriting array with basic indexer: {slicer=}") + new_data = data.draw( + npst.arrays(shape=np.shape(model_array[slicer]), dtype=model_array.dtype) + ) + model_array[slicer] = new_data + store_array[slicer] = new_data + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def resize_array(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + model_array = zarr.open_array(path=array, store=self.model) + store_array = zarr.open_array(path=array, store=self.store) + ndim = model_array.ndim + new_shape = tuple( + 0 if oldsize == 0 else newsize + for newsize, oldsize in zip( + data.draw(npst.array_shapes(max_dims=ndim, min_dims=ndim, min_side=0)), + model_array.shape, + strict=True, + ) + ) + + note(f"resizing array from {model_array.shape} to {new_shape}") + model_array.resize(new_shape) + store_array.resize(new_shape) + + @precondition(lambda self: bool(self.all_arrays) or bool(self.all_groups)) + @rule(data=st.data()) + def delete_dir(self, data: DataObject) -> None: + path = self.draw_directory(data) + note(f"delete_dir with {path=!r}") + self._sync(self.model.delete_dir(path)) + self._sync(self.store.delete_dir(path)) + + matches = set() + for node in self.all_groups | self.all_arrays: + if node.startswith(path): + matches.add(node) + self.all_groups = self.all_groups - matches + self.all_arrays = self.all_arrays - matches + # @precondition(lambda self: bool(self.all_groups)) # @precondition(lambda self: bool(self.all_arrays)) # @rule(data=st.data()) @@ -135,6 +271,7 @@ def add_array( # self.model.rename(from_group, new_path) # self.repo.store.rename(from_group, new_path) + @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.all_arrays) >= 1) @rule(data=st.data()) def delete_array_using_del(self, data: DataObject) -> None: @@ -149,6 +286,7 @@ def delete_array_using_del(self, data: DataObject) -> None: del group[array_name] self.all_arrays.remove(array_path) + @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.all_groups) >= 2) # fixme don't delete root @rule(data=st.data()) def delete_group_using_del(self, data: DataObject) -> None: @@ -213,13 +351,19 @@ def delete_group_using_del(self, data: DataObject) -> None: # self.check_group_arrays(group) # t1 = time.time() # note(f"Checks took {t1 - t0} sec.") - @invariant() def check_list_prefix_from_root(self) -> None: model_list = self._sync_iter(self.model.list_prefix("")) store_list = self._sync_iter(self.store.list_prefix("")) - note(f"Checking {len(model_list)} keys") - assert sorted(model_list) == sorted(store_list) + note(f"Checking {len(model_list)} expected keys vs {len(store_list)} actual keys") + assert sorted(model_list) == sorted(store_list), ( + sorted(model_list), + sorted(store_list), + ) + + # check that our internal state matches that of the store and model + assert all(f"{path}/zarr.json" in model_list for path in self.all_groups | self.all_arrays) + assert all(f"{path}/zarr.json" in store_list for path in self.all_groups | self.all_arrays) class SyncStoreWrapper(zarr.core.sync.SyncMixin): @@ -284,6 +428,10 @@ def supports_partial_writes(self) -> bool: def supports_writes(self) -> bool: return self.store.supports_writes + @property + def supports_deletes(self) -> bool: + return self.store.supports_deletes + class ZarrStoreStateMachine(RuleBasedStateMachine): """ " @@ -313,16 +461,16 @@ def __init__(self, store: Store) -> None: def init_store(self) -> None: self.store.clear() - @rule(key=zarr_keys, data=st.binary(min_size=0, max_size=MAX_BINARY_SIZE)) - def set(self, key: str, data: DataObject) -> None: - note(f"(set) Setting {key!r} with {data}") + @rule(key=zarr_keys(), data=st.binary(min_size=0, max_size=MAX_BINARY_SIZE)) + def set(self, key: str, data: bytes) -> None: + note(f"(set) Setting {key!r} with {data!r}") assert not self.store.read_only data_buf = cpu.Buffer.from_bytes(data) self.store.set(key, data_buf) self.model[key] = data_buf @precondition(lambda self: len(self.model.keys()) > 0) - @rule(key=zarr_keys, data=st.data()) + @rule(key=zarr_keys(), data=st.data()) def get(self, key: str, data: DataObject) -> None: key = data.draw( st.sampled_from(sorted(self.model.keys())) @@ -332,7 +480,7 @@ def get(self, key: str, data: DataObject) -> None: # to bytes here necessary because data_buf set to model in set() assert self.model[key] == store_value - @rule(key=zarr_keys, data=st.data()) + @rule(key=zarr_keys(), data=st.data()) def get_invalid_zarr_keys(self, key: str, data: DataObject) -> None: note("(get_invalid)") assume(key not in self.model) @@ -355,9 +503,8 @@ def get_partial_values(self, data: DataObject) -> None: model_vals_ls = [] for key, byte_range in key_range: - start = byte_range[0] or 0 - step = byte_range[1] - stop = start + step if step is not None else None + start = byte_range.start + stop = byte_range.end model_vals_ls.append(self.model[key][start:stop]) assert all( @@ -367,6 +514,7 @@ def get_partial_values(self, data: DataObject) -> None: model_vals_ls, ) + @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.model.keys()) > 0) @rule(data=st.data()) def delete(self, data: DataObject) -> None: @@ -396,7 +544,7 @@ def is_empty(self) -> None: # make sure they either both are or both aren't empty (same state) assert self.store.is_empty("") == (not self.model) - @rule(key=zarr_keys) + @rule(key=zarr_keys()) def exists(self, key: str) -> None: note("(exists)") diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index ada028c273..970329f393 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -2,6 +2,7 @@ import asyncio import pickle +from abc import abstractmethod from typing import TYPE_CHECKING, Generic, TypeVar from zarr.storage import WrapperStore @@ -9,15 +10,21 @@ if TYPE_CHECKING: from typing import Any - from zarr.abc.store import ByteRangeRequest + from zarr.abc.store import ByteRequest from zarr.core.buffer.core import BufferPrototype import pytest -from zarr.abc.store import ByteRangeRequest, Store +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.core.sync import _collect_aiterator -from zarr.storage._utils import _normalize_interval_index +from zarr.storage._utils import _normalize_byte_range_index from zarr.testing.utils import assert_bytes_equal __all__ = ["StoreTests"] @@ -31,30 +38,53 @@ class StoreTests(Generic[S, B]): store_cls: type[S] buffer_cls: type[B] + @abstractmethod async def set(self, store: S, key: str, value: Buffer) -> None: """ Insert a value into a storage backend, with a specific key. - This should not not use any store methods. Bypassing the store methods allows them to be + This should not use any store methods. Bypassing the store methods allows them to be tested. """ - raise NotImplementedError + ... + @abstractmethod async def get(self, store: S, key: str) -> Buffer: """ Retrieve a value from a storage backend, by key. - This should not not use any store methods. Bypassing the store methods allows them to be + This should not use any store methods. Bypassing the store methods allows them to be tested. """ + ... - raise NotImplementedError + @abstractmethod + @pytest.fixture + def store_kwargs(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + """Kwargs for instantiating a store""" + ... + + @abstractmethod + def test_store_repr(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_writes(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_partial_writes(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_listing(self, store: S) -> None: ... + + @pytest.fixture + def open_kwargs(self, store_kwargs: dict[str, Any]) -> dict[str, Any]: + return store_kwargs @pytest.fixture - def store_kwargs(self) -> dict[str, Any]: - return {"read_only": False} + async def store(self, open_kwargs: dict[str, Any]) -> Store: + return await self.store_cls.open(**open_kwargs) @pytest.fixture - async def store(self, store_kwargs: dict[str, Any]) -> Store: - return await self.store_cls.open(**store_kwargs) + async def store_not_open(self, store_kwargs: dict[str, Any]) -> Store: + return self.store_cls(**store_kwargs) def test_store_type(self, store: S) -> None: assert isinstance(store, Store) @@ -69,9 +99,16 @@ def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None: store2 = self.store_cls(**store_kwargs) assert store == store2 - def test_serializable_store(self, store: S) -> None: - foo = pickle.dumps(store) - assert pickle.loads(foo) == store + async def test_serializable_store(self, store: S) -> None: + new_store: S = pickle.loads(pickle.dumps(store)) + assert new_store == store + assert new_store.read_only == store.read_only + # quickly roundtrip data to a key to test that new store works + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "foo" + await store.set(key, data_buf) + observed = await store.get(key, prototype=default_buffer_prototype()) + assert_bytes_equal(observed, data_buf) def test_store_read_only(self, store: S) -> None: assert not store.read_only @@ -80,55 +117,132 @@ def test_store_read_only(self, store: S) -> None: store.read_only = False # type: ignore[misc] @pytest.mark.parametrize("read_only", [True, False]) - async def test_store_open_read_only( - self, store_kwargs: dict[str, Any], read_only: bool - ) -> None: - store_kwargs["read_only"] = read_only - store = await self.store_cls.open(**store_kwargs) + async def test_store_open_read_only(self, open_kwargs: dict[str, Any], read_only: bool) -> None: + open_kwargs["read_only"] = read_only + store = await self.store_cls.open(**open_kwargs) assert store._is_open assert store.read_only == read_only - async def test_read_only_store_raises(self, store_kwargs: dict[str, Any]) -> None: - kwargs = {**store_kwargs, "read_only": True} + async def test_store_context_manager(self, open_kwargs: dict[str, Any]) -> None: + # Test that the context manager closes the store + with await self.store_cls.open(**open_kwargs) as store: + assert store._is_open + # Test trying to open an already open store + with pytest.raises(ValueError, match="store is already open"): + await store._open() + assert not store._is_open + + async def test_read_only_store_raises(self, open_kwargs: dict[str, Any]) -> None: + kwargs = {**open_kwargs, "read_only": True} store = await self.store_cls.open(**kwargs) assert store.read_only # set - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): await store.set("foo", self.buffer_cls.from_bytes(b"bar")) # delete - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): await store.delete("foo") - def test_store_repr(self, store: S) -> None: - raise NotImplementedError + async def test_with_read_only_store(self, open_kwargs: dict[str, Any]) -> None: + kwargs = {**open_kwargs, "read_only": True} + store = await self.store_cls.open(**kwargs) + assert store.read_only - def test_store_supports_writes(self, store: S) -> None: - raise NotImplementedError + # Test that you cannot write to a read-only store + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) - def test_store_supports_partial_writes(self, store: S) -> None: - raise NotImplementedError + # Check if the store implements with_read_only + try: + writer = store.with_read_only(read_only=False) + except NotImplementedError: + # Test that stores that do not implement with_read_only raise NotImplementedError with the correct message + with pytest.raises( + NotImplementedError, + match=f"with_read_only is not implemented for the {type(store)} store type.", + ): + store.with_read_only(read_only=False) + return + + # Test that you can write to a new store copy + assert not writer._is_open + assert not writer.read_only + await writer.set("foo", self.buffer_cls.from_bytes(b"bar")) + await writer.delete("foo") + + # Test that you cannot write to the original store + assert store.read_only + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.delete("foo") - def test_store_supports_listing(self, store: S) -> None: - raise NotImplementedError + # Test that you cannot write to a read-only store copy + reader = store.with_read_only(read_only=True) + assert reader.read_only + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await reader.set("foo", self.buffer_cls.from_bytes(b"bar")) + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await reader.delete("foo") @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) - @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) - @pytest.mark.parametrize("byte_range", [None, (0, None), (1, None), (1, 2), (None, 1)]) - async def test_get( - self, store: S, key: str, data: bytes, byte_range: tuple[int | None, int | None] | None - ) -> None: + @pytest.mark.parametrize( + ("data", "byte_range"), + [ + (b"\x01\x02\x03\x04", None), + (b"\x01\x02\x03\x04", RangeByteRequest(1, 4)), + (b"\x01\x02\x03\x04", OffsetByteRequest(1)), + (b"\x01\x02\x03\x04", SuffixByteRequest(1)), + (b"", None), + ], + ) + async def test_get(self, store: S, key: str, data: bytes, byte_range: ByteRequest) -> None: """ Ensure that data can be read from the store using the store.get method. """ data_buf = self.buffer_cls.from_bytes(data) await self.set(store, key, data_buf) observed = await store.get(key, prototype=default_buffer_prototype(), byte_range=byte_range) - start, length = _normalize_interval_index(data_buf, interval=byte_range) - expected = data_buf[start : start + length] + start, stop = _normalize_byte_range_index(data_buf, byte_range=byte_range) + expected = data_buf[start:stop] assert_bytes_equal(observed, expected) + async def test_get_not_open(self, store_not_open: S) -> None: + """ + Ensure that data can be read from the store that isn't yet open using the store.get method. + """ + assert not store_not_open._is_open + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await self.set(store_not_open, key, data_buf) + observed = await store_not_open.get(key, prototype=default_buffer_prototype()) + assert_bytes_equal(observed, data_buf) + + async def test_get_raises(self, store: S) -> None: + """ + Ensure that a ValueError is raise for invalid byte range syntax + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + await self.set(store, "c/0", data_buf) + with pytest.raises((ValueError, TypeError), match=r"Unexpected byte_range, got.*"): + await store.get("c/0", prototype=default_buffer_prototype(), byte_range=(0, 2)) # type: ignore[arg-type] + async def test_get_many(self, store: S) -> None: """ Ensure that multiple keys can be retrieved at once with the _get_many method. @@ -151,6 +265,37 @@ async def test_get_many(self, store: S) -> None: expected_kvs = sorted(((k, b) for k, b in zip(keys, values, strict=False))) assert observed_kvs == expected_kvs + @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) + @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) + async def test_getsize(self, store: S, key: str, data: bytes) -> None: + """ + Test the result of store.getsize(). + """ + data_buf = self.buffer_cls.from_bytes(data) + expected = len(data_buf) + await self.set(store, key, data_buf) + observed = await store.getsize(key) + assert observed == expected + + async def test_getsize_prefix(self, store: S) -> None: + """ + Test the result of store.getsize_prefix(). + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + keys = ["c/0/0", "c/0/1", "c/1/0", "c/1/1"] + keys_values = [(k, data_buf) for k in keys] + await store._set_many(keys_values) + expected = len(data_buf) * len(keys) + observed = await store.getsize_prefix("c") + assert observed == expected + + async def test_getsize_raises(self, store: S) -> None: + """ + Test that getsize() raise a FileNotFoundError if the key doesn't exist. + """ + with pytest.raises(FileNotFoundError): + await store.getsize("c/1000") + @pytest.mark.parametrize("key", ["zarr.json", "c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) async def test_set(self, store: S, key: str, data: bytes) -> None: @@ -163,6 +308,17 @@ async def test_set(self, store: S, key: str, data: bytes) -> None: observed = await self.get(store, key) assert_bytes_equal(observed, data_buf) + async def test_set_not_open(self, store_not_open: S) -> None: + """ + Ensure that data can be written to the store that's not yet open using the store.set method. + """ + assert not store_not_open._is_open + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await store_not_open.set(key, data_buf) + observed = await self.get(store_not_open, key) + assert_bytes_equal(observed, data_buf) + async def test_set_many(self, store: S) -> None: """ Test that a dict of key : value pairs can be inserted into the store via the @@ -179,13 +335,17 @@ async def test_set_many(self, store: S) -> None: "key_ranges", [ [], - [("zarr.json", (0, 1))], - [("c/0", (0, 1)), ("zarr.json", (0, None))], - [("c/0/0", (0, 1)), ("c/0/1", (None, 2)), ("c/0/2", (0, 3))], + [("zarr.json", RangeByteRequest(0, 2))], + [("c/0", RangeByteRequest(0, 2)), ("zarr.json", None)], + [ + ("c/0/0", RangeByteRequest(0, 2)), + ("c/0/1", SuffixByteRequest(2)), + ("c/0/2", OffsetByteRequest(2)), + ], ], ) async def test_get_partial_values( - self, store: S, key_ranges: list[tuple[str, tuple[int | None, int | None]]] + self, store: S, key_ranges: list[tuple[str, ByteRequest]] ) -> None: # put all of the data for key, _ in key_ranges: @@ -294,6 +454,37 @@ async def test_list_prefix(self, store: S) -> None: expected = tuple(sorted(expected)) assert observed == expected + async def test_list_empty_path(self, store: S) -> None: + """ + Verify that list and list_prefix work correctly when path is an empty string, + i.e. no unwanted replacement occurs. + """ + data = self.buffer_cls.from_bytes(b"") + store_dict = { + "foo/bar/zarr.json": data, + "foo/bar/c/1": data, + "foo/baz/c/0": data, + } + await store._set_many(store_dict.items()) + + # Test list() + observed_list = await _collect_aiterator(store.list()) + observed_list_sorted = sorted(observed_list) + expected_list_sorted = sorted(store_dict.keys()) + assert observed_list_sorted == expected_list_sorted + + # Test list_prefix() with an empty prefix + observed_prefix_empty = await _collect_aiterator(store.list_prefix("")) + observed_prefix_empty_sorted = sorted(observed_prefix_empty) + expected_prefix_empty_sorted = sorted(store_dict.keys()) + assert observed_prefix_empty_sorted == expected_prefix_empty_sorted + + # Test list_prefix() with a non-empty prefix + observed_prefix = await _collect_aiterator(store.list_prefix("foo/bar/")) + observed_prefix_sorted = sorted(observed_prefix) + expected_prefix_sorted = sorted(k for k in store_dict if k.startswith("foo/bar/")) + assert observed_prefix_sorted == expected_prefix_sorted + async def test_list_dir(self, store: S) -> None: root = "foo" store_dict = { @@ -367,7 +558,7 @@ async def set(self, key: str, value: Buffer) -> None: await self._store.set(key, value) async def get( - self, key: str, prototype: BufferPrototype, byte_range: ByteRangeRequest | None = None + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: """ Add latency to the ``get`` method. @@ -380,7 +571,7 @@ async def get( The key to get prototype : BufferPrototype The BufferPrototype to use. - byte_range : ByteRangeRequest, optional + byte_range : ByteRequest, optional An optional byte range. Returns diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 1bde01b8f9..5e070b5387 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,17 +1,28 @@ -from typing import Any +import math +import sys +from collections.abc import Callable, Mapping +from typing import Any, Literal import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np -from hypothesis import given, settings # noqa: F401 +import numpy.typing as npt +from hypothesis import event from hypothesis.strategies import SearchStrategy import zarr +from zarr.abc.store import RangeByteRequest, Store +from zarr.codecs.bytes import BytesCodec from zarr.core.array import Array -from zarr.core.common import ZarrFormat +from zarr.core.chunk_grids import RegularChunkGrid +from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding +from zarr.core.common import JSON, ZarrFormat +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync from zarr.storage import MemoryStore, StoreLike from zarr.storage._common import _dereference_path +from zarr.storage._utils import normalize_path # Copied from Xarray _attr_keys = st.text(st.characters(), min_size=1) @@ -22,21 +33,31 @@ ) -def v3_dtypes() -> st.SearchStrategy[np.dtype]: +@st.composite +def keys(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: + return draw(st.lists(node_names, min_size=1, max_size=max_num_nodes).map("/".join)) + + +@st.composite +def paths(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: + return draw(st.just("/") | keys(max_num_nodes=max_num_nodes)) + + +def v3_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return ( npst.boolean_dtypes() | npst.integer_dtypes(endianness="=") | npst.unsigned_integer_dtypes(endianness="=") | npst.floating_dtypes(endianness="=") | npst.complex_number_dtypes(endianness="=") - # | npst.byte_string_dtypes(endianness="=") - # | npst.unicode_string_dtypes() - # | npst.datetime64_dtypes() - # | npst.timedelta64_dtypes() + | npst.byte_string_dtypes(endianness="=") + | npst.unicode_string_dtypes(endianness="=") + | npst.datetime64_dtypes(endianness="=") + | npst.timedelta64_dtypes(endianness="=") ) -def v2_dtypes() -> st.SearchStrategy[np.dtype]: +def v2_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return ( npst.boolean_dtypes() | npst.integer_dtypes(endianness="=") @@ -46,10 +67,30 @@ def v2_dtypes() -> st.SearchStrategy[np.dtype]: | npst.byte_string_dtypes(endianness="=") | npst.unicode_string_dtypes(endianness="=") | npst.datetime64_dtypes(endianness="=") - # | npst.timedelta64_dtypes() + | npst.timedelta64_dtypes(endianness="=") + ) + + +def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]: + """Generate UTF-8-safe text constrained to max_len of dtype.""" + # account for utf-32 encoding (i.e. 4 bytes/character) + max_len = max(1, dtype.itemsize // 4) + + return st.text( + alphabet=st.characters( + exclude_categories=["Cs"], # Avoid *technically allowed* surrogates + min_codepoint=32, + ), + min_size=1, + max_size=max_len, ) +def clear_store(x: Store) -> Store: + sync(x.clear()) + return x + + # From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names # 1. must not be the empty string ("") # 2. must not include the character "/" @@ -58,68 +99,157 @@ def v2_dtypes() -> st.SearchStrategy[np.dtype]: zarr_key_chars = st.sampled_from( ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ) -node_names = st.text(zarr_key_chars, min_size=1).filter( - lambda t: t not in (".", "..") and not t.startswith("__") +node_names = ( + st.text(zarr_key_chars, min_size=1) + .filter(lambda t: t not in (".", "..") and not t.startswith("__")) + .filter(lambda name: name.lower() != "zarr.json") +) +short_node_names = ( + st.text(zarr_key_chars, max_size=3, min_size=1) + .filter(lambda t: t not in (".", "..") and not t.startswith("__")) + .filter(lambda name: name.lower() != "zarr.json") ) array_names = node_names -attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) -keys = st.lists(node_names, min_size=1).map("/".join) -paths = st.just("/") | keys +attrs: st.SearchStrategy[Mapping[str, JSON] | None] = st.none() | st.dictionaries( + _attr_keys, _attr_values +) # st.builds will only call a new store constructor for different keyword arguments # i.e. stores.examples() will always return the same object per Store class. # So we map a clear to reset the store. -stores = st.builds(MemoryStore, st.just({})).map(lambda x: sync(x.clear())) +stores = st.builds(MemoryStore, st.just({})).map(clear_store) compressors = st.sampled_from([None, "default"]) -zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([2, 3]) -array_shapes = npst.array_shapes(max_dims=4, min_side=0) +zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([3, 2]) +# We de-prioritize arrays having dim sizes 0, 1, 2 +array_shapes = npst.array_shapes(max_dims=4, min_side=3, max_side=5) | npst.array_shapes( + max_dims=4, min_side=0 +) + + +@st.composite +def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[None | str] | None: + simple_text = st.text(zarr_key_chars, min_size=0) + return draw(st.none() | st.lists(st.none() | simple_text, min_size=ndim, max_size=ndim)) # type: ignore[arg-type] -@st.composite # type: ignore[misc] +@st.composite +def array_metadata( + draw: st.DrawFn, + *, + array_shapes: Callable[..., st.SearchStrategy[tuple[int, ...]]] = npst.array_shapes, + zarr_formats: st.SearchStrategy[Literal[2, 3]] = zarr_formats, + attributes: SearchStrategy[Mapping[str, JSON] | None] = attrs, +) -> ArrayV2Metadata | ArrayV3Metadata: + zarr_format = draw(zarr_formats) + # separator = draw(st.sampled_from(['/', '\\'])) + shape = draw(array_shapes()) + ndim = len(shape) + chunk_shape = draw(array_shapes(min_dims=ndim, max_dims=ndim)) + np_dtype = draw(v3_dtypes()) + dtype = get_data_type_from_native_dtype(np_dtype) + fill_value = draw(npst.from_dtype(np_dtype)) + if zarr_format == 2: + return ArrayV2Metadata( + shape=shape, + chunks=chunk_shape, + dtype=dtype, + fill_value=fill_value, + order=draw(st.sampled_from(["C", "F"])), + attributes=draw(attributes), # type: ignore[arg-type] + dimension_separator=draw(st.sampled_from([".", "/"])), + filters=None, + compressor=None, + ) + else: + return ArrayV3Metadata( + shape=shape, + data_type=dtype, + chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), + fill_value=fill_value, + attributes=draw(attributes), # type: ignore[arg-type] + dimension_names=draw(dimension_names(ndim=ndim)), + chunk_key_encoding=DefaultChunkKeyEncoding(separator="/"), # FIXME + codecs=[BytesCodec()], + storage_transformers=(), + ) + + +@st.composite def numpy_arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, + dtype: np.dtype[Any] | None = None, zarr_formats: st.SearchStrategy[ZarrFormat] = zarr_formats, -) -> Any: +) -> npt.NDArray[Any]: """ Generate numpy arrays that can be saved in the provided Zarr format. """ zarr_format = draw(zarr_formats) - return draw(npst.arrays(dtype=v3_dtypes() if zarr_format == 3 else v2_dtypes(), shape=shapes)) + if dtype is None: + dtype = draw(v3_dtypes() if zarr_format == 3 else v2_dtypes()) + if np.issubdtype(dtype, np.str_): + safe_unicode_strings = safe_unicode_for_dtype(dtype) + return draw(npst.arrays(dtype=dtype, shape=shapes, elements=safe_unicode_strings)) + return draw(npst.arrays(dtype=dtype, shape=shapes)) -@st.composite # type: ignore[misc] -def np_array_and_chunks( - draw: st.DrawFn, *, arrays: st.SearchStrategy[np.ndarray] = numpy_arrays -) -> tuple[np.ndarray, tuple[int, ...]]: # type: ignore[type-arg] - """A hypothesis strategy to generate small sized random arrays. - Returns: a tuple of the array and a suitable random chunking for it. - """ - array = draw(arrays) +@st.composite +def chunk_shapes(draw: st.DrawFn, *, shape: tuple[int, ...]) -> tuple[int, ...]: # We want this strategy to shrink towards arrays with smaller number of chunks # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks numchunks = draw( - st.tuples( - *[st.integers(min_value=0 if size == 0 else 1, max_value=size) for size in array.shape] - ) + st.tuples(*[st.integers(min_value=0 if size == 0 else 1, max_value=size) for size in shape]) ) # 2. and now generate the chunks tuple chunks = tuple( size // nchunks if nchunks > 0 else 0 - for size, nchunks in zip(array.shape, numchunks, strict=True) + for size, nchunks in zip(shape, numchunks, strict=True) ) - return (array, chunks) + + for c in chunks: + event("chunk size", c) + + if any((c != 0 and s % c != 0) for s, c in zip(shape, chunks, strict=True)): + event("smaller last chunk") + + return chunks + + +@st.composite +def shard_shapes( + draw: st.DrawFn, *, shape: tuple[int, ...], chunk_shape: tuple[int, ...] +) -> tuple[int, ...]: + # We want this strategy to shrink towards arrays with smaller number of shards + # shards must be an integral number of chunks + assert all(c != 0 for c in chunk_shape) + numchunks = tuple(s // c for s, c in zip(shape, chunk_shape, strict=True)) + multiples = tuple(draw(st.integers(min_value=1, max_value=nc)) for nc in numchunks) + return tuple(m * c for m, c in zip(multiples, chunk_shape, strict=True)) + + +@st.composite +def np_array_and_chunks( + draw: st.DrawFn, + *, + arrays: st.SearchStrategy[npt.NDArray[Any]] = numpy_arrays(), # noqa: B008 +) -> tuple[np.ndarray, tuple[int, ...]]: # type: ignore[type-arg] + """A hypothesis strategy to generate small sized random arrays. + + Returns: a tuple of the array and a suitable random chunking for it. + """ + array = draw(arrays) + return (array, draw(chunk_shapes(shape=array.shape))) -@st.composite # type: ignore[misc] +@st.composite def arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, compressors: st.SearchStrategy = compressors, stores: st.SearchStrategy[StoreLike] = stores, - paths: st.SearchStrategy[str | None] = paths, + paths: st.SearchStrategy[str] = paths(), # noqa: B008 array_names: st.SearchStrategy = array_names, arrays: st.SearchStrategy | None = None, attrs: st.SearchStrategy = attrs, @@ -132,7 +262,12 @@ def arrays( zarr_format = draw(zarr_formats) if arrays is None: arrays = numpy_arrays(shapes=shapes, zarr_formats=st.just(zarr_format)) - nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) + nparray = draw(arrays) + chunk_shape = draw(chunk_shapes(shape=nparray.shape)) + if zarr_format == 3 and all(c > 0 for c in chunk_shape): + shard_shape = draw(st.none() | shard_shapes(shape=nparray.shape, chunk_shape=chunk_shape)) + else: + shard_shape = None # test that None works too. fill_value = draw(st.one_of([st.none(), npst.from_dtype(nparray.dtype)])) # compressor = draw(compressors) @@ -145,7 +280,8 @@ def arrays( a = root.create_array( array_path, shape=nparray.shape, - chunks=chunks, + chunks=chunk_shape, + shards=shard_shape, dtype=nparray.dtype, attributes=attributes, # compressor=compressor, # FIXME @@ -156,10 +292,12 @@ def arrays( if a.metadata.zarr_format == 3: assert a.fill_value is not None assert a.name is not None + assert a.path == normalize_path(array_path) + assert a.name == "/" + a.path assert isinstance(root[array_path], Array) assert nparray.shape == a.shape - assert chunks == a.chunks - assert array_path == a.path, (path, name, array_path, a.name, a.path) + assert chunk_shape == a.chunks + assert shard_shape == a.shards assert a.basename == name, (a.basename, name) assert dict(a.attrs) == expected_attrs @@ -168,38 +306,136 @@ def arrays( return a +@st.composite +def simple_arrays( + draw: st.DrawFn, + *, + shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, +) -> Any: + return draw( + arrays( + shapes=shapes, + paths=paths(max_num_nodes=2), + array_names=short_node_names, + attrs=st.none(), + compressors=st.sampled_from([None, "default"]), + ) + ) + + def is_negative_slice(idx: Any) -> bool: return isinstance(idx, slice) and idx.step is not None and idx.step < 0 -@st.composite # type: ignore[misc] -def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs: Any) -> Any: +@st.composite +def end_slices(draw: st.DrawFn, *, shape: tuple[int, ...]) -> Any: + """ + A strategy that slices ranges that include the last chunk. + This is intended to stress-test handling of a possibly smaller last chunk. + """ + slicers = [] + for size in shape: + start = draw(st.integers(min_value=size // 2, max_value=size - 1)) + length = draw(st.integers(min_value=0, max_value=size - start)) + slicers.append(slice(start, start + length)) + event("drawing end slice") + return tuple(slicers) + + +@st.composite +def basic_indices( + draw: st.DrawFn, + *, + shape: tuple[int, ...], + min_dims: int = 0, + max_dims: int | None = None, + allow_newaxis: bool = False, + allow_ellipsis: bool = True, +) -> Any: """Basic indices without unsupported negative slices.""" - return draw( - npst.basic_indices(shape=shape, **kwargs).filter( - lambda idxr: ( - not ( - is_negative_slice(idxr) - or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) - ) + strategy = npst.basic_indices( + shape=shape, + min_dims=min_dims, + max_dims=max_dims, + allow_newaxis=allow_newaxis, + allow_ellipsis=allow_ellipsis, + ).filter( + lambda idxr: ( + not ( + is_negative_slice(idxr) + or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) # type: ignore[redundant-expr] ) ) ) + if math.prod(shape) >= 3: + strategy = end_slices(shape=shape) | strategy + return draw(strategy) + + +@st.composite +def orthogonal_indices( + draw: st.DrawFn, *, shape: tuple[int, ...] +) -> tuple[tuple[np.ndarray[Any, Any], ...], tuple[np.ndarray[Any, Any], ...]]: + """ + Strategy that returns + (1) a tuple of integer arrays used for orthogonal indexing of Zarr arrays. + (2) an tuple of integer arrays that can be used for equivalent indexing of numpy arrays + """ + zindexer = [] + npindexer = [] + ndim = len(shape) + for axis, size in enumerate(shape): + val = draw( + npst.integer_array_indices( + shape=(size,), result_shape=npst.array_shapes(min_side=1, max_side=size, max_dims=1) + ) + | basic_indices(min_dims=1, shape=(size,), allow_ellipsis=False) + .map(lambda x: (x,) if not isinstance(x, tuple) else x) # bare ints, slices + .filter(bool) # skip empty tuple + ) + (idxr,) = val + if isinstance(idxr, int): + idxr = np.array([idxr]) + zindexer.append(idxr) + if isinstance(idxr, slice): + idxr = np.arange(*idxr.indices(size)) + elif isinstance(idxr, (tuple, int)): + idxr = np.array(idxr) + newshape = [1] * ndim + newshape[axis] = idxr.size + npindexer.append(idxr.reshape(newshape)) + + # casting the output of broadcast_arrays is needed for numpy 1.25 + return tuple(zindexer), tuple(np.broadcast_arrays(*npindexer)) def key_ranges( - keys: SearchStrategy = node_names, max_size: int | None = None -) -> SearchStrategy[list[int]]: + keys: SearchStrategy[str] = node_names, max_size: int = sys.maxsize +) -> SearchStrategy[list[tuple[str, RangeByteRequest]]]: """ Function to generate key_ranges strategy for get_partial_values() returns list strategy w/ form:: - [(key, (range_start, range_step)), - (key, (range_start, range_step)),...] + [(key, (range_start, range_end)), + (key, (range_start, range_end)),...] """ - byte_ranges = st.tuples( - st.none() | st.integers(min_value=0, max_value=max_size), - st.none() | st.integers(min_value=0, max_value=max_size), + + def make_request(start: int, length: int) -> RangeByteRequest: + return RangeByteRequest(start, end=min(start + length, max_size)) + + byte_ranges = st.builds( + make_request, + start=st.integers(min_value=0, max_value=max_size), + length=st.integers(min_value=0, max_value=max_size), ) key_tuple = st.tuples(keys, byte_ranges) return st.lists(key_tuple, min_size=1, max_size=10) + + +@st.composite +def chunk_paths(draw: st.DrawFn, ndim: int, numblocks: tuple[int, ...], subset: bool = True) -> str: + blockidx = draw( + st.tuples(*tuple(st.integers(min_value=0, max_value=max(0, b - 1)) for b in numblocks)) + ) + subset_slicer = slice(draw(st.integers(min_value=0, max_value=ndim))) if subset else slice(None) + return "/".join(map(str, blockidx[subset_slicer])) diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index c7b6e7939c..afc15d742c 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,7 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Coroutine -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, TypeVar, cast import pytest @@ -31,20 +30,20 @@ def has_cupy() -> bool: try: import cupy - return cast(bool, cupy.cuda.runtime.getDeviceCount() > 0) + return cast("bool", cupy.cuda.runtime.getDeviceCount() > 0) except ImportError: return False except cupy.cuda.runtime.CUDARuntimeError: return False -T_Callable = TypeVar("T_Callable", bound=Callable[[], Coroutine[Any, Any, None]]) +T = TypeVar("T") # Decorator for GPU tests -def gpu_test(func: T_Callable) -> T_Callable: +def gpu_test(func: T) -> T: return cast( - T_Callable, + "T", pytest.mark.gpu( pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")( func diff --git a/tests/conftest.py b/tests/conftest.py index e9cd2b8120..4d300a1fd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import pathlib from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -11,6 +12,21 @@ from zarr import AsyncGroup, config from zarr.abc.store import Store +from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation +from zarr.core.array import ( + _parse_chunk_encoding_v2, + _parse_chunk_encoding_v3, + _parse_chunk_key_encoding, +) +from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition +from zarr.core.common import JSON, DimensionNames, parse_shapelike +from zarr.core.config import config as zarr_config +from zarr.core.dtype import ( + get_data_type_from_native_dtype, +) +from zarr.core.dtype.common import HasItemSize +from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import sync from zarr.storage import FsspecStore, LocalStore, MemoryStore, StorePath, ZipStore @@ -20,7 +36,11 @@ from _pytest.compat import LEGACY_PATH - from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat + from zarr.abc.codec import Codec + from zarr.core.array import CompressorsLike, FiltersLike, SerializerLike, ShardsLike + from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike + from zarr.core.common import ChunkCoords, MemoryOrder, ShapeLike, ZarrFormat + from zarr.core.dtype.wrapper import ZDType async def parse_store( @@ -75,6 +95,14 @@ async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: return await parse_store(param, str(tmpdir)) +@pytest.fixture +async def store2(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: + """Fixture to create a second store for testing copy operations between stores""" + param = request.param + store2_path = tmpdir.mkdir("store2") + return await parse_store(param, str(store2_path)) + + @pytest.fixture(params=["local", "memory", "zip"]) def sync_store(request: pytest.FixtureRequest, tmp_path: LEGACY_PATH) -> Store: result = sync(parse_store(request.param, str(tmp_path))) @@ -147,15 +175,270 @@ def zarr_format(request: pytest.FixtureRequest) -> ZarrFormat: raise ValueError(msg) +def pytest_addoption(parser: Any) -> None: + parser.addoption( + "--run-slow-hypothesis", + action="store_true", + default=False, + help="run slow hypothesis tests", + ) + + +def pytest_collection_modifyitems(config: Any, items: Any) -> None: + if config.getoption("--run-slow-hypothesis"): + return + skip_slow_hyp = pytest.mark.skip(reason="need --run-slow-hypothesis option to run") + for item in items: + if "slow_hypothesis" in item.keywords: + item.add_marker(skip_slow_hyp) + + settings.register_profile( - "ci", - max_examples=1000, - deadline=None, + "default", + parent=settings.get_profile("default"), + max_examples=300, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], + deadline=None, + verbosity=Verbosity.verbose, ) settings.register_profile( - "local", + "ci", + parent=settings.get_profile("ci"), max_examples=300, + derandomize=True, # more like regression testing + deadline=None, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], - verbosity=Verbosity.verbose, ) +settings.register_profile( + "nightly", + max_examples=500, + parent=settings.get_profile("ci"), + derandomize=False, + stateful_step_count=100, +) + +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) + + +# TODO: uncomment these overrides when we can get mypy to accept them +""" +@overload +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"], + shards: None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: Literal[2], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: None, +) -> ArrayV2Metadata: ... + + +@overload +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"], + shards: ShardsLike | None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: None, + zarr_format: Literal[3], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV3Metadata: ... +""" + + +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any = 0, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, +) -> ArrayV2Metadata | ArrayV3Metadata: + """ + Create array metadata + """ + dtype_parsed = get_data_type_from_native_dtype(dtype) + shape_parsed = parse_shapelike(shape) + chunk_key_encoding_parsed = _parse_chunk_key_encoding( + chunk_key_encoding, zarr_format=zarr_format + ) + item_size = 1 + if isinstance(dtype_parsed, HasItemSize): + item_size = dtype_parsed.item_size + shard_shape_parsed, chunk_shape_parsed = _auto_partition( + array_shape=shape_parsed, + shard_shape=shards, + chunk_shape=chunks, + item_size=item_size, + ) + + if order is None: + order_parsed = zarr_config.get("array.order") + else: + order_parsed = order + chunks_out: tuple[int, ...] + + if zarr_format == 2: + filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( + compressor=compressors, filters=filters, dtype=dtype_parsed + ) + return ArrayV2Metadata( + shape=shape_parsed, + dtype=dtype_parsed, + chunks=chunk_shape_parsed, + order=order_parsed, + dimension_separator=chunk_key_encoding_parsed.separator, + fill_value=fill_value, + compressor=compressor_parsed, + filters=filters_parsed, + attributes=attributes, + ) + elif zarr_format == 3: + array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( + compressors=compressors, + filters=filters, + serializer=serializer, + dtype=dtype_parsed, + ) + + sub_codecs: tuple[Codec, ...] = (*array_array, array_bytes, *bytes_bytes) + codecs_out: tuple[Codec, ...] + if shard_shape_parsed is not None: + index_location = None + if isinstance(shards, dict): + index_location = ShardingCodecIndexLocation(shards.get("index_location", None)) + if index_location is None: + index_location = ShardingCodecIndexLocation.end + sharding_codec = ShardingCodec( + chunk_shape=chunk_shape_parsed, + codecs=sub_codecs, + index_location=index_location, + ) + sharding_codec.validate( + shape=chunk_shape_parsed, + dtype=dtype_parsed, + chunk_grid=RegularChunkGrid(chunk_shape=shard_shape_parsed), + ) + codecs_out = (sharding_codec,) + chunks_out = shard_shape_parsed + else: + chunks_out = chunk_shape_parsed + codecs_out = sub_codecs + + return ArrayV3Metadata( + shape=shape_parsed, + data_type=dtype_parsed, + chunk_grid=RegularChunkGrid(chunk_shape=chunks_out), + chunk_key_encoding=chunk_key_encoding_parsed, + fill_value=fill_value, + codecs=codecs_out, + attributes=attributes, + dimension_names=dimension_names, + ) + + raise ValueError(f"Invalid Zarr format: {zarr_format}") + + +# TODO: uncomment these overrides when we can get mypy to accept them +""" +@overload +def meta_from_array( + array: np.ndarray[Any, Any], + chunks: ChunkCoords | Literal["auto"], + shards: None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: Literal[2], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV2Metadata: ... + + +@overload +def meta_from_array( + array: np.ndarray[Any, Any], + chunks: ChunkCoords | Literal["auto"], + shards: ShardsLike | None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: None, + zarr_format: Literal[3], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV3Metadata: ... + +""" + + +def meta_from_array( + array: np.ndarray[Any, Any], + *, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any = 0, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, +) -> ArrayV3Metadata | ArrayV2Metadata: + """ + Create array metadata from an array + """ + return create_array_metadata( + shape=array.shape, + dtype=array.dtype, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + ) + + +def skip_object_dtype(dtype: ZDType[Any, Any]) -> None: + if dtype.dtype_cls is type(np.dtype("O")): + msg = ( + f"{dtype} uses the numpy object data type, which is not a valid target for data " + "type resolution" + ) + pytest.skip(msg) diff --git a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt index eee724c912..7eb0eb7c86 100644 --- a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +++ b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt @@ -12,3 +12,5 @@ another_buffer = package_with_entrypoint:TestEntrypointGroup.Buffer another_ndbuffer = package_with_entrypoint:TestEntrypointGroup.NDBuffer [zarr.codec_pipeline] another_pipeline = package_with_entrypoint:TestEntrypointGroup.Pipeline +[zarr.data_type] +new_data_type = package_with_entrypoint:TestDataType \ No newline at end of file diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py index b818adf8ea..e0d8a52c4d 100644 --- a/tests/package_with_entrypoint/__init__.py +++ b/tests/package_with_entrypoint/__init__.py @@ -1,13 +1,23 @@ -from collections.abc import Iterable +from __future__ import annotations -from numpy import ndarray +from typing import TYPE_CHECKING, Any, Literal, Self + +import numpy as np +import numpy.typing as npt import zarr.core.buffer -from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecOutput, CodecPipeline +from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline from zarr.codecs import BytesCodec -from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer -from zarr.core.common import BytesLike +from zarr.core.dtype.common import DataTypeValidationError, DTypeJSON, DTypeSpec_V2 +from zarr.core.dtype.npy.bool import Bool + +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import ClassVar, Literal + + from zarr.core.array_spec import ArraySpec + from zarr.core.common import ZarrFormat class TestEntrypointCodec(ArrayBytesCodec): @@ -16,14 +26,14 @@ class TestEntrypointCodec(ArrayBytesCodec): async def encode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], - ) -> Iterable[CodecOutput | None]: - pass + ) -> Iterable[Buffer | None]: + return [None] async def decode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], - ) -> ndarray: - pass + ) -> npt.NDArray[Any]: + return np.array(1) def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: return input_byte_length @@ -35,13 +45,13 @@ def __init__(self, batch_size: int = 1) -> None: async def encode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] - ) -> BytesLike: - pass + ) -> Iterable[Buffer | None]: + return [None] async def decode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] - ) -> ndarray: - pass + ) -> Iterable[NDBuffer | None]: + return np.array(1) class TestEntrypointBuffer(Buffer): @@ -64,3 +74,28 @@ class NDBuffer(zarr.core.buffer.NDBuffer): class Pipeline(CodecPipeline): pass + + +class TestDataType(Bool): + """ + This is a "data type" that serializes to "test" + """ + + _zarr_v3_name: ClassVar[Literal["test"]] = "test" # type: ignore[assignment] + + @classmethod + def from_json(cls, data: DTypeJSON, *, zarr_format: Literal[2, 3]) -> Self: + if zarr_format == 2 and data == {"name": cls._zarr_v3_name, "object_codec_id": None}: + return cls() + if zarr_format == 3 and data == cls._zarr_v3_name: + return cls() + raise DataTypeValidationError( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}" + ) + + def to_json(self, zarr_format: ZarrFormat) -> str | DTypeSpec_V2: # type: ignore[override] + if zarr_format == 2: + return {"name": self._zarr_v3_name, "object_codec_id": None} + if zarr_format == 3: + return self._zarr_v3_name + raise ValueError("zarr_format must be 2 or 3") diff --git a/tests/test_api.py b/tests/test_api.py index aacd558f2a..2a95d7b97c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,4 +1,18 @@ -import pathlib +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +import zarr.codecs +import zarr.storage + +if TYPE_CHECKING: + import pathlib + + from zarr.abc.store import Store + from zarr.core.common import JSON, MemoryOrder, ZarrFormat + +import contextlib import warnings from typing import Literal @@ -8,25 +22,30 @@ import zarr import zarr.api.asynchronous +import zarr.api.synchronous import zarr.core.group from zarr import Array, Group -from zarr.abc.store import Store from zarr.api.synchronous import ( create, create_array, create_group, + from_array, group, load, - open, open_group, save, save_array, save_group, ) -from zarr.core.common import JSON, MemoryOrder, ZarrFormat +from zarr.core.buffer import NDArrayLike from zarr.errors import MetadataValidationError -from zarr.storage import MemoryStore +from zarr.storage import LocalStore, MemoryStore, ZipStore from zarr.storage._utils import normalize_path +from zarr.testing.utils import gpu_test + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path def test_create(memory_store: Store) -> None: @@ -60,13 +79,19 @@ def test_create(memory_store: Store) -> None: # TODO: parametrize over everything this function takes @pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_create_array(store: Store) -> None: +def test_create_array(store: Store, zarr_format: ZarrFormat) -> None: attrs: dict[str, JSON] = {"foo": 100} # explicit type annotation to avoid mypy error shape = (10, 10) path = "foo" data_val = 1 array_w = create_array( - store, name=path, shape=shape, attributes=attrs, chunks=shape, dtype="uint8" + store, + name=path, + shape=shape, + attributes=attrs, + chunks=shape, + dtype="uint8", + zarr_format=zarr_format, ) array_w[:] = data_val assert array_w.shape == shape @@ -75,18 +100,27 @@ def test_create_array(store: Store) -> None: @pytest.mark.parametrize("write_empty_chunks", [True, False]) -def test_write_empty_chunks_warns(write_empty_chunks: bool) -> None: +def test_write_empty_chunks_warns(write_empty_chunks: bool, zarr_format: ZarrFormat) -> None: """ Test that using the `write_empty_chunks` kwarg on array access will raise a warning. """ match = "The `write_empty_chunks` keyword argument .*" with pytest.warns(RuntimeWarning, match=match): _ = zarr.array( - data=np.arange(10), shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks + data=np.arange(10), + shape=(10,), + dtype="uint8", + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, ) with pytest.warns(RuntimeWarning, match=match): - _ = zarr.create(shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks) + _ = zarr.create( + shape=(10,), + dtype="uint8", + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + ) @pytest.mark.parametrize("path", ["foo", "/", "/foo", "///foo/bar"]) @@ -103,32 +137,41 @@ def test_open_normalized_path( assert node.path == normalize_path(path) -async def test_open_array(memory_store: MemoryStore) -> None: +async def test_open_array(memory_store: MemoryStore, zarr_format: ZarrFormat) -> None: store = memory_store # open array, create if doesn't exist - z = open(store=store, shape=100) + z = zarr.api.synchronous.open(store=store, shape=100, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (100,) # open array, overwrite # store._store_dict = {} store = MemoryStore() - z = open(store=store, shape=200) + z = zarr.api.synchronous.open(store=store, shape=200, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (200,) # open array, read-only store_cls = type(store) ro_store = await store_cls.open(store_dict=store._store_dict, read_only=True) - z = open(store=ro_store, mode="r") + z = zarr.api.synchronous.open(store=ro_store, mode="r") assert isinstance(z, Array) assert z.shape == (200,) assert z.read_only # path not found with pytest.raises(FileNotFoundError): - open(store="doesnotexist", mode="r") + zarr.api.synchronous.open(store="doesnotexist", mode="r", zarr_format=zarr_format) + + +@pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True) +def test_v2_and_v3_exist_at_same_path(store: Store) -> None: + zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=3) + zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=2) + msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store}. Zarr v3 will be used." + with pytest.warns(UserWarning, match=re.escape(msg)): + zarr.open(store=store) @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -151,9 +194,9 @@ async def test_open_group(memory_store: MemoryStore) -> None: assert "foo" in g # open group, overwrite - # g = open_group(store=store) - # assert isinstance(g, Group) - # assert "foo" not in g + g = open_group(store=store, mode="w") + assert isinstance(g, Group) + assert "foo" not in g # open group, read-only store_cls = type(store) @@ -186,22 +229,23 @@ async def test_open_group_unspecified_version( @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("n_args", [10, 1, 0]) @pytest.mark.parametrize("n_kwargs", [10, 1, 0]) -def test_save(store: Store, n_args: int, n_kwargs: int) -> None: +@pytest.mark.parametrize("path", [None, "some_path"]) +def test_save(store: Store, n_args: int, n_kwargs: int, path: None | str) -> None: data = np.arange(10) args = [np.arange(10) for _ in range(n_args)] kwargs = {f"arg_{i}": data for i in range(n_kwargs)} if n_kwargs == 0 and n_args == 0: with pytest.raises(ValueError): - save(store) + save(store, path=path) elif n_args == 1 and n_kwargs == 0: - save(store, *args) - array = open(store) + save(store, *args, path=path) + array = zarr.api.synchronous.open(store, path=path) assert isinstance(array, Array) assert_array_equal(array[:], data) else: - save(store, *args, **kwargs) # type: ignore[arg-type] - group = open(store) + save(store, *args, path=path, **kwargs) # type: ignore [arg-type] + group = zarr.api.synchronous.open(store, path=path) assert isinstance(group, Group) for array in group.array_values(): assert_array_equal(array[:], data) @@ -235,7 +279,9 @@ def test_open_with_mode_r(tmp_path: pathlib.Path) -> None: z2 = zarr.open(store=tmp_path, mode="r") assert isinstance(z2, Array) assert z2.fill_value == 1 - assert (z2[:] == 1).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() with pytest.raises(ValueError): z2[:] = 3 @@ -247,7 +293,9 @@ def test_open_with_mode_r_plus(tmp_path: pathlib.Path) -> None: zarr.ones(store=tmp_path, shape=(3, 3)) z2 = zarr.open(store=tmp_path, mode="r+") assert isinstance(z2, Array) - assert (z2[:] == 1).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() z2[:] = 3 @@ -263,7 +311,9 @@ async def test_open_with_mode_a(tmp_path: pathlib.Path) -> None: arr[...] = 1 z2 = zarr.open(store=tmp_path, mode="a") assert isinstance(z2, Array) - assert (z2[:] == 1).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() z2[:] = 3 @@ -275,7 +325,9 @@ def test_open_with_mode_w(tmp_path: pathlib.Path) -> None: arr[...] = 3 z2 = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) assert isinstance(z2, Array) - assert not (z2[:] == 3).all() + result = z2[:] + assert isinstance(result, NDArrayLike) + assert not (result == 3).all() z2[:] = 3 @@ -288,7 +340,6 @@ def test_open_with_mode_w_minus(tmp_path: pathlib.Path) -> None: zarr.open(store=tmp_path, mode="w-") -@pytest.mark.parametrize("zarr_format", [2, 3]) def test_array_order(zarr_format: ZarrFormat) -> None: arr = zarr.ones(shape=(2, 2), order=None, zarr_format=zarr_format) expected = zarr.config.get("array.order") @@ -304,17 +355,15 @@ def test_array_order(zarr_format: ZarrFormat) -> None: @pytest.mark.parametrize("order", ["C", "F"]) -@pytest.mark.parametrize("zarr_format", [2, 3]) def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None: with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) - expected = order or zarr.config.get("array.order") - assert arr.order == expected + assert arr.order == order vals = np.asarray(arr) - if expected == "C": + if order == "C": assert vals.flags.c_contiguous - elif expected == "F": + elif order == "F": assert vals.flags.f_contiguous else: raise AssertionError @@ -336,8 +385,8 @@ def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) - # assert "LazyLoader: " in repr(loader) -def test_load_array(memory_store: Store) -> None: - store = memory_store +def test_load_array(sync_store: Store) -> None: + store = sync_store foo = np.arange(100) bar = np.arange(100, 0, -1) save(store, foo=foo, bar=bar) @@ -352,6 +401,38 @@ def test_load_array(memory_store: Store) -> None: assert_array_equal(bar, array) +@pytest.mark.parametrize("path", ["data", None]) +@pytest.mark.parametrize("load_read_only", [True, False, None]) +def test_load_zip(tmp_path: pathlib.Path, path: str | None, load_read_only: bool | None) -> None: + file = tmp_path / "test.zip" + data = np.arange(100).reshape(10, 10) + + with ZipStore(file, mode="w", read_only=False) as zs: + save(zs, data, path=path) + with ZipStore(file, mode="r", read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + with ZipStore(file, read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + + +@pytest.mark.parametrize("path", ["data", None]) +@pytest.mark.parametrize("load_read_only", [True, False]) +def test_load_local(tmp_path: pathlib.Path, path: str | None, load_read_only: bool) -> None: + file = tmp_path / "test.zip" + data = np.arange(100).reshape(10, 10) + + with LocalStore(file, read_only=False) as zs: + save(zs, data, path=path) + with LocalStore(file, read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + + def test_tree() -> None: pytest.importorskip("rich") g1 = zarr.group() @@ -1035,7 +1116,7 @@ def test_tree() -> None: def test_open_positional_args_deprecated() -> None: store = MemoryStore() with pytest.warns(FutureWarning, match="pass"): - open(store, "w", shape=(1,)) + zarr.api.synchronous.open(store, "w", shape=(1,)) def test_save_array_positional_args_deprecated() -> None: @@ -1076,21 +1157,30 @@ def test_open_falls_back_to_open_group() -> None: assert group.attrs == {"key": "value"} -async def test_open_falls_back_to_open_group_async() -> None: +async def test_open_falls_back_to_open_group_async(zarr_format: ZarrFormat) -> None: # https://github.com/zarr-developers/zarr-python/issues/2309 store = MemoryStore() - await zarr.api.asynchronous.open_group(store, attributes={"key": "value"}) + await zarr.api.asynchronous.open_group( + store, attributes={"key": "value"}, zarr_format=zarr_format + ) group = await zarr.api.asynchronous.open(store=store) assert isinstance(group, zarr.core.group.AsyncGroup) + assert group.metadata.zarr_format == zarr_format assert group.attrs == {"key": "value"} -def test_open_mode_write_creates_group(tmp_path: pathlib.Path) -> None: +@pytest.mark.parametrize("mode", ["r", "r+", "w", "a"]) +def test_open_modes_creates_group(tmp_path: pathlib.Path, mode: str) -> None: # https://github.com/zarr-developers/zarr-python/issues/2490 - zarr_dir = tmp_path / "test.zarr" - group = zarr.open(zarr_dir, mode="w") - assert isinstance(group, Group) + zarr_dir = tmp_path / f"mode-{mode}-test.zarr" + if mode in ["r", "r+"]: + # Expect FileNotFoundError to be raised if 'r' or 'r+' mode + with pytest.raises(FileNotFoundError): + zarr.open(store=zarr_dir, mode=mode) + else: + group = zarr.open(store=zarr_dir, mode=mode) + assert isinstance(group, Group) async def test_metadata_validation_error() -> None: @@ -1098,13 +1188,13 @@ async def test_metadata_validation_error() -> None: MetadataValidationError, match="Invalid value for 'zarr_format'. Expected '2, 3, or None'. Got '3.0'.", ): - await zarr.api.asynchronous.open_group(zarr_format="3.0") # type: ignore[arg-type] + await zarr.api.asynchronous.open_group(zarr_format="3.0") # type: ignore [arg-type] with pytest.raises( MetadataValidationError, match="Invalid value for 'zarr_format'. Expected '2, 3, or None'. Got '3.0'.", ): - await zarr.api.asynchronous.open_array(shape=(1,), zarr_format="3.0") # type: ignore[arg-type] + await zarr.api.asynchronous.open_array(shape=(1,), zarr_format="3.0") # type: ignore [arg-type] @pytest.mark.parametrize( @@ -1112,12 +1202,173 @@ async def test_metadata_validation_error() -> None: ["local", "memory", "zip"], indirect=True, ) -def test_open_array_with_mode_r_plus(store: Store) -> None: +def test_open_array_with_mode_r_plus(store: Store, zarr_format: ZarrFormat) -> None: # 'r+' means read/write (must exist) with pytest.raises(FileNotFoundError): - zarr.open_array(store=store, mode="r+") - zarr.ones(store=store, shape=(3, 3)) + zarr.open_array(store=store, mode="r+", zarr_format=zarr_format) + zarr.ones(store=store, shape=(3, 3), zarr_format=zarr_format) z2 = zarr.open_array(store=store, mode="r+") assert isinstance(z2, Array) - assert (z2[:] == 1).all() + assert z2.metadata.zarr_format == zarr_format + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() z2[:] = 3 + + +def test_api_exports() -> None: + """ + Test that the sync API and the async API export the same objects + """ + assert zarr.api.asynchronous.__all__ == zarr.api.synchronous.__all__ + + +@gpu_test +@pytest.mark.parametrize( + "store", + ["local", "memory", "zip"], + indirect=True, +) +@pytest.mark.parametrize("zarr_format", [None, 2, 3]) +def test_gpu_basic(store: Store, zarr_format: ZarrFormat | None) -> None: + import cupy as cp + + if zarr_format == 2: + # Without this, the zstd codec attempts to convert the cupy + # array to bytes. + compressors = None + else: + compressors = "auto" + + with zarr.config.enable_gpu(): + src = cp.random.uniform(size=(100, 100)) # allocate on the device + z = zarr.create_array( + store, + name="a", + shape=src.shape, + chunks=(10, 10), + dtype=src.dtype, + overwrite=True, + zarr_format=zarr_format, + compressors=compressors, + ) + z[:10, :10] = src[:10, :10] + + result = z[:10, :10] + # assert_array_equal doesn't check the type + assert isinstance(result, type(src)) + cp.testing.assert_array_equal(result, src[:10, :10]) + + +def test_v2_without_compressor() -> None: + # Make sure it's possible to set no compressor for v2 arrays + arr = zarr.create(store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=None) + assert arr.compressors == () + + +def test_v2_with_v3_compressor() -> None: + # Check trying to create a v2 array with a v3 compressor fails + with pytest.raises( + ValueError, + match="Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. Use a numcodecs codec directly instead.", + ): + zarr.create( + store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=zarr.codecs.BloscCodec() + ) + + +def add_empty_file(path: Path) -> Path: + fpath = path / "a.txt" + fpath.touch() + return fpath + + +@pytest.mark.parametrize("create_function", [create_array, from_array]) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_no_overwrite_array(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + create_function(store=store, data=np.ones(shape=(1,)), overwrite=overwrite) + if overwrite: + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +@pytest.mark.parametrize("create_function", [create_group, group]) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_no_overwrite_group(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + create_function(store=store, overwrite=overwrite) + if overwrite: + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +@pytest.mark.parametrize("open_func", [zarr.open, open_group]) +@pytest.mark.parametrize("mode", ["r", "r+", "a", "w", "w-"]) +def test_no_overwrite_open(tmp_path: Path, open_func: Callable, mode: str) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + with contextlib.suppress(FileExistsError, FileNotFoundError, ValueError): + open_func(store=store, mode=mode) + if mode == "w": + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +def test_no_overwrite_load(tmp_path: Path) -> None: + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + with contextlib.suppress(NotImplementedError): + zarr.load(store) + assert existing_fpath.exists() + + +@pytest.mark.parametrize( + "f", + [ + zarr.array, + zarr.create, + zarr.create_array, + zarr.ones, + zarr.ones_like, + zarr.empty, + zarr.empty_like, + zarr.full, + zarr.full_like, + zarr.zeros, + zarr.zeros_like, + ], +) +def test_auto_chunks(f: Callable[..., Array]) -> None: + # Make sure chunks are set automatically across the public API + # TODO: test shards with this test too + shape = (1000, 1000) + dtype = np.uint8 + kwargs = {"shape": shape, "dtype": dtype} + array = np.zeros(shape, dtype=dtype) + store = zarr.storage.MemoryStore() + + if f in [zarr.full, zarr.full_like]: + kwargs["fill_value"] = 0 + if f in [zarr.array]: + kwargs["data"] = array + if f in [zarr.empty_like, zarr.full_like, zarr.empty_like, zarr.ones_like, zarr.zeros_like]: + kwargs["a"] = array + if f in [zarr.create_array]: + kwargs["store"] = store + + a = f(**kwargs) + assert a.chunks == (500, 500) diff --git a/tests/test_array.py b/tests/test_array.py index 410b2e58d0..28ea812967 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1,49 +1,69 @@ import dataclasses +import inspect import json import math +import multiprocessing as mp import pickle import re +import sys from itertools import accumulate from typing import TYPE_CHECKING, Any, Literal +from unittest import mock import numcodecs import numpy as np +import numpy.typing as npt import pytest +from packaging.version import Version import zarr.api.asynchronous +import zarr.api.synchronous as sync_api +from tests.conftest import skip_object_dtype from zarr import Array, AsyncArray, Group +from zarr.abc.store import Store from zarr.codecs import ( BytesCodec, GzipCodec, TransposeCodec, - VLenBytesCodec, - VLenUTF8Codec, ZstdCodec, ) from zarr.core._info import ArrayInfo from zarr.core.array import ( CompressorsLike, FiltersLike, - _get_default_chunk_encoding_v2, - _get_default_chunk_encoding_v3, _parse_chunk_encoding_v2, _parse_chunk_encoding_v3, chunks_initialized, create_array, ) -from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype from zarr.core.buffer.cpu import NDBuffer from zarr.core.chunk_grids import _auto_partition +from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams from zarr.core.common import JSON, MemoryOrder, ZarrFormat +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype.common import ENDIANNESS_STR, EndiannessStr +from zarr.core.dtype.npy.common import NUMPY_ENDIANNESS_STR, endianness_from_numpy_str +from zarr.core.dtype.npy.float import Float32, Float64 +from zarr.core.dtype.npy.int import Int16, UInt8 +from zarr.core.dtype.npy.string import VariableLengthUTF8 +from zarr.core.dtype.npy.structured import ( + Structured, +) +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 +from zarr.core.dtype.wrapper import ZDType from zarr.core.group import AsyncGroup -from zarr.core.indexing import ceildiv -from zarr.core.metadata.v3 import DataType +from zarr.core.indexing import BasicIndexer, ceildiv +from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.storage import LocalStore, MemoryStore, StorePath +from .test_dtype.conftest import zdtype_examples + if TYPE_CHECKING: from zarr.core.array_spec import ArrayConfigLike + from zarr.core.metadata.v3 import ArrayV3Metadata @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @@ -143,7 +163,7 @@ def test_array_name_properties_no_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: arr = zarr.create_array( - store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" + store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype=">i4" ) assert arr.path == "" assert arr.name == "/" @@ -169,34 +189,45 @@ def test_array_name_properties_with_group( assert spam.basename == "spam" +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("specifiy_fill_value", [True, False]) -@pytest.mark.parametrize("dtype_str", ["bool", "uint8", "complex64"]) -def test_array_v3_fill_value_default( - store: MemoryStore, specifiy_fill_value: bool, dtype_str: str +@pytest.mark.parametrize( + "zdtype", zdtype_examples, ids=tuple(str(type(v)) for v in zdtype_examples) +) +def test_array_fill_value_default( + store: MemoryStore, specifiy_fill_value: bool, zdtype: ZDType[Any, Any] ) -> None: """ Test that creating an array with the fill_value parameter set to None, or unspecified, - results in the expected fill_value attribute of the array, i.e. 0 cast to the array's dtype. + results in the expected fill_value attribute of the array, i.e. the default value of the dtype """ shape = (10,) - default_fill_value = 0 if specifiy_fill_value: arr = zarr.create_array( store=store, shape=shape, - dtype=dtype_str, + dtype=zdtype, zarr_format=3, chunks=shape, fill_value=None, ) else: - arr = zarr.create_array( - store=store, shape=shape, dtype=dtype_str, zarr_format=3, chunks=shape - ) + arr = zarr.create_array(store=store, shape=shape, dtype=zdtype, zarr_format=3, chunks=shape) + expected_fill_value = zdtype.default_scalar() + if isinstance(expected_fill_value, np.datetime64 | np.timedelta64): + if np.isnat(expected_fill_value): + assert np.isnat(arr.fill_value) + elif isinstance(expected_fill_value, np.floating | np.complexfloating): + if np.isnan(expected_fill_value): + assert np.isnan(arr.fill_value) + else: + assert arr.fill_value == expected_fill_value + # A simpler check would be to ensure that arr.fill_value.dtype == arr.dtype + # But for some numpy data types (namely, U), scalars might not have length. An empty string + # scalar from a `>U4` array would have dtype `>U`, and arr.fill_value.dtype == arr.dtype will fail. - assert arr.fill_value == np.dtype(dtype_str).type(default_fill_value) - assert arr.fill_value.dtype == arr.dtype + assert type(arr.fill_value) is type(np.array([arr.fill_value], dtype=arr.dtype)[0]) @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -219,10 +250,13 @@ def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str assert arr.fill_value.dtype == arr.dtype -def test_create_positional_args_deprecated() -> None: - store = MemoryStore() - with pytest.warns(FutureWarning, match="Pass"): - zarr.Array.create(store, (2, 2), dtype="f8") +async def test_create_deprecated() -> None: + with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning, match=re.escape("Pass shape=(2, 2) as keyword args")): + await zarr.AsyncArray.create(MemoryStore(), (2, 2), dtype="f8") # type: ignore[call-overload] + with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning, match=re.escape("Pass shape=(2, 2) as keyword args")): + zarr.Array.create(MemoryStore(), (2, 2), dtype="f8") def test_selection_positional_args_deprecated() -> None: @@ -313,24 +347,47 @@ def test_serializable_sync_array(store: LocalStore, zarr_format: ZarrFormat) -> @pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_storage_transformers(store: MemoryStore) -> None: +@pytest.mark.parametrize("zarr_format", [2, 3, "invalid"]) +def test_storage_transformers(store: MemoryStore, zarr_format: ZarrFormat | str) -> None: """ Test that providing an actual storage transformer produces a warning and otherwise passes through """ - metadata_dict: dict[str, JSON] = { - "zarr_format": 3, - "node_type": "array", - "shape": (10,), - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, - "data_type": "uint8", - "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "/"}}, - "codecs": (BytesCodec().to_dict(),), - "fill_value": 0, - "storage_transformers": ({"test": "should_raise"}), - } - match = "Arrays with storage transformers are not supported in zarr-python at this time." - with pytest.raises(ValueError, match=match): + metadata_dict: dict[str, JSON] + if zarr_format == 3: + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": "uint8", + "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "/"}}, + "codecs": (BytesCodec().to_dict(),), + "fill_value": 0, + "storage_transformers": ({"test": "should_raise"}), + } + else: + metadata_dict = { + "zarr_format": zarr_format, + "shape": (10,), + "chunks": (1,), + "dtype": "|u1", + "dimension_separator": ".", + "codecs": (BytesCodec().to_dict(),), + "fill_value": 0, + "order": "C", + "storage_transformers": ({"test": "should_raise"}), + } + if zarr_format == 3: + match = "Arrays with storage transformers are not supported in zarr-python at this time." + with pytest.raises(ValueError, match=match): + Array.from_dict(StorePath(store), data=metadata_dict) + elif zarr_format == 2: + # no warning Array.from_dict(StorePath(store), data=metadata_dict) + else: + match = f"Invalid zarr_format: {zarr_format}. Expected 2 or 3" + with pytest.raises(ValueError, match=match): + Array.from_dict(StorePath(store), data=metadata_dict) @pytest.mark.parametrize("test_cls", [Array, AsyncArray[Any]]) @@ -379,12 +436,13 @@ async def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]] assert observed == expected -async def test_chunks_initialized() -> None: +@pytest.mark.parametrize("path", ["", "foo"]) +async def test_chunks_initialized(path: str) -> None: """ Test that chunks_initialized accurately returns the keys of stored chunks. """ store = MemoryStore() - arr = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="i4") + arr = zarr.create_array(store, name=path, shape=(100,), chunks=(10,), dtype="i4") chunks_accumulated = tuple( accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_chunk_keys())) @@ -399,13 +457,13 @@ async def test_chunks_initialized() -> None: def test_nbytes_stored() -> None: arr = zarr.create(shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec()]) result = arr.nbytes_stored() - assert result == 366 # the size of the metadata document. This is a fragile test. + assert result == 502 # the size of the metadata document. This is a fragile test. arr[:50] = 1 result = arr.nbytes_stored() - assert result == 566 # the size with 5 chunks filled. + assert result == 702 # the size with 5 chunks filled. arr[50:] = 2 result = arr.nbytes_stored() - assert result == 766 # the size with all chunks filled. + assert result == 902 # the size with all chunks filled. async def test_nbytes_stored_async() -> None: @@ -413,55 +471,13 @@ async def test_nbytes_stored_async() -> None: shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec()] ) result = await arr.nbytes_stored() - assert result == 366 # the size of the metadata document. This is a fragile test. + assert result == 502 # the size of the metadata document. This is a fragile test. await arr.setitem(slice(50), 1) result = await arr.nbytes_stored() - assert result == 566 # the size with 5 chunks filled. + assert result == 702 # the size with 5 chunks filled. await arr.setitem(slice(50, 100), 2) result = await arr.nbytes_stored() - assert result == 766 # the size with all chunks filled. - - -def test_default_fill_values() -> None: - a = zarr.Array.create(MemoryStore(), shape=5, chunk_shape=5, dtype=" None: - with pytest.raises(ValueError, match="At least one ArrayBytesCodec is required."): - Array.create(MemoryStore(), shape=5, chunks=5, dtype=" None: ) a = np.arange(105, dtype="i4") z[:] = a + result = z[:] + assert isinstance(result, NDArrayLike) assert (105,) == z.shape - assert (105,) == z[:].shape + assert (105,) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10,) == z.chunks - np.testing.assert_array_equal(a, z[:]) + np.testing.assert_array_equal(a, result) z.resize(205) + result = z[:] + assert isinstance(result, NDArrayLike) assert (205,) == z.shape - assert (205,) == z[:].shape + assert (205,) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10,) == z.chunks np.testing.assert_array_equal(a, z[:105]) np.testing.assert_array_equal(np.zeros(100, dtype="i4"), z[105:]) z.resize(55) + result = z[:] + assert isinstance(result, NDArrayLike) assert (55,) == z.shape - assert (55,) == z[:].shape + assert (55,) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10,) == z.chunks - np.testing.assert_array_equal(a[:55], z[:]) + np.testing.assert_array_equal(a[:55], result) # via shape setter new_shape = (105,) z.shape = new_shape + result = z[:] + assert isinstance(result, NDArrayLike) assert new_shape == z.shape - assert new_shape == z[:].shape + assert new_shape == result.shape @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -691,44 +720,54 @@ def test_resize_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: ) a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) z[:] = a + result = z[:] + assert isinstance(result, NDArrayLike) assert (105, 105) == z.shape - assert (105, 105) == z[:].shape + assert (105, 105) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks - np.testing.assert_array_equal(a, z[:]) + np.testing.assert_array_equal(a, result) z.resize((205, 205)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (205, 205) == z.shape - assert (205, 205) == z[:].shape + assert (205, 205) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a, z[:105, :105]) np.testing.assert_array_equal(np.zeros((100, 205), dtype="i4"), z[105:, :]) np.testing.assert_array_equal(np.zeros((205, 100), dtype="i4"), z[:, 105:]) z.resize((55, 55)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (55, 55) == z.shape - assert (55, 55) == z[:].shape + assert (55, 55) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks - np.testing.assert_array_equal(a[:55, :55], z[:]) + np.testing.assert_array_equal(a[:55, :55], result) z.resize((55, 1)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (55, 1) == z.shape - assert (55, 1) == z[:].shape + assert (55, 1) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks - np.testing.assert_array_equal(a[:55, :1], z[:]) + np.testing.assert_array_equal(a[:55, :1], result) z.resize((1, 55)) + result = z[:] + assert isinstance(result, NDArrayLike) assert (1, 55) == z.shape - assert (1, 55) == z[:].shape + assert (1, 55) == result.shape assert np.dtype("i4") == z.dtype - assert np.dtype("i4") == z[:].dtype + assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a[:1, :10], z[:, :10]) np.testing.assert_array_equal(np.zeros((1, 55 - 10), dtype="i4"), z[:, 10:55]) @@ -736,8 +775,10 @@ def test_resize_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: # via shape setter new_shape = (105, 105) z.shape = new_shape + result = z[:] + assert isinstance(result, NDArrayLike) assert new_shape == z.shape - assert new_shape == z[:].shape + assert new_shape == result.shape @pytest.mark.parametrize("store", ["memory"], indirect=True) @@ -823,66 +864,6 @@ def test_append_bad_shape(store: MemoryStore, zarr_format: ZarrFormat) -> None: z.append(b) -@pytest.mark.parametrize("order", ["C", "F", None]) -@pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_array_create_metadata_order_v2( - order: MemoryOrder | None, zarr_format: int, store: MemoryStore -) -> None: - """ - Test that the ``order`` attribute in zarr v2 array metadata is set correctly via the ``order`` - keyword argument to ``Array.create``. When ``order`` is ``None``, the value of the - ``array.order`` config is used. - """ - arr = zarr.create_array(store=store, shape=(2, 2), order=order, zarr_format=2, dtype="i4") - - expected = order or zarr.config.get("array.order") - assert arr.metadata.zarr_format == 2 # guard for mypy - assert arr.metadata.order == expected - - -@pytest.mark.parametrize("order_config", ["C", "F", None]) -@pytest.mark.parametrize("store", ["memory"], indirect=True) -def test_array_create_order( - order_config: MemoryOrder | None, - zarr_format: ZarrFormat, - store: MemoryStore, -) -> None: - """ - Test that the arrays generated by array indexing have a memory order defined by the config order - value - """ - config: ArrayConfigLike = {} - if order_config is None: - config = {} - expected = zarr.config.get("array.order") - else: - config = {"order": order_config} - expected = order_config - - arr = zarr.create_array( - store=store, shape=(2, 2), zarr_format=zarr_format, dtype="i4", config=config - ) - - vals = np.asarray(arr) - if expected == "C": - assert vals.flags.c_contiguous - elif expected == "F": - assert vals.flags.f_contiguous - else: - raise AssertionError - - -@pytest.mark.parametrize("write_empty_chunks", [True, False]) -def test_write_empty_chunks_config(write_empty_chunks: bool) -> None: - """ - Test that the value of write_empty_chunks is sensitive to the global config when not set - explicitly - """ - with zarr.config.set({"array.write_empty_chunks": write_empty_chunks}): - arr = zarr.create_array({}, shape=(2, 2), dtype="i4") - assert arr._async_array._config.write_empty_chunks == write_empty_chunks - - @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("write_empty_chunks", [True, False]) @pytest.mark.parametrize("fill_value", [0, 5]) @@ -981,285 +962,811 @@ def test_auto_partition_auto_shards( expected_shards += (cs,) auto_shards, _ = _auto_partition( - array_shape=array_shape, chunk_shape=chunk_shape, shard_shape="auto", dtype=dtype + array_shape=array_shape, + chunk_shape=chunk_shape, + shard_shape="auto", + item_size=dtype.itemsize, ) assert auto_shards == expected_shards -def test_chunks_and_shards() -> None: - store = StorePath(MemoryStore()) - shape = (100, 100) - chunks = (5, 5) - shards = (10, 10) +@pytest.mark.parametrize("store", ["memory"], indirect=True) +class TestCreateArray: + @staticmethod + def test_chunks_and_shards(store: Store) -> None: + spath = StorePath(store) + shape = (100, 100) + chunks = (5, 5) + shards = (10, 10) + + arr_v3 = zarr.create_array(store=spath / "v3", shape=shape, chunks=chunks, dtype="i4") + assert arr_v3.chunks == chunks + assert arr_v3.shards is None + + arr_v3_sharding = zarr.create_array( + store=spath / "v3_sharding", + shape=shape, + chunks=chunks, + shards=shards, + dtype="i4", + ) + assert arr_v3_sharding.chunks == chunks + assert arr_v3_sharding.shards == shards - arr_v3 = zarr.create_array(store=store / "v3", shape=shape, chunks=chunks, dtype="i4") - assert arr_v3.chunks == chunks - assert arr_v3.shards is None + arr_v2 = zarr.create_array( + store=spath / "v2", shape=shape, chunks=chunks, zarr_format=2, dtype="i4" + ) + assert arr_v2.chunks == chunks + assert arr_v2.shards is None + + @staticmethod + @pytest.mark.parametrize("dtype", zdtype_examples) + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + def test_default_fill_value(dtype: ZDType[Any, Any], store: Store) -> None: + """ + Test that the fill value of an array is set to the default value for the dtype object + """ + a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype) + if isinstance(dtype, DateTime64 | TimeDelta64) and np.isnat(a.fill_value): + assert np.isnat(dtype.default_scalar()) + else: + assert a.fill_value == dtype.default_scalar() + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", zdtype_examples) + def test_dtype_forms(dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat) -> None: + """ + Test that the same array is produced from a ZDType instance, a numpy dtype, or a numpy string + """ + skip_object_dtype(dtype) + a = zarr.create_array( + store, name="a", shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format + ) - arr_v3_sharding = zarr.create_array( - store=store / "v3_sharding", - shape=shape, - chunks=chunks, - shards=shards, - dtype="i4", + b = zarr.create_array( + store, + name="b", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype(), + zarr_format=zarr_format, + ) + assert a.dtype == b.dtype + + # Structured dtypes do not have a numpy string representation that uniquely identifies them + if not isinstance(dtype, Structured): + if isinstance(dtype, VariableLengthUTF8): + # in numpy 2.3, StringDType().str becomes the string 'StringDType()' which numpy + # does not accept as a string representation of the dtype. + c = zarr.create_array( + store, + name="c", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype().char, + zarr_format=zarr_format, + ) + else: + c = zarr.create_array( + store, + name="c", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype().str, + zarr_format=zarr_format, + ) + assert a.dtype == c.dtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", zdtype_examples) + def test_dtype_roundtrip( + dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat + ) -> None: + """ + Test that creating an array, then opening it, gets the same array. + """ + skip_object_dtype(dtype) + a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format) + b = zarr.open_array(store) + assert a.dtype == b.dtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", ["uint8", "float32", "U3", "S4", "V1"]) + @pytest.mark.parametrize( + "compressors", + [ + "auto", + None, + (), + (ZstdCodec(level=3),), + (ZstdCodec(level=3), GzipCodec(level=0)), + ZstdCodec(level=3), + {"name": "zstd", "configuration": {"level": 3}}, + ({"name": "zstd", "configuration": {"level": 3}},), + ], ) - assert arr_v3_sharding.chunks == chunks - assert arr_v3_sharding.shards == shards + @pytest.mark.parametrize( + "filters", + [ + "auto", + None, + (), + ( + TransposeCodec( + order=[ + 0, + ] + ), + ), + ( + TransposeCodec( + order=[ + 0, + ] + ), + TransposeCodec( + order=[ + 0, + ] + ), + ), + TransposeCodec( + order=[ + 0, + ] + ), + {"name": "transpose", "configuration": {"order": [0]}}, + ({"name": "transpose", "configuration": {"order": [0]}},), + ], + ) + @pytest.mark.parametrize(("chunks", "shards"), [((6,), None), ((3,), (6,))]) + async def test_v3_chunk_encoding( + store: MemoryStore, + compressors: CompressorsLike, + filters: FiltersLike, + dtype: str, + chunks: tuple[int, ...], + shards: tuple[int, ...] | None, + ) -> None: + """ + Test various possibilities for the compressors and filters parameter to create_array + """ + arr = await create_array( + store=store, + dtype=dtype, + shape=(12,), + chunks=chunks, + shards=shards, + zarr_format=3, + filters=filters, + compressors=compressors, + ) + filters_expected, _, compressors_expected = _parse_chunk_encoding_v3( + filters=filters, + compressors=compressors, + serializer="auto", + dtype=arr._zdtype, + ) + assert arr.filters == filters_expected + assert arr.compressors == compressors_expected + + @staticmethod + @pytest.mark.parametrize("name", ["v2", "default", "invalid"]) + @pytest.mark.parametrize("separator", [".", "/"]) + async def test_chunk_key_encoding( + name: str, separator: Literal[".", "/"], zarr_format: ZarrFormat, store: MemoryStore + ) -> None: + chunk_key_encoding = ChunkKeyEncodingParams(name=name, separator=separator) # type: ignore[typeddict-item] + error_msg = "" + if name == "invalid": + error_msg = "Unknown chunk key encoding." + if zarr_format == 2 and name == "default": + error_msg = "Invalid chunk key encoding. For Zarr format 2 arrays, the `name` field of the chunk key encoding must be 'v2'." + if error_msg: + with pytest.raises(ValueError, match=re.escape(error_msg)): + arr = await create_array( + store=store, + dtype="uint8", + shape=(10,), + chunks=(1,), + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + ) + else: + arr = await create_array( + store=store, + dtype="uint8", + shape=(10,), + chunks=(1,), + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + ) + if isinstance(arr.metadata, ArrayV2Metadata): + assert arr.metadata.dimension_separator == separator + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ({"serializer": "bytes"}, "Zarr format 2 arrays do not support `serializer`."), + ({"dimension_names": ["test"]}, "Zarr format 2 arrays do not support dimension names."), + ], + ) + async def test_create_array_invalid_v2_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + with pytest.raises(ValueError, match=re.escape(error_msg)): + await zarr.api.asynchronous.create_array( + store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs + ) + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ( + {"dimension_names": ["test"]}, + "dimension_names cannot be used for arrays with zarr_format 2.", + ), + ( + {"chunk_key_encoding": {"name": "default", "separator": "/"}}, + "chunk_key_encoding cannot be used for arrays with zarr_format 2. Use dimension_separator instead.", + ), + ( + {"codecs": "bytes"}, + "codecs cannot be used for arrays with zarr_format 2. Use filters and compressor instead.", + ), + ], + ) + async def test_create_invalid_v2_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + with pytest.raises(ValueError, match=re.escape(error_msg)): + await zarr.api.asynchronous.create( + store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs + ) - arr_v2 = zarr.create_array( - store=store / "v2", shape=shape, chunks=chunks, zarr_format=2, dtype="i4" + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ( + {"chunk_shape": (1,), "chunks": (2,)}, + "Only one of chunk_shape or chunks can be provided.", + ), + ( + {"dimension_separator": "/"}, + "dimension_separator cannot be used for arrays with zarr_format 3. Use chunk_key_encoding instead.", + ), + ( + {"filters": []}, + "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead", + ), + ( + {"compressor": "blosc"}, + "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead", + ), + ], ) - assert arr_v2.chunks == chunks - assert arr_v2.shards is None + async def test_invalid_v3_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + kwargs.setdefault("chunks", (1,)) + with pytest.raises(ValueError, match=re.escape(error_msg)): + zarr.create(store=store, dtype="uint8", shape=(10,), zarr_format=3, **kwargs) + + @staticmethod + @pytest.mark.parametrize("dtype", ["uint8", "float32", "str", "U10", "S10", ">M8[10s]"]) + @pytest.mark.parametrize( + "compressors", + [ + "auto", + None, + numcodecs.Zstd(level=3), + (), + (numcodecs.Zstd(level=3),), + ], + ) + @pytest.mark.parametrize( + "filters", ["auto", None, numcodecs.GZip(level=1), (numcodecs.GZip(level=1),)] + ) + async def test_v2_chunk_encoding( + store: MemoryStore, compressors: CompressorsLike, filters: FiltersLike, dtype: str + ) -> None: + arr = await create_array( + store=store, + dtype=dtype, + shape=(10,), + zarr_format=2, + compressors=compressors, + filters=filters, + ) + filters_expected, compressor_expected = _parse_chunk_encoding_v2( + filters=filters, compressor=compressors, dtype=get_data_type_from_native_dtype(dtype) + ) + assert arr.metadata.zarr_format == 2 # guard for mypy + assert arr.metadata.compressor == compressor_expected + assert arr.metadata.filters == filters_expected + + # Normalize for property getters + compressor_expected = () if compressor_expected is None else (compressor_expected,) + filters_expected = () if filters_expected is None else filters_expected + + assert arr.compressors == compressor_expected + assert arr.filters == filters_expected + + @staticmethod + @pytest.mark.parametrize("dtype", [UInt8(), Float32(), VariableLengthUTF8()]) + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + async def test_default_filters_compressors( + store: MemoryStore, dtype: UInt8 | Float32 | VariableLengthUTF8, zarr_format: ZarrFormat + ) -> None: + """ + Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with ``filters`` and ``compressors`` unspecified. + """ + + arr = await create_array( + store=store, + dtype=dtype, # type: ignore[arg-type] + shape=(10,), + zarr_format=zarr_format, + ) + sig = inspect.signature(create_array) -def test_create_array_default_fill_values() -> None: - a = zarr.create_array(MemoryStore(), shape=(5,), chunks=(5,), dtype=" None: + """ + Test that creating a Zarr v2 array with ``shard_shape`` set to a non-None value raises an error. + """ + msg = re.escape( + "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. Got `shard_shape=(5,)` instead." + ) + with pytest.raises(ValueError, match=msg): + _ = await create_array( + store=store, + dtype="uint8", + shape=(10,), + shards=(5,), + zarr_format=2, + ) - c = zarr.create_array(MemoryStore(), shape=(5,), chunks=(5,), dtype="i") - assert c.fill_value == 0 + @staticmethod + @pytest.mark.parametrize("impl", ["sync", "async"]) + async def test_with_data(impl: Literal["sync", "async"], store: Store) -> None: + """ + Test that we can invoke ``create_array`` with a ``data`` parameter. + """ + data = np.arange(10) + name = "foo" + arr: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array + if impl == "sync": + arr = sync_api.create_array(store, name=name, data=data) + stored = arr[:] + elif impl == "async": + arr = await create_array(store, name=name, data=data, zarr_format=3) + stored = await arr._get_selection( + BasicIndexer(..., shape=arr.shape, chunk_grid=arr.metadata.chunk_grid), + prototype=default_buffer_prototype(), + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + assert np.array_equal(stored, data) + + @staticmethod + async def test_with_data_invalid_params(store: Store) -> None: + """ + Test that failing to specify data AND shape / dtype results in a ValueError + """ + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype=None) + + # we catch shape=None first, so specifying a dtype should raise the same exception as before + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype="uint8") + + with pytest.raises(ValueError, match="dtype was not specified"): + await create_array(store, data=None, shape=(10, 10)) + + @staticmethod + async def test_data_ignored_params(store: Store) -> None: + """ + Test that specifying data AND shape AND dtype results in a ValueError + """ + data = np.arange(10) + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=None, overwrite=True) + + # we catch shape first, so specifying a dtype should raise the same warning as before + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=data.dtype, overwrite=True) + + with pytest.raises( + ValueError, match="The data parameter was used, but the dtype parameter was also used." + ): + await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) + + @staticmethod + @pytest.mark.parametrize("order", ["C", "F", None]) + @pytest.mark.parametrize("with_config", [True, False]) + def test_order( + order: MemoryOrder | None, + with_config: bool, + zarr_format: ZarrFormat, + store: MemoryStore, + ) -> None: + """ + Test that the arrays generated by array indexing have a memory order defined by the config order + value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly. + """ + config: ArrayConfigLike | None = {} + if order is None: + config = {} + expected = zarr.config.get("array.order") + else: + config = {"order": order} + expected = order - d = zarr.create_array(MemoryStore(), shape=(5,), chunks=(5,), dtype="f") - assert d.fill_value == 0.0 + if not with_config: + # Test without passing config parameter + config = None + arr = zarr.create_array( + store=store, + shape=(2, 2), + zarr_format=zarr_format, + dtype="i4", + order=order, + config=config, + ) + assert arr.order == expected + if zarr_format == 2: + assert arr.metadata.zarr_format == 2 + assert arr.metadata.order == expected + + vals = np.asarray(arr) + if expected == "C": + assert vals.flags.c_contiguous + elif expected == "F": + assert vals.flags.f_contiguous + else: + raise AssertionError + + @staticmethod + @pytest.mark.parametrize("write_empty_chunks", [True, False]) + async def test_write_empty_chunks_config(write_empty_chunks: bool, store: Store) -> None: + """ + Test that the value of write_empty_chunks is sensitive to the global config when not set + explicitly + """ + with zarr.config.set({"array.write_empty_chunks": write_empty_chunks}): + arr = await create_array(store, shape=(2, 2), dtype="i4") + assert arr._config.write_empty_chunks == write_empty_chunks + + @staticmethod + @pytest.mark.parametrize("path", [None, "", "/", "/foo", "foo", "foo/bar"]) + async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) -> None: + arr = await create_array( + store, shape=(2, 2), dtype="i4", name=path, zarr_format=zarr_format + ) + if path is None: + expected_path = "" + elif path.startswith("/"): + expected_path = path.lstrip("/") + else: + expected_path = path + assert arr.path == expected_path + assert arr.name == "/" + expected_path + + # test that implicit groups were created + path_parts = expected_path.split("/") + if len(path_parts) > 1: + *parents, _ = ["", *accumulate(path_parts, lambda x, y: "/".join([x, y]))] # noqa: FLY002 + for parent_path in parents: + # this will raise if these groups were not created + _ = await zarr.api.asynchronous.open_group( + store=store, path=parent_path, zarr_format=zarr_format + ) + + @staticmethod + @pytest.mark.parametrize("endianness", ENDIANNESS_STR) + def test_default_endianness( + store: Store, zarr_format: ZarrFormat, endianness: EndiannessStr + ) -> None: + """ + Test that that endianness is correctly set when creating an array when not specifying a serializer + """ + dtype = Int16(endianness=endianness) + arr = zarr.create_array(store=store, shape=(1,), dtype=dtype, zarr_format=zarr_format) + byte_order: str = arr[:].dtype.byteorder # type: ignore[union-attr] + assert byte_order in NUMPY_ENDIANNESS_STR + assert endianness_from_numpy_str(byte_order) == endianness # type: ignore[arg-type] -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) -@pytest.mark.parametrize("empty_value", [None, ()]) -async def test_create_array_no_filters_compressors( - store: MemoryStore, dtype: str, empty_value: Any + +@pytest.mark.parametrize("value", [1, 1.4, "a", b"a", np.array(1)]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_scalar_array(value: Any, zarr_format: ZarrFormat) -> None: + arr = zarr.array(value, zarr_format=zarr_format) + assert arr[...] == value + assert arr.shape == () + assert arr.ndim == 0 + assert isinstance(arr[()], NDArrayLikeOrScalar) + + +@pytest.mark.parametrize("store", ["local"], indirect=True) +@pytest.mark.parametrize("store2", ["local"], indirect=["store2"]) +@pytest.mark.parametrize("src_format", [2, 3]) +@pytest.mark.parametrize("new_format", [2, 3, None]) +async def test_creation_from_other_zarr_format( + store: Store, + store2: Store, + src_format: ZarrFormat, + new_format: ZarrFormat | None, ) -> None: - """ - Test that the default ``filters`` and ``compressors`` are removed when ``create_array`` is invoked. - """ + if src_format == 2: + src = zarr.create( + (50, 50), chunks=(10, 10), store=store, zarr_format=src_format, dimension_separator="/" + ) + else: + src = zarr.create( + (50, 50), + chunks=(10, 10), + store=store, + zarr_format=src_format, + chunk_key_encoding=("default", "."), + ) - # v2 - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=2, - compressors=empty_value, - filters=empty_value, + src[:] = np.arange(50 * 50).reshape((50, 50)) + result = zarr.from_array( + store=store2, + data=src, + zarr_format=new_format, ) - # Test metadata explicitly - assert arr.metadata.zarr_format == 2 # guard for mypy - # The v2 metadata stores None and () separately - assert arr.metadata.filters == empty_value - # The v2 metadata does not allow tuple for compressor, therefore it is turned into None - assert arr.metadata.compressor is None - - assert arr.filters == () - assert arr.compressors == () - - # v3 - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - compressors=empty_value, - filters=empty_value, + np.testing.assert_array_equal(result[:], src[:]) + assert result.fill_value == src.fill_value + assert result.dtype == src.dtype + assert result.chunks == src.chunks + expected_format = src_format if new_format is None else new_format + assert result.metadata.zarr_format == expected_format + if src_format == new_format: + assert result.metadata == src.metadata + + result2 = zarr.array( + data=src, + store=store2, + overwrite=True, + zarr_format=new_format, ) - assert arr.metadata.zarr_format == 3 # guard for mypy - if dtype == "str": - assert arr.metadata.codecs == (VLenUTF8Codec(),) - assert arr.serializer == VLenUTF8Codec() - else: - assert arr.metadata.codecs == (BytesCodec(),) - assert arr.serializer == BytesCodec() - - -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) -@pytest.mark.parametrize( - "compressors", - [ - "auto", - None, - (), - (ZstdCodec(level=3),), - (ZstdCodec(level=3), GzipCodec(level=0)), - ZstdCodec(level=3), - {"name": "zstd", "configuration": {"level": 3}}, - ({"name": "zstd", "configuration": {"level": 3}},), - ], -) -@pytest.mark.parametrize( - "filters", - [ - "auto", - None, - (), - ( - TransposeCodec( - order=[ - 0, - ] - ), - ), - ( - TransposeCodec( - order=[ - 0, - ] - ), - TransposeCodec( - order=[ - 0, - ] - ), - ), - TransposeCodec( - order=[ - 0, - ] - ), - {"name": "transpose", "configuration": {"order": [0]}}, - ({"name": "transpose", "configuration": {"order": [0]}},), - ], -) -@pytest.mark.parametrize(("chunks", "shards"), [((6,), None), ((3,), (6,))]) -async def test_create_array_v3_chunk_encoding( - store: MemoryStore, - compressors: CompressorsLike, - filters: FiltersLike, - dtype: str, - chunks: tuple[int, ...], - shards: tuple[int, ...] | None, + np.testing.assert_array_equal(result2[:], src[:]) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) +@pytest.mark.parametrize("store2", ["local", "memory", "zip"], indirect=["store2"]) +@pytest.mark.parametrize("src_chunks", [(40, 10), (11, 50)]) +@pytest.mark.parametrize("new_chunks", [(40, 10), (11, 50)]) +async def test_from_array( + store: Store, + store2: Store, + src_chunks: tuple[int, int], + new_chunks: tuple[int, int], + zarr_format: ZarrFormat, ) -> None: - """ - Test various possibilities for the compressors and filters parameter to create_array - """ - arr = await create_array( + src_fill_value = 2 + src_dtype = np.dtype("uint8") + src_attributes = None + + src = zarr.create( + (100, 10), + chunks=src_chunks, + dtype=src_dtype, store=store, - dtype=dtype, - shape=(12,), - chunks=chunks, - shards=shards, - zarr_format=3, - filters=filters, - compressors=compressors, + fill_value=src_fill_value, + attributes=src_attributes, ) - filters_expected, _, compressors_expected = _parse_chunk_encoding_v3( - filters=filters, compressors=compressors, serializer="auto", dtype=np.dtype(dtype) + src[:] = np.arange(1000).reshape((100, 10)) + + new_fill_value = 3 + new_attributes: dict[str, JSON] = {"foo": "bar"} + + result = zarr.from_array( + data=src, + store=store2, + chunks=new_chunks, + fill_value=new_fill_value, + attributes=new_attributes, ) - assert arr.filters == filters_expected - assert arr.compressors == compressors_expected + np.testing.assert_array_equal(result[:], src[:]) + assert result.fill_value == new_fill_value + assert result.dtype == src_dtype + assert result.attrs == new_attributes + assert result.chunks == new_chunks -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) + +@pytest.mark.parametrize("store", ["local"], indirect=True) +@pytest.mark.parametrize("chunks", ["keep", "auto"]) +@pytest.mark.parametrize("write_data", [True, False]) @pytest.mark.parametrize( - "compressors", + "src", [ - "auto", - None, - numcodecs.Zstd(level=3), - (), - (numcodecs.Zstd(level=3),), + np.arange(1000).reshape(10, 10, 10), + zarr.ones((10, 10, 10)), + 5, + [1, 2, 3], + [[1, 2, 3], [4, 5, 6]], ], -) -@pytest.mark.parametrize( - "filters", ["auto", None, numcodecs.GZip(level=1), (numcodecs.GZip(level=1),)] -) -async def test_create_array_v2_chunk_encoding( - store: MemoryStore, compressors: CompressorsLike, filters: FiltersLike, dtype: str +) # add other npt.ArrayLike? +async def test_from_array_arraylike( + store: Store, + chunks: Literal["auto", "keep"] | tuple[int, int], + write_data: bool, + src: Array | npt.ArrayLike, ) -> None: - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=2, - compressors=compressors, - filters=filters, + fill_value = 42 + result = zarr.from_array( + store, data=src, chunks=chunks, write_data=write_data, fill_value=fill_value ) - filters_expected, compressor_expected = _parse_chunk_encoding_v2( - filters=filters, compressor=compressors, dtype=np.dtype(dtype) + if write_data: + np.testing.assert_array_equal(result[...], np.array(src)) + else: + np.testing.assert_array_equal(result[...], np.full_like(src, fill_value)) + + +async def test_orthogonal_set_total_slice() -> None: + """Ensure that a whole chunk overwrite does not read chunks""" + store = MemoryStore() + array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1) + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[0, slice(4, 10)] = np.arange(6) + + array = zarr.create_array( + store, shape=(20, 21), chunks=(1, 2), dtype=int, fill_value=-1, overwrite=True ) - assert arr.metadata.zarr_format == 2 # guard for mypy - assert arr.metadata.compressor == compressor_expected - assert arr.metadata.filters == filters_expected + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[0, :] = np.arange(21) - # Normalize for property getters - compressor_expected = () if compressor_expected is None else (compressor_expected,) - filters_expected = () if filters_expected is None else filters_expected + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[:] = 1 - assert arr.compressors == compressor_expected - assert arr.filters == filters_expected +@pytest.mark.skipif( + Version(numcodecs.__version__) < Version("0.15.1"), + reason="codec configuration is overwritten on older versions. GH2800", +) +def test_roundtrip_numcodecs() -> None: + store = MemoryStore() -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"]) -async def test_create_array_v3_default_filters_compressors(store: MemoryStore, dtype: str) -> None: - """ - Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with - ``zarr_format`` = 3 and ``filters`` and ``compressors`` are not specified. - """ - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=3, - ) - expected_filters, expected_serializer, expected_compressors = _get_default_chunk_encoding_v3( - np_dtype=np.dtype(dtype) + compressors = [ + {"name": "numcodecs.shuffle", "configuration": {"elementsize": 2}}, + {"name": "numcodecs.zlib", "configuration": {"level": 4}}, + ] + filters = [ + { + "name": "numcodecs.fixedscaleoffset", + "configuration": { + "scale": 100.0, + "offset": 0.0, + "dtype": " None: - """ - Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with - ``zarr_format`` = 2 and ``filters`` and ``compressors`` are not specified. - """ - arr = await create_array( - store=store, - dtype=dtype, - shape=(10,), - zarr_format=2, - ) - expected_filters, expected_compressors = _get_default_chunk_encoding_v2( - np_dtype=np.dtype(dtype) - ) - assert arr.metadata.zarr_format == 2 # guard for mypy - assert arr.metadata.filters == expected_filters - assert arr.metadata.compressor == expected_compressors - # Normalize for property getters - expected_filters = () if expected_filters is None else expected_filters - expected_compressors = () if expected_compressors is None else (expected_compressors,) - assert arr.filters == expected_filters - assert arr.compressors == expected_compressors +def _index_array(arr: Array, index: Any) -> Any: + return arr[index] -@pytest.mark.parametrize("store", ["memory"], indirect=True) -async def test_create_array_v2_no_shards(store: MemoryStore) -> None: +@pytest.mark.parametrize( + "method", + [ + pytest.param( + "fork", + marks=pytest.mark.skipif( + sys.platform in ("win32", "darwin"), reason="fork not supported on Windows or OSX" + ), + ), + "spawn", + pytest.param( + "forkserver", + marks=pytest.mark.skipif( + sys.platform == "win32", reason="forkserver not supported on Windows" + ), + ), + ], +) +@pytest.mark.parametrize("store", ["local"], indirect=True) +def test_multiprocessing(store: Store, method: Literal["fork", "spawn", "forkserver"]) -> None: """ - Test that creating a Zarr v2 array with ``shard_shape`` set to a non-None value raises an error. + Test that arrays can be pickled and indexed in child processes """ - msg = re.escape( - "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. Got `shard_shape=(5,)` instead." + data = np.arange(100) + arr = zarr.create_array(store=store, data=data) + ctx = mp.get_context(method) + with ctx.Pool() as pool: + results = pool.starmap(_index_array, [(arr, slice(len(data)))]) + assert all(np.array_equal(r, data) for r in results) + + +async def test_sharding_coordinate_selection() -> None: + store = MemoryStore() + g = zarr.open_group(store, mode="w") + arr = g.create_array( + name="a", + shape=(2, 3, 4), + chunks=(1, 2, 2), + overwrite=True, + dtype=np.float32, + shards=(2, 4, 4), ) - with pytest.raises(ValueError, match=msg): - _ = await create_array( - store=store, - dtype="uint8", - shape=(10,), - shards=(5,), - zarr_format=2, - ) + arr[:] = np.arange(2 * 3 * 4).reshape((2, 3, 4)) + result = arr[1, [0, 1]] # type: ignore[index] + assert isinstance(result, NDArrayLike) + assert (result == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all() -async def test_scalar_array() -> None: - arr = zarr.array(1.5) - assert arr[...] == 1.5 - assert arr[()] == 1.5 - assert arr.shape == () +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_array_repr(store: Store) -> None: + shape = (2, 3, 4) + dtype = "uint8" + arr = zarr.create_array(store, shape=shape, dtype=dtype) + assert str(arr) == f"" diff --git a/tests/test_attributes.py b/tests/test_attributes.py index b26db5df89..127b2dbc36 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -1,3 +1,5 @@ +import pytest + import zarr.core import zarr.core.attributes import zarr.storage @@ -20,3 +22,63 @@ def test_asdict() -> None: ) result = attrs.asdict() assert result == {"a": 1, "b": 2} + + +def test_update_attributes_preserves_existing() -> None: + """ + Test that `update_attributes` only updates the specified attributes + and preserves existing ones. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + z.attrs["a"] = [] + z.attrs["b"] = 3 + assert dict(z.attrs) == {"a": [], "b": 3} + + z.update_attributes({"a": [3, 4], "c": 4}) + assert dict(z.attrs) == {"a": [3, 4], "b": 3, "c": 4} + + +def test_update_empty_attributes() -> None: + """ + Ensure updating when initial attributes are empty works. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + assert dict(z.attrs) == {} + z.update_attributes({"a": [3, 4], "c": 4}) + assert dict(z.attrs) == {"a": [3, 4], "c": 4} + + +def test_update_no_changes() -> None: + """ + Ensure updating when no new or modified attributes does not alter existing ones. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + z.attrs["a"] = [] + z.attrs["b"] = 3 + + z.update_attributes({}) + assert dict(z.attrs) == {"a": [], "b": 3} + + +@pytest.mark.parametrize("group", [True, False]) +def test_del_works(group: bool) -> None: + store = zarr.storage.MemoryStore() + z: zarr.Group | zarr.Array + if group: + z = zarr.create_group(store) + else: + z = zarr.create_array(store=store, shape=10, dtype=int) + assert dict(z.attrs) == {} + z.update_attributes({"a": [3, 4], "c": 4}) + del z.attrs["a"] + assert dict(z.attrs) == {"c": 4} + + z2: zarr.Group | zarr.Array + if group: + z2 = zarr.open_group(store) + else: + z2 = zarr.open_array(store) + assert dict(z2.attrs) == {"c": 4} diff --git a/tests/test_buffer.py b/tests/test_buffer.py index baef0b8109..93b116e908 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -6,12 +6,13 @@ import pytest import zarr +from zarr.abc.buffer import ArrayLike, BufferPrototype, NDArrayLike +from zarr.buffer import cpu, gpu from zarr.codecs.blosc import BloscCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec -from zarr.core.buffer import ArrayLike, BufferPrototype, NDArrayLike, cpu, gpu from zarr.storage import MemoryStore, StorePath from zarr.testing.buffer import ( NDBufferUsingTestNDArrayLike, @@ -30,6 +31,8 @@ cp = None +import zarr.api.asynchronous + if TYPE_CHECKING: import types @@ -64,7 +67,7 @@ async def test_async_array_prototype() -> None: got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=my_prototype) # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol # The test passes, so it clearly does. - assert isinstance(got, TestNDArrayLike) # type: ignore[unreachable] + assert isinstance(got, TestNDArrayLike) assert np.array_equal(expect, got) # type: ignore[unreachable] @@ -117,7 +120,7 @@ async def test_codecs_use_of_prototype() -> None: got = await a.getitem(selection=(slice(0, 10), slice(0, 10)), prototype=my_prototype) # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol # The test passes, so it clearly does. - assert isinstance(got, TestNDArrayLike) # type: ignore[unreachable] + assert isinstance(got, TestNDArrayLike) assert np.array_equal(expect, got) # type: ignore[unreachable] @@ -146,8 +149,54 @@ async def test_codecs_use_of_gpu_prototype() -> None: assert cp.array_equal(expect, got) +@gpu_test +@pytest.mark.asyncio +async def test_sharding_use_of_gpu_prototype() -> None: + with zarr.config.enable_gpu(): + expect = cp.zeros((10, 10), dtype="uint16", order="F") + + a = await zarr.api.asynchronous.create_array( + StorePath(MemoryStore()) / "test_codecs_use_of_gpu_prototype", + shape=expect.shape, + chunks=(5, 5), + shards=(10, 10), + dtype=expect.dtype, + fill_value=0, + ) + expect[:] = cp.arange(100).reshape(10, 10) + + await a.setitem( + selection=(slice(0, 10), slice(0, 10)), + value=expect[:], + prototype=gpu.buffer_prototype, + ) + got = await a.getitem( + selection=(slice(0, 10), slice(0, 10)), prototype=gpu.buffer_prototype + ) + assert isinstance(got, cp.ndarray) + assert cp.array_equal(expect, got) + + def test_numpy_buffer_prototype() -> None: buffer = cpu.buffer_prototype.buffer.create_zero_length() ndbuffer = cpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) assert isinstance(buffer.as_array_like(), np.ndarray) assert isinstance(ndbuffer.as_ndarray_like(), np.ndarray) + with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): + ndbuffer.as_scalar() + + +@gpu_test +def test_gpu_buffer_prototype() -> None: + buffer = gpu.buffer_prototype.buffer.create_zero_length() + ndbuffer = gpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=cp.dtype("int64")) + assert isinstance(buffer.as_array_like(), cp.ndarray) + assert isinstance(ndbuffer.as_ndarray_like(), cp.ndarray) + with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): + ndbuffer.as_scalar() + + +# TODO: the same test for other buffer classes +def test_cpu_buffer_as_scalar() -> None: + buf = cpu.buffer_prototype.nd_buffer.create(shape=(), dtype="int64") + assert buf.as_scalar() == buf.as_ndarray_like()[()] # type: ignore[index] diff --git a/tests/test_codecs/test_blosc.py b/tests/test_codecs/test_blosc.py index c1c5c92329..6e6e9df383 100644 --- a/tests/test_codecs/test_blosc.py +++ b/tests/test_codecs/test_blosc.py @@ -1,7 +1,9 @@ import json +import numcodecs import numpy as np import pytest +from packaging.version import Version import zarr from zarr.abc.store import Store @@ -54,3 +56,20 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None: assert blosc_configuration_json["shuffle"] == "bitshuffle" else: assert blosc_configuration_json["shuffle"] == "shuffle" + + +async def test_typesize() -> None: + a = np.arange(1000000, dtype=np.uint64) + codecs = [zarr.codecs.BytesCodec(), zarr.codecs.BloscCodec()] + z = zarr.array(a, chunks=(10000), codecs=codecs) + data = await z.store.get("c/0", prototype=default_buffer_prototype()) + assert data is not None + bytes = data.to_bytes() + size = len(bytes) + msg = f"Blosc size mismatch. First 10 bytes: {bytes[:20]!r} and last 10 bytes: {bytes[-20:]!r}" + if Version(numcodecs.__version__) >= Version("0.16.0"): + expected_size = 402 + assert size == expected_size, msg + else: + expected_size = 10216 + assert size == expected_size, msg diff --git a/tests/test_codecs/test_codecs.py b/tests/test_codecs/test_codecs.py index e36a332440..468f395254 100644 --- a/tests/test_codecs/test_codecs.py +++ b/tests/test_codecs/test_codecs.py @@ -2,7 +2,7 @@ import json from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np import pytest @@ -18,32 +18,33 @@ TransposeCodec, ) from zarr.core.buffer import default_buffer_prototype -from zarr.core.indexing import Selection, morton_order_iter +from zarr.core.indexing import BasicSelection, morton_order_iter +from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.storage import StorePath if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.buffer.core import NDArrayLike - from zarr.core.common import MemoryOrder + from zarr.core.buffer.core import NDArrayLikeOrScalar + from zarr.core.common import ChunkCoords, MemoryOrder @dataclass(frozen=True) class _AsyncArrayProxy: - array: AsyncArray + array: AsyncArray[Any] - def __getitem__(self, selection: Selection) -> _AsyncArraySelectionProxy: + def __getitem__(self, selection: BasicSelection) -> _AsyncArraySelectionProxy: return _AsyncArraySelectionProxy(self.array, selection) @dataclass(frozen=True) class _AsyncArraySelectionProxy: - array: AsyncArray - selection: Selection + array: AsyncArray[Any] + selection: BasicSelection - async def get(self) -> NDArrayLike: + async def get(self) -> NDArrayLikeOrScalar: return await self.array.getitem(self.selection) - async def set(self, value: np.ndarray) -> None: + async def set(self, value: np.ndarray[Any, Any]) -> None: return await self.array.setitem(self.selection, value) @@ -101,6 +102,7 @@ async def test_order( read_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, read_data) + assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] @@ -142,6 +144,7 @@ def test_order_implicit( read_data = a[:, :] assert np.array_equal(data, read_data) + assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] @@ -209,7 +212,7 @@ def test_morton() -> None: [3, 2, 1, 6, 4, 5, 2], ], ) -def test_morton2(shape) -> None: +def test_morton2(shape: ChunkCoords) -> None: order = list(morton_order_iter(shape)) for i, x in enumerate(order): assert x not in order[:i] # no duplicates @@ -263,7 +266,10 @@ async def test_dimension_names(store: Store) -> None: dimension_names=("x", "y"), ) - assert (await zarr.api.asynchronous.open_array(store=spath)).metadata.dimension_names == ( + assert isinstance( + meta := (await zarr.api.asynchronous.open_array(store=spath)).metadata, ArrayV3Metadata + ) + assert meta.dimension_names == ( "x", "y", ) @@ -277,7 +283,8 @@ async def test_dimension_names(store: Store) -> None: fill_value=0, ) - assert (await AsyncArray.open(spath2)).metadata.dimension_names is None + assert isinstance(meta := (await AsyncArray.open(spath2)).metadata, ArrayV3Metadata) + assert meta.dimension_names is None zarr_json_buffer = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype()) assert zarr_json_buffer is not None assert "dimension_names" not in json.loads(zarr_json_buffer.to_bytes()) diff --git a/tests/test_codecs/test_endian.py b/tests/test_codecs/test_endian.py index c0c4dd4e75..ab64afb1b8 100644 --- a/tests/test_codecs/test_endian.py +++ b/tests/test_codecs/test_endian.py @@ -11,6 +11,7 @@ from .test_codecs import _AsyncArrayProxy +@pytest.mark.filterwarnings("ignore:The endianness of the requested serializer") @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("endian", ["big", "little"]) async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: @@ -32,6 +33,7 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: assert np.array_equal(data, readback_data) +@pytest.mark.filterwarnings("ignore:The endianness of the requested serializer") @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("dtype_input_endian", [">u2", " None: - bstrings = [b"hello", b"world", b"this", b"is", b"a", b"test"] - data = np.array(bstrings).reshape((2, 3)) - assert data.dtype == "|S5" - - sp = StorePath(store, path="string") - a = zarr.create_array( - sp, - shape=data.shape, - chunks=data.shape, - dtype=data.dtype, - fill_value=b"", - compressors=compressor, - ) - assert isinstance(a.metadata, ArrayV3Metadata) # needed for mypy - - # should also work if input array is an object array, provided we explicitly specified - # a bytesting-like dtype when creating the Array - if as_object_array: - data = data.astype("O") - a[:, :] = data - assert np.array_equal(data, a[:, :]) - assert a.metadata.data_type == DataType.bytes - assert a.dtype == "O" - - # test round trip - b = Array.open(sp) - assert isinstance(b.metadata, ArrayV3Metadata) # needed for mypy - assert np.array_equal(data, b[:, :]) - assert b.metadata.data_type == DataType.bytes - assert a.dtype == "O" + assert b.metadata.data_type == get_data_type_from_native_dtype(data.dtype) + assert a.dtype == data.dtype diff --git a/tests/test_config.py b/tests/test_config.py index c552ace840..ed778a02ae 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ import os from collections.abc import Iterable -from typing import Any +from typing import TYPE_CHECKING, Any from unittest import mock from unittest.mock import Mock @@ -10,7 +10,7 @@ import zarr import zarr.api from zarr import zeros -from zarr.abc.codec import CodecInput, CodecOutput, CodecPipeline +from zarr.abc.codec import CodecPipeline from zarr.abc.store import ByteSetter, Store from zarr.codecs import ( BloscCodec, @@ -19,10 +19,13 @@ GzipCodec, ShardingCodec, ) +from zarr.core.array import create_array from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer +from zarr.core.buffer.core import Buffer from zarr.core.codec_pipeline import BatchedCodecPipeline from zarr.core.config import BadConfigError, config +from zarr.core.dtype import Int8, VariableLengthUTF8 from zarr.core.indexing import SelectorTuple from zarr.registry import ( fully_qualified_name, @@ -43,66 +46,66 @@ TestNDArrayLike, ) +if TYPE_CHECKING: + from zarr.core.dtype.wrapper import ZDType + def test_config_defaults_set() -> None: # regression test for available defaults - assert config.defaults == [ - { - "default_zarr_format": 3, - "array": { - "order": "C", - "write_empty_chunks": False, - "v2_default_compressor": { - "numeric": {"id": "zstd", "level": 0, "checksum": False}, - "string": {"id": "zstd", "level": 0, "checksum": False}, - "bytes": {"id": "zstd", "level": 0, "checksum": False}, - }, - "v2_default_filters": { - "numeric": None, - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], + assert ( + config.defaults + == [ + { + "default_zarr_format": 3, + "array": { + "order": "C", + "write_empty_chunks": False, + "v2_default_compressor": { + "default": {"id": "zstd", "level": 0, "checksum": False}, + "variable-length-string": {"id": "zstd", "level": 0, "checksum": False}, + }, + "v2_default_filters": { + "default": None, + "variable-length-string": [{"id": "vlen-utf8"}], + }, + "v3_default_filters": {"default": [], "variable-length-string": []}, + "v3_default_serializer": { + "default": {"name": "bytes", "configuration": {"endian": "little"}}, + "variable-length-string": {"name": "vlen-utf8"}, + }, + "v3_default_compressors": { + "default": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, + ], + "variable-length-string": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}} + ], + }, }, - "v3_default_filters": {"numeric": [], "string": [], "bytes": []}, - "v3_default_serializer": { - "numeric": {"name": "bytes", "configuration": {"endian": "little"}}, - "string": {"name": "vlen-utf8"}, - "bytes": {"name": "vlen-bytes"}, + "async": {"concurrency": 10, "timeout": None}, + "threading": {"max_workers": None}, + "json_indent": 2, + "codec_pipeline": { + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", + "batch_size": 1, }, - "v3_default_compressors": { - "numeric": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], - "string": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], - "bytes": [ - {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, - ], + "codecs": { + "blosc": "zarr.codecs.blosc.BloscCodec", + "gzip": "zarr.codecs.gzip.GzipCodec", + "zstd": "zarr.codecs.zstd.ZstdCodec", + "bytes": "zarr.codecs.bytes.BytesCodec", + "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 + "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", + "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", + "transpose": "zarr.codecs.transpose.TransposeCodec", + "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", + "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", }, - }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, - "json_indent": 2, - "codec_pipeline": { - "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", - "batch_size": 1, - }, - "buffer": "zarr.core.buffer.cpu.Buffer", - "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", - "codecs": { - "blosc": "zarr.codecs.blosc.BloscCodec", - "gzip": "zarr.codecs.gzip.GzipCodec", - "zstd": "zarr.codecs.zstd.ZstdCodec", - "bytes": "zarr.codecs.bytes.BytesCodec", - "endian": "zarr.codecs.bytes.BytesCodec", - "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", - "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", - "transpose": "zarr.codecs.transpose.TransposeCodec", - "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", - "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", - }, - } - ] + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", + } + ] + ) assert config.get("array.order") == "C" assert config.get("async.concurrency") == 10 assert config.get("async.timeout") is None @@ -143,7 +146,7 @@ def test_config_codec_pipeline_class(store: Store) -> None: class MockCodecPipeline(BatchedCodecPipeline): async def write( self, - batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]], + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: @@ -173,7 +176,7 @@ async def write( class MockEnvCodecPipeline(CodecPipeline): pass - register_pipeline(MockEnvCodecPipeline) + register_pipeline(MockEnvCodecPipeline) # type: ignore[type-abstract] with mock.patch.dict( os.environ, {"ZARR_CODEC_PIPELINE__PATH": fully_qualified_name(MockEnvCodecPipeline)} @@ -190,10 +193,9 @@ def test_config_codec_implementation(store: Store) -> None: _mock = Mock() class MockBloscCodec(BloscCodec): - async def _encode_single( - self, chunk_data: CodecInput, chunk_spec: ArraySpec - ) -> CodecOutput | None: + async def _encode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer | None: _mock.call() + return None register_codec("blosc", MockBloscCodec) with config.set({"codecs.blosc": fully_qualified_name(MockBloscCodec)}): @@ -222,9 +224,6 @@ class NewBloscCodec(BloscCodec): @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_ndbuffer_implementation(store: Store) -> None: - # has default value - assert fully_qualified_name(get_ndbuffer_class()) == config.defaults[0]["ndbuffer"] - # set custom ndbuffer with TestNDArrayLike implementation register_ndbuffer(NDBufferUsingTestNDArrayLike) with config.set({"ndbuffer": fully_qualified_name(NDBufferUsingTestNDArrayLike)}): @@ -242,9 +241,9 @@ def test_config_ndbuffer_implementation(store: Store) -> None: def test_config_buffer_implementation() -> None: # has default value - assert fully_qualified_name(get_buffer_class()) == config.defaults[0]["buffer"] + assert config.defaults[0]["buffer"] == "zarr.buffer.cpu.Buffer" - arr = zeros(shape=(100), store=StoreExpectingTestBuffer()) + arr = zeros(shape=(100,), store=StoreExpectingTestBuffer()) # AssertionError of StoreExpectingTestBuffer when not using my buffer with pytest.raises(AssertionError): @@ -277,6 +276,27 @@ def test_config_buffer_implementation() -> None: assert np.array_equal(arr_Crc32c[:], data2d) +def test_config_buffer_backwards_compatibility() -> None: + # This should warn once zarr.core is private + # https://github.com/zarr-developers/zarr-python/issues/2621 + with zarr.config.set( + {"buffer": "zarr.core.buffer.cpu.Buffer", "ndbuffer": "zarr.core.buffer.cpu.NDBuffer"} + ): + get_buffer_class() + get_ndbuffer_class() + + +@pytest.mark.gpu +def test_config_buffer_backwards_compatibility_gpu() -> None: + # This should warn once zarr.core is private + # https://github.com/zarr-developers/zarr-python/issues/2621 + with zarr.config.set( + {"buffer": "zarr.core.buffer.gpu.Buffer", "ndbuffer": "zarr.core.buffer.gpu.NDBuffer"} + ): + get_buffer_class() + get_ndbuffer_class() + + @pytest.mark.filterwarnings("error") def test_warning_on_missing_codec_config() -> None: class NewCodec(BytesCodec): @@ -303,28 +323,29 @@ class NewCodec2(BytesCodec): get_codec_class("new_codec") -@pytest.mark.parametrize("dtype", ["int", "bytes", "str"]) -async def test_default_codecs(dtype: str) -> None: - with config.set( - { - "array.v3_default_compressors": { # test setting non-standard codecs - "numeric": [ - {"name": "gzip", "configuration": {"level": 5}}, - ], - "string": [ - {"name": "gzip", "configuration": {"level": 5}}, - ], - "bytes": [ - {"name": "gzip", "configuration": {"level": 5}}, - ], - } - } - ): - arr = await zarr.api.asynchronous.create_array( +@pytest.mark.parametrize("dtype_category", ["variable-length-string", "default"]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +async def test_default_codecs(dtype_category: str) -> None: + """ + Test that the default compressors are sensitive to the current setting of the config. + """ + zdtype: ZDType[Any, Any] + if dtype_category == "variable-length-string": + zdtype = VariableLengthUTF8() + else: + zdtype = Int8() + expected_compressors = (GzipCodec(),) + new_conf = { + f"array.v3_default_compressors.{dtype_category}": [ + c.to_dict() for c in expected_compressors + ] + } + with config.set(new_conf): + arr = await create_array( shape=(100,), chunks=(100,), - dtype=np.dtype(dtype), + dtype=zdtype, zarr_format=3, store=MemoryStore(), ) - assert arr.compressors == (GzipCodec(),) + assert arr.compressors == expected_compressors diff --git a/tests/test_dtype/__init__.py b/tests/test_dtype/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_dtype/conftest.py b/tests/test_dtype/conftest.py new file mode 100644 index 0000000000..0be1c60088 --- /dev/null +++ b/tests/test_dtype/conftest.py @@ -0,0 +1,71 @@ +# Generate a collection of zdtype instances for use in testing. +import warnings +from typing import Any + +import numpy as np + +from zarr.core.dtype import data_type_registry +from zarr.core.dtype.common import HasLength +from zarr.core.dtype.npy.structured import Structured +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 +from zarr.core.dtype.wrapper import ZDType + +zdtype_examples: tuple[ZDType[Any, Any], ...] = () +for wrapper_cls in data_type_registry.contents.values(): + # The Structured dtype has to be constructed with some actual fields + if wrapper_cls is Structured: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zdtype_examples += ( + wrapper_cls.from_native_dtype(np.dtype([("a", np.float64), ("b", np.int8)])), + ) + elif issubclass(wrapper_cls, HasLength): + zdtype_examples += (wrapper_cls(length=1),) + elif issubclass(wrapper_cls, DateTime64 | TimeDelta64): + zdtype_examples += (wrapper_cls(unit="s", scale_factor=10),) + else: + zdtype_examples += (wrapper_cls(),) + + +def pytest_generate_tests(metafunc: Any) -> None: + """ + This is a pytest hook to parametrize class-scoped fixtures. + + This hook allows us to define class-scoped fixtures as class attributes and then + generate the parametrize calls for pytest. This allows the fixtures to be + reused across multiple tests within the same class. + + For example, if you had a regular pytest class like this: + + class TestClass: + @pytest.mark.parametrize("param_a", [1, 2, 3]) + def test_method(self, param_a): + ... + + Child classes inheriting from ``TestClass`` would not be able to override the ``param_a`` fixture + + this implementation of ``pytest_generate_tests`` allows you to define class-scoped fixtures as + class attributes, which allows the following to work: + + class TestExample: + param_a = [1, 2, 3] + + def test_example(self, param_a): + ... + + # this class will have its test_example method parametrized with the values of TestB.param_a + class TestB(TestExample): + param_a = [1, 2, 100, 10] + + """ + # Iterate over all the fixtures defined in the class + # and parametrize them with the values defined in the class + # This allows us to define class-scoped fixtures as class attributes + # and then generate the parametrize calls for pytest + for fixture_name in metafunc.fixturenames: + if hasattr(metafunc.cls, fixture_name): + params = getattr(metafunc.cls, fixture_name) + if len(params) == 0: + msg = f"{metafunc.cls}.{fixture_name} is empty. Please provide a non-empty sequence of values." + raise ValueError(msg) + metafunc.parametrize(fixture_name, params, scope="class") diff --git a/tests/test_dtype/test_npy/test_bool.py b/tests/test_dtype/test_npy/test_bool.py new file mode 100644 index 0000000000..010dec2e47 --- /dev/null +++ b/tests/test_dtype/test_npy/test_bool.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.bool import Bool + + +class TestBool(BaseTestZDType): + test_cls = Bool + + valid_dtype = (np.dtype(np.bool_),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype(np.uint16), + ) + valid_json_v2 = ({"name": "|b1", "object_codec_id": None},) + valid_json_v3 = ("bool",) + invalid_json_v2 = ( + "|b1", + "bool", + "|f8", + ) + invalid_json_v3 = ( + "|b1", + "|f8", + {"name": "bool", "configuration": {"endianness": "little"}}, + ) + + scalar_v2_params = ((Bool(), True), (Bool(), False)) + scalar_v3_params = ((Bool(), True), (Bool(), False)) + + cast_value_params = ( + (Bool(), "true", np.True_), + (Bool(), True, np.True_), + (Bool(), False, np.False_), + (Bool(), np.True_, np.True_), + (Bool(), np.False_, np.False_), + ) + item_size_params = (Bool(),) diff --git a/tests/test_dtype/test_npy/test_bytes.py b/tests/test_dtype/test_npy/test_bytes.py new file mode 100644 index 0000000000..b7c16f573e --- /dev/null +++ b/tests/test_dtype/test_npy/test_bytes.py @@ -0,0 +1,154 @@ +import numpy as np +import pytest + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.common import UnstableSpecificationWarning +from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes + + +class TestNullTerminatedBytes(BaseTestZDType): + test_cls = NullTerminatedBytes + valid_dtype = (np.dtype("|S10"), np.dtype("|S4")) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|U10"), + ) + valid_json_v2 = ( + {"name": "|S0", "object_codec_id": None}, + {"name": "|S2", "object_codec_id": None}, + {"name": "|S4", "object_codec_id": None}, + ) + valid_json_v3 = ({"name": "null_terminated_bytes", "configuration": {"length_bytes": 10}},) + invalid_json_v2 = ( + "|S", + "|U10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, + {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, + ) + + scalar_v2_params = ( + (NullTerminatedBytes(length=0), ""), + (NullTerminatedBytes(length=2), "YWI="), + (NullTerminatedBytes(length=4), "YWJjZA=="), + ) + scalar_v3_params = ( + (NullTerminatedBytes(length=0), ""), + (NullTerminatedBytes(length=2), "YWI="), + (NullTerminatedBytes(length=4), "YWJjZA=="), + ) + cast_value_params = ( + (NullTerminatedBytes(length=0), "", np.bytes_("")), + (NullTerminatedBytes(length=2), "ab", np.bytes_("ab")), + (NullTerminatedBytes(length=4), "abcdefg", np.bytes_("abcd")), + ) + item_size_params = ( + NullTerminatedBytes(length=0), + NullTerminatedBytes(length=4), + NullTerminatedBytes(length=10), + ) + + +class TestRawBytes(BaseTestZDType): + test_cls = RawBytes + valid_dtype = (np.dtype("|V10"),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|S10"), + ) + valid_json_v2 = ({"name": "|V10", "object_codec_id": None},) + valid_json_v3 = ( + {"name": "raw_bytes", "configuration": {"length_bytes": 0}}, + {"name": "raw_bytes", "configuration": {"length_bytes": 8}}, + ) + + invalid_json_v2 = ( + "|V", + "|S10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "r10"}, + {"name": "r-80"}, + ) + + scalar_v2_params = ( + (RawBytes(length=0), ""), + (RawBytes(length=2), "YWI="), + (RawBytes(length=4), "YWJjZA=="), + ) + scalar_v3_params = ( + (RawBytes(length=0), ""), + (RawBytes(length=2), "YWI="), + (RawBytes(length=4), "YWJjZA=="), + ) + cast_value_params = ( + (RawBytes(length=0), b"", np.void(b"")), + (RawBytes(length=2), b"ab", np.void(b"ab")), + (RawBytes(length=4), b"abcd", np.void(b"abcd")), + ) + item_size_params = ( + RawBytes(length=0), + RawBytes(length=4), + RawBytes(length=10), + ) + + +class TestVariableLengthBytes(BaseTestZDType): + test_cls = VariableLengthBytes + valid_dtype = (np.dtype("|O"),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|U10"), + ) + valid_json_v2 = ({"name": "|O", "object_codec_id": "vlen-bytes"},) + valid_json_v3 = ("variable_length_bytes",) + invalid_json_v2 = ( + "|S", + "|U10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, + {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, + ) + + scalar_v2_params = ( + (VariableLengthBytes(), ""), + (VariableLengthBytes(), "YWI="), + (VariableLengthBytes(), "YWJjZA=="), + ) + scalar_v3_params = ( + (VariableLengthBytes(), ""), + (VariableLengthBytes(), "YWI="), + (VariableLengthBytes(), "YWJjZA=="), + ) + cast_value_params = ( + (VariableLengthBytes(), "", b""), + (VariableLengthBytes(), "ab", b"ab"), + (VariableLengthBytes(), "abcdefg", b"abcdefg"), + ) + item_size_params = ( + VariableLengthBytes(), + VariableLengthBytes(), + VariableLengthBytes(), + ) + + +@pytest.mark.parametrize( + "zdtype", [NullTerminatedBytes(length=10), RawBytes(length=10), VariableLengthBytes()] +) +def test_unstable_dtype_warning( + zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, +) -> None: + """ + Test that we get a warning when serializing a dtype without a zarr v3 spec to json + when zarr_format is 3 + """ + with pytest.raises(UnstableSpecificationWarning): + zdtype.to_json(zarr_format=3) diff --git a/tests/test_dtype/test_npy/test_common.py b/tests/test_dtype/test_npy/test_common.py new file mode 100644 index 0000000000..d39d308112 --- /dev/null +++ b/tests/test_dtype/test_npy/test_common.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import base64 +import math +import re +import sys +from typing import TYPE_CHECKING, Any, get_args + +import numpy as np +import pytest + +from zarr.core.dtype.common import ENDIANNESS_STR, JSONFloatV2, SpecialFloatStrings +from zarr.core.dtype.npy.common import ( + NumpyEndiannessStr, + bytes_from_json, + bytes_to_json, + check_json_bool, + check_json_complex_float_v2, + check_json_complex_float_v3, + check_json_float_v2, + check_json_float_v3, + check_json_int, + check_json_str, + complex_float_to_json_v2, + complex_float_to_json_v3, + endianness_from_numpy_str, + endianness_to_numpy_str, + float_from_json_v2, + float_from_json_v3, + float_to_json_v2, + float_to_json_v3, +) + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +def nan_equal(a: object, b: object) -> bool: + """ + Convenience function for equality comparison between two values ``a`` and ``b``, that might both + be NaN. Returns True if both ``a`` and ``b`` are NaN, otherwise returns a == b + """ + if math.isnan(a) and math.isnan(b): # type: ignore[arg-type] + return True + return a == b + + +json_float_v2_roundtrip_cases: tuple[tuple[JSONFloatV2, float | np.floating[Any]], ...] = ( + ("Infinity", float("inf")), + ("Infinity", np.inf), + ("-Infinity", float("-inf")), + ("-Infinity", -np.inf), + ("NaN", float("nan")), + ("NaN", np.nan), + (1.0, 1.0), +) + +json_float_v3_cases = json_float_v2_roundtrip_cases + + +@pytest.mark.parametrize( + ("data", "expected"), + [(">", "big"), ("<", "little"), ("=", sys.byteorder), ("|", None), ("err", "")], +) +def test_endianness_from_numpy_str(data: str, expected: str | None) -> None: + """ + Test that endianness_from_numpy_str correctly converts a numpy str literal to a human-readable literal value. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data in get_args(NumpyEndiannessStr): + assert endianness_from_numpy_str(data) == expected # type: ignore[arg-type] + else: + msg = f"Invalid endianness: {data!r}. Expected one of {get_args(NumpyEndiannessStr)}" + with pytest.raises(ValueError, match=re.escape(msg)): + endianness_from_numpy_str(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), + [("big", ">"), ("little", "<"), (None, "|"), ("err", "")], +) +def test_endianness_to_numpy_str(data: str | None, expected: str) -> None: + """ + Test that endianness_to_numpy_str correctly converts a human-readable literal value to a numpy str literal. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data in ENDIANNESS_STR: + assert endianness_to_numpy_str(data) == expected # type: ignore[arg-type] + else: + msg = f"Invalid endianness: {data!r}. Expected one of {ENDIANNESS_STR}" + with pytest.raises(ValueError, match=re.escape(msg)): + endianness_to_numpy_str(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), json_float_v2_roundtrip_cases + (("SHOULD_ERR", ""),) +) +def test_float_from_json_v2(data: JSONFloatV2 | str, expected: float | str) -> None: + """ + Test that float_from_json_v2 correctly converts a JSON string representation of a float to a float. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data != "SHOULD_ERR": + assert nan_equal(float_from_json_v2(data), expected) # type: ignore[arg-type] + else: + msg = f"could not convert string to float: {data!r}" + with pytest.raises(ValueError, match=msg): + float_from_json_v2(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), json_float_v3_cases + (("SHOULD_ERR", ""), ("0x", "")) +) +def test_float_from_json_v3(data: JSONFloatV2 | str, expected: float | str) -> None: + """ + Test that float_from_json_v3 correctly converts a JSON string representation of a float to a float. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data == "SHOULD_ERR": + msg = ( + f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" + " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." + ) + with pytest.raises(ValueError, match=msg): + float_from_json_v3(data) + elif data == "0x": + msg = ( + f"Invalid hexadecimal float value: {data!r}. " + "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" + ) + + with pytest.raises(ValueError, match=msg): + float_from_json_v3(data) + else: + assert nan_equal(float_from_json_v3(data), expected) + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("expected", "data"), json_float_v2_roundtrip_cases) +def test_float_to_json_v2(data: float | np.floating[Any], expected: JSONFloatV2) -> None: + """ + Test that floats are JSON-encoded properly for zarr v2 + """ + observed = float_to_json_v2(data) + assert observed == expected + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("expected", "data"), json_float_v3_cases) +def test_float_to_json_v3(data: float | np.floating[Any], expected: JSONFloatV2) -> None: + """ + Test that floats are JSON-encoded properly for zarr v3 + """ + observed = float_to_json_v3(data) + assert observed == expected + + +def test_bytes_from_json(zarr_format: ZarrFormat) -> None: + """ + Test that a string is interpreted as base64-encoded bytes using the ascii alphabet. + This test takes zarr_format as a parameter but doesn't actually do anything with it, because at + present there is no zarr-format-specific logic in the code being tested, but such logic may + exist in the future. + """ + data = "\00" + assert bytes_from_json(data, zarr_format=zarr_format) == base64.b64decode(data.encode("ascii")) + + +def test_bytes_to_json(zarr_format: ZarrFormat) -> None: + """ + Test that bytes are encoded with base64 using the ascii alphabet. + + This test takes zarr_format as a parameter but doesn't actually do anything with it, because at + present there is no zarr-format-specific logic in the code being tested, but such logic may + exist in the future. + """ + + data = b"asdas" + assert bytes_to_json(data, zarr_format=zarr_format) == base64.b64encode(data).decode("ascii") + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v2_roundtrip_cases) +def test_complex_to_json_v2( + float_data: float | np.floating[Any], json_expected: JSONFloatV2 +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v2 format. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v2(cplx_npy) == (json_expected, json_expected) + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) +def test_complex_to_json_v3( + float_data: float | np.floating[Any], json_expected: JSONFloatV2 +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v3 format. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v3(cplx_npy) == (json_expected, json_expected) + + +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) +def test_complex_float_to_json( + float_data: float | np.floating[Any], json_expected: JSONFloatV2, zarr_format: ZarrFormat +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v2 or v3 formats, depending + on the ``zarr_format`` keyword argument. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + if zarr_format == 2: + assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v2(cplx_npy) == ( + json_expected, + json_expected, + ) + elif zarr_format == 3: + assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v3(cplx_npy) == ( + json_expected, + json_expected, + ) + else: + raise ValueError("zarr_format must be 2 or 3") # pragma: no cover + + +check_json_float_cases = get_args(SpecialFloatStrings) + (1.0, 2) + + +@pytest.mark.parametrize("data", check_json_float_cases) +def test_check_json_float_v2_valid(data: JSONFloatV2 | int) -> None: + assert check_json_float_v2(data) + + +def test_check_json_float_v2_invalid() -> None: + assert not check_json_float_v2("invalid") + + +@pytest.mark.parametrize("data", check_json_float_cases) +def test_check_json_float_v3_valid(data: JSONFloatV2 | int) -> None: + assert check_json_float_v3(data) + + +def test_check_json_float_v3_invalid() -> None: + assert not check_json_float_v3("invalid") + + +check_json_complex_float_true_cases: tuple[list[JSONFloatV2], ...] = ( + [0.0, 1.0], + [0.0, 1.0], + [-1.0, "NaN"], + ["Infinity", 1.0], + ["Infinity", "NaN"], +) + +check_json_complex_float_false_cases: tuple[object, ...] = ( + 0.0, + "foo", + [0.0], + [1.0, 2.0, 3.0], + [1.0, "_infinity_"], + {"hello": 1.0}, +) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_v2_true(data: JSON) -> None: + assert check_json_complex_float_v2(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_v2_false(data: JSON) -> None: + assert not check_json_complex_float_v2(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_v3_true(data: JSON) -> None: + assert check_json_complex_float_v3(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_v3_false(data: JSON) -> None: + assert not check_json_complex_float_v3(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_true(data: JSON, zarr_format: ZarrFormat) -> None: + if zarr_format == 2: + assert check_json_complex_float_v2(data) + elif zarr_format == 3: + assert check_json_complex_float_v3(data) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_false(data: JSON, zarr_format: ZarrFormat) -> None: + if zarr_format == 2: + assert not check_json_complex_float_v2(data) + elif zarr_format == 3: + assert not check_json_complex_float_v3(data) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +def test_check_json_int() -> None: + assert check_json_int(0) + assert not check_json_int(1.0) + + +def test_check_json_str() -> None: + assert check_json_str("0") + assert not check_json_str(1.0) + + +def test_check_json_bool() -> None: + assert check_json_bool(True) + assert check_json_bool(False) + assert not check_json_bool(1.0) + assert not check_json_bool("True") diff --git a/tests/test_dtype/test_npy/test_complex.py b/tests/test_dtype/test_npy/test_complex.py new file mode 100644 index 0000000000..b6a1e799eb --- /dev/null +++ b/tests/test_dtype/test_npy/test_complex.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import math + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.complex import Complex64, Complex128 + + +class _BaseTestFloat(BaseTestZDType): + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + +class TestComplex64(_BaseTestFloat): + test_cls = Complex64 + valid_dtype = (np.dtype(">c8"), np.dtype("c8", "object_codec_id": None}, + {"name": "c16"), np.dtype("c16", "object_codec_id": None}, + {"name": " bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + hex_string_params: tuple[tuple[str, float], ...] = () + + def test_hex_encoding(self, hex_string_params: tuple[str, float]) -> None: + """ + Test that hexadecimal strings can be read as NaN values + """ + hex_string, expected = hex_string_params + zdtype = self.test_cls() + observed = zdtype.from_json_scalar(hex_string, zarr_format=3) + assert self.scalar_equals(observed, expected) + + +class TestFloat16(_BaseTestFloat): + test_cls = Float16 + valid_dtype = (np.dtype(">f2"), np.dtype("f2", "object_codec_id": None}, + {"name": "f4"), np.dtype("f4", "object_codec_id": None}, + {"name": "f8"), np.dtype("f8", "object_codec_id": None}, + {"name": "i1", + "int8", + "|f8", + ) + invalid_json_v3 = ( + "|i1", + "|f8", + {"name": "int8", "configuration": {"endianness": "little"}}, + ) + + scalar_v2_params = ((Int8(), 1), (Int8(), -1)) + scalar_v3_params = ((Int8(), 1), (Int8(), -1)) + cast_value_params = ( + (Int8(), 1, np.int8(1)), + (Int8(), -1, np.int8(-1)), + ) + item_size_params = (Int8(),) + + +class TestInt16(BaseTestZDType): + test_cls = Int16 + scalar_type = np.int16 + valid_dtype = (np.dtype(">i2"), np.dtype("i2", "object_codec_id": None}, + {"name": "i4"), np.dtype("i4", "object_codec_id": None}, + {"name": "i8"), np.dtype("i8", "object_codec_id": None}, + {"name": "u2"), np.dtype("u2", "object_codec_id": None}, + {"name": "u4"), np.dtype("u4", "object_codec_id": None}, + {"name": "u8"), np.dtype("u8", "object_codec_id": None}, + {"name": "U10"), np.dtype("U10", "object_codec_id": None}, + {"name": " None: + """ + Test that we get a warning when serializing a dtype without a zarr v3 spec to json + when zarr_format is 3 + """ + with pytest.raises(UnstableSpecificationWarning): + zdtype.to_json(zarr_format=3) diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py new file mode 100644 index 0000000000..e9c9ab11d0 --- /dev/null +++ b/tests/test_dtype/test_npy/test_structured.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype import ( + Float16, + Float64, + Int32, + Int64, + Structured, +) + + +class TestStructured(BaseTestZDType): + test_cls = Structured + valid_dtype = ( + np.dtype([("field1", np.int32), ("field2", np.float64)]), + np.dtype([("field1", np.int64), ("field2", np.int32)]), + ) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|S10"), + ) + valid_json_v2 = ( + {"name": [["field1", ">i4"], ["field2", ">f8"]], "object_codec_id": None}, + {"name": [["field1", ">i8"], ["field2", ">i4"]], "object_codec_id": None}, + ) + valid_json_v3 = ( + { + "name": "structured", + "configuration": { + "fields": [ + ["field1", "int32"], + ["field2", "float64"], + ] + }, + }, + { + "name": "structured", + "configuration": { + "fields": [ + [ + "field1", + { + "name": "numpy.datetime64", + "configuration": {"unit": "s", "scale_factor": 1}, + }, + ], + [ + "field2", + {"name": "fixed_length_utf32", "configuration": {"length_bytes": 32}}, + ], + ] + }, + }, + ) + invalid_json_v2 = ( + [("field1", "|i1"), ("field2", "|f8")], + [("field1", "|S10"), ("field2", "|f8")], + ) + invalid_json_v3 = ( + { + "name": "structured", + "configuration": { + "fields": [ + ("field1", {"name": "int32", "configuration": {"endianness": "invalid"}}), + ("field2", {"name": "float64", "configuration": {"endianness": "big"}}), + ] + }, + }, + {"name": "invalid_name"}, + ) + + scalar_v2_params = ( + (Structured(fields=(("field1", Int32()), ("field2", Float64()))), "AQAAAAAAAAAAAPA/"), + (Structured(fields=(("field1", Float16()), ("field2", Int32()))), "AQAAAAAA"), + ) + scalar_v3_params = ( + (Structured(fields=(("field1", Int32()), ("field2", Float64()))), "AQAAAAAAAAAAAPA/"), + (Structured(fields=(("field1", Int64()), ("field2", Int32()))), "AQAAAAAAAAAAAPA/"), + ) + + cast_value_params = ( + ( + Structured(fields=(("field1", Int32()), ("field2", Float64()))), + (1, 2.0), + np.array((1, 2.0), dtype=[("field1", np.int32), ("field2", np.float64)]), + ), + ( + Structured(fields=(("field1", Int64()), ("field2", Int32()))), + (3, 4.5), + np.array((3, 4.5), dtype=[("field1", np.int64), ("field2", np.int32)]), + ), + ) + + def scalar_equals(self, scalar1: Any, scalar2: Any) -> bool: + if hasattr(scalar1, "shape") and hasattr(scalar2, "shape"): + return np.array_equal(scalar1, scalar2) + return super().scalar_equals(scalar1, scalar2) + + item_size_params = ( + Structured(fields=(("field1", Int32()), ("field2", Float64()))), + Structured(fields=(("field1", Int64()), ("field2", Int32()))), + ) diff --git a/tests/test_dtype/test_npy/test_time.py b/tests/test_dtype/test_npy/test_time.py new file mode 100644 index 0000000000..e201be5cf6 --- /dev/null +++ b/tests/test_dtype/test_npy/test_time.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import re +from typing import get_args + +import numpy as np +import pytest + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.common import DateTimeUnit +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64, datetime_from_int + + +class _TestTimeBase(BaseTestZDType): + def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # This method gets overridden here to support the equivalency between NaT and + # -9223372036854775808 fill values + nat_scalars = (-9223372036854775808, "NaT") + if scalar1 in nat_scalars and scalar2 in nat_scalars: + return True + return scalar1 == scalar2 + + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + +class TestDateTime64(_TestTimeBase): + test_cls = DateTime64 + valid_dtype = (np.dtype("datetime64[10ns]"), np.dtype("datetime64[us]"), np.dtype("datetime64")) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("timedelta64[ns]"), + ) + valid_json_v2 = ( + {"name": ">M8", "object_codec_id": None}, + {"name": ">M8[s]", "object_codec_id": None}, + {"name": "m8", "object_codec_id": None}, + {"name": ">m8[s]", "object_codec_id": None}, + {"name": " None: + """ + Test that an invalid unit raises a ValueError. + """ + unit = "invalid" + msg = f"unit must be one of ('Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'μs', 'ns', 'ps', 'fs', 'as', 'generic'), got {unit!r}." + with pytest.raises(ValueError, match=re.escape(msg)): + DateTime64(unit=unit) # type: ignore[arg-type] + with pytest.raises(ValueError, match=re.escape(msg)): + TimeDelta64(unit=unit) # type: ignore[arg-type] + + +def test_time_scale_factor_too_low() -> None: + """ + Test that an invalid unit raises a ValueError. + """ + scale_factor = 0 + msg = f"scale_factor must be > 0, got {scale_factor}." + with pytest.raises(ValueError, match=msg): + DateTime64(scale_factor=scale_factor) + with pytest.raises(ValueError, match=msg): + TimeDelta64(scale_factor=scale_factor) + + +def test_time_scale_factor_too_high() -> None: + """ + Test that an invalid unit raises a ValueError. + """ + scale_factor = 2**31 + msg = f"scale_factor must be < 2147483648, got {scale_factor}." + with pytest.raises(ValueError, match=msg): + DateTime64(scale_factor=scale_factor) + with pytest.raises(ValueError, match=msg): + TimeDelta64(scale_factor=scale_factor) + + +@pytest.mark.parametrize("unit", get_args(DateTimeUnit)) +@pytest.mark.parametrize("scale_factor", [1, 10]) +@pytest.mark.parametrize("value", [0, 1, 10]) +def test_datetime_from_int(unit: DateTimeUnit, scale_factor: int, value: int) -> None: + """ + Test datetime_from_int. + """ + expected = np.int64(value).view(f"datetime64[{scale_factor}{unit}]") + assert datetime_from_int(value, unit=unit, scale_factor=scale_factor) == expected diff --git a/tests/test_dtype/test_wrapper.py b/tests/test_dtype/test_wrapper.py new file mode 100644 index 0000000000..8f461f1a77 --- /dev/null +++ b/tests/test_dtype/test_wrapper.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +import pytest + +from zarr.core.dtype.common import DTypeSpec_V2, DTypeSpec_V3, HasItemSize + +if TYPE_CHECKING: + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +""" +class _TestZDTypeSchema: + # subclasses define the URL for the schema, if available + schema_url: ClassVar[str] = "" + + @pytest.fixture(scope="class") + def get_schema(self) -> object: + response = requests.get(self.schema_url) + response.raise_for_status() + return json_schema.loads(response.text) + + def test_schema(self, schema: json_schema.Schema) -> None: + assert schema.is_valid(self.test_cls.to_json(zarr_format=2)) +""" + + +class BaseTestZDType: + """ + A base class for testing ZDType subclasses. This class works in conjunction with the custom + pytest collection function ``pytest_generate_tests`` defined in conftest.py, which applies the + following procedure when generating tests: + + At test generation time, for each test fixture referenced by a method on this class + pytest will look for an attribute with the same name as that fixture. Pytest will assume that + this class attribute is a tuple of values to be used for generating a parametrized test fixture. + + This means that child classes can, by using different values for these class attributes, have + customized test parametrization. + + Attributes + ---------- + test_cls : type[ZDType[TBaseDType, TBaseScalar]] + The ZDType subclass being tested. + scalar_type : ClassVar[type[TBaseScalar]] + The expected scalar type for the ZDType. + valid_dtype : ClassVar[tuple[TBaseDType, ...]] + A tuple of valid numpy dtypes for the ZDType. + invalid_dtype : ClassVar[tuple[TBaseDType, ...]] + A tuple of invalid numpy dtypes for the ZDType. + valid_json_v2 : ClassVar[tuple[str | dict[str, object] | list[object], ...]] + A tuple of valid JSON representations for Zarr format version 2. + invalid_json_v2 : ClassVar[tuple[str | dict[str, object] | list[object], ...]] + A tuple of invalid JSON representations for Zarr format version 2. + valid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] + A tuple of valid JSON representations for Zarr format version 3. + invalid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] + A tuple of invalid JSON representations for Zarr format version 3. + cast_value_params : ClassVar[tuple[tuple[Any, Any, Any], ...]] + A tuple of (dtype, value, expected) tuples for testing ZDType.cast_value. + """ + + test_cls: type[ZDType[TBaseDType, TBaseScalar]] + scalar_type: ClassVar[type[TBaseScalar]] + valid_dtype: ClassVar[tuple[TBaseDType, ...]] = () + invalid_dtype: ClassVar[tuple[TBaseDType, ...]] = () + + valid_json_v2: ClassVar[tuple[DTypeSpec_V2, ...]] = () + invalid_json_v2: ClassVar[tuple[str | dict[str, object] | list[object], ...]] = () + + valid_json_v3: ClassVar[tuple[DTypeSpec_V3, ...]] = () + invalid_json_v3: ClassVar[tuple[str | dict[str, object], ...]] = () + + # for testing scalar round-trip serialization, we need a tuple of (data type json, scalar json) + # pairs. the first element of the pair is used to create a dtype instance, and the second + # element is the json serialization of the scalar that we want to round-trip. + + scalar_v2_params: ClassVar[tuple[tuple[Any, Any], ...]] = () + scalar_v3_params: ClassVar[tuple[tuple[Any, Any], ...]] = () + cast_value_params: ClassVar[tuple[tuple[Any, Any, Any], ...]] + item_size_params: ClassVar[tuple[ZDType[Any, Any], ...]] + + def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # An equality check for json-encoded scalars. This defaults to regular equality, + # but some classes may need to override this for special cases + return scalar1 == scalar2 + + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # An equality check for scalars. This defaults to regular equality, + # but some classes may need to override this for special cases + return scalar1 == scalar2 + + def test_check_dtype_valid(self, valid_dtype: TBaseDType) -> None: + assert self.test_cls._check_native_dtype(valid_dtype) + + def test_check_dtype_invalid(self, invalid_dtype: object) -> None: + assert not self.test_cls._check_native_dtype(invalid_dtype) # type: ignore[arg-type] + + def test_from_dtype_roundtrip(self, valid_dtype: Any) -> None: + zdtype = self.test_cls.from_native_dtype(valid_dtype) + assert zdtype.to_native_dtype() == valid_dtype + + def test_from_json_roundtrip_v2(self, valid_json_v2: DTypeSpec_V2) -> None: + zdtype = self.test_cls.from_json(valid_json_v2, zarr_format=2) + assert zdtype.to_json(zarr_format=2) == valid_json_v2 + + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + def test_from_json_roundtrip_v3(self, valid_json_v3: DTypeSpec_V3) -> None: + zdtype = self.test_cls.from_json(valid_json_v3, zarr_format=3) + assert zdtype.to_json(zarr_format=3) == valid_json_v3 + + def test_scalar_roundtrip_v2(self, scalar_v2_params: tuple[ZDType[Any, Any], Any]) -> None: + zdtype, scalar_json = scalar_v2_params + scalar = zdtype.from_json_scalar(scalar_json, zarr_format=2) + assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=2)) + + def test_scalar_roundtrip_v3(self, scalar_v3_params: tuple[ZDType[Any, Any], Any]) -> None: + zdtype, scalar_json = scalar_v3_params + scalar = zdtype.from_json_scalar(scalar_json, zarr_format=3) + assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=3)) + + def test_cast_value(self, cast_value_params: tuple[ZDType[Any, Any], Any, Any]) -> None: + zdtype, value, expected = cast_value_params + observed = zdtype.cast_scalar(value) + assert self.scalar_equals(expected, observed) + + def test_item_size(self, item_size_params: ZDType[Any, Any]) -> None: + """ + Test that the item_size attribute matches the numpy dtype itemsize attribute, for dtypes + with a fixed scalar size. + """ + if isinstance(item_size_params, HasItemSize): + assert item_size_params.item_size == item_size_params.to_native_dtype().itemsize + else: + pytest.skip(f"Dtype {item_size_params} does not implement HasItemSize") diff --git a/tests/test_dtype_registry.py b/tests/test_dtype_registry.py new file mode 100644 index 0000000000..c7d5f90065 --- /dev/null +++ b/tests/test_dtype_registry.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any, get_args + +import numpy as np +import pytest + +import zarr +from tests.conftest import skip_object_dtype +from zarr.core.config import config +from zarr.core.dtype import ( + AnyDType, + Bool, + DataTypeRegistry, + DateTime64, + FixedLengthUTF32, + Int8, + Int16, + TBaseDType, + TBaseScalar, + ZDType, + data_type_registry, + get_data_type_from_json, + parse_data_type, +) + +if TYPE_CHECKING: + from collections.abc import Generator + + from zarr.core.common import ZarrFormat + +from .test_dtype.conftest import zdtype_examples + + +@pytest.fixture +def data_type_registry_fixture() -> DataTypeRegistry: + return DataTypeRegistry() + + +class TestRegistry: + @staticmethod + def test_register(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that registering a dtype in a data type registry works. + """ + data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) + assert data_type_registry_fixture.get(Bool._zarr_v3_name) == Bool + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), Bool) + + @staticmethod + def test_override(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that registering a new dtype with the same name works (overriding the previous one). + """ + data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) + + class NewBool(Bool): + def default_scalar(self) -> np.bool_: + return np.True_ + + data_type_registry_fixture.register(NewBool._zarr_v3_name, NewBool) + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), NewBool) + + @staticmethod + @pytest.mark.parametrize( + ("wrapper_cls", "dtype_str"), [(Bool, "bool"), (FixedLengthUTF32, "|U4")] + ) + def test_match_dtype( + data_type_registry_fixture: DataTypeRegistry, + wrapper_cls: type[ZDType[TBaseDType, TBaseScalar]], + dtype_str: str, + ) -> None: + """ + Test that match_dtype resolves a numpy dtype into an instance of the correspond wrapper for that dtype. + """ + data_type_registry_fixture.register(wrapper_cls._zarr_v3_name, wrapper_cls) + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype(dtype_str)), wrapper_cls) + + @staticmethod + def test_unregistered_dtype(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that match_dtype raises an error if the dtype is not registered. + """ + outside_dtype_name = "int8" + outside_dtype = np.dtype(outside_dtype_name) + msg = f"No Zarr data type found that matches dtype '{outside_dtype!r}'" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_dtype(outside_dtype) + + with pytest.raises(KeyError): + data_type_registry_fixture.get(outside_dtype_name) + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_registered_dtypes_match_dtype(zdtype: ZDType[TBaseDType, TBaseScalar]) -> None: + """ + Test that the registered dtypes can be retrieved from the registry. + """ + skip_object_dtype(zdtype) + assert data_type_registry.match_dtype(zdtype.to_native_dtype()) == zdtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_registered_dtypes_match_json( + zdtype: ZDType[TBaseDType, TBaseScalar], zarr_format: ZarrFormat + ) -> None: + assert ( + data_type_registry.match_json( + zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format + ) + == zdtype + ) + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_match_dtype_unique( + zdtype: ZDType[Any, Any], + data_type_registry_fixture: DataTypeRegistry, + zarr_format: ZarrFormat, + ) -> None: + """ + Test that the match_dtype method uniquely specifies a registered data type. We create a local registry + that excludes the data type class being tested, and ensure that an instance of the wrapped data type + fails to match anything in the registry + """ + skip_object_dtype(zdtype) + for _cls in get_args(AnyDType): + if _cls is not type(zdtype): + data_type_registry_fixture.register(_cls._zarr_v3_name, _cls) + + dtype_instance = zdtype.to_native_dtype() + + msg = f"No Zarr data type found that matches dtype '{dtype_instance!r}'" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_dtype(dtype_instance) + + instance_dict = zdtype.to_json(zarr_format=zarr_format) + msg = f"No Zarr data type found that matches {instance_dict!r}" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_json(instance_dict, zarr_format=zarr_format) + + +# this is copied from the registry tests -- we should deduplicate +here = str(Path(__file__).parent.absolute()) + + +@pytest.fixture +def set_path() -> Generator[None, None, None]: + sys.path.append(here) + zarr.registry._collect_entrypoints() + yield + sys.path.remove(here) + registries = zarr.registry._collect_entrypoints() + for registry in registries: + registry.lazy_load_list.clear() + config.reset() + + +@pytest.mark.usefixtures("set_path") +def test_entrypoint_dtype(zarr_format: ZarrFormat) -> None: + from package_with_entrypoint import TestDataType + + data_type_registry.lazy_load() + instance = TestDataType() + dtype_json = instance.to_json(zarr_format=zarr_format) + assert get_data_type_from_json(dtype_json, zarr_format=zarr_format) == instance + data_type_registry.unregister(TestDataType._zarr_v3_name) + + +@pytest.mark.parametrize( + ("dtype_params", "expected", "zarr_format"), + [ + ("int8", Int8(), 3), + (Int8(), Int8(), 3), + (">i2", Int16(endianness="big"), 2), + ("datetime64[10s]", DateTime64(unit="s", scale_factor=10), 2), + ( + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, + DateTime64(unit="s", scale_factor=10), + 3, + ), + ], +) +def test_parse_data_type( + dtype_params: Any, expected: ZDType[Any, Any], zarr_format: ZarrFormat +) -> None: + """ + Test that parse_data_type accepts alternative representations of ZDType instances, and resolves + those inputs to the expected ZDType instance. + """ + observed = parse_data_type(dtype_params, zarr_format=zarr_format) + assert observed == expected diff --git a/tests/test_group.py b/tests/test_group.py index 19a9f9c9bb..60a1fcb9bf 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1,14 +1,17 @@ from __future__ import annotations import contextlib +import inspect import operator import pickle +import re +import time import warnings from typing import TYPE_CHECKING, Any, Literal import numpy as np import pytest -from numcodecs import Zstd +from numcodecs import Blosc import zarr import zarr.api.asynchronous @@ -16,16 +19,37 @@ import zarr.storage from zarr import Array, AsyncArray, AsyncGroup, Group from zarr.abc.store import Store +from zarr.core import sync_group from zarr.core._info import GroupInfo from zarr.core.buffer import default_buffer_prototype -from zarr.core.group import ConsolidatedMetadata, GroupMetadata -from zarr.core.sync import sync -from zarr.errors import ContainsArrayError, ContainsGroupError -from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore, make_store_path +from zarr.core.config import config as zarr_config +from zarr.core.dtype.common import unpack_dtype_json +from zarr.core.dtype.npy.int import UInt8 +from zarr.core.group import ( + ConsolidatedMetadata, + GroupMetadata, + ImplicitGroupMarker, + _build_metadata_v3, + _get_roots, + _parse_hierarchy_dict, + create_hierarchy, + create_nodes, + create_rooted_hierarchy, + get_node, +) +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.core.sync import _collect_aiterator, sync +from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError +from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore +from zarr.storage._common import make_store_path +from zarr.storage._utils import _join_paths, normalize_path +from zarr.testing.store import LatencyStore -from .conftest import parse_store +from .conftest import meta_from_array, parse_store if TYPE_CHECKING: + from collections.abc import Callable + from _pytest.compat import LEGACY_PATH from zarr.core.common import JSON, ZarrFormat @@ -108,24 +132,27 @@ async def test_create_creates_parents(store: Store, zarr_format: ZarrFormat) -> assert g.attrs == {} -def test_group_name_properties(store: Store, zarr_format: ZarrFormat) -> None: +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("root_name", ["", "/", "a", "/a"]) +@pytest.mark.parametrize("branch_name", ["foo", "/foo", "foo/bar", "/foo/bar"]) +def test_group_name_properties( + store: Store, zarr_format: ZarrFormat, root_name: str, branch_name: str +) -> None: """ - Test basic properties of groups + Test that the path, name, and basename attributes of a group and its subgroups are consistent """ - root = Group.from_store(store=store, zarr_format=zarr_format) - assert root.path == "" - assert root.name == "/" - assert root.basename == "" - - foo = root.create_group("foo") - assert foo.path == "foo" - assert foo.name == "/foo" - assert foo.basename == "foo" - - bar = root.create_group("foo/bar") - assert bar.path == "foo/bar" - assert bar.name == "/foo/bar" - assert bar.basename == "bar" + root = Group.from_store(store=StorePath(store=store, path=root_name), zarr_format=zarr_format) + assert root.path == normalize_path(root_name) + assert root.name == "/" + root.path + assert root.basename == root.path + + branch = root.create_group(branch_name) + if root.path == "": + assert branch.path == normalize_path(branch_name) + else: + assert branch.path == "/".join([root.path, normalize_path(branch_name)]) + assert branch.name == "/" + branch.path + assert branch.basename == branch_name.split("/")[-1] @pytest.mark.parametrize("consolidated_metadata", [True, False]) @@ -350,7 +377,7 @@ def test_group_getitem(store: Store, zarr_format: ZarrFormat, consolidated: bool ) with pytest.raises(KeyError): - # We've chosen to trust the consolidted metadata, which doesn't + # We've chosen to trust the consolidated metadata, which doesn't # contain this array group["subgroup/subarray"] @@ -469,7 +496,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat expected_groups = list(zip(expected_group_keys, expected_group_values, strict=False)) fill_value = 3 - dtype = "uint8" + dtype = UInt8() expected_group_values[0].create_group("subgroup") expected_group_values[0].create_array( @@ -490,13 +517,13 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat metadata = { "subarray": { "attributes": {}, - "dtype": dtype, + "dtype": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), "fill_value": fill_value, "shape": (1,), "chunks": (1,), "order": "C", "filters": None, - "compressor": Zstd(level=0), + "compressor": Blosc(), "zarr_format": zarr_format, }, "subgroup": { @@ -526,7 +553,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat {"configuration": {"endian": "little"}, "name": "bytes"}, {"configuration": {}, "name": "zstd"}, ), - "data_type": dtype, + "data_type": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), "fill_value": fill_value, "node_type": "array", "shape": (1,), @@ -582,7 +609,10 @@ def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: assert group.attrs == attrs new_attrs = {"bar": 100} new_group = group.update_attributes(new_attrs) - assert new_group.attrs == new_attrs + + updated_attrs = attrs.copy() + updated_attrs.update(new_attrs) + assert new_group.attrs == updated_attrs async def test_group_update_attributes_async(store: Store, zarr_format: ZarrFormat) -> None: @@ -598,11 +628,13 @@ async def test_group_update_attributes_async(store: Store, zarr_format: ZarrForm @pytest.mark.parametrize("method", ["create_array", "array"]) +@pytest.mark.parametrize("name", ["a", "/a"]) def test_group_create_array( store: Store, zarr_format: ZarrFormat, overwrite: bool, method: Literal["create_array", "array"], + name: str, ) -> None: """ Test `Group.from_store` @@ -613,24 +645,26 @@ def test_group_create_array( data = np.arange(np.prod(shape)).reshape(shape).astype(dtype) if method == "create_array": - array = group.create_array(name="array", shape=shape, dtype=dtype) + array = group.create_array(name=name, shape=shape, dtype=dtype) array[:] = data elif method == "array": with pytest.warns(DeprecationWarning): - array = group.array(name="array", shape=shape, dtype=dtype) - array[:] = data + array = group.array(name=name, data=data, shape=shape, dtype=dtype) else: raise AssertionError if not overwrite: if method == "create_array": with pytest.raises(ContainsArrayError): - a = group.create_array(name="array", shape=shape, dtype=dtype) + a = group.create_array(name=name, shape=shape, dtype=dtype) a[:] = data elif method == "array": with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): - a = group.array(name="array", shape=shape, dtype=dtype) + a = group.array(name=name, shape=shape, dtype=dtype) a[:] = data + + assert array.path == normalize_path(name) + assert array.name == "/" + array.path assert array.shape == shape assert array.dtype == np.dtype(dtype) assert np.array_equal(array[:], data) @@ -921,20 +955,23 @@ async def test_asyncgroup_delitem(store: Store, zarr_format: ZarrFormat) -> None raise AssertionError +@pytest.mark.parametrize("name", ["a", "/a"]) async def test_asyncgroup_create_group( store: Store, + name: str, zarr_format: ZarrFormat, ) -> None: agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) - sub_node_path = "sub_group" attributes = {"foo": 999} - subnode = await agroup.create_group(name=sub_node_path, attributes=attributes) + subgroup = await agroup.create_group(name=name, attributes=attributes) - assert isinstance(subnode, AsyncGroup) - assert subnode.attrs == attributes - assert subnode.store_path.path == sub_node_path - assert subnode.store_path.store == store - assert subnode.metadata.zarr_format == zarr_format + assert isinstance(subgroup, AsyncGroup) + assert subgroup.path == normalize_path(name) + assert subgroup.name == "/" + subgroup.path + assert subgroup.attrs == attributes + assert subgroup.store_path.path == subgroup.path + assert subgroup.store_path.store == store + assert subgroup.metadata.zarr_format == zarr_format async def test_asyncgroup_create_array( @@ -987,7 +1024,9 @@ async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrForma ) agroup_new_attributes = await agroup.update_attributes(attributes_new) - assert agroup_new_attributes.attrs == attributes_new + attributes_updated = attributes_old.copy() + attributes_updated.update(attributes_new) + assert agroup_new_attributes.attrs == attributes_updated @pytest.mark.parametrize("store", ["local"], indirect=["store"]) @@ -1440,11 +1479,587 @@ def test_delitem_removes_children(store: Store, zarr_format: ZarrFormat) -> None g1["0/0"] -@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) -def test_deprecated_compressor(store: Store) -> None: - g = zarr.group(store=store, zarr_format=2) - with pytest.warns(UserWarning, match="The `compressor` argument is deprecated.*"): - a = g.create_array( - "foo", shape=(100,), chunks=(10,), dtype="i4", compressor={"id": "blosc"} +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_nodes( + impl: Literal["async", "sync"], store: Store, zarr_format: ZarrFormat +) -> None: + """ + Ensure that ``create_nodes`` can create a zarr hierarchy from a model of that + hierarchy in dict form. Note that this creates an incomplete Zarr hierarchy. + """ + node_spec = { + "group": GroupMetadata(attributes={"foo": 10}), + "group/array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "group/array_1": meta_from_array(np.arange(4), zarr_format=zarr_format), + "group/subgroup/array_0": meta_from_array(np.arange(4), zarr_format=zarr_format), + "group/subgroup/array_1": meta_from_array(np.arange(5), zarr_format=zarr_format), + } + if impl == "sync": + observed_nodes = dict(sync_group.create_nodes(store=store, nodes=node_spec)) + elif impl == "async": + observed_nodes = dict(await _collect_aiterator(create_nodes(store=store, nodes=node_spec))) + else: + raise ValueError(f"Invalid impl: {impl}") + + assert node_spec == {k: v.metadata for k, v in observed_nodes.items()} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_create_nodes_concurrency_limit(store: MemoryStore) -> None: + """ + Test that the execution time of create_nodes can be constrained by the async concurrency + configuration setting. + """ + set_latency = 0.02 + num_groups = 10 + groups = {str(idx): GroupMetadata() for idx in range(num_groups)} + + latency_store = LatencyStore(store, set_latency=set_latency) + + # check how long it takes to iterate over the groups + # if create_nodes is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + with zarr_config.set({"async.concurrency": 1}): + start = time.time() + _ = tuple(sync_group.create_nodes(store=latency_store, nodes=groups)) + elapsed = time.time() - start + assert elapsed > num_groups * set_latency + + +@pytest.mark.parametrize( + ("a_func", "b_func"), + [ + (zarr.core.group.AsyncGroup.create_hierarchy, zarr.core.group.Group.create_hierarchy), + (zarr.core.group.create_hierarchy, zarr.core.sync_group.create_hierarchy), + (zarr.core.group.create_nodes, zarr.core.sync_group.create_nodes), + (zarr.core.group.create_rooted_hierarchy, zarr.core.sync_group.create_rooted_hierarchy), + (zarr.core.group.get_node, zarr.core.sync_group.get_node), + ], +) +def test_consistent_signatures( + a_func: Callable[[object], object], b_func: Callable[[object], object] +) -> None: + """ + Ensure that pairs of functions have consistent signatures + """ + base_sig = inspect.signature(a_func) + test_sig = inspect.signature(b_func) + assert test_sig.parameters == base_sig.parameters + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy( + impl: Literal["async", "sync"], store: Store, overwrite: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that ``create_hierarchy`` can create a complete Zarr hierarchy, even if the input describes + an incomplete one. + """ + + hierarchy_spec = { + "group": GroupMetadata(attributes={"path": "group"}, zarr_format=zarr_format), + "group/array_0": meta_from_array( + np.arange(3), attributes={"path": "group/array_0"}, zarr_format=zarr_format + ), + "group/subgroup/array_0": meta_from_array( + np.arange(4), attributes={"path": "group/subgroup/array_0"}, zarr_format=zarr_format + ), + } + pre_existing_nodes = { + "group/extra": GroupMetadata(zarr_format=zarr_format, attributes={"path": "group/extra"}), + "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "root"}), + } + # we expect create_hierarchy to insert a group that was missing from the hierarchy spec + expected_meta = hierarchy_spec | {"group/subgroup": GroupMetadata(zarr_format=zarr_format)} + + # initialize the group with some nodes + _ = dict(sync_group.create_nodes(store=store, nodes=pre_existing_nodes)) + + if impl == "sync": + created = dict( + sync_group.create_hierarchy(store=store, nodes=hierarchy_spec, overwrite=overwrite) + ) + elif impl == "async": + created = { + k: v + async for k, v in create_hierarchy( + store=store, nodes=hierarchy_spec, overwrite=overwrite + ) + } + else: + raise ValueError(f"Invalid impl: {impl}") + if not overwrite: + extra_group = sync_group.get_node(store=store, path="group/extra", zarr_format=zarr_format) + assert extra_group.metadata.attributes == {"path": "group/extra"} + else: + with pytest.raises(FileNotFoundError): + await get_node(store=store, path="group/extra", zarr_format=zarr_format) + assert expected_meta == {k: v.metadata for k, v in created.items()} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("extant_node", ["array", "group"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_existing_nodes( + impl: Literal["async", "sync"], + store: Store, + extant_node: Literal["array", "group"], + zarr_format: ZarrFormat, +) -> None: + """ + Test that create_hierarchy with overwrite = False will not overwrite an existing array or group, + and raises an exception instead. + """ + extant_node_path = "node" + + if extant_node == "array": + extant_metadata = meta_from_array( + np.zeros(4), zarr_format=zarr_format, attributes={"extant": True} + ) + new_metadata = meta_from_array(np.zeros(4), zarr_format=zarr_format) + err_cls = ContainsArrayError + else: + extant_metadata = GroupMetadata(zarr_format=zarr_format, attributes={"extant": True}) + new_metadata = GroupMetadata(zarr_format=zarr_format) + err_cls = ContainsGroupError + + # write the extant metadata + tuple(sync_group.create_nodes(store=store, nodes={extant_node_path: extant_metadata})) + + msg = f"{extant_node} exists in store {store!r} at path {extant_node_path!r}." + # ensure that we cannot invoke create_hierarchy with overwrite=False here + if impl == "sync": + with pytest.raises(err_cls, match=re.escape(msg)): + tuple( + sync_group.create_hierarchy( + store=store, nodes={"node": new_metadata}, overwrite=False + ) + ) + elif impl == "async": + with pytest.raises(err_cls, match=re.escape(msg)): + tuple( + [ + x + async for x in create_hierarchy( + store=store, nodes={"node": new_metadata}, overwrite=False + ) + ] + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + # ensure that the extant metadata was not overwritten + assert ( + await get_node(store=store, path=extant_node_path, zarr_format=zarr_format) + ).metadata.attributes == {"extant": True} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("group_path", ["", "foo"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_group_create_hierarchy( + store: Store, + zarr_format: ZarrFormat, + overwrite: bool, + group_path: str, + impl: Literal["async", "sync"], +) -> None: + """ + Test that the Group.create_hierarchy method creates specified nodes and returns them in a dict. + Also test that off-target nodes are not deleted, and that the root group is not deleted + """ + root_attrs = {"root": True} + g = sync_group.create_rooted_hierarchy( + store=store, + nodes={group_path: GroupMetadata(zarr_format=zarr_format, attributes=root_attrs)}, + ) + node_spec = { + "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + "a/b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a/b"}), + "a/b/c": meta_from_array( + np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/b/c"} + ), + } + # This node should be kept if overwrite is True + extant_spec = {"b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "b"})} + if impl == "async": + extant_created = dict( + await _collect_aiterator(g._async_group.create_hierarchy(extant_spec, overwrite=False)) + ) + nodes_created = dict( + await _collect_aiterator( + g._async_group.create_hierarchy(node_spec, overwrite=overwrite) + ) + ) + elif impl == "sync": + extant_created = dict(g.create_hierarchy(extant_spec, overwrite=False)) + nodes_created = dict(g.create_hierarchy(node_spec, overwrite=overwrite)) + + all_members = dict(g.members(max_depth=None)) + for k, v in node_spec.items(): + assert all_members[k].metadata == v == nodes_created[k].metadata + + # if overwrite is True, the extant nodes should be erased + for k, v in extant_spec.items(): + if overwrite: + assert k in all_members + else: + assert all_members[k].metadata == v == extant_created[k].metadata + # ensure that we left the root group as-is + assert ( + sync_group.get_node(store=store, path=group_path, zarr_format=zarr_format).attrs.asdict() + == root_attrs + ) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_group_create_hierarchy_no_root( + store: Store, zarr_format: ZarrFormat, overwrite: bool +) -> None: + """ + Test that the Group.create_hierarchy method will error if the dict provided contains a root. + """ + g = Group.from_store(store, zarr_format=zarr_format) + tree = { + "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + } + with pytest.raises( + ValueError, match="It is an error to use this method to create a root node. " + ): + _ = dict(g.create_hierarchy(tree, overwrite=overwrite)) + + +class TestParseHierarchyDict: + """ + Tests for the function that parses dicts of str : Metadata pairs, ensuring that the output models a + valid Zarr hierarchy + """ + + @staticmethod + def test_normed_keys() -> None: + """ + Test that keys get normalized properly + """ + + nodes = { + "a": GroupMetadata(), + "/b": GroupMetadata(), + "": GroupMetadata(), + "/a//c////": GroupMetadata(), + } + observed = _parse_hierarchy_dict(data=nodes) + expected = {normalize_path(k): v for k, v in nodes.items()} + assert observed == expected + + @staticmethod + def test_empty() -> None: + """ + Test that an empty dict passes through + """ + assert _parse_hierarchy_dict(data={}) == {} + + @staticmethod + def test_implicit_groups() -> None: + """ + Test that implicit groups were added as needed. + """ + requested = {"a/b/c": GroupMetadata()} + expected = requested | { + "": ImplicitGroupMarker(), + "a": ImplicitGroupMarker(), + "a/b": ImplicitGroupMarker(), + } + observed = _parse_hierarchy_dict(data=requested) + assert observed == expected + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_create_hierarchy_invalid_mixed_zarr_format( + store: Store, zarr_format: ZarrFormat +) -> None: + """ + Test that ``Group.create_hierarchy`` will raise an error if the zarr_format of the nodes is + different from the parent group. + """ + other_format = 2 if zarr_format == 3 else 3 + g = Group.from_store(store, zarr_format=other_format) + tree = { + "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + "a/b": meta_from_array(np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/c"}), + } + + msg = "The zarr_format of the nodes must be the same as the parent group." + with pytest.raises(ValueError, match=msg): + _ = tuple(g.create_hierarchy(tree)) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("defect", ["array/array", "array/group"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_invalid_nested( + impl: Literal["async", "sync"], store: Store, defect: tuple[str, str], zarr_format: ZarrFormat +) -> None: + """ + Test that create_hierarchy will not create a Zarr array that contains a Zarr group + or Zarr array. + """ + if defect == "array/array": + hierarchy_spec = { + "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "array_0/subarray": meta_from_array(np.arange(4), zarr_format=zarr_format), + } + elif defect == "array/group": + hierarchy_spec = { + "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "array_0/subgroup": GroupMetadata(attributes={"foo": 10}, zarr_format=zarr_format), + } + + msg = "Only Zarr groups can contain other nodes." + if impl == "sync": + with pytest.raises(ValueError, match=msg): + tuple(sync_group.create_hierarchy(store=store, nodes=hierarchy_spec)) + elif impl == "async": + with pytest.raises(ValueError, match=msg): + await _collect_aiterator(create_hierarchy(store=store, nodes=hierarchy_spec)) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_invalid_mixed_format( + impl: Literal["async", "sync"], store: Store +) -> None: + """ + Test that create_hierarchy will not create a Zarr group that contains a both Zarr v2 and + Zarr v3 nodes. + """ + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + "The following keys map to Zarr v2 nodes: ['v2']. " + "The following keys map to Zarr v3 nodes: ['v3']." + "Ensure that all nodes have the same Zarr format." + ) + nodes = { + "v2": GroupMetadata(zarr_format=2), + "v3": GroupMetadata(zarr_format=3), + } + if impl == "sync": + with pytest.raises(ValueError, match=re.escape(msg)): + tuple( + sync_group.create_hierarchy( + store=store, + nodes=nodes, + ) + ) + elif impl == "async": + with pytest.raises(ValueError, match=re.escape(msg)): + await _collect_aiterator( + create_hierarchy( + store=store, + nodes=nodes, + ) + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + +@pytest.mark.parametrize("store", ["memory", "local"], indirect=True) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("root_key", ["", "root"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_group( + impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str +) -> None: + """ + Test that the _create_rooted_hierarchy can create a group. + """ + root_meta = {root_key: GroupMetadata(zarr_format=zarr_format, attributes={"path": root_key})} + group_names = ["a", "a/b"] + array_names = ["a/b/c", "a/b/d"] + + # just to ensure that we don't use the same name twice in tests + assert set(group_names) & set(array_names) == set() + + groups_expected_meta = { + _join_paths([root_key, node_name]): GroupMetadata( + zarr_format=zarr_format, attributes={"path": node_name} ) - assert a.metadata.compressor.codec_id == "blosc" + for node_name in group_names + } + + arrays_expected_meta = { + _join_paths([root_key, node_name]): meta_from_array(np.zeros(4), zarr_format=zarr_format) + for node_name in array_names + } + + nodes_create = root_meta | groups_expected_meta | arrays_expected_meta + if impl == "sync": + g = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create) + assert isinstance(g, Group) + members = g.members(max_depth=None) + elif impl == "async": + g = await create_rooted_hierarchy(store=store, nodes=nodes_create) + assert isinstance(g, AsyncGroup) + members = await _collect_aiterator(g.members(max_depth=None)) + else: + raise ValueError(f"Unknown implementation: {impl}") + + assert g.metadata.attributes == {"path": root_key} + + members_observed_meta = {k: v.metadata for k, v in members} + members_expected_meta_relative = { + k.removeprefix(root_key).lstrip("/"): v + for k, v in (groups_expected_meta | arrays_expected_meta).items() + } + assert members_observed_meta == members_expected_meta_relative + + +@pytest.mark.parametrize("store", ["memory", "local"], indirect=True) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("root_key", ["", "root"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_array( + impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str +) -> None: + """ + Test that _create_rooted_hierarchy can create an array. + """ + + root_meta = { + root_key: meta_from_array( + np.arange(3), zarr_format=zarr_format, attributes={"path": root_key} + ) + } + nodes_create = root_meta + + if impl == "sync": + a = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) + assert isinstance(a, Array) + elif impl == "async": + a = await create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) + assert isinstance(a, AsyncArray) + else: + raise ValueError(f"Invalid impl: {impl}") + assert a.metadata.attributes == {"path": root_key} + + +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_invalid(impl: Literal["async", "sync"]) -> None: + """ + Ensure _create_rooted_hierarchy will raise a ValueError if the input does not contain + a root node. + """ + zarr_format = 3 + nodes = { + "a": GroupMetadata(zarr_format=zarr_format), + "b": GroupMetadata(zarr_format=zarr_format), + } + msg = "The input does not specify a root node. " + if impl == "sync": + with pytest.raises(ValueError, match=msg): + sync_group.create_rooted_hierarchy(store=store, nodes=nodes) + elif impl == "async": + with pytest.raises(ValueError, match=msg): + await create_rooted_hierarchy(store=store, nodes=nodes) + else: + raise ValueError(f"Invalid impl: {impl}") + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_members_performance(store: Store) -> None: + """ + Test that the execution time of Group.members is less than the number of members times the + latency for accessing each member. + """ + get_latency = 0.1 + + # use the input store to create some groups + group_create = zarr.group(store=store) + num_groups = 10 + + # Create some groups + for i in range(num_groups): + group_create.create_group(f"group{i}") + + latency_store = LatencyStore(store, get_latency=get_latency) + # create a group with some latency on get operations + group_read = zarr.group(store=latency_store) + + # check how long it takes to iterate over the groups + # if .members is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + start = time.time() + _ = group_read.members() + elapsed = time.time() - start + + assert elapsed < (num_groups * get_latency) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_members_concurrency_limit(store: MemoryStore) -> None: + """ + Test that the execution time of Group.members can be constrained by the async concurrency + configuration setting. + """ + get_latency = 0.02 + + # use the input store to create some groups + group_create = zarr.group(store=store) + num_groups = 10 + + # Create some groups + for i in range(num_groups): + group_create.create_group(f"group{i}") + + latency_store = LatencyStore(store, get_latency=get_latency) + # create a group with some latency on get operations + group_read = zarr.group(store=latency_store) + + # check how long it takes to iterate over the groups + # if .members is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + with zarr_config.set({"async.concurrency": 1}): + start = time.time() + _ = group_read.members() + elapsed = time.time() - start + + assert elapsed > num_groups * get_latency + + +@pytest.mark.parametrize("option", ["array", "group", "invalid"]) +def test_build_metadata_v3(option: Literal["array", "group", "invalid"]) -> None: + """ + Test that _build_metadata_v3 returns the correct metadata for a v3 array or group + """ + match option: + case "array": + metadata_dict = meta_from_array(np.arange(10), zarr_format=3).to_dict() + assert _build_metadata_v3(metadata_dict) == ArrayV3Metadata.from_dict(metadata_dict) + case "group": + metadata_dict = GroupMetadata(attributes={"foo": 10}, zarr_format=3).to_dict() + assert _build_metadata_v3(metadata_dict) == GroupMetadata.from_dict(metadata_dict) + case "invalid": + metadata_dict = GroupMetadata(zarr_format=3).to_dict() + metadata_dict.pop("node_type") + # TODO: fix the error message + msg = "Invalid value for 'node_type'. Expected 'array or group'. Got 'nothing (the key is missing)'." + with pytest.raises(MetadataValidationError, match=re.escape(msg)): + _build_metadata_v3(metadata_dict) + + +@pytest.mark.parametrize("roots", [("",), ("a", "b")]) +def test_get_roots(roots: tuple[str, ...]): + root_nodes = {k: GroupMetadata(attributes={"name": k}) for k in roots} + child_nodes = { + _join_paths([k, "foo"]): GroupMetadata(attributes={"name": _join_paths([k, "foo"])}) + for k in roots + } + data = root_nodes | child_nodes + assert set(_get_roots(data)) == set(roots) diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 30d0d75f22..b1707c88a3 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -424,6 +424,18 @@ def test_orthogonal_indexing_fallback_on_getitem_2d( np.testing.assert_array_equal(z[index], expected_result) +@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") +def test_setitem_repeated_index(): + array = zarr.array(data=np.zeros((4,)), chunks=(1,)) + indexer = np.array([-1, -1, 0, 0]) + array.oindex[(indexer,)] = [0, 1, 2, 3] + np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) + + indexer = np.array([-1, 0, 0, -1]) + array.oindex[(indexer,)] = [0, 1, 2, 3] + np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) + + Index = list[int] | tuple[slice | int | list[int], ...] @@ -815,6 +827,25 @@ def test_set_orthogonal_selection_1d(store: StorePath) -> None: _test_set_orthogonal_selection(v, a, z, selection) +def test_set_item_1d_last_two_chunks(store: StorePath): + # regression test for GH2849 + g = zarr.open_group(store=store, zarr_format=3, mode="w") + a = g.create_array("bar", shape=(10,), chunks=(3,), dtype=int) + data = np.array([7, 8, 9]) + a[slice(7, 10)] = data + np.testing.assert_array_equal(a[slice(7, 10)], data) + + z = zarr.open_group(store=store, mode="w") + z.create_array("zoo", dtype=float, shape=()) + z["zoo"][...] = np.array(1) # why doesn't [:] work? + np.testing.assert_equal(z["zoo"][()], np.array(1)) + + z = zarr.open_group(store=store, mode="w") + z.create_array("zoo", dtype=float, shape=()) + z["zoo"][...] = 1 # why doesn't [:] work? + np.testing.assert_equal(z["zoo"][()], np.array(1)) + + def _test_set_orthogonal_selection_2d( v: npt.NDArray[np.int_], a: npt.NDArray[np.int_], @@ -1871,7 +1902,7 @@ def test_iter_grid( """ Test that iter_grid works as expected for 1, 2, and 3 dimensions. """ - grid_shape = (5,) * ndim + grid_shape = (10, 5, 7)[:ndim] if origin_0d is not None: origin_kwarg = origin_0d * ndim @@ -1953,3 +1984,13 @@ def test_vectorized_indexing_incompatible_shape(store) -> None: ) with pytest.raises(ValueError, match="Attempting to set"): arr[np.array([1, 2]), np.array([1, 2])] = np.array([[-1, -2], [-3, -4]]) + + +def test_iter_chunk_regions(): + chunks = (2, 3) + a = zarr.create((10, 10), chunks=chunks) + a[:] = 1 + for region in a._iter_chunk_regions(): + assert_array_equal(a[region], np.ones_like(a[region])) + a[region] = 0 + assert_array_equal(a[region], np.zeros_like(a[region])) diff --git a/tests/test_info.py b/tests/test_info.py index db0fd0ef76..0abaff9ae7 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -1,11 +1,11 @@ import textwrap -import numpy as np import pytest from zarr.codecs.bytes import BytesCodec from zarr.core._info import ArrayInfo, GroupInfo, human_readable_size from zarr.core.common import ZarrFormat +from zarr.core.dtype.npy.int import Int32 ZARR_FORMATS = [2, 3] @@ -53,7 +53,8 @@ def test_group_info_complete(zarr_format: ZarrFormat) -> None: def test_array_info(zarr_format: ZarrFormat) -> None: info = ArrayInfo( _zarr_format=zarr_format, - _data_type=np.dtype("int32"), + _data_type=Int32(), + _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), _order="C", @@ -65,7 +66,8 @@ def test_array_info(zarr_format: ZarrFormat) -> None: assert result == textwrap.dedent(f"""\ Type : Array Zarr format : {zarr_format} - Data type : int32 + Data type : Int32(endianness='little') + Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) Order : C @@ -91,7 +93,8 @@ def test_array_info_complete( ) = bytes_things info = ArrayInfo( _zarr_format=zarr_format, - _data_type=np.dtype("int32"), + _data_type=Int32(), + _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), _order="C", @@ -106,7 +109,8 @@ def test_array_info_complete( assert result == textwrap.dedent(f"""\ Type : Array Zarr format : {zarr_format} - Data type : int32 + Data type : Int32(endianness='little') + Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) Order : C diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 2731abada4..395e036db2 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from numcodecs import Zstd +from numcodecs import Blosc import zarr.api.asynchronous import zarr.api.synchronous @@ -17,7 +17,8 @@ open, open_consolidated, ) -from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer import cpu, default_buffer_prototype +from zarr.core.dtype import parse_data_type from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV3Metadata from zarr.core.metadata.v2 import ArrayV2Metadata @@ -476,10 +477,34 @@ async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat): with pytest.raises(ValueError): await zarr.api.asynchronous.open_consolidated(store, zarr_format=None) + @pytest.fixture + async def v2_consolidated_metadata_empty_dataset( + self, memory_store: zarr.storage.MemoryStore + ) -> AsyncGroup: + zgroup_bytes = cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) + zmetadata_bytes = cpu.Buffer.from_bytes( + b'{"metadata":{".zgroup":{"zarr_format":2}},"zarr_consolidated_format":1}' + ) + return AsyncGroup._from_bytes_v2( + None, zgroup_bytes, zattrs_bytes=None, consolidated_metadata_bytes=zmetadata_bytes + ) + + async def test_consolidated_metadata_backwards_compatibility( + self, v2_consolidated_metadata_empty_dataset + ): + """ + Test that consolidated metadata handles a missing .zattrs key. This is necessary for backwards compatibility with zarr-python 2.x. See https://github.com/zarr-developers/zarr-python/issues/2694 + """ + store = zarr.storage.MemoryStore() + await zarr.api.asynchronous.open(store=store, zarr_format=2) + await zarr.api.asynchronous.consolidate_metadata(store) + result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2) + assert result.metadata == v2_consolidated_metadata_empty_dataset.metadata + async def test_consolidated_metadata_v2(self): store = zarr.storage.MemoryStore() g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2) - dtype = "uint8" + dtype = parse_data_type("uint8", zarr_format=2) await g.create_array(name="a", shape=(1,), attributes={"key": "a"}, dtype=dtype) g1 = await g.create_group(name="g1", attributes={"key": "g1"}) await g1.create_group(name="g2", attributes={"key": "g2"}) @@ -498,7 +523,7 @@ async def test_consolidated_metadata_v2(self): attributes={"key": "a"}, chunks=(1,), fill_value=0, - compressor=Zstd(level=0), + compressor=Blosc(), order="C", ), "g1": GroupMetadata( @@ -549,3 +574,108 @@ async def test_use_consolidated_false( assert len([x async for x in good.members()]) == 2 assert good.metadata.consolidated_metadata assert sorted(good.metadata.consolidated_metadata.metadata) == ["a", "b"] + + async def test_stale_child_metadata_ignored(self, memory_store: zarr.storage.MemoryStore): + # https://github.com/zarr-developers/zarr-python/issues/2921 + # When consolidating metadata, we should ignore any (possibly stale) metadata + # from previous consolidations, *including at child nodes*. + root = await zarr.api.asynchronous.group(store=memory_store, zarr_format=3) + await root.create_group("foo") + await zarr.api.asynchronous.consolidate_metadata(memory_store, path="foo") + await root.create_group("foo/bar/spam") + + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + reopened = await zarr.api.asynchronous.open_consolidated(store=memory_store, zarr_format=3) + result = [x[0] async for x in reopened.members(max_depth=None)] + expected = ["foo", "foo/bar", "foo/bar/spam"] + assert result == expected + + async def test_use_consolidated_for_children_members( + self, memory_store: zarr.storage.MemoryStore + ): + # A test that has *unconsolidated* metadata at the root group, but discovers + # a child group with consolidated metadata. + + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + # Consolidate metadata at "a/b" + await zarr.api.asynchronous.consolidate_metadata(memory_store, path="a/b") + + # Add a new group a/b/c, that's not present in the CM at "a/b" + await root.create_group("a/b/c") + + # Now according to the consolidated metadata, "a" has children ["b"] + # but according to the unconsolidated metadata, "a" has children ["b", "c"] + group = await zarr.api.asynchronous.open_group(store=memory_store, path="a") + with pytest.warns(UserWarning, match="Object at 'c' not found"): + result = sorted([x[0] async for x in group.members(max_depth=None)]) + expected = ["b"] + assert result == expected + + result = sorted( + [x[0] async for x in group.members(max_depth=None, use_consolidated_for_children=False)] + ) + expected = ["b", "b/c"] + assert result == expected + + +@pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf]) +async def test_consolidated_metadata_encodes_special_chars( + memory_store: Store, zarr_format: ZarrFormat, fill_value: float +): + root = await group(store=memory_store, zarr_format=zarr_format) + _time = await root.create_array("time", shape=(12,), dtype=np.float64, fill_value=fill_value) + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + root = await group(store=memory_store, zarr_format=zarr_format) + root_buffer = root.metadata.to_buffer_dict(default_buffer_prototype()) + + if zarr_format == 2: + root_metadata = json.loads(root_buffer[".zmetadata"].to_bytes().decode("utf-8"))["metadata"] + elif zarr_format == 3: + root_metadata = json.loads(root_buffer["zarr.json"].to_bytes().decode("utf-8"))[ + "consolidated_metadata" + ]["metadata"] + + expected_fill_value = _time._zdtype.to_json_scalar(fill_value, zarr_format=2) + + if zarr_format == 2: + assert root_metadata["time/.zarray"]["fill_value"] == expected_fill_value + elif zarr_format == 3: + assert root_metadata["time"]["fill_value"] == expected_fill_value + + +class NonConsolidatedStore(zarr.storage.MemoryStore): + """A store that doesn't support consolidated metadata""" + + @property + def supports_consolidated_metadata(self) -> bool: + return False + + +async def test_consolidate_metadata_raises_for_self_consolidating_stores(): + """Verify calling consolidate_metadata on a non supporting stores raises an error.""" + + memory_store = NonConsolidatedStore() + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + + with pytest.raises(TypeError, match="doesn't support consolidated metadata"): + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + +async def test_open_group_in_non_consolidating_stores(): + memory_store = NonConsolidatedStore() + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + + # Opening a group without consolidatedion works as expected + await AsyncGroup.open(memory_store, use_consolidated=False) + + # let the Store opt out of consolidation + await AsyncGroup.open(memory_store, use_consolidated=None) + + # Opening a group with use_consolidated=True should fail + with pytest.raises(ValueError, match="doesn't support consolidated metadata"): + await AsyncGroup.open(memory_store, use_consolidated=True) diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 69dbd4645b..a2894529aa 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -9,6 +9,9 @@ import zarr.api.asynchronous import zarr.storage from zarr.core.buffer import cpu +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype.npy.float import Float32, Float64 +from zarr.core.dtype.npy.int import Int16 from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV2Metadata from zarr.core.metadata.v2 import parse_zarr_format @@ -18,8 +21,6 @@ from zarr.abc.codec import Codec -import numcodecs - def test_parse_zarr_format_valid() -> None: assert parse_zarr_format(2) == 2 @@ -32,8 +33,8 @@ def test_parse_zarr_format_invalid(data: Any) -> None: @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) -@pytest.mark.parametrize("filters", [None, (), (numcodecs.GZip(),)]) -@pytest.mark.parametrize("compressor", [None, numcodecs.GZip()]) +@pytest.mark.parametrize("filters", [None, [{"id": "gzip", "level": 1}]]) +@pytest.mark.parametrize("compressor", [None, {"id": "gzip", "level": 1}]) @pytest.mark.parametrize("fill_value", [None, 0, 1]) @pytest.mark.parametrize("order", ["C", "F"]) @pytest.mark.parametrize("dimension_separator", [".", "/", None]) @@ -80,6 +81,24 @@ def test_metadata_to_dict( assert observed == expected +def test_filters_empty_tuple_warns() -> None: + metadata_dict = { + "zarr_format": 2, + "shape": (1,), + "chunks": (1,), + "dtype": "|u1", + "order": "C", + "compressor": None, + "filters": (), + "fill_value": 0, + } + with pytest.warns( + UserWarning, match="Found an empty list of filters in the array metadata document." + ): + meta = ArrayV2Metadata.from_dict(metadata_dict) + assert meta.filters is None + + class TestConsolidated: @pytest.fixture async def v2_consolidated_metadata( @@ -109,7 +128,7 @@ async def v2_consolidated_metadata( "chunks": [730], "compressor": None, "dtype": " None: expected = ArrayV2Metadata( attributes={"key": "value"}, shape=(8,), - dtype="float64", + dtype=Float64(), chunks=(8,), fill_value=0.0, order="C", ) assert result == expected + + +def test_zstd_checksum() -> None: + arr = zarr.create_array( + {}, + shape=(10,), + chunks=(10,), + dtype="int32", + compressors={"id": "zstd", "level": 5, "checksum": False}, + zarr_format=2, + ) + metadata = json.loads( + arr.metadata.to_buffer_dict(default_buffer_prototype())[".zarray"].to_bytes() + ) + assert "checksum" not in metadata["compressor"] + + +@pytest.mark.parametrize("fill_value", [np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))]) +def test_structured_dtype_fill_value_serialization(tmp_path, fill_value): + zarr_format = 2 + group_path = tmp_path / "test.zarr" + root_group = zarr.open_group(group_path, mode="w", zarr_format=zarr_format) + dtype = np.dtype([("foo", "i4"), ("bar", "i4")]) + root_group.create_array( + name="structured_dtype", + shape=(100, 100), + chunks=(100, 100), + dtype=dtype, + fill_value=fill_value, + ) + + zarr.consolidate_metadata(root_group.store, zarr_format=zarr_format) + root_group = zarr.open_group(group_path, mode="r") + observed = root_group.metadata.consolidated_metadata.metadata["structured_dtype"].fill_value + assert observed == fill_value diff --git a/tests/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py index ef527f42ef..4f385afa6d 100644 --- a/tests/test_metadata/test_v3.py +++ b/tests/test_metadata/test_v3.py @@ -10,16 +10,17 @@ from zarr.codecs.bytes import BytesCodec from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding -from zarr.core.group import parse_node_type +from zarr.core.config import config +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype.npy.string import _NUMPY_SUPPORTS_VLEN_STRING +from zarr.core.dtype.npy.time import DateTime64 +from zarr.core.group import GroupMetadata, parse_node_type from zarr.core.metadata.v3 import ( ArrayV3Metadata, - DataType, - default_fill_value, parse_dimension_names, - parse_fill_value, parse_zarr_format, ) -from zarr.errors import MetadataValidationError +from zarr.errors import MetadataValidationError, NodeTypeValidationError if TYPE_CHECKING: from collections.abc import Sequence @@ -53,15 +54,27 @@ ) complex_dtypes = ("complex64", "complex128") -vlen_dtypes = ("string", "bytes") - -dtypes = (*bool_dtypes, *int_dtypes, *float_dtypes, *complex_dtypes, *vlen_dtypes) +flexible_dtypes = ("str", "bytes", "void") +if _NUMPY_SUPPORTS_VLEN_STRING: + vlen_string_dtypes = ("T",) +else: + vlen_string_dtypes = ("O",) + +dtypes = ( + *bool_dtypes, + *int_dtypes, + *float_dtypes, + *complex_dtypes, + *flexible_dtypes, + *vlen_string_dtypes, +) @pytest.mark.parametrize("data", [None, 1, 2, 4, 5, "3"]) def test_parse_zarr_format_invalid(data: Any) -> None: with pytest.raises( - ValueError, match=f"Invalid value for 'zarr_format'. Expected '3'. Got '{data}'." + MetadataValidationError, + match=f"Invalid value for 'zarr_format'. Expected '3'. Got '{data}'.", ): parse_zarr_format(data) @@ -87,7 +100,8 @@ def test_parse_node_type_invalid(node_type: Any) -> None: @pytest.mark.parametrize("data", [None, "group"]) def test_parse_node_type_array_invalid(data: Any) -> None: with pytest.raises( - ValueError, match=f"Invalid value for 'node_type'. Expected 'array'. Got '{data}'." + NodeTypeValidationError, + match=f"Invalid value for 'node_type'. Expected 'array'. Got '{data}'.", ): parse_node_type_array(data) @@ -107,90 +121,19 @@ def parse_dimension_names_valid(data: Sequence[str] | None) -> None: assert parse_dimension_names(data) == data -@pytest.mark.parametrize("dtype_str", dtypes) -def test_default_fill_value(dtype_str: str) -> None: - """ - Test that parse_fill_value(None, dtype) results in the 0 value for the given dtype. - """ - dtype = DataType(dtype_str) - fill_value = default_fill_value(dtype) - if dtype == DataType.string: - assert fill_value == "" - elif dtype == DataType.bytes: - assert fill_value == b"" - else: - assert fill_value == dtype.to_numpy().type(0) - - -@pytest.mark.parametrize( - ("fill_value", "dtype_str"), - [ - (True, "bool"), - (False, "bool"), - (-8, "int8"), - (0, "int16"), - (1e10, "uint64"), - (-999, "float32"), - (1e32, "float64"), - (float("NaN"), "float64"), - (np.nan, "float64"), - (np.inf, "float64"), - (-1 * np.inf, "float64"), - (0j, "complex64"), - ], -) -def test_parse_fill_value_valid(fill_value: Any, dtype_str: str) -> None: - """ - Test that parse_fill_value(fill_value, dtype) casts fill_value to the given dtype. - """ - parsed = parse_fill_value(fill_value, dtype_str) - - if np.isnan(fill_value): - assert np.isnan(parsed) - else: - assert parsed == DataType(dtype_str).to_numpy().type(fill_value) - - -@pytest.mark.parametrize("fill_value", ["not a valid value"]) -@pytest.mark.parametrize("dtype_str", [*int_dtypes, *float_dtypes, *complex_dtypes]) -def test_parse_fill_value_invalid_value(fill_value: Any, dtype_str: str) -> None: - """ - Test that parse_fill_value(fill_value, dtype) raises ValueError for invalid values. - This test excludes bool because the bool constructor takes anything. - """ - with pytest.raises(ValueError): - parse_fill_value(fill_value, dtype_str) - - -@pytest.mark.parametrize("fill_value", [[1.0, 0.0], [0, 1], complex(1, 1), np.complex64(0)]) +@pytest.mark.parametrize("fill_value", [[1.0, 0.0], [0, 1]]) @pytest.mark.parametrize("dtype_str", [*complex_dtypes]) -def test_parse_fill_value_complex(fill_value: Any, dtype_str: str) -> None: +def test_jsonify_fill_value_complex(fill_value: Any, dtype_str: str) -> None: """ Test that parse_fill_value(fill_value, dtype) correctly handles complex values represented as length-2 sequences """ - dtype = DataType(dtype_str) - if isinstance(fill_value, list): - expected = dtype.to_numpy().type(complex(*fill_value)) - else: - expected = dtype.to_numpy().type(fill_value) - assert expected == parse_fill_value(fill_value, dtype_str) - - -@pytest.mark.parametrize("fill_value", [[1.0, 0.0, 3.0], [0, 1, 3], [1]]) -@pytest.mark.parametrize("dtype_str", [*complex_dtypes]) -def test_parse_fill_value_complex_invalid(fill_value: Any, dtype_str: str) -> None: - """ - Test that parse_fill_value(fill_value, dtype) correctly rejects sequences with length not - equal to 2 - """ - match = ( - f"Got an invalid fill value for complex data type {dtype_str}." - f"Expected a sequence with 2 elements, but {fill_value} has " - f"length {len(fill_value)}." - ) - with pytest.raises(ValueError, match=re.escape(match)): - parse_fill_value(fill_value=fill_value, dtype=dtype_str) + zarr_format = 3 + dtype = get_data_type_from_native_dtype(dtype_str) + expected = dtype.to_native_dtype().type(complex(*fill_value)) + observed = dtype.from_json_scalar(fill_value, zarr_format=zarr_format) + assert observed == expected + assert dtype.to_json_scalar(observed, zarr_format=zarr_format) == tuple(fill_value) @pytest.mark.parametrize("fill_value", [{"foo": 10}]) @@ -200,8 +143,9 @@ def test_parse_fill_value_invalid_type(fill_value: Any, dtype_str: str) -> None: Test that parse_fill_value(fill_value, dtype) raises TypeError for invalid non-sequential types. This test excludes bool because the bool constructor takes anything. """ - with pytest.raises(ValueError, match=r"fill value .* is not valid for dtype .*"): - parse_fill_value(fill_value, dtype_str) + dtype_instance = get_data_type_from_native_dtype(dtype_str) + with pytest.raises(TypeError, match=f"Invalid type: {fill_value}"): + dtype_instance.from_json_scalar(fill_value, zarr_format=3) @pytest.mark.parametrize( @@ -220,14 +164,14 @@ def test_parse_fill_value_invalid_type_sequence(fill_value: Any, dtype_str: str) This test excludes bool because the bool constructor takes anything, and complex because complex values can be created from length-2 sequences. """ - match = f"Cannot parse non-string sequence {fill_value} as a scalar with type {dtype_str}" - with pytest.raises(TypeError, match=re.escape(match)): - parse_fill_value(fill_value, dtype_str) + dtype_instance = get_data_type_from_native_dtype(dtype_str) + with pytest.raises(TypeError, match=re.escape(f"Invalid type: {fill_value}")): + dtype_instance.from_json_scalar(fill_value, zarr_format=3) @pytest.mark.parametrize("chunk_grid", ["regular"]) @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) -@pytest.mark.parametrize("codecs", [[BytesCodec()]]) +@pytest.mark.parametrize("codecs", [[BytesCodec(endian=None)]]) @pytest.mark.parametrize("fill_value", [0, 1]) @pytest.mark.parametrize("chunk_key_encoding", ["v2", "default"]) @pytest.mark.parametrize("dimension_separator", [".", "/", None]) @@ -244,7 +188,7 @@ def test_metadata_to_dict( storage_transformers: tuple[dict[str, JSON]] | None, ) -> None: shape = (1, 2, 3) - data_type = DataType.uint8 + data_type_str = "uint8" if chunk_grid == "regular": cgrid = {"name": "regular", "configuration": {"chunk_shape": (1, 1, 1)}} @@ -268,7 +212,7 @@ def test_metadata_to_dict( "node_type": "array", "shape": shape, "chunk_grid": cgrid, - "data_type": data_type, + "data_type": data_type_str, "chunk_key_encoding": cke, "codecs": tuple(c.to_dict() for c in codecs), "fill_value": fill_value, @@ -304,50 +248,40 @@ def test_metadata_to_dict( assert observed == expected -# @pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) -# @pytest.mark.parametrize("precision", ["ns", "D"]) -# async def test_datetime_metadata(fill_value: int, precision: str) -> None: -# metadata_dict = { -# "zarr_format": 3, -# "node_type": "array", -# "shape": (1,), -# "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, -# "data_type": f" None: +@pytest.mark.parametrize("indent", [2, 4, None]) +def test_json_indent(indent: int): + with config.set({"json_indent": indent}): + m = GroupMetadata() + d = m.to_buffer_dict(default_buffer_prototype())["zarr.json"].to_bytes() + assert d == json.dumps(json.loads(d), indent=indent).encode() + + +@pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) +@pytest.mark.parametrize("precision", ["ns", "D"]) +async def test_datetime_metadata(fill_value: int, precision: str) -> None: + dtype = DateTime64(unit=precision) metadata_dict = { "zarr_format": 3, "node_type": "array", "shape": (1,), "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, - "data_type": " None: metadata_dict = { @@ -357,10 +291,11 @@ async def test_invalid_fill_value_raises(data_type: str, fill_value: float) -> N "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, "data_type": data_type, "chunk_key_encoding": {"name": "default", "separator": "."}, - "codecs": (), + "codecs": ({"name": "bytes"},), "fill_value": fill_value, # this is not a valid fill value for uint8 } - with pytest.raises(ValueError, match=r"fill value .* is not valid for dtype .*"): + # multiple things can go wrong here, so we don't match on the error message. + with pytest.raises(TypeError): ArrayV3Metadata.from_dict(metadata_dict) @@ -388,17 +323,3 @@ async def test_special_float_fill_values(fill_value: str) -> None: elif fill_value == "-Infinity": assert np.isneginf(m.fill_value) assert d["fill_value"] == "-Infinity" - - -@pytest.mark.parametrize("dtype_str", dtypes) -def test_dtypes(dtype_str: str) -> None: - dt = DataType(dtype_str) - np_dtype = dt.to_numpy() - if dtype_str not in vlen_dtypes: - # we can round trip "normal" dtypes - assert dt == DataType.from_numpy(np_dtype) - assert dt.byte_count == np_dtype.itemsize - assert dt.has_endianness == (dt.byte_count > 1) - else: - # return type for vlen types may vary depending on numpy version - assert dt.byte_count is None diff --git a/tests/test_properties.py b/tests/test_properties.py index 678dcae89c..b8d50ef0b1 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,41 +1,153 @@ +import json +import numbers +from typing import Any + import numpy as np import pytest from numpy.testing import assert_array_equal +from zarr.core.buffer import default_buffer_prototype + pytest.importorskip("hypothesis") -import hypothesis.extra.numpy as npst # noqa: E402 -import hypothesis.strategies as st # noqa: E402 -from hypothesis import given # noqa: E402 +import hypothesis.extra.numpy as npst +import hypothesis.strategies as st +from hypothesis import assume, given, settings + +from zarr.abc.store import Store +from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.sync import sync +from zarr.testing.strategies import ( + array_metadata, + arrays, + basic_indices, + numpy_arrays, + orthogonal_indices, + simple_arrays, + stores, + zarr_formats, +) + + +def deep_equal(a: Any, b: Any) -> bool: + """Deep equality check with handling of special cases for array metadata classes""" + if isinstance(a, (complex, np.complexfloating)) and isinstance( + b, (complex, np.complexfloating) + ): + a_real, a_imag = float(a.real), float(a.imag) + b_real, b_imag = float(b.real), float(b.imag) + if np.isnan(a_real) and np.isnan(b_real): + real_eq = True + else: + real_eq = a_real == b_real + if np.isnan(a_imag) and np.isnan(b_imag): + imag_eq = True + else: + imag_eq = a_imag == b_imag + return real_eq and imag_eq + + if isinstance(a, (float, np.floating)) and isinstance(b, (float, np.floating)): + if np.isnan(a) and np.isnan(b): + return True + return a == b + + if isinstance(a, np.datetime64) and isinstance(b, np.datetime64): + if np.isnat(a) and np.isnat(b): + return True + return a == b -from zarr.testing.strategies import arrays, basic_indices, numpy_arrays, zarr_formats # noqa: E402 + if isinstance(a, np.ndarray) and isinstance(b, np.ndarray): + if a.shape != b.shape: + return False + return all(deep_equal(x, y) for x, y in zip(a.flat, b.flat, strict=False)) + if isinstance(a, dict) and isinstance(b, dict): + if set(a.keys()) != set(b.keys()): + return False + return all(deep_equal(a[k], b[k]) for k in a) + if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): + if len(a) != len(b): + return False + return all(deep_equal(x, y) for x, y in zip(a, b, strict=False)) + + return a == b + + +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data(), zarr_format=zarr_formats) -def test_roundtrip(data: st.DataObject, zarr_format: int) -> None: +def test_array_roundtrip(data: st.DataObject, zarr_format: int) -> None: nparray = data.draw(numpy_arrays(zarr_formats=st.just(zarr_format))) zarray = data.draw(arrays(arrays=st.just(nparray), zarr_formats=st.just(zarr_format))) assert_array_equal(nparray, zarray[:]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +@given(array=arrays()) +def test_array_creates_implicit_groups(array): + path = array.path + ancestry = path.split("/")[:-1] + for i in range(len(ancestry)): + parent = "/".join(ancestry[: i + 1]) + if array.metadata.zarr_format == 2: + assert ( + sync(array.store.get(f"{parent}/.zgroup", prototype=default_buffer_prototype())) + is not None + ) + elif array.metadata.zarr_format == 3: + assert ( + sync(array.store.get(f"{parent}/zarr.json", prototype=default_buffer_prototype())) + is not None + ) + + +# this decorator removes timeout; not ideal but it should avoid intermittent CI failures + + +@settings(deadline=None) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data()) def test_basic_indexing(data: st.DataObject) -> None: - zarray = data.draw(arrays()) + zarray = data.draw(simple_arrays()) nparray = zarray[:] indexer = data.draw(basic_indices(shape=nparray.shape)) actual = zarray[indexer] assert_array_equal(nparray[indexer], actual) - new_data = np.ones_like(actual) + new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) zarray[indexer] = new_data nparray[indexer] = new_data assert_array_equal(nparray, zarray[:]) @given(data=st.data()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_oindex(data: st.DataObject) -> None: + # integer_array_indices can't handle 0-size dimensions. + zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) + nparray = zarray[:] + + zindexer, npindexer = data.draw(orthogonal_indices(shape=nparray.shape)) + actual = zarray.oindex[zindexer] + assert_array_equal(nparray[npindexer], actual) + + assume(zarray.shards is None) # GH2834 + for idxr in npindexer: + if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr).size: + # behaviour of setitem with repeated indices is not guaranteed in practice + assume(False) + new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) + nparray[npindexer] = new_data + zarray.oindex[zindexer] = new_data + assert_array_equal(nparray, zarray[:]) + + +@given(data=st.data()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_vindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. - zarray = data.draw(arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) + zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) nparray = zarray[:] indexer = data.draw( @@ -46,6 +158,70 @@ def test_vindex(data: st.DataObject) -> None: actual = zarray.vindex[indexer] assert_array_equal(nparray[indexer], actual) + # FIXME! + # when the indexer is such that a value gets overwritten multiple times, + # I think the output depends on chunking. + # new_data = data.draw(npst.arrays(shape=st.just(actual.shape), dtype=nparray.dtype)) + # nparray[indexer] = new_data + # zarray.vindex[indexer] = new_data + # assert_array_equal(nparray, zarray[:]) + + +@given(store=stores, meta=array_metadata()) # type: ignore[misc] +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +async def test_roundtrip_array_metadata_from_store( + store: Store, meta: ArrayV2Metadata | ArrayV3Metadata +) -> None: + """ + Verify that the I/O for metadata in a store are lossless. + + This test serializes an ArrayV2Metadata or ArrayV3Metadata object to a dict + of buffers via `to_buffer_dict`, writes each buffer to a store under keys + prefixed with "0/", and then reads them back. The test asserts that each + retrieved buffer exactly matches the original buffer. + """ + asdict = meta.to_buffer_dict(prototype=default_buffer_prototype()) + for key, expected in asdict.items(): + await store.set(f"0/{key}", expected) + actual = await store.get(f"0/{key}", prototype=default_buffer_prototype()) + assert actual == expected + + +@given(data=st.data(), zarr_format=zarr_formats) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: int) -> None: + """ + Verify that JSON serialization and deserialization of metadata is lossless. + + For Zarr v2: + - The metadata is split into two JSON documents (one for array data and one + for attributes). The test merges the attributes back before deserialization. + For Zarr v3: + - All metadata is stored in a single JSON document. No manual merger is necessary. + + The test then converts both the original and round-tripped metadata objects + into dictionaries using `dataclasses.asdict` and uses a deep equality check + to verify that the roundtrip has preserved all fields (including special + cases like NaN, Infinity, complex numbers, and datetime values). + """ + metadata = data.draw(array_metadata(zarr_formats=st.just(zarr_format))) + buffer_dict = metadata.to_buffer_dict(prototype=default_buffer_prototype()) + + if zarr_format == 2: + zarray_dict = json.loads(buffer_dict[ZARRAY_JSON].to_bytes().decode()) + zattrs_dict = json.loads(buffer_dict[ZATTRS_JSON].to_bytes().decode()) + # zattrs and zarray are separate in v2, we have to add attributes back prior to `from_dict` + zarray_dict["attributes"] = zattrs_dict + metadata_roundtripped = ArrayV2Metadata.from_dict(zarray_dict) + else: + zarray_dict = json.loads(buffer_dict[ZARR_JSON].to_bytes().decode()) + metadata_roundtripped = ArrayV3Metadata.from_dict(zarray_dict) + + orig = metadata.to_dict() + rt = metadata_roundtripped.to_dict() + + assert deep_equal(orig, rt), f"Roundtrip mismatch:\nOriginal: {orig}\nRoundtripped: {rt}" + # @st.composite # def advanced_indices(draw, *, shape): @@ -69,3 +245,92 @@ def test_vindex(data: st.DataObject) -> None: # nparray = data.draw(np_arrays) # zarray = data.draw(arrays(arrays=st.just(nparray))) # assert_array_equal(nparray, zarray[:]) + + +def serialized_complex_float_is_valid( + serialized: tuple[numbers.Real | str, numbers.Real | str], +) -> bool: + """ + Validate that the serialized representation of a complex float conforms to the spec. + + The specification requires that a serialized complex float must be either: + - A JSON number, or + - One of the strings "NaN", "Infinity", or "-Infinity". + + Args: + serialized: The value produced by JSON serialization for a complex floating point number. + + Returns: + bool: True if the serialized value is valid according to the spec, False otherwise. + """ + return ( + isinstance(serialized, tuple) + and len(serialized) == 2 + and all(serialized_float_is_valid(x) for x in serialized) + ) + + +def serialized_float_is_valid(serialized: numbers.Real | str) -> bool: + """ + Validate that the serialized representation of a float conforms to the spec. + + The specification requires that a serialized float must be either: + - A JSON number, or + - One of the strings "NaN", "Infinity", or "-Infinity". + + Args: + serialized: The value produced by JSON serialization for a floating point number. + + Returns: + bool: True if the serialized value is valid according to the spec, False otherwise. + """ + if isinstance(serialized, numbers.Real): + return True + return serialized in ("NaN", "Infinity", "-Infinity") + + +@given(meta=array_metadata()) # type: ignore[misc] +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_array_metadata_meets_spec(meta: ArrayV2Metadata | ArrayV3Metadata) -> None: + """ + Validate that the array metadata produced by the library conforms to the relevant spec (V2 vs V3). + + For ArrayV2Metadata: + - Ensures that 'zarr_format' is 2. + - Verifies that 'filters' is either None or a tuple (and not an empty tuple). + For ArrayV3Metadata: + - Ensures that 'zarr_format' is 3. + + For both versions: + - If the dtype is a floating point of some kind, verifies of fill values: + * NaN is serialized as the string "NaN" + * Positive Infinity is serialized as the string "Infinity" + * Negative Infinity is serialized as the string "-Infinity" + * Other fill values are preserved as-is. + - If the dtype is a complex number of some kind, verifies that each component of the fill + value (real and imaginary) satisfies the serialization rules for floating point numbers. + - If the dtype is a datetime of some kind, verifies that `NaT` values are serialized as "NaT". + + Note: + This test validates spec-compliance for array metadata serialization. + It is a work-in-progress and should be expanded as further edge cases are identified. + """ + asdict_dict = meta.to_dict() + + # version-specific validations + if isinstance(meta, ArrayV2Metadata): + assert asdict_dict["filters"] != () + assert asdict_dict["filters"] is None or isinstance(asdict_dict["filters"], tuple) + assert asdict_dict["zarr_format"] == 2 + else: + assert asdict_dict["zarr_format"] == 3 + + # version-agnostic validations + dtype_native = meta.dtype.to_native_dtype() + if dtype_native.kind == "f": + assert serialized_float_is_valid(asdict_dict["fill_value"]) + elif dtype_native.kind == "c": + # fill_value should be a two-element array [real, imag]. + assert serialized_complex_float_is_valid(asdict_dict["fill_value"]) + elif dtype_native.kind in ("M", "m") and np.isnat(meta.fill_value): + assert asdict_dict["fill_value"] == -9223372036854775808 diff --git a/tests/test_regression/__init__.py b/tests/test_regression/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_regression/scripts/__init__.py b/tests/test_regression/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_regression/scripts/v2.18.py b/tests/test_regression/scripts/v2.18.py new file mode 100644 index 0000000000..39e1c5210c --- /dev/null +++ b/tests/test_regression/scripts/v2.18.py @@ -0,0 +1,81 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr==2.18", +# "numcodecs==0.15" +# ] +# /// + +import argparse + +import zarr +from zarr._storage.store import BaseStore + + +def copy_group( + *, node: zarr.hierarchy.Group, store: zarr.storage.BaseStore, path: str, overwrite: bool +) -> zarr.hierarchy.Group: + result = zarr.group(store=store, path=path, overwrite=overwrite) + result.attrs.put(node.attrs.asdict()) + for key, child in node.items(): + child_path = f"{path}/{key}" + if isinstance(child, zarr.hierarchy.Group): + copy_group(node=child, store=store, path=child_path, overwrite=overwrite) + elif isinstance(child, zarr.core.Array): + copy_array(node=child, store=store, overwrite=overwrite, path=child_path) + return result + + +def copy_array( + *, node: zarr.core.Array, store: BaseStore, path: str, overwrite: bool +) -> zarr.core.Array: + result = zarr.create( + shape=node.shape, + dtype=node.dtype, + fill_value=node.fill_value, + chunks=node.chunks, + compressor=node.compressor, + filters=node.filters, + order=node.order, + dimension_separator=node._dimension_separator, + store=store, + path=path, + overwrite=overwrite, + ) + result.attrs.put(node.attrs.asdict()) + result[:] = node[:] + return result + + +def copy_node( + node: zarr.hierarchy.Group | zarr.core.Array, store: BaseStore, path: str, overwrite: bool +) -> zarr.hierarchy.Group | zarr.core.Array: + if isinstance(node, zarr.hierarchy.Group): + return copy_group(node=node, store=store, path=path, overwrite=overwrite) + elif isinstance(node, zarr.core.Array): + return copy_array(node=node, store=store, path=path, overwrite=overwrite) + else: + raise TypeError(f"Unexpected node type: {type(node)}") # pragma: no cover + + +def cli() -> None: + parser = argparse.ArgumentParser( + description="Copy a zarr hierarchy from one location to another" + ) + parser.add_argument("source", type=str, help="Path to the source zarr hierarchy") + parser.add_argument("destination", type=str, help="Path to the destination zarr hierarchy") + args = parser.parse_args() + + src, dst = args.source, args.destination + root_src = zarr.open(src, mode="r") + result = copy_node(node=root_src, store=zarr.NestedDirectoryStore(dst), path="", overwrite=True) + + print(f"successfully created {result} at {dst}") + + +def main() -> None: + cli() + + +if __name__ == "__main__": + main() diff --git a/tests/test_regression/test_regression.py b/tests/test_regression/test_regression.py new file mode 100644 index 0000000000..34c48a6933 --- /dev/null +++ b/tests/test_regression/test_regression.py @@ -0,0 +1,156 @@ +import subprocess +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from typing import TYPE_CHECKING + +import numcodecs +import numpy as np +import pytest +from numcodecs import LZ4, LZMA, Blosc, GZip, VLenBytes, VLenUTF8, Zstd + +import zarr +from zarr.core.array import Array +from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding +from zarr.core.dtype.npy.bytes import VariableLengthBytes +from zarr.core.dtype.npy.string import VariableLengthUTF8 +from zarr.storage import LocalStore + +if TYPE_CHECKING: + from zarr.core.dtype import ZDTypeLike + + +def runner_installed() -> bool: + """ + Check if a PEP-723 compliant python script runner is installed. + """ + try: + subprocess.check_output(["uv", "--version"]) + return True # noqa: TRY300 + except FileNotFoundError: + return False + + +@dataclass(kw_only=True) +class ArrayParams: + values: np.ndarray[tuple[int], np.dtype[np.generic]] + fill_value: np.generic | str | int | bytes + filters: tuple[numcodecs.abc.Codec, ...] = () + compressor: numcodecs.abc.Codec + + +basic_codecs = GZip(), Blosc(), LZ4(), LZMA(), Zstd() +basic_dtypes = "|b", ">i2", ">i4", ">f4", ">f8", "c8", "c16", "M8[10us]", "m8[4ps]" +string_dtypes = "U4" +bytes_dtypes = ">S1", "V10", " Array: + dest = tmp_path / "in" + store = LocalStore(dest) + array_params: ArrayParams = request.param + compressor = array_params.compressor + chunk_key_encoding = V2ChunkKeyEncoding(separator="/") + dtype: ZDTypeLike + if array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenUTF8(),): + dtype = VariableLengthUTF8() # type: ignore[assignment] + elif array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenBytes(),): + dtype = VariableLengthBytes() + else: + dtype = array_params.values.dtype + z = zarr.create_array( + store, + shape=array_params.values.shape, + dtype=dtype, + chunks=array_params.values.shape, + compressors=compressor, + filters=array_params.filters, + fill_value=array_params.fill_value, + order="C", + chunk_key_encoding=chunk_key_encoding, + write_data=True, + zarr_format=2, + ) + z[:] = array_params.values + return z + + +# TODO: make this dynamic based on the installed scripts +script_paths = [Path(__file__).resolve().parent / "scripts" / "v2.18.py"] + + +@pytest.mark.skipif(not runner_installed(), reason="no python script runner installed") +@pytest.mark.parametrize( + "source_array", array_cases, indirect=True, ids=tuple(map(str, array_cases)) +) +@pytest.mark.parametrize("script_path", script_paths) +def test_roundtrip(source_array: Array, tmp_path: Path, script_path: Path) -> None: + out_path = tmp_path / "out" + copy_op = subprocess.run( + [ + "uv", + "run", + script_path, + str(source_array.store).removeprefix("file://"), + str(out_path), + ], + capture_output=True, + text=True, + ) + assert copy_op.returncode == 0 + out_array = zarr.open_array(store=out_path, mode="r", zarr_format=2) + assert source_array.metadata.to_dict() == out_array.metadata.to_dict() + assert np.array_equal(source_array[:], out_array[:]) diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 5ab299442d..e9c9319ad3 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -4,9 +4,80 @@ import pytest from _pytest.compat import LEGACY_PATH -from zarr.core.common import AccessModeLiteral -from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath, make_store_path -from zarr.storage._utils import normalize_path +import zarr +from zarr import Group +from zarr.core.common import AccessModeLiteral, ZarrFormat +from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath +from zarr.storage._common import contains_array, contains_group, make_store_path +from zarr.storage._utils import ( + _join_paths, + _normalize_path_keys, + _normalize_paths, + _relativize_path, + normalize_path, +) + + +@pytest.fixture( + params=["none", "temp_dir_str", "temp_dir_path", "store_path", "memory_store", "dict"] +) +def store_like(request): + if request.param == "none": + yield None + elif request.param == "temp_dir_str": + with tempfile.TemporaryDirectory() as temp_dir: + yield temp_dir + elif request.param == "temp_dir_path": + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + elif request.param == "store_path": + yield StorePath(store=MemoryStore(store_dict={}), path="/") + elif request.param == "memory_store": + yield MemoryStore(store_dict={}) + elif request.param == "dict": + yield {} + + +@pytest.mark.parametrize("path", ["foo", "foo/bar"]) +@pytest.mark.parametrize("write_group", [True, False]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_contains_group( + local_store, path: str, write_group: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that the contains_group method correctly reports the existence of a group. + """ + root = Group.from_store(store=local_store, zarr_format=zarr_format) + if write_group: + root.create_group(path) + store_path = StorePath(local_store, path=path) + assert await contains_group(store_path, zarr_format=zarr_format) == write_group + + +@pytest.mark.parametrize("path", ["foo", "foo/bar"]) +@pytest.mark.parametrize("write_array", [True, False]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_contains_array( + local_store, path: str, write_array: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that the contains array method correctly reports the existence of an array. + """ + root = Group.from_store(store=local_store, zarr_format=zarr_format) + if write_array: + root.create_array(path, shape=(100,), chunks=(10,), dtype="i4") + store_path = StorePath(local_store, path=path) + assert await contains_array(store_path, zarr_format=zarr_format) == write_array + + +@pytest.mark.parametrize("func", [contains_array, contains_group]) +async def test_contains_invalid_format_raises(local_store, func: callable) -> None: + """ + Test contains_group and contains_array raise errors for invalid zarr_formats + """ + store_path = StorePath(local_store) + with pytest.raises(ValueError): + assert await func(store_path, zarr_format="3.0") @pytest.mark.parametrize("path", [None, "", "bar"]) @@ -55,10 +126,18 @@ async def test_make_store_path_store_path( assert Path(store_path.store.root) == Path(tmpdir) path_normalized = normalize_path(path) assert store_path.path == (store_like / path_normalized).path - assert store_path.read_only == ro +@pytest.mark.parametrize("modes", [(True, "w"), (False, "x")]) +async def test_store_path_invalid_mode_raises(tmpdir: LEGACY_PATH, modes: tuple) -> None: + """ + Test that ValueErrors are raise for invalid mode. + """ + with pytest.raises(ValueError): + await StorePath.open(LocalStore(str(tmpdir), read_only=modes[0]), path=None, mode=modes[1]) + + async def test_make_store_path_invalid() -> None: """ Test that invalid types raise TypeError @@ -69,21 +148,12 @@ async def test_make_store_path_invalid() -> None: async def test_make_store_path_fsspec(monkeypatch) -> None: pytest.importorskip("fsspec") + pytest.importorskip("requests") + pytest.importorskip("aiohttp") store_path = await make_store_path("http://foo.com/bar") assert isinstance(store_path.store, FsspecStore) -@pytest.mark.parametrize( - "store_like", - [ - None, - tempfile.TemporaryDirectory().name, - Path(tempfile.TemporaryDirectory().name), - StorePath(store=MemoryStore(store_dict={}), path="/"), - MemoryStore(store_dict={}), - {}, - ], -) async def test_make_store_path_storage_options_raises(store_like: StoreLike) -> None: with pytest.raises(TypeError, match="storage_options"): await make_store_path(store_like, storage_options={"foo": "bar"}) @@ -122,3 +192,79 @@ def test_normalize_path_none(): def test_normalize_path_invalid(path: str): with pytest.raises(ValueError): normalize_path(path) + + +@pytest.mark.parametrize("paths", [("", "foo"), ("foo", "bar")]) +def test_join_paths(paths: tuple[str, str]) -> None: + """ + Test that _join_paths joins paths in a way that is robust to an empty string + """ + observed = _join_paths(paths) + if paths[0] == "": + assert observed == paths[1] + else: + assert observed == "/".join(paths) + + +class TestNormalizePaths: + @staticmethod + def test_valid() -> None: + """ + Test that path normalization works as expected + """ + paths = ["a", "b", "c", "d", "", "//a///b//"] + assert _normalize_paths(paths) == tuple(normalize_path(p) for p in paths) + + @staticmethod + @pytest.mark.parametrize("paths", [("", "/"), ("///a", "a")]) + def test_invalid(paths: tuple[str, str]) -> None: + """ + Test that name collisions after normalization raise a ``ValueError`` + """ + msg = ( + f"After normalization, the value '{paths[1]}' collides with '{paths[0]}'. " + f"Both '{paths[1]}' and '{paths[0]}' normalize to the same value: '{normalize_path(paths[0])}'. " + f"You should use either '{paths[1]}' or '{paths[0]}', but not both." + ) + with pytest.raises(ValueError, match=msg): + _normalize_paths(paths) + + +def test_normalize_path_keys(): + """ + Test that ``_normalize_path_keys`` just applies the normalize_path function to each key of its + input + """ + data = {"a": 10, "//b": 10} + assert _normalize_path_keys(data) == {normalize_path(k): v for k, v in data.items()} + + +@pytest.mark.parametrize( + ("path", "prefix", "expected"), + [ + ("a", "", "a"), + ("a/b/c", "a/b", "c"), + ("a/b/c", "a", "b/c"), + ], +) +def test_relativize_path_valid(path: str, prefix: str, expected: str) -> None: + """ + Test the normal behavior of the _relativize_path function. Prefixes should be removed from the + path argument. + """ + assert _relativize_path(path=path, prefix=prefix) == expected + + +def test_relativize_path_invalid() -> None: + path = "a/b/c" + prefix = "b" + msg = f"The first component of {path} does not start with {prefix}." + with pytest.raises(ValueError, match=msg): + _relativize_path(path="a/b/c", prefix="b") + + +def test_invalid_open_mode() -> None: + store = MemoryStore() + zarr.create((100,), store=store, zarr_format=2, path="a") + with pytest.raises(ValueError, match="Store is not read-only but mode is 'r'"): + zarr.open_array(store=store, path="a", zarr_format=2, mode="r") diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index b307f2cdf4..1a989525e3 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -2,28 +2,51 @@ import json import os -from typing import TYPE_CHECKING +import re +from typing import TYPE_CHECKING, Any +import numpy as np import pytest -from botocore.session import Session +from packaging.version import parse as parse_version import zarr.api.asynchronous +from zarr import Array +from zarr.abc.store import OffsetByteRequest from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.sync import _collect_aiterator, sync from zarr.storage import FsspecStore +from zarr.storage._fsspec import _make_async from zarr.testing.store import StoreTests if TYPE_CHECKING: + import pathlib from collections.abc import Generator + from pathlib import Path import botocore.client + import s3fs + from zarr.core.common import JSON + + +# Warning filter due to https://github.com/boto/boto3/issues/3889 +pytestmark = [ + pytest.mark.filterwarnings( + re.escape("ignore:datetime.datetime.utcnow() is deprecated:DeprecationWarning") + ), + # TODO: fix these warnings + pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning"), + pytest.mark.filterwarnings( + "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" + ), +] fsspec = pytest.importorskip("fsspec") s3fs = pytest.importorskip("s3fs") requests = pytest.importorskip("requests") moto_server = pytest.importorskip("moto.moto_server.threaded_moto_server") moto = pytest.importorskip("moto") +botocore = pytest.importorskip("botocore") # ### amended from s3fs ### # test_bucket_name = "test" @@ -50,7 +73,7 @@ def s3_base() -> Generator[None, None, None]: def get_boto3_client() -> botocore.client.BaseClient: # NB: we use the sync botocore client for setup - session = Session() + session = botocore.session.Session() return session.create_client("s3", endpoint_url=endpoint_url) @@ -95,10 +118,13 @@ async def test_basic() -> None: data = b"hello" await store.set("foo", cpu.Buffer.from_bytes(data)) assert await store.exists("foo") - assert (await store.get("foo", prototype=default_buffer_prototype())).to_bytes() == data + buf = await store.get("foo", prototype=default_buffer_prototype()) + assert buf is not None + assert buf.to_bytes() == data out = await store.get_partial_values( - prototype=default_buffer_prototype(), key_ranges=[("foo", (1, None))] + prototype=default_buffer_prototype(), key_ranges=[("foo", OffsetByteRequest(1))] ) + assert out[0] is not None assert out[0].to_bytes() == data[1:] @@ -107,14 +133,19 @@ class TestFsspecStoreS3(StoreTests[FsspecStore, cpu.Buffer]): buffer_cls = cpu.Buffer @pytest.fixture - def store_kwargs(self, request) -> dict[str, str | bool]: - fs, path = fsspec.url_to_fs( + def store_kwargs(self) -> dict[str, str | bool]: + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + fs, path = url_to_fs( f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=True ) return {"fs": fs, "path": path} @pytest.fixture - def store(self, store_kwargs: dict[str, str | bool]) -> FsspecStore: + async def store(self, store_kwargs: dict[str, Any]) -> FsspecStore: return self.store_cls(**store_kwargs) async def get(self, store: FsspecStore, key: str) -> Buffer: @@ -149,7 +180,11 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: "anon": False, } - meta = {"attributes": {"key": "value"}, "zarr_format": 3, "node_type": "group"} + meta: dict[str, JSON] = { + "attributes": {"key": "value"}, + "zarr_format": 3, + "node_type": "group", + } await store.set( "zarr.json", @@ -160,7 +195,11 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value"} - meta["attributes"]["key"] = "value-2" + meta = { + "attributes": {"key": "value-2"}, + "zarr_format": 3, + "node_type": "group", + } await store.set( "directory-2/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), @@ -170,7 +209,11 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value-2"} - meta["attributes"]["key"] = "value-3" + meta = { + "attributes": {"key": "value-3"}, + "zarr_format": 3, + "node_type": "group", + } await store.set( "directory-3/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), @@ -180,6 +223,10 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: ) assert dict(group.attrs) == {"key": "value-3"} + @pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.03.01"), + reason="Prior bug in from_upath", + ) def test_from_upath(self) -> None: upath = pytest.importorskip("upath") path = upath.UPath( @@ -193,7 +240,7 @@ def test_from_upath(self) -> None: assert result.fs.asynchronous assert result.path == f"{test_bucket_name}/foo/bar" - def test_init_raises_if_path_has_scheme(self, store_kwargs) -> None: + def test_init_raises_if_path_has_scheme(self, store_kwargs: dict[str, Any]) -> None: # regression test for https://github.com/zarr-developers/zarr-python/issues/2342 store_kwargs["path"] = "s3://" + store_kwargs["path"] with pytest.raises( @@ -202,15 +249,180 @@ def test_init_raises_if_path_has_scheme(self, store_kwargs) -> None: self.store_cls(**store_kwargs) def test_init_warns_if_fs_asynchronous_is_false(self) -> None: - fs, path = fsspec.url_to_fs( + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + fs, path = url_to_fs( f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=False ) store_kwargs = {"fs": fs, "path": path} with pytest.warns(UserWarning, match=r".* was not created with `asynchronous=True`.*"): self.store_cls(**store_kwargs) - async def test_empty_nonexistent_path(self, store_kwargs) -> None: + async def test_empty_nonexistent_path(self, store_kwargs: dict[str, Any]) -> None: # regression test for https://github.com/zarr-developers/zarr-python/pull/2343 store_kwargs["path"] += "/abc" store = await self.store_cls.open(**store_kwargs) assert await store.is_empty("") + + async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: + store.supports_deletes = False + with pytest.raises( + NotImplementedError, + match="This method is only available for stores that support deletes.", + ): + await store.delete_dir("test_prefix") + + +def array_roundtrip(store: FsspecStore) -> None: + """ + Round trip an array using a Zarr store + + Args: + store: FsspecStore + """ + data = np.ones((3, 3)) + arr = zarr.create_array(store=store, overwrite=True, data=data) + assert isinstance(arr, Array) + # Read set values + arr2 = zarr.open_array(store=store) + assert isinstance(arr2, Array) + np.testing.assert_array_equal(arr[:], data) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_wrap_sync_filesystem(tmp_path: pathlib.Path) -> None: + """The local fs is not async so we should expect it to be wrapped automatically""" + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + store = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) + assert isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl + array_roundtrip(store) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) >= parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_wrap_sync_filesystem_raises(tmp_path: pathlib.Path) -> None: + """The local fs is not async so we should expect it to be wrapped automatically""" + with pytest.raises(ImportError, match="The filesystem .*"): + FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_no_wrap_async_filesystem() -> None: + """An async fs should not be wrapped automatically; fsspec's s3 filesystem is such an fs""" + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + store = FsspecStore.from_url( + f"s3://{test_bucket_name}/foo/spam/", + storage_options={"endpoint_url": endpoint_url, "anon": False, "asynchronous": True}, + read_only=False, + ) + assert not isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl + array_roundtrip(store) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_open_fsmap_file(tmp_path: pathlib.Path) -> None: + min_fsspec_with_async_wrapper = parse_version("2024.12.0") + current_version = parse_version(fsspec.__version__) + + fs = fsspec.filesystem("file", auto_mkdir=True) + mapper = fs.get_mapper(tmp_path) + + if current_version < min_fsspec_with_async_wrapper: + # Expect ImportError for older versions + with pytest.raises( + ImportError, + match=r"The filesystem .* is synchronous, and the required AsyncFileSystemWrapper is not available.*", + ): + array_roundtrip(mapper) + else: + # Newer versions should work + array_roundtrip(mapper) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_open_fsmap_file_raises(tmp_path: pathlib.Path) -> None: + fsspec = pytest.importorskip("fsspec.implementations.local") + fs = fsspec.LocalFileSystem(auto_mkdir=False) + mapper = fs.get_mapper(tmp_path) + with pytest.raises(ValueError, match="LocalFilesystem .*"): + array_roundtrip(mapper) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +def test_open_fsmap_s3(asynchronous: bool) -> None: + s3_filesystem = s3fs.S3FileSystem( + asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False + ) + mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") + array_roundtrip(mapper) + + +def test_open_s3map_raises() -> None: + with pytest.raises(TypeError, match="Unsupported type for store_like:.*"): + zarr.open(store=0, mode="w", shape=(3, 3)) + s3_filesystem = s3fs.S3FileSystem(asynchronous=True, endpoint_url=endpoint_url, anon=False) + mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") + with pytest.raises( + ValueError, match="'path' was provided but is not used for FSMap store_like objects" + ): + zarr.open(store=mapper, path="bar", mode="w", shape=(3, 3)) + with pytest.raises( + ValueError, + match="'storage_options was provided but is not used for FSMap store_like objects", + ): + zarr.open(store=mapper, storage_options={"anon": True}, mode="w", shape=(3, 3)) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +def test_make_async(asynchronous: bool) -> None: + s3_filesystem = s3fs.S3FileSystem( + asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False + ) + fs = _make_async(s3_filesystem) + assert fs.asynchronous + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +async def test_delete_dir_wrapped_filesystem(tmp_path: Path) -> None: + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + from fsspec.implementations.local import LocalFileSystem + + wrapped_fs = AsyncFileSystemWrapper(LocalFileSystem(auto_mkdir=True)) + store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmp_path}/test/path") + + assert isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.asynchronous + + await store.set("zarr.json", cpu.Buffer.from_bytes(b"root")) + await store.set("foo-bar/zarr.json", cpu.Buffer.from_bytes(b"root")) + await store.set("foo/zarr.json", cpu.Buffer.from_bytes(b"bar")) + await store.set("foo/c/0", cpu.Buffer.from_bytes(b"chunk")) + await store.delete_dir("foo") + assert await store.exists("zarr.json") + assert await store.exists("foo-bar/zarr.json") + assert not await store.exists("foo/zarr.json") + assert not await store.exists("foo/c/0") diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py index 22597a2c3f..7974d0d633 100644 --- a/tests/test_store/test_local.py +++ b/tests/test_store/test_local.py @@ -1,16 +1,17 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import pathlib +import re +import numpy as np import pytest import zarr +from zarr import create_array from zarr.core.buffer import Buffer, cpu from zarr.storage import LocalStore from zarr.testing.store import StoreTests - -if TYPE_CHECKING: - import pathlib +from zarr.testing.utils import assert_bytes_equal class TestLocalStore(StoreTests[LocalStore, cpu.Buffer]): @@ -27,7 +28,7 @@ async def set(self, store: LocalStore, key: str, value: Buffer) -> None: (store.root / key).write_bytes(value.to_bytes()) @pytest.fixture - def store_kwargs(self, tmpdir) -> dict[str, str]: + def store_kwargs(self, tmpdir: str) -> dict[str, str]: return {"root": str(tmpdir)} def test_store_repr(self, store: LocalStore) -> None: @@ -47,9 +48,64 @@ async def test_empty_with_empty_subdir(self, store: LocalStore) -> None: (store.root / "foo/bar").mkdir(parents=True) assert await store.is_empty("") - def test_creates_new_directory(self, tmp_path: pathlib.Path): + def test_creates_new_directory(self, tmp_path: pathlib.Path) -> None: target = tmp_path.joinpath("a", "b", "c") assert not target.exists() store = self.store_cls(root=target) zarr.group(store=store) + + def test_invalid_root_raises(self) -> None: + """ + Test that a TypeError is raised when a non-str/Path type is used for the `root` argument + """ + with pytest.raises( + TypeError, + match=r"'root' must be a string or Path instance. Got an instance of instead.", + ): + LocalStore(root=0) # type: ignore[arg-type] + + async def test_get_with_prototype_default(self, store: LocalStore) -> None: + """ + Ensure that data can be read via ``store.get`` if the prototype keyword argument is unspecified, i.e. set to ``None``. + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await self.set(store, key, data_buf) + observed = await store.get(key, prototype=None) + assert_bytes_equal(observed, data_buf) + + @pytest.mark.parametrize("ndim", [0, 1, 3]) + @pytest.mark.parametrize( + "destination", ["destination", "foo/bar/destintion", pathlib.Path("foo/bar/destintion")] + ) + async def test_move( + self, tmp_path: pathlib.Path, ndim: int, destination: pathlib.Path | str + ) -> None: + origin = tmp_path / "origin" + if isinstance(destination, str): + destination = str(tmp_path / destination) + else: + destination = tmp_path / destination + + print(type(destination)) + store = await LocalStore.open(root=origin) + shape = (4,) * ndim + chunks = (2,) * ndim + data = np.arange(4**ndim) + if ndim > 0: + data = data.reshape(*shape) + array = create_array(store, data=data, chunks=chunks or "auto") + + await store.move(destination) + + assert store.root == pathlib.Path(destination) + assert pathlib.Path(destination).exists() + assert not origin.exists() + assert np.array_equal(array[...], data) + + store2 = await LocalStore.open(root=origin) + with pytest.raises( + FileExistsError, match=re.escape(f"Destination root {destination} already exists") + ): + await store2.move(destination) diff --git a/tests/test_store/test_logging.py b/tests/test_store/test_logging.py index b32a214db5..1a89dca874 100644 --- a/tests/test_store/test_logging.py +++ b/tests/test_store/test_logging.py @@ -1,17 +1,87 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING import pytest import zarr -from zarr.core.buffer import default_buffer_prototype -from zarr.storage import LoggingStore +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.storage import LocalStore, LoggingStore +from zarr.testing.store import StoreTests if TYPE_CHECKING: + from _pytest.compat import LEGACY_PATH + from zarr.abc.store import Store +class TestLoggingStore(StoreTests[LoggingStore, cpu.Buffer]): + store_cls = LoggingStore + buffer_cls = cpu.Buffer + + async def get(self, store: LoggingStore, key: str) -> Buffer: + return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) + + async def set(self, store: LoggingStore, key: str, value: Buffer) -> None: + parent = (store._store.root / key).parent + if not parent.exists(): + parent.mkdir(parents=True) + (store._store.root / key).write_bytes(value.to_bytes()) + + @pytest.fixture + def store_kwargs(self, tmpdir: LEGACY_PATH) -> dict[str, str]: + return {"store": LocalStore(str(tmpdir)), "log_level": "DEBUG"} + + @pytest.fixture + def open_kwargs(self, tmpdir) -> dict[str, str]: + return {"store_cls": LocalStore, "root": str(tmpdir), "log_level": "DEBUG"} + + @pytest.fixture + def store(self, store_kwargs: str | dict[str, Buffer] | None) -> LoggingStore: + return self.store_cls(**store_kwargs) + + def test_store_supports_writes(self, store: LoggingStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: LoggingStore) -> None: + assert store.supports_partial_writes + + def test_store_supports_listing(self, store: LoggingStore) -> None: + assert store.supports_listing + + def test_store_repr(self, store: LoggingStore) -> None: + assert f"{store!r}" == f"LoggingStore(LocalStore, 'file://{store._store.root.as_posix()}')" + + def test_store_str(self, store: LoggingStore) -> None: + assert str(store) == f"logging-file://{store._store.root.as_posix()}" + + async def test_default_handler(self, local_store, capsys) -> None: + # Store and then remove existing handlers to enter default handler code path + handlers = logging.getLogger().handlers[:] + for h in handlers: + logging.getLogger().removeHandler(h) + # Test logs are sent to stdout + wrapped = LoggingStore(store=local_store) + buffer = default_buffer_prototype().buffer + res = await wrapped.set("foo/bar/c/0", buffer.from_bytes(b"\x01\x02\x03\x04")) + assert res is None + captured = capsys.readouterr() + assert len(captured) == 2 + assert "Calling LocalStore.set" in captured.out + assert "Finished LocalStore.set" in captured.out + # Restore handlers + for h in handlers: + logging.getLogger().addHandler(h) + + def test_is_open_setter_raises(self, store: LoggingStore) -> None: + "Test that a user cannot change `_is_open` without opening the underlying store." + with pytest.raises( + NotImplementedError, match="LoggingStore must be opened via the `_open` method" + ): + store._is_open = True + + @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) async def test_logging_store(store: Store, caplog) -> None: wrapped = LoggingStore(store=store, log_level="DEBUG") diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py index ba38889b52..4fc3f6e698 100644 --- a/tests/test_store/test_memory.py +++ b/tests/test_store/test_memory.py @@ -1,13 +1,26 @@ from __future__ import annotations +import re +from typing import TYPE_CHECKING, Any + +import numpy as np +import numpy.typing as npt import pytest +import zarr from zarr.core.buffer import Buffer, cpu, gpu from zarr.storage import GpuMemoryStore, MemoryStore from zarr.testing.store import StoreTests from zarr.testing.utils import gpu_test +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + +# TODO: work out where this warning is coming from and fix it +@pytest.mark.filterwarnings( + re.escape("ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited") +) class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]): store_cls = MemoryStore buffer_cls = cpu.Buffer @@ -19,16 +32,16 @@ async def get(self, store: MemoryStore, key: str) -> Buffer: return store._store_dict[key] @pytest.fixture(params=[None, True]) - def store_kwargs( - self, request: pytest.FixtureRequest - ) -> dict[str, str | dict[str, Buffer] | None]: - kwargs = {"store_dict": None} + def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: + kwargs: dict[str, Any] if request.param is True: - kwargs["store_dict"] = {} + kwargs = {"store_dict": {}} + else: + kwargs = {"store_dict": None} return kwargs @pytest.fixture - def store(self, store_kwargs: str | dict[str, Buffer] | None) -> MemoryStore: + async def store(self, store_kwargs: dict[str, Any]) -> MemoryStore: return self.store_cls(**store_kwargs) def test_store_repr(self, store: MemoryStore) -> None: @@ -43,32 +56,53 @@ def test_store_supports_listing(self, store: MemoryStore) -> None: def test_store_supports_partial_writes(self, store: MemoryStore) -> None: assert store.supports_partial_writes - def test_list_prefix(self, store: MemoryStore) -> None: + async def test_list_prefix(self, store: MemoryStore) -> None: assert True - + @pytest.mark.parametrize("dtype", ["uint8", "float32", "int64"]) + @pytest.mark.parametrize("zarr_format", [2, 3]) + async def test_deterministic_size( + self, store: MemoryStore, dtype: npt.DTypeLike, zarr_format: ZarrFormat + ) -> None: + a = zarr.empty( + store=store, + shape=(3,), + chunks=(1000,), + dtype=dtype, + zarr_format=zarr_format, + overwrite=True, + ) + a[...] = 1 + a.resize((1000,)) + + np.testing.assert_array_equal(a[:3], 1) + np.testing.assert_array_equal(a[3:], 0) + + +# TODO: fix this warning +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @gpu_test class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): store_cls = GpuMemoryStore buffer_cls = gpu.Buffer - async def set(self, store: GpuMemoryStore, key: str, value: Buffer) -> None: + async def set(self, store: GpuMemoryStore, key: str, value: gpu.Buffer) -> None: # type: ignore[override] store._store_dict[key] = value async def get(self, store: MemoryStore, key: str) -> Buffer: return store._store_dict[key] @pytest.fixture(params=[None, True]) - def store_kwargs( - self, request: pytest.FixtureRequest - ) -> dict[str, str | dict[str, Buffer] | None]: - kwargs = {"store_dict": None} + def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: + kwargs: dict[str, Any] if request.param is True: - kwargs["store_dict"] = {} + kwargs = {"store_dict": {}} + else: + kwargs = {"store_dict": None} return kwargs @pytest.fixture - def store(self, store_kwargs: str | dict[str, gpu.Buffer] | None) -> GpuMemoryStore: + async def store(self, store_kwargs: dict[str, Any]) -> GpuMemoryStore: return self.store_cls(**store_kwargs) def test_store_repr(self, store: GpuMemoryStore) -> None: @@ -83,15 +117,15 @@ def test_store_supports_listing(self, store: GpuMemoryStore) -> None: def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: assert store.supports_partial_writes - def test_list_prefix(self, store: GpuMemoryStore) -> None: + async def test_list_prefix(self, store: GpuMemoryStore) -> None: assert True def test_dict_reference(self, store: GpuMemoryStore) -> None: - store_dict = {} + store_dict: dict[str, Any] = {} result = GpuMemoryStore(store_dict=store_dict) assert result._store_dict is store_dict - def test_from_dict(self): + def test_from_dict(self) -> None: d = { "a": gpu.Buffer.from_bytes(b"aaaa"), "b": cpu.Buffer.from_bytes(b"bbbb"), diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py new file mode 100644 index 0000000000..4d9e8fcc1f --- /dev/null +++ b/tests/test_store/test_object.py @@ -0,0 +1,86 @@ +# ruff: noqa: E402 +from typing import Any + +import pytest + +obstore = pytest.importorskip("obstore") + +from hypothesis.stateful import ( + run_state_machine_as_test, +) +from obstore.store import LocalStore, MemoryStore + +from zarr.core.buffer import Buffer, cpu +from zarr.storage import ObjectStore +from zarr.testing.stateful import ZarrHierarchyStateMachine +from zarr.testing.store import StoreTests + + +class TestObjectStore(StoreTests[ObjectStore, cpu.Buffer]): + store_cls = ObjectStore + buffer_cls = cpu.Buffer + + @pytest.fixture + def store_kwargs(self, tmpdir) -> dict[str, Any]: + store = LocalStore(prefix=tmpdir) + return {"store": store, "read_only": False} + + @pytest.fixture + def store(self, store_kwargs: dict[str, str | bool]) -> ObjectStore: + return self.store_cls(**store_kwargs) + + async def get(self, store: ObjectStore, key: str) -> Buffer: + assert isinstance(store.store, LocalStore) + new_local_store = LocalStore(prefix=store.store.prefix) + return self.buffer_cls.from_bytes(obstore.get(new_local_store, key).bytes()) + + async def set(self, store: ObjectStore, key: str, value: Buffer) -> None: + assert isinstance(store.store, LocalStore) + new_local_store = LocalStore(prefix=store.store.prefix) + obstore.put(new_local_store, key, value.to_bytes()) + + def test_store_repr(self, store: ObjectStore) -> None: + from fnmatch import fnmatch + + pattern = "ObjectStore(object_store://LocalStore(*))" + assert fnmatch(f"{store!r}", pattern) + + def test_store_supports_writes(self, store: ObjectStore) -> None: + assert store.supports_writes + + async def test_store_supports_partial_writes(self, store: ObjectStore) -> None: + assert not store.supports_partial_writes + with pytest.raises(NotImplementedError): + await store.set_partial_values([("foo", 0, b"\x01\x02\x03\x04")]) + + def test_store_supports_listing(self, store: ObjectStore) -> None: + assert store.supports_listing + + def test_store_equal(self, store: ObjectStore) -> None: + """Test store equality""" + # Test equality against a different instance type + assert store != 0 + # Test equality against a different store type + new_memory_store = ObjectStore(MemoryStore()) + assert store != new_memory_store + # Test equality against a read only store + new_local_store = ObjectStore(LocalStore(prefix=store.store.prefix), read_only=True) + assert store != new_local_store + # Test two memory stores cannot be equal + second_memory_store = ObjectStore(MemoryStore()) + assert new_memory_store != second_memory_store + + def test_store_init_raises(self) -> None: + """Test __init__ raises appropriate error for improper store type""" + with pytest.raises(TypeError): + ObjectStore("path/to/store") + + +@pytest.mark.slow_hypothesis +def test_zarr_hierarchy(): + sync_store = ObjectStore(MemoryStore()) + + def mk_test_instance_sync() -> ZarrHierarchyStateMachine: + return ZarrHierarchyStateMachine(sync_store) + + run_state_machine_as_test(mk_test_instance_sync) diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py index ae10ca8d79..c0997c3df3 100644 --- a/tests/test_store/test_stateful.py +++ b/tests/test_store/test_stateful.py @@ -1,25 +1,29 @@ # Stateful tests for arbitrary Zarr stores. import pytest from hypothesis.stateful import ( - Settings, run_state_machine_as_test, ) from zarr.abc.store import Store -from zarr.storage import LocalStore, MemoryStore, ZipStore +from zarr.storage import LocalStore, ZipStore from zarr.testing.stateful import ZarrHierarchyStateMachine, ZarrStoreStateMachine +pytestmark = [ + pytest.mark.slow_hypothesis, + # TODO: work out where this warning is coming from and fix + pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning"), +] + +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_zarr_hierarchy(sync_store: Store): def mk_test_instance_sync() -> ZarrHierarchyStateMachine: return ZarrHierarchyStateMachine(sync_store) if isinstance(sync_store, ZipStore): pytest.skip(reason="ZipStore does not support delete") - if isinstance(sync_store, MemoryStore): - run_state_machine_as_test( - mk_test_instance_sync, settings=Settings(report_multiple_bugs=False) - ) + + run_state_machine_as_test(mk_test_instance_sync) def test_zarr_store(sync_store: Store) -> None: @@ -28,6 +32,10 @@ def mk_test_instance_sync() -> None: if isinstance(sync_store, ZipStore): pytest.skip(reason="ZipStore does not support delete") + if isinstance(sync_store, LocalStore): - pytest.skip(reason="This test has errors") - run_state_machine_as_test(mk_test_instance_sync, settings=Settings(report_multiple_bugs=True)) + # This test uses arbitrary keys, which are passed to `set` and `delete`. + # It assumes that `set` and `delete` are the only two operations that modify state. + # But LocalStore, directories can hang around even after a key is delete-d. + pytest.skip(reason="Test isn't suitable for LocalStore.") + run_state_machine_as_test(mk_test_instance_sync) diff --git a/tests/test_store/test_wrapper.py b/tests/test_store/test_wrapper.py index 489bcd5a7a..c6edd4f4dd 100644 --- a/tests/test_store/test_wrapper.py +++ b/tests/test_store/test_wrapper.py @@ -5,13 +5,81 @@ import pytest from zarr.core.buffer.cpu import Buffer, buffer_prototype -from zarr.storage import WrapperStore +from zarr.storage import LocalStore, WrapperStore +from zarr.testing.store import StoreTests if TYPE_CHECKING: + from _pytest.compat import LEGACY_PATH + from zarr.abc.store import Store from zarr.core.buffer.core import BufferPrototype +# TODO: fix this warning +@pytest.mark.filterwarnings( + "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" +) +class TestWrapperStore(StoreTests[WrapperStore, Buffer]): + store_cls = WrapperStore + buffer_cls = Buffer + + async def get(self, store: WrapperStore, key: str) -> Buffer: + return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) + + async def set(self, store: WrapperStore, key: str, value: Buffer) -> None: + parent = (store._store.root / key).parent + if not parent.exists(): + parent.mkdir(parents=True) + (store._store.root / key).write_bytes(value.to_bytes()) + + @pytest.fixture + def store_kwargs(self, tmpdir: LEGACY_PATH) -> dict[str, str]: + return {"store": LocalStore(str(tmpdir))} + + @pytest.fixture + def open_kwargs(self, tmpdir) -> dict[str, str]: + return {"store_cls": LocalStore, "root": str(tmpdir)} + + def test_store_supports_writes(self, store: WrapperStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: WrapperStore) -> None: + assert store.supports_partial_writes + + def test_store_supports_listing(self, store: WrapperStore) -> None: + assert store.supports_listing + + def test_store_repr(self, store: WrapperStore) -> None: + assert f"{store!r}" == f"WrapperStore(LocalStore, 'file://{store._store.root.as_posix()}')" + + def test_store_str(self, store: WrapperStore) -> None: + assert str(store) == f"wrapping-file://{store._store.root.as_posix()}" + + def test_check_writeable(self, store: WrapperStore) -> None: + """ + Test _check_writeable() runs without errors. + """ + store._check_writable() + + def test_close(self, store: WrapperStore) -> None: + "Test store can be closed" + store.close() + assert not store._is_open + + def test_is_open_setter_raises(self, store: WrapperStore) -> None: + """ + Test that a user cannot change `_is_open` without opening the underlying store. + """ + with pytest.raises( + NotImplementedError, match="WrapperStore must be opened via the `_open` method" + ): + store._is_open = True + + +# TODO: work out where warning is coming from and fix +@pytest.mark.filterwarnings( + "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" +) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) async def test_wrapped_set(store: Store, capsys: pytest.CaptureFixture[str]) -> None: # define a class that prints when it sets @@ -29,6 +97,7 @@ async def set(self, key: str, value: Buffer) -> None: assert await store_wrapped.get(key, buffer_prototype) == value +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) async def test_wrapped_get(store: Store, capsys: pytest.CaptureFixture[str]) -> None: # define a class that prints when it sets diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index a83327d99a..24b25ed315 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shutil import tempfile import zipfile from typing import TYPE_CHECKING @@ -9,20 +10,31 @@ import pytest import zarr +from zarr import create_array from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.core.group import Group from zarr.storage import ZipStore from zarr.testing.store import StoreTests if TYPE_CHECKING: + from pathlib import Path from typing import Any +# TODO: work out where this is coming from and fix +pytestmark = [ + pytest.mark.filterwarnings( + "ignore:coroutine method 'aclose' of 'ZipStore.list' was never awaited:RuntimeWarning" + ) +] + + class TestZipStore(StoreTests[ZipStore, cpu.Buffer]): store_cls = ZipStore buffer_cls = cpu.Buffer @pytest.fixture - def store_kwargs(self, request) -> dict[str, str | bool]: + def store_kwargs(self) -> dict[str, str | bool]: fd, temp_path = tempfile.mkstemp() os.close(fd) os.unlink(temp_path) @@ -30,12 +42,14 @@ def store_kwargs(self, request) -> dict[str, str | bool]: return {"path": temp_path, "mode": "w", "read_only": False} async def get(self, store: ZipStore, key: str) -> Buffer: - return store._get(key, prototype=default_buffer_prototype()) + buf = store._get(key, prototype=default_buffer_prototype()) + assert buf is not None + return buf async def set(self, store: ZipStore, key: str, value: Buffer) -> None: return store._set(key, value) - def test_store_read_only(self, store: ZipStore, store_kwargs: dict[str, Any]) -> None: + def test_store_read_only(self, store: ZipStore) -> None: assert not store.read_only async def test_read_only_store_raises(self, store_kwargs: dict[str, Any]) -> None: @@ -64,6 +78,8 @@ def test_store_supports_partial_writes(self, store: ZipStore) -> None: def test_store_supports_listing(self, store: ZipStore) -> None: assert store.supports_listing + # TODO: fix this warning + @pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") def test_api_integration(self, store: ZipStore) -> None: root = zarr.open_group(store=store, mode="a") @@ -97,7 +113,7 @@ def test_api_integration(self, store: ZipStore) -> None: async def test_store_open_read_only( self, store_kwargs: dict[str, Any], read_only: bool ) -> None: - if read_only == "r": + if read_only: # create an empty zipfile with zipfile.ZipFile(store_kwargs["path"], mode="w"): pass @@ -111,3 +127,31 @@ async def test_zip_open_mode_translation( kws = {**store_kwargs, "mode": zip_mode} store = await self.store_cls.open(**kws) assert store.read_only == read_only + + def test_externally_zipped_store(self, tmp_path: Path) -> None: + # See: https://github.com/zarr-developers/zarr-python/issues/2757 + zarr_path = tmp_path / "foo.zarr" + root = zarr.open_group(store=zarr_path, mode="w") + root.require_group("foo") + assert isinstance(foo := root["foo"], Group) # noqa: RUF018 + foo["bar"] = np.array([1]) + shutil.make_archive(str(zarr_path), "zip", zarr_path) + zip_path = tmp_path / "foo.zarr.zip" + zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") + assert list(zipped.keys()) == list(root.keys()) + assert isinstance(group := zipped["foo"], Group) + assert list(group.keys()) == list(group.keys()) + + async def test_move(self, tmp_path: Path) -> None: + origin = tmp_path / "origin.zip" + destination = tmp_path / "some_folder" / "destination.zip" + + store = await ZipStore.open(path=origin, mode="a") + array = create_array(store, data=np.arange(10)) + + await store.move(str(destination)) + + assert store.path == destination + assert destination.exists() + assert not origin.exists() + assert np.array_equal(array[...], np.arange(10)) diff --git a/tests/test_strings.py b/tests/test_strings.py deleted file mode 100644 index dca0570a25..0000000000 --- a/tests/test_strings.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Tests for the strings module.""" - -import numpy as np -import pytest - -from zarr.core.strings import _NUMPY_SUPPORTS_VLEN_STRING, _STRING_DTYPE, cast_to_string_dtype - - -def test_string_defaults() -> None: - if _NUMPY_SUPPORTS_VLEN_STRING: - assert _STRING_DTYPE == np.dtypes.StringDType() - else: - assert _STRING_DTYPE == np.dtypes.ObjectDType() - - -def test_cast_to_string_dtype() -> None: - d1 = np.array(["a", "b", "c"]) - assert d1.dtype == np.dtype(" None: foo.assert_not_awaited() +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_calling_sync_from_within_a_running_loop( sync_loop: asyncio.AbstractEventLoop | None, @@ -148,11 +150,20 @@ def test_open_positional_args_deprecate(): @pytest.mark.parametrize("workers", [None, 1, 2]) -def test_get_executor(clean_state, workers) -> None: +def test_threadpool_executor(clean_state, workers: int | None) -> None: with zarr.config.set({"threading.max_workers": workers}): - e = _get_executor() - if workers is not None and workers != 0: - assert e._max_workers == workers + _ = zarr.zeros(shape=(1,)) # trigger executor creation + assert loop != [None] # confirm loop was created + if workers is None: + # confirm no executor was created if no workers were specified + # (this is the default behavior) + assert loop[0]._default_executor is None + else: + # confirm executor was created and attached to loop as the default executor + # note: python doesn't have a direct way to get the default executor so we + # use the private attribute + assert _get_executor() is loop[0]._default_executor + assert _get_executor()._max_workers == workers def test_cleanup_resources_idempotent() -> None: diff --git a/tests/test_v2.py b/tests/test_v2.py index 72127f4ede..29f031663f 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -1,22 +1,31 @@ import json -from collections.abc import Iterator +from pathlib import Path from typing import Any, Literal +import numcodecs.abc import numcodecs.vlen import numpy as np import pytest from numcodecs import Delta from numcodecs.blosc import Blosc +from numcodecs.zstd import Zstd import zarr import zarr.core.buffer import zarr.storage from zarr import config +from zarr.abc.store import Store +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype import FixedLengthUTF32, Structured, VariableLengthUTF8 +from zarr.core.dtype.npy.bytes import NullTerminatedBytes +from zarr.core.dtype.wrapper import ZDType +from zarr.core.group import Group +from zarr.core.sync import sync from zarr.storage import MemoryStore, StorePath @pytest.fixture -async def store() -> Iterator[StorePath]: +async def store() -> StorePath: return StorePath(await MemoryStore.open()) @@ -36,33 +45,6 @@ def test_simple(store: StorePath) -> None: assert np.array_equal(data, a[:, :]) -@pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize( - ("dtype", "fill_value"), - [ - ("bool", False), - ("int64", 0), - ("float64", 0.0), - ("|S1", b""), - ("|U1", ""), - ("object", ""), - (str, ""), - ], -) -def test_implicit_fill_value(store: MemoryStore, dtype: str, fill_value: Any) -> None: - arr = zarr.create(store=store, shape=(4,), fill_value=None, zarr_format=2, dtype=dtype) - assert arr.metadata.fill_value is None - assert arr.metadata.to_dict()["fill_value"] is None - result = arr[:] - if dtype is str: - # special case - numpy_dtype = np.dtype(object) - else: - numpy_dtype = np.dtype(dtype) - expected = np.full(arr.shape, fill_value, dtype=numpy_dtype) - np.testing.assert_array_equal(result, expected) - - def test_codec_pipeline() -> None: # https://github.com/zarr-developers/zarr-python/issues/2243 store = MemoryStore() @@ -80,8 +62,17 @@ def test_codec_pipeline() -> None: np.testing.assert_array_equal(result, expected) -@pytest.mark.parametrize("dtype", ["|S", "|V"]) -async def test_v2_encode_decode(dtype): +@pytest.mark.parametrize( + ("dtype", "expected_dtype", "fill_value", "fill_value_json"), + [ + ("|S1", "|S1", b"X", "WA=="), + ("|V1", "|V1", b"X", "WA=="), + ("|V10", "|V10", b"X", "WAAAAAAAAAAAAA=="), + ], +) +async def test_v2_encode_decode( + dtype: str, expected_dtype: str, fill_value: bytes, fill_value_json: str +) -> None: with config.set( { "array.v2_default_filters.bytes": [{"id": "vlen-bytes"}], @@ -91,11 +82,7 @@ async def test_v2_encode_decode(dtype): store = zarr.storage.MemoryStore() g = zarr.group(store=store, zarr_format=2) g.create_array( - name="foo", - shape=(3,), - chunks=(3,), - dtype=dtype, - fill_value=b"X", + name="foo", shape=(3,), chunks=(3,), dtype=dtype, fill_value=fill_value, compressor=None ) result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype()) @@ -105,9 +92,9 @@ async def test_v2_encode_decode(dtype): expected = { "chunks": [3], "compressor": None, - "dtype": f"{dtype}0", - "fill_value": "WA==", - "filters": [{"id": "vlen-bytes"}], + "dtype": expected_dtype, + "fill_value": fill_value_json, + "filters": None, "order": "C", "shape": [3], "zarr_format": 2, @@ -116,41 +103,27 @@ async def test_v2_encode_decode(dtype): assert serialized == expected data = zarr.open_array(store=store, path="foo")[:] - expected = np.full((3,), b"X", dtype=dtype) - np.testing.assert_equal(data, expected) + np.testing.assert_equal(data, np.full((3,), b"X", dtype=dtype)) -@pytest.mark.parametrize("dtype_value", [["|S", b"Y"], ["|U", "Y"], ["O", b"Y"]]) -def test_v2_encode_decode_with_data(dtype_value): - dtype, value = dtype_value - with config.set( - { - "array.v2_default_filters": { - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], - }, - } - ): - expected = np.full((3,), value, dtype=dtype) - a = zarr.create( - shape=(3,), - zarr_format=2, - dtype=dtype, - ) - a[:] = expected - data = a[:] - np.testing.assert_equal(data, expected) - - -@pytest.mark.parametrize("dtype", [str, "str"]) -async def test_create_dtype_str(dtype: Any) -> None: - arr = zarr.create(shape=3, dtype=dtype, zarr_format=2) - assert arr.dtype.kind == "O" - assert arr.metadata.to_dict()["dtype"] == "|O" - assert arr.metadata.filters == (numcodecs.vlen.VLenBytes(),) - arr[:] = [b"a", b"bb", b"ccc"] - result = arr[:] - np.testing.assert_array_equal(result, np.array([b"a", b"bb", b"ccc"], dtype="object")) +@pytest.mark.parametrize( + ("dtype", "value"), + [ + (NullTerminatedBytes(length=1), b"Y"), + (FixedLengthUTF32(length=1), "Y"), + (VariableLengthUTF8(), "Y"), + ], +) +def test_v2_encode_decode_with_data(dtype: ZDType[Any, Any], value: str) -> None: + expected = np.full((3,), value, dtype=dtype.to_native_dtype()) + a = zarr.create( + shape=(3,), + zarr_format=2, + dtype=dtype, + ) + a[:] = expected + data = a[:] + np.testing.assert_equal(data, expected) @pytest.mark.parametrize("filters", [[], [numcodecs.Delta(dtype=" None: np.testing.assert_array_equal(result, array_fixture) -@pytest.mark.parametrize("array_order", ["C", "F"]) -@pytest.mark.parametrize("data_order", ["C", "F"]) -def test_v2_non_contiguous(array_order: Literal["C", "F"], data_order: Literal["C", "F"]) -> None: +@pytest.mark.filterwarnings("ignore") +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_create_array_defaults(store: Store) -> None: + """ + Test that passing compressor=None results in no compressor. Also test that the default value of the compressor + parameter does produce a compressor. + """ + g = zarr.open(store, mode="w", zarr_format=2) + assert isinstance(g, Group) + arr = g.create_array("one", dtype="i8", shape=(1,), chunks=(1,), compressor=None) + assert arr._async_array.compressor is None + assert not (arr.filters) + arr = g.create_array("two", dtype="i8", shape=(1,), chunks=(1,)) + assert arr._async_array.compressor is not None + assert not (arr.filters) + arr = g.create_array("three", dtype="i8", shape=(1,), chunks=(1,), compressor=Zstd()) + assert arr._async_array.compressor is not None + assert not (arr.filters) + with pytest.raises(ValueError): + g.create_array( + "four", dtype="i8", shape=(1,), chunks=(1,), compressor=None, compressors=None + ) + + +@pytest.mark.parametrize("numpy_order", ["C", "F"]) +@pytest.mark.parametrize("zarr_order", ["C", "F"]) +def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal["C", "F"]) -> None: + """ + Make sure zarr v2 arrays save data using the memory order given to the zarr array, + not the memory order of the original numpy array. + """ + store = MemoryStore() arr = zarr.create_array( - MemoryStore({}), + store, shape=(10, 8), chunks=(3, 3), fill_value=np.nan, dtype="float64", zarr_format=2, + filters=None, + compressors=None, overwrite=True, - order=array_order, + order=zarr_order, ) - # Non-contiguous write - a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=data_order) - arr[slice(6, 9, None), slice(3, 6, None)] = a[ - slice(6, 9, None), slice(3, 6, None) - ] # The slice on the RHS is important + # Non-contiguous write, using numpy memory order + a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=numpy_order) + arr[6:9, 3:6] = a[6:9, 3:6] # The slice on the RHS is important + np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6]) + + buf = sync(store.get("2.1", default_buffer_prototype())) + assert buf is not None np.testing.assert_array_equal( - arr[slice(6, 9, None), slice(3, 6, None)], a[slice(6, 9, None), slice(3, 6, None)] + a[6:9, 3:6], + np.frombuffer(buf.to_bytes(), dtype="float64").reshape((3, 3), order=zarr_order), ) + # After writing and reading from zarr array, order should be same as zarr order + sub_arr = arr[6:9, 3:6] + assert isinstance(sub_arr, np.ndarray) + if zarr_order == "F": + assert (sub_arr).flags.f_contiguous + else: + assert (sub_arr).flags.c_contiguous + # Contiguous write + store = MemoryStore() arr = zarr.create_array( - MemoryStore({}), + store, shape=(10, 8), chunks=(3, 3), fill_value=np.nan, dtype="float64", zarr_format=2, + compressors=None, + filters=None, overwrite=True, - order=array_order, + order=zarr_order, ) - # Contiguous write - a = np.arange(9).reshape((3, 3), order=data_order) - if data_order == "F": - assert a.flags.f_contiguous + a = np.arange(9).reshape((3, 3), order=numpy_order) + arr[6:9, 3:6] = a + np.testing.assert_array_equal(arr[6:9, 3:6], a) + # After writing and reading from zarr array, order should be same as zarr order + sub_arr = arr[6:9, 3:6] + assert isinstance(sub_arr, np.ndarray) + if zarr_order == "F": + assert (sub_arr).flags.f_contiguous else: - assert a.flags.c_contiguous - arr[slice(6, 9, None), slice(3, 6, None)] = a - np.testing.assert_array_equal(arr[slice(6, 9, None), slice(3, 6, None)], a) + assert (sub_arr).flags.c_contiguous -def test_default_compressor_deprecation_warning(): +def test_default_compressor_deprecation_warning() -> None: with pytest.warns(DeprecationWarning, match="default_compressor is deprecated"): - zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" + zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" # type: ignore[attr-defined] + + +@pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"]) +def test_structured_dtype_roundtrip(fill_value: float | bytes, tmp_path: Path) -> None: + a = np.array( + [(b"aaa", 1, 4.2), (b"bbb", 2, 8.4), (b"ccc", 3, 12.6)], + dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], + ) + array_path = tmp_path / "data.zarr" + za = zarr.create( + shape=(3,), + store=array_path, + chunks=(2,), + fill_value=fill_value, + zarr_format=2, + dtype=a.dtype, + ) + if fill_value is not None: + assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() + za[...] = a + za = zarr.open_array(store=array_path) + assert (a == za[:]).all() @pytest.mark.parametrize( - "dtype_expected", + ( + "fill_value", + "dtype", + "expected_result", + ), [ - ["b", "zstd", None], - ["i", "zstd", None], - ["f", "zstd", None], - ["|S1", "zstd", "vlen-bytes"], - ["|U1", "zstd", "vlen-utf8"], + ( + ("Alice", 30), + np.dtype([("name", "U10"), ("age", "i4")]), + np.array([("Alice", 30)], dtype=[("name", "U10"), ("age", "i4")])[0], + ), + ( + ["Bob", 25], + np.dtype([("name", "U10"), ("age", "i4")]), + np.array([("Bob", 25)], dtype=[("name", "U10"), ("age", "i4")])[0], + ), + ( + b"\x01\x00\x00\x00\x02\x00\x00\x00", + np.dtype([("x", "i4"), ("y", "i4")]), + np.array([(1, 2)], dtype=[("x", "i4"), ("y", "i4")])[0], + ), + ], + ids=[ + "tuple_input", + "list_input", + "bytes_input", ], ) -def test_default_filters_and_compressor(dtype_expected: Any) -> None: - with config.set( - { - "array.v2_default_compressor": { - "numeric": {"id": "zstd", "level": "0"}, - "string": {"id": "zstd", "level": "0"}, - "bytes": {"id": "zstd", "level": "0"}, - }, - "array.v2_default_filters": { - "numeric": [], - "string": [{"id": "vlen-utf8"}], - "bytes": [{"id": "vlen-bytes"}], - }, - } - ): - dtype, expected_compressor, expected_filter = dtype_expected - arr = zarr.create(shape=(3,), path="foo", store={}, zarr_format=2, dtype=dtype) - assert arr.metadata.compressor.codec_id == expected_compressor - if expected_filter is not None: - assert arr.metadata.filters[0].codec_id == expected_filter +def test_parse_structured_fill_value_valid( + fill_value: Any, dtype: np.dtype[Any], expected_result: Any +) -> None: + zdtype = Structured.from_native_dtype(dtype) + result = zdtype.cast_scalar(fill_value) + assert result.dtype == expected_result.dtype + assert result == expected_result + if isinstance(expected_result, np.void): + for name in expected_result.dtype.names or []: + assert result[name] == expected_result[name] + + +@pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"]) +def test_other_dtype_roundtrip(fill_value: None | bytes, tmp_path: Path) -> None: + a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7") + array_path = tmp_path / "data.zarr" + za = zarr.create( + shape=(3,), + store=array_path, + chunks=(2,), + fill_value=fill_value, + zarr_format=2, + dtype=a.dtype, + ) + if fill_value is not None: + assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() + za[...] = a + za = zarr.open_array(store=array_path) + assert (a == za[:]).all() diff --git a/tests/test_zarr.py b/tests/test_zarr.py index 2aa62e4231..f49873132e 100644 --- a/tests/test_zarr.py +++ b/tests/test_zarr.py @@ -1,3 +1,5 @@ +import pytest + import zarr @@ -9,3 +11,19 @@ def test_exports() -> None: for export in __all__: getattr(zarr, export) + + +def test_print_debug_info(capsys: pytest.CaptureFixture[str]) -> None: + """ + Ensure that print_debug_info does not raise an error + """ + from importlib.metadata import version + + from zarr import __version__, print_debug_info + + print_debug_info() + captured = capsys.readouterr() + # test that at least some of what we expect is + # printed out + assert f"zarr: {__version__}" in captured.out + assert f"numpy: {version('numpy')}" in captured.out